Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_sendInviteEmailToggle

.github/scripts/close_duplicate_issues.py

@@ -42,7 +42,9 @@ def gh(*args: str) -> str:
 def fetch_open_issues(repo: str | None) -> list[dict]:
     """Fetch all open issues (excluding PRs) via gh api --paginate."""
     if repo:
-        endpoint = f"repos/{repo}/issues?state=open&per_page=100&sort=created&direction=asc"
+        endpoint = (
+            f"repos/{repo}/issues?state=open&per_page=100&sort=created&direction=asc"
+        )
     else:
         endpoint = "repos/{owner}/{repo}/issues?state=open&per_page=100&sort=created&direction=asc"
     cmd = ["api", "--paginate", endpoint]
@@ -71,7 +73,9 @@ def close_as_duplicate(
     repo_args = ["--repo", repo] if repo else []
 
     if dry_run:
-        print(f"  [DRY RUN] Would close #{issue_number} as duplicate of #{duplicate_of}")
+        print(
+            f"  [DRY RUN] Would close #{issue_number} as duplicate of #{duplicate_of}"
+        )
         return
 
     # Add comment
@@ -115,7 +119,9 @@ def find_duplicate(
     return None
 
 
-def scan_all(issues: list[dict], threshold: float, repo: str | None, dry_run: bool) -> int:
+def scan_all(
+    issues: list[dict], threshold: float, repo: str | None, dry_run: bool
+) -> int:
     """Compare every issue against all older issues. Returns count of duplicates found."""
     # Sort oldest first
     issues.sort(key=lambda i: i["number"])
@@ -144,7 +150,11 @@ def scan_all(issues: list[dict], threshold: float, repo: str | None, dry_run: bo
 
 
 def check_single(
-    issue_number: int, issues: list[dict], threshold: float, repo: str | None, dry_run: bool
+    issue_number: int,
+    issues: list[dict],
+    threshold: float,
+    repo: str | None,
+    dry_run: bool,
 ) -> bool:
     """Check a single issue against all older open issues. Returns True if duplicate found."""
     target = None
@@ -178,13 +188,23 @@ def check_single(
 
 
 def main() -> None:
-    parser = argparse.ArgumentParser(description="Detect and close duplicate GitHub issues")
+    parser = argparse.ArgumentParser(
+        description="Detect and close duplicate GitHub issues"
+    )
     mode = parser.add_mutually_exclusive_group(required=True)
     mode.add_argument("--scan", action="store_true", help="Scan all open issues")
     mode.add_argument("--issue-number", type=int, help="Check a single issue number")
-    parser.add_argument("--threshold", type=float, default=0.85, help="Similarity threshold (0-1)")
-    parser.add_argument("--close", action="store_true", help="Actually close duplicates (default is dry-run)")
-    parser.add_argument("--repo", type=str, help="Repository (owner/repo). Auto-detected if omitted.")
+    parser.add_argument(
+        "--threshold", type=float, default=0.85, help="Similarity threshold (0-1)"
+    )
+    parser.add_argument(
+        "--close",
+        action="store_true",
+        help="Actually close duplicates (default is dry-run)",
+    )
+    parser.add_argument(
+        "--repo", type=str, help="Repository (owner/repo). Auto-detected if omitted."
+    )
     args = parser.parse_args()
 
     dry_run = not args.close
@@ -200,7 +220,9 @@ def main() -> None:
         count = scan_all(issues, args.threshold, args.repo, dry_run)
         print(f"\nTotal duplicates {'found' if dry_run else 'closed'}: {count}")
     else:
-        found = check_single(args.issue_number, issues, args.threshold, args.repo, dry_run)
+        found = check_single(
+            args.issue_number, issues, args.threshold, args.repo, dry_run
+        )
         sys.exit(0 if found else 0)  # Always exit 0; finding no dup is not an error
 
 

.github/scripts/scan_keywords.py

@@ -67,14 +67,13 @@ def send_webhook(webhook_url: str, payload: dict) -> None:
 def _excerpt(text: str, max_len: int = 400) -> str:
     if not text:
         return ""
-    
+
     # Keep original formatting
     if len(text) <= max_len:
         return text
     return text[: max_len - 1] + "…"
 
 
-
 def main() -> int:
     event = read_event_payload()
     if not event:
@@ -87,8 +86,19 @@ def main() -> int:
 
     # Keywords from env or defaults
     keywords_env = os.environ.get("KEYWORDS", "")
-    default_keywords = ["azure", "openai", "bedrock", "vertexai", "vertex ai", "anthropic"]
-    keywords = [k.strip() for k in keywords_env.split(",")] if keywords_env else default_keywords
+    default_keywords = [
+        "azure",
+        "openai",
+        "bedrock",
+        "vertexai",
+        "vertex ai",
+        "anthropic",
+    ]
+    keywords = (
+        [k.strip() for k in keywords_env.split(",")]
+        if keywords_env
+        else default_keywords
+    )
 
     matches = detect_keywords(combined_text, keywords)
     found = bool(matches)
@@ -129,5 +139,3 @@ def main() -> int:
 
 if __name__ == "__main__":
     raise SystemExit(main())
-
-

.github/workflows/_test-unit-services-base.yml

@@ -32,41 +32,39 @@ on:
         required: false
         type: boolean
         default: false
+      dist:
+        description: "pytest-xdist distribution mode (loadscope|load|worksteal|loadfile|no)"
+        required: false
+        type: string
+        default: "loadscope"
       artifact-name:
         description: "Unique name for the coverage artifact (must be unique per run)"
         required: false
         type: string
         default: "run"
-    secrets:
-      DATABASE_URL:
-        required: false
-      POSTGRES_USER:
-        required: false
-      POSTGRES_PASSWORD:
-        required: false
 
 permissions:
   contents: read
 
+# The postgres service container below is spawned per-job on localhost and
+# destroyed with the job. Nothing outside the runner can reach it. The
+# user/password/database here are not secrets — they're bootstrap values
+# for a throwaway container — so we hardcode them instead of attaching
+# every matrix shard to a GHA environment just to read three "secrets"
+# (which also produces a "temporarily deployed to …" notification on the
+# PR timeline per shard per push).
 jobs:
   run:
     name: Run tests
     runs-on: ubuntu-latest
     timeout-minutes: ${{ inputs.timeout-minutes }}
-    # Environment is derived from the enable-* flags, not caller-controllable.
-    # This prevents callers from passing arbitrary environment names to bypass secret scoping.
-    environment: >-
-      ${{
-        inputs.enable-postgres && 'integration-postgres' ||
-        ''
-      }}
 
     services:
       postgres:
         image: postgres@sha256:705a5d5b5836f3fcba0d02c4d281e6a7dd9ed2dd4078640f08a1e1e9896e097d # postgres:14
         env:
-          POSTGRES_USER: ${{ secrets.POSTGRES_USER }}
-          POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}
+          POSTGRES_USER: litellm
+          POSTGRES_PASSWORD: litellm
           POSTGRES_DB: litellm_test
         ports:
           - 5432:5432
@@ -114,7 +112,7 @@ jobs:
       - name: Run Prisma migrations
         if: ${{ inputs.enable-postgres }}
         env:
-          DATABASE_URL: ${{ secrets.DATABASE_URL }}
+          DATABASE_URL: "postgresql://litellm:litellm@localhost:5432/litellm_test"
         run: |
           uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
 
@@ -124,7 +122,8 @@ jobs:
           MAX_FAILURES: ${{ inputs.max-failures }}
           WORKERS: ${{ inputs.workers }}
           RERUNS: ${{ inputs.reruns }}
-          DATABASE_URL: ${{ inputs.enable-postgres && secrets.DATABASE_URL || '' }}
+          DIST: ${{ inputs.dist }}
+          DATABASE_URL: ${{ inputs.enable-postgres && 'postgresql://litellm:litellm@localhost:5432/litellm_test' || '' }}
         run: |
           if [ "${WORKERS}" = "0" ]; then
             uv run --no-sync pytest ${TEST_PATH:?} \
@@ -143,7 +142,7 @@ jobs:
               -n "${WORKERS}" \
               --reruns "${RERUNS}" \
               --reruns-delay 1 \
-              --dist=loadscope \
+              --dist="${DIST}" \
               --durations=20 \
               --cov=litellm \
               --cov-report=xml:coverage.xml \

.github/workflows/check_duplicate_issues.yml

@@ -39,7 +39,7 @@ jobs:
         if: github.event.action == 'opened'
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
-          python-version: "3.11"
+          python-version: "3.12"
 
       - name: Auto-close if high-confidence duplicate
         if: github.event.action == 'opened'

.github/workflows/create-release-branch.yml

@@ -0,0 +1,65 @@
+name: Create Release Branch
+
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: "Release tag (e.g. v1.83.0-stable) — branch will be named release/<tag>"
+        required: true
+        type: string
+      commit_hash:
+        description: "Full 40-char commit SHA the branch should point to"
+        required: true
+        type: string
+  workflow_call:
+    inputs:
+      tag:
+        description: "Release tag"
+        required: true
+        type: string
+      commit_hash:
+        description: "Full 40-char commit SHA the branch should point to"
+        required: true
+        type: string
+
+permissions: {}
+
+jobs:
+  create-branch:
+    name: Create Release Branch
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - name: Validate inputs
+        env:
+          TAG: ${{ inputs.tag }}
+          COMMIT_HASH: ${{ inputs.commit_hash }}
+        run: |
+          if ! echo "${COMMIT_HASH}" | grep -qE '^[0-9a-f]{40}$'; then
+            echo "::error::commit_hash must be a full 40-character commit SHA"
+            exit 1
+          fi
+          if ! echo "${TAG}" | grep -qE '^v[0-9]+\.[0-9]+\.[0-9]+'; then
+            echo "::error::tag must start with vX.Y.Z"
+            exit 1
+          fi
+
+      - name: Create release branch
+        env:
+          TAG: ${{ inputs.tag }}
+          COMMIT_HASH: ${{ inputs.commit_hash }}
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+        with:
+          script: |
+            const tag = process.env.TAG;
+            const commitHash = process.env.COMMIT_HASH;
+            const branchName = `release/${tag}`;
+
+            await github.rest.git.createRef({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              ref: `refs/heads/${branchName}`,
+              sha: commitHash,
+            });
+            core.info(`Created branch ${branchName} at ${commitHash}`);

.github/workflows/create-release.yml

@@ -102,6 +102,17 @@ jobs:
                 body: updatedBody,
                 draft: false,
               });
+
             } catch (error) {
               core.setFailed(error.message);
             }
+
+  create-branch:
+    name: Create Release Branch
+    needs: release
+    permissions:
+      contents: write
+    uses: ./.github/workflows/create-release-branch.yml
+    with:
+      tag: ${{ inputs.tag }}
+      commit_hash: ${{ inputs.commit_hash }}

.github/workflows/llm-translation-testing.yml

@@ -29,7 +29,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
-          python-version: "3.11"
+          python-version: "3.12"
 
       - name: Set up uv
         uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7

.github/workflows/scan_duplicate_issues.yml

@@ -29,7 +29,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
-          python-version: "3.13"
+          python-version: "3.12"
 
       - name: Scan for duplicate issues
         env:

.github/workflows/test-code-quality.yml

@@ -0,0 +1,128 @@
+name: Code Quality Checks
+
+on:
+  pull_request:
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  code-quality:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Checkout litellm-docs into docs/my-website (for documentation_tests)
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          repository: BerriAI/litellm-docs
+          path: docs/my-website
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+        with:
+          version: "0.10.9"
+
+      - name: Cache uv dependencies
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          path: |
+            ~/.cache/uv
+            .venv
+          key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-uv-
+
+      - name: Install dependencies
+        run: uv sync --frozen --all-groups --all-extras
+
+      - name: check_licenses
+        run: uv run --no-sync python ./tests/code_coverage_tests/check_licenses.py
+
+      - name: check_provider_folders_documented
+        run: uv run --no-sync python ./tests/code_coverage_tests/check_provider_folders_documented.py
+
+      - name: router_code_coverage
+        run: uv run --no-sync python ./tests/code_coverage_tests/router_code_coverage.py
+
+      - name: test_chat_completion_imports
+        run: uv run --no-sync python ./tests/code_coverage_tests/test_chat_completion_imports.py
+
+      - name: info_log_check
+        run: uv run --no-sync python ./tests/code_coverage_tests/info_log_check.py
+
+      - name: check_guardrail_apply_decorator
+        run: uv run --no-sync python ./tests/code_coverage_tests/check_guardrail_apply_decorator.py
+
+      - name: test_ban_set_verbose
+        run: uv run --no-sync python ./tests/code_coverage_tests/test_ban_set_verbose.py
+
+      - name: code_qa_check_tests
+        run: uv run --no-sync python ./tests/code_coverage_tests/code_qa_check_tests.py
+
+      - name: check_get_model_cost_key_performance
+        run: uv run --no-sync python ./tests/code_coverage_tests/check_get_model_cost_key_performance.py
+
+      - name: test_proxy_types_import
+        run: uv run --no-sync python ./tests/code_coverage_tests/test_proxy_types_import.py
+
+      - name: callback_manager_test
+        run: uv run --no-sync python ./tests/code_coverage_tests/callback_manager_test.py
+
+      - name: recursive_detector
+        run: uv run --no-sync python ./tests/code_coverage_tests/recursive_detector.py
+
+      - name: test_router_strategy_async
+        run: uv run --no-sync python ./tests/code_coverage_tests/test_router_strategy_async.py
+
+      - name: litellm_logging_code_coverage
+        run: uv run --no-sync python ./tests/code_coverage_tests/litellm_logging_code_coverage.py
+
+      - name: ensure_async_clients_test
+        run: uv run --no-sync python ./tests/code_coverage_tests/ensure_async_clients_test.py
+
+      - name: enforce_llms_folder_style
+        run: uv run --no-sync python ./tests/code_coverage_tests/enforce_llms_folder_style.py
+
+      - name: prevent_key_leaks_in_exceptions
+        run: uv run --no-sync python ./tests/code_coverage_tests/prevent_key_leaks_in_exceptions.py
+
+      - name: check_unsafe_enterprise_import
+        run: uv run --no-sync python ./tests/code_coverage_tests/check_unsafe_enterprise_import.py
+
+      - name: ban_copy_deepcopy_kwargs
+        run: uv run --no-sync python ./tests/code_coverage_tests/ban_copy_deepcopy_kwargs.py
+
+      - name: check_fastuuid_usage
+        run: uv run --no-sync python ./tests/code_coverage_tests/check_fastuuid_usage.py
+
+      - name: memory_test
+        run: uv run --no-sync python ./tests/code_coverage_tests/memory_test.py
+
+      - name: documentation_test_env_keys
+        run: uv run --no-sync python ./tests/documentation_tests/test_env_keys.py
+
+      - name: documentation_test_router_settings
+        run: uv run --no-sync python ./tests/documentation_tests/test_router_settings.py
+
+      - name: documentation_test_api_docs
+        run: uv run --no-sync python ./tests/documentation_tests/test_api_docs.py

.github/workflows/test-linting.yml

@@ -2,7 +2,11 @@ name: LiteLLM Linting
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
 
 permissions:
   contents: read

.github/workflows/test-litellm-ui-build.yml

@@ -4,7 +4,11 @@ permissions:
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
 
 jobs:
   build-ui:

.github/workflows/test-mcp.yml

@@ -2,7 +2,11 @@ name: LiteLLM MCP Tests (folder - tests/mcp_tests)
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
 
 permissions:
   contents: read

.github/workflows/test-model-map.yaml

@@ -2,7 +2,11 @@ name: Validate model_prices_and_context_window.json
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
 
 permissions:
   contents: read

.github/workflows/test-semgrep.yml

@@ -0,0 +1,39 @@
+name: Semgrep
+
+on:
+  pull_request:
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  semgrep:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+        with:
+          version: "0.10.9"
+
+      - name: Run Semgrep (custom rules)
+        run: uv tool run --from 'semgrep==1.157.0' semgrep scan --config .semgrep/rules . --error

.github/workflows/test-unit-caching-redis.yml

@@ -0,0 +1,38 @@
+name: "Unit Tests: Caching (Redis)"
+
+# Uses cloud Redis credentials — only runs on trusted branches, not PRs.
+# This prevents external PRs from accessing Redis credentials.
+on:
+  push:
+    branches: [main, "litellm_*"]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  caching-redis:
+    uses: ./.github/workflows/_test-unit-services-base.yml
+    with:
+      # Redis-only tests that do NOT require provider API keys.
+      # Tests needing API keys (test_caching.py, test_caching_ssl.py, test_prometheus_service.py,
+      # test_router_caching.py) are in Phase 3 integration workflows.
+      test-path: >-
+        tests/local_testing/test_dual_cache.py
+        tests/local_testing/test_redis_batch_optimizations.py
+        tests/local_testing/test_router_utils.py
+      workers: 2
+      reruns: 2
+      timeout-minutes: 20
+      enable-redis: true
+      enable-postgres: false
+    secrets:
+      REDIS_HOST: ${{ secrets.REDIS_HOST }}
+      REDIS_PORT: ${{ secrets.REDIS_PORT }}
+      REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD }}
+      DATABASE_URL: ${{ secrets.DATABASE_URL }}
+      POSTGRES_USER: ${{ secrets.POSTGRES_USER }}
+      POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}

.github/workflows/test-unit-core-utils.yml

@@ -2,7 +2,11 @@ name: "Unit Tests: Core Utilities"
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
 
 permissions:
   contents: read

.github/workflows/test-unit-documentation.yml

@@ -2,7 +2,11 @@ name: "Unit Tests: Documentation Validation"
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
 
 permissions:
   contents: read
@@ -21,6 +25,13 @@ jobs:
         with:
           persist-credentials: false
 
+      - name: Checkout litellm-docs into docs/my-website (for documentation_tests)
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          repository: BerriAI/litellm-docs
+          path: docs/my-website
+          persist-credentials: false
+
       - name: Set up Python
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:

.github/workflows/test-unit-enterprise-routing.yml

@@ -2,7 +2,11 @@ name: "Unit Tests: Enterprise, Google GenAI & Routing"
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
 
 permissions:
   contents: read

.github/workflows/test-unit-integrations.yml

@@ -2,7 +2,11 @@ name: "Unit Tests: Integrations (Callbacks & Logging)"
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
 
 permissions:
   contents: read

.github/workflows/test-unit-llm-providers.yml

@@ -2,7 +2,11 @@ name: "Unit Tests: LLM Provider Transformations"
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
 
 permissions:
   contents: read

.github/workflows/test-unit-misc.yml

@@ -2,7 +2,11 @@ name: "Unit Tests: MCP, Secrets, Containers & Misc"
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
 
 permissions:
   contents: read

.github/workflows/test-unit-proxy-auth.yml

@@ -2,7 +2,11 @@ name: "Unit Tests: Proxy Auth & Key Management"
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
 
 permissions:
   contents: read

.github/workflows/test-unit-proxy-db.yml

@@ -3,7 +3,7 @@ name: "Unit Tests: Proxy DB Operations"
 # Uses DATABASE_URL secret — only runs on trusted branches, not PRs.
 on:
   push:
-    branches: [main, "litellm_*"]
+    branches: [main, "litellm_**"]
 
 permissions:
   contents: read
@@ -12,8 +12,74 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+# Semantic matrix: each shard groups tests by concern (auth, server, logging, …)
+# rather than alphabetical letter ranges. Adding a new test file means adding it
+# to whichever group it belongs to, not reshuffling slices.
+#
+# Design targets:
+#   * Every shard runs in <= 7 minutes of wall-clock on the default runner.
+#     Most of a shard's time is pytest plugin load + xdist worker imports +
+#     pytest-cov instrumentation, not the tests themselves. Keeping per-shard
+#     work low and matching worker count to runner cores is what controls it.
+#   * workers: 4 matches the 4-core ubuntu-latest runner. -n 8 on 4 cores
+#     oversubscribes 2x and workers fight for CPU during their cold-start
+#     imports (measured ~441% CPU for -n 8 locally, i.e. ~55% effective).
+#   * test_key_generate_prisma.py stays serial (workers=0) — it has event-loop
+#     conflicts with the logging worker when run in parallel.
+#   * test_proxy_utils.py runs as a single shard with --dist=worksteal so
+#     xdist balances its 188 parametrized cases across workers instead of
+#     pinning the whole file to one worker (the default --dist=loadscope
+#     behavior for single-file targets).
+#   * test_db_schema_migration.py is isolated because one test in it
+#     (test_aaaasschema_migration_check) takes ~170s — by itself it
+#     determines the shard's wall-clock floor.
 jobs:
+  # Fast guard — fails the workflow if a test_*.py file under
+  # tests/proxy_unit_tests/ is not referenced by any matrix entry below.
+  # The semantic-shard design (no catch-all "remaining" bucket) relies on
+  # every test file being explicitly assigned; this guard prevents a new
+  # file from silently dropping out of CI.
+  assert-shard-coverage:
+    runs-on: ubuntu-latest
+    timeout-minutes: 2
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+      - name: Assert every test_*.py is in a matrix shard
+        run: |
+          python3 - <<'PY'
+          import pathlib, sys, yaml
+          wf = yaml.safe_load(open(".github/workflows/test-unit-proxy-db.yml"))
+          matrix = wf["jobs"]["proxy-db"]["strategy"]["matrix"]["include"]
+          referenced = set()
+          for entry in matrix:
+              for token in entry["test-path"].split():
+                  if token.startswith("tests/proxy_unit_tests/"):
+                      referenced.add(pathlib.PurePosixPath(token).name)
+          actual = {p.name for p in pathlib.Path("tests/proxy_unit_tests").iterdir()
+                    if p.name.startswith("test_") and (p.suffix == ".py" or p.is_dir())
+                    and p.name != "test_configs"}
+          orphans = sorted(actual - referenced)
+          if orphans:
+              print("ERROR: the following files/dirs under tests/proxy_unit_tests/")
+              print("       are not assigned to any shard in test-unit-proxy-db.yml:")
+              for o in orphans:
+                  print(f"  - {o}")
+              print()
+              print("Add each to whichever semantic shard it belongs to.")
+              sys.exit(1)
+          print(f"OK: all {len(actual)} files assigned to a shard.")
+          PY
+
   proxy-db:
+    needs: assert-shard-coverage
+    # Display only the semantic shard name in the checks UI instead of GHA's
+    # default "proxy-db (key-generation, tests/proxy_unit_tests/…, 0, loadscope, 20)"
+    # which includes every matrix field and gets truncated past the test-path.
+    name: ${{ matrix.test-group }}
     permissions:
       contents: read
       id-token: write
@@ -22,19 +88,146 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          # Key generation tests must NOT run in parallel (event loop conflicts with logging worker)
+          # Must run serially — event-loop conflict with the logging worker.
           - test-group: key-generation
             test-path: "tests/proxy_unit_tests/test_key_generate_prisma.py"
             workers: 0
-            timeout: 30
-          - test-group: auth-checks
-            test-path: "tests/proxy_unit_tests/test_auth_checks.py tests/proxy_unit_tests/test_user_api_key_auth.py"
-            workers: 8
+            dist: loadscope
             timeout: 20
-          - test-group: remaining
-            test-path: "tests/proxy_unit_tests --ignore=tests/proxy_unit_tests/test_key_generate_prisma.py --ignore=tests/proxy_unit_tests/test_auth_checks.py --ignore=tests/proxy_unit_tests/test_user_api_key_auth.py"
-            workers: 8
-            timeout: 30
+
+          # ---- auth: split into 2 shards ----
+          - test-group: auth-checks
+            test-path: >-
+              tests/proxy_unit_tests/test_auth_checks.py
+              tests/proxy_unit_tests/test_user_api_key_auth.py
+            workers: 4
+            dist: loadscope
+            timeout: 15
+          - test-group: jwt-and-keys
+            test-path: >-
+              tests/proxy_unit_tests/test_jwt.py
+              tests/proxy_unit_tests/test_jwt_key_mapping.py
+              tests/proxy_unit_tests/test_proxy_custom_auth.py
+              tests/proxy_unit_tests/test_key_generate_dynamodb.py
+              tests/proxy_unit_tests/test_deployed_proxy_keygen.py
+            workers: 4
+            dist: loadscope
+            timeout: 15
+
+          # ---- test_proxy_utils.py, single shard, worksteal distribution ----
+          - test-group: proxy-utils
+            test-path: "tests/proxy_unit_tests/test_proxy_utils.py"
+            workers: 4
+            dist: worksteal
+            timeout: 15
+
+          # ---- proxy server: split into 2 shards ----
+          - test-group: proxy-server-core
+            test-path: >-
+              tests/proxy_unit_tests/test_proxy_server.py
+              tests/proxy_unit_tests/test_proxy_server_keys.py
+              tests/proxy_unit_tests/test_proxy_server_caching.py
+              tests/proxy_unit_tests/test_proxy_server_langfuse.py
+              tests/proxy_unit_tests/test_proxy_server_spend.py
+              tests/proxy_unit_tests/test_aproxy_startup.py
+            workers: 4
+            dist: loadscope
+            timeout: 15
+          - test-group: proxy-runtime
+            test-path: >-
+              tests/proxy_unit_tests/test_proxy_config_unit_test.py
+              tests/proxy_unit_tests/test_proxy_routes.py
+              tests/proxy_unit_tests/test_proxy_gunicorn.py
+              tests/proxy_unit_tests/test_server_root_path.py
+              tests/proxy_unit_tests/test_proxy_pass_user_config.py
+              tests/proxy_unit_tests/test_proxy_token_counter.py
+            workers: 4
+            dist: loadscope
+            timeout: 15
+
+          # ---- logging: split into 2 shards ----
+          - test-group: custom-logging
+            test-path: >-
+              tests/proxy_unit_tests/test_custom_callback_input.py
+              tests/proxy_unit_tests/test_custom_logger_s3_gcs.py
+              tests/proxy_unit_tests/test_proxy_custom_logger.py
+            workers: 4
+            dist: loadscope
+            timeout: 15
+          - test-group: logging-misc
+            test-path: >-
+              tests/proxy_unit_tests/test_proxy_reject_logging.py
+              tests/proxy_unit_tests/test_audit_logs_proxy.py
+              tests/proxy_unit_tests/test_search_api_logging.py
+            workers: 4
+            dist: loadscope
+            timeout: 15
+
+          # ---- db-and-spend: isolate the 170s schema-migration test ----
+          # test_db_schema_migration.py has exactly one test, and that test
+          # is mostly waiting on `prisma migrate deploy` / `prisma migrate
+          # diff` subprocesses (~170s). It does no CPU-bound Python work
+          # inside the test. Running with workers=0 (serial, no xdist)
+          # skips the 4-worker cold-start cost we'd otherwise pay for a
+          # single test, saving ~4 minutes of wall-clock.
+          - test-group: schema-migration
+            test-path: "tests/proxy_unit_tests/test_db_schema_migration.py"
+            workers: 0
+            dist: loadscope
+            timeout: 15
+          - test-group: db-and-spend
+            test-path: >-
+              tests/proxy_unit_tests/test_prisma_client_backoff_retry.py
+              tests/proxy_unit_tests/test_db_schema_changes.py
+              tests/proxy_unit_tests/test_e2e_pod_lock_manager.py
+              tests/proxy_unit_tests/test_skills_db.py
+              tests/proxy_unit_tests/test_update_daily_tag_spend.py
+              tests/proxy_unit_tests/test_update_spend.py
+              tests/proxy_unit_tests/test_project_endpoints_prisma.py
+              tests/proxy_unit_tests/test_proxy_encrypt_decrypt.py
+            workers: 4
+            dist: loadscope
+            timeout: 15
+
+          # ---- guardrails + budget + hooks: split into 2 ----
+          - test-group: guardrails-hooks
+            test-path: >-
+              tests/proxy_unit_tests/test_proxy_setting_guardrails.py
+              tests/proxy_unit_tests/test_banned_keyword_list.py
+              tests/proxy_unit_tests/test_unit_test_proxy_hooks.py
+            workers: 4
+            dist: loadscope
+            timeout: 15
+          - test-group: budgets
+            test-path: >-
+              tests/proxy_unit_tests/test_default_end_user_budget_simple.py
+              tests/proxy_unit_tests/test_unit_test_max_model_budget_limiter.py
+              tests/proxy_unit_tests/test_zero_cost_model_budget_bypass.py
+            workers: 4
+            dist: loadscope
+            timeout: 15
+
+          - test-group: endpoints-and-responses
+            test-path: >-
+              tests/proxy_unit_tests/test_blog_posts_endpoint.py
+              tests/proxy_unit_tests/test_models_fallback_endpoint.py
+              tests/proxy_unit_tests/test_google_endpoint_routing.py
+              tests/proxy_unit_tests/test_google_gemini_proxy_request.py
+              tests/proxy_unit_tests/test_get_favicon.py
+              tests/proxy_unit_tests/test_get_image.py
+              tests/proxy_unit_tests/test_ui_path_detection.py
+              tests/proxy_unit_tests/test_prompt_test_endpoint.py
+              tests/proxy_unit_tests/test_check_batch_cost.py
+              tests/proxy_unit_tests/test_check_responses_cost.py
+              tests/proxy_unit_tests/test_response_polling_handler.py
+              tests/proxy_unit_tests/test_response_polling_pre_call_checks.py
+              tests/proxy_unit_tests/test_realtime_cache.py
+              tests/proxy_unit_tests/test_proxy_exception_mapping.py
+              tests/proxy_unit_tests/test_custom_tokenizer_bug.py
+              tests/proxy_unit_tests/test_model_response_typing
+            workers: 4
+            dist: loadscope
+            timeout: 15
     uses: ./.github/workflows/_test-unit-services-base.yml
     with:
       test-path: ${{ matrix.test-path }}
@@ -42,8 +235,5 @@ jobs:
       reruns: 2
       timeout-minutes: ${{ matrix.timeout }}
       enable-postgres: true
+      dist: ${{ matrix.dist }}
       artifact-name: proxy-db-${{ matrix.test-group }}
-    secrets:
-      DATABASE_URL: ${{ secrets.DATABASE_URL }}
-      POSTGRES_USER: ${{ secrets.POSTGRES_USER }}
-      POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}

.github/workflows/test-unit-proxy-endpoints.yml

@@ -2,7 +2,11 @@ name: "Unit Tests: Proxy API Endpoints"
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
 
 permissions:
   contents: read
@@ -32,6 +36,8 @@ jobs:
         tests/test_litellm/proxy/health_endpoints
         tests/test_litellm/proxy/public_endpoints
         tests/test_litellm/proxy/prompts
+        tests/test_litellm/proxy/rag_endpoints
+        tests/test_litellm/proxy/realtime_endpoints
         tests/test_litellm/proxy/ui_crud_endpoints
       workers: 2
       reruns: 2

.github/workflows/test-unit-proxy-infra.yml

@@ -2,7 +2,11 @@ name: "Unit Tests: Proxy Infrastructure"
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
 
 permissions:
   contents: read

.github/workflows/test-unit-proxy-legacy.yml

@@ -2,7 +2,11 @@ name: "Unit Tests: Proxy Legacy Tests"
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
 
 permissions:
   contents: read

.github/workflows/test-unit-responses-caching-types.yml

@@ -2,7 +2,11 @@ name: "Unit Tests: Responses, Caching & Types"
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
 
 permissions:
   contents: read

.github/workflows/test-unit-security.yml

@@ -1,9 +1,11 @@
 name: "Unit Tests: Security"
 
-# Uses DATABASE_URL secret — only runs on trusted branches, not PRs.
+# Kept push-only (was previously required by DATABASE_URL secret scoping;
+# now the postgres credentials are ephemeral localhost values but the
+# push-trigger stays to match the proxy-db workflow cadence).
 on:
   push:
-    branches: [main, "litellm_*"]
+    branches: [main, "litellm_**"]
 
 permissions:
   contents: read
@@ -24,7 +26,3 @@ jobs:
       timeout-minutes: 20
       enable-postgres: true
       artifact-name: security
-    secrets:
-      DATABASE_URL: ${{ secrets.DATABASE_URL }}
-      POSTGRES_USER: ${{ secrets.POSTGRES_USER }}
-      POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}

.github/workflows/test_server_root_path.yml

@@ -4,7 +4,11 @@ permissions:
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
 
 jobs:
   test-server-root-path:

AGENTS.md

@@ -23,10 +23,11 @@ LiteLLM is a unified interface for 100+ LLMs that:
 
 ### Key Directories
 - `tests/` - Comprehensive test suites
-- `docs/my-website/` - Documentation website
 - `ui/litellm-dashboard/` - Admin dashboard UI
 - `enterprise/` - Enterprise-specific features
 
+Documentation lives in the separate [BerriAI/litellm-docs](https://github.com/BerriAI/litellm-docs) repository and is served at [docs.litellm.ai](https://docs.litellm.ai).
+
 ## DEVELOPMENT GUIDELINES
 
 ### MAKING CODE CHANGES
@@ -218,8 +219,8 @@ When opening issues or pull requests, follow these templates:
 
 ## HELPFUL RESOURCES
 
-- Main documentation: https://docs.litellm.ai/
-- Provider-specific docs in `docs/my-website/docs/providers/`
+- Main documentation: https://docs.litellm.ai/ (source: [BerriAI/litellm-docs](https://github.com/BerriAI/litellm-docs))
+- Provider-specific docs: https://docs.litellm.ai/docs/providers/
 - Admin UI for testing proxy features
 
 ## WHEN IN DOUBT

CLAUDE.md

@@ -2,6 +2,10 @@
 
 This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
 
+## Documentation
+
+Documentation lives in a separate repository: [BerriAI/litellm-docs](https://github.com/BerriAI/litellm-docs). It is served at [docs.litellm.ai](https://docs.litellm.ai). Do not create or edit documentation files in this repository — open doc PRs against `BerriAI/litellm-docs` instead.
+
 ## Development Commands
 
 ### Installation
@@ -110,7 +114,7 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components:
 - When wiring a new UI entity type to an existing backend endpoint, verify the backend API contract (single value vs. array, required vs. optional params) and ensure the UI controls match — e.g., use a single-select dropdown when the backend accepts a single value, not a multi-select
 
 ### UI Component Library
-- **Always use `antd` for new UI components** — we are migrating off of `@tremor/react`. Do not introduce new `Badge`, `Text`, `Card`, `Grid`, `Title`, or other imports from `@tremor/react` in any new or modified file. Use `antd` equivalents: `Tag` for labels, plain `<span>`/`<div>` with Tailwind classes (or `Typography.Text`) for text, `Card` from `antd`, etc. Note that `antd` has no `"yellow"` Tag color — use `"gold"` for amber/yellow.
+- **Always use `antd` for new UI components** — we are migrating off of `@tremor/react`. Do not introduce new `Badge`, `Text`, `Card`, `Grid`, `Title`, or other imports from `@tremor/react` in any new or modified file. Use `antd` equivalents: `Tag` for labels, `Typography.Text` / `Typography.Title` / `Typography.Paragraph` for textual content (avoid plain text-only `<span>`, `<p>`, `<h*>` when Typography fits), and `Card` from `antd`. Note that `antd` has no `"yellow"` Tag color — use `"gold"` for amber/yellow.
 
 ### MCP OAuth / OpenAPI Transport Mapping
 - `TRANSPORT.OPENAPI` is a UI-only concept. The backend only accepts `"http"`, `"sse"`, or `"stdio"`. Always map it to `"http"` before any API call (including pre-OAuth temp-session calls).

Dockerfile

@@ -27,10 +27,8 @@ RUN apk add --no-cache \
     npm \
     libsndfile
 
-ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
-    UV_PROJECT_ENVIRONMENT=/app/.venv \
+ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
     UV_LINK_MODE=copy \
-    XDG_CACHE_HOME=/app/.cache \
     PATH="/app/.venv/bin:${PATH}"
 
 # Copy dependency metadata first for layer caching
@@ -94,11 +92,14 @@ RUN apk add --no-cache bash openssl tzdata nodejs npm python3 libsndfile supervi
     { apk del --no-cache npm 2>/dev/null || true; }
 
 WORKDIR /app
-ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
-    XDG_CACHE_HOME=/app/.cache \
-    PATH="/app/.venv/bin:${PATH}"
+ENV PATH="/app/.venv/bin:${PATH}"
 
 COPY --from=builder /app /app
+# Prisma binaries live in $HOME/.cache (default prisma-python location),
+# which is /root/.cache here. Copy them from the builder so they survive
+# deployments that volume-mount /app/.cache (e.g. readOnlyRootFilesystem
+# + emptyDir) — otherwise the mount would shadow the baked-in query engine.
+COPY --from=builder /root/.cache /root/.cache
 
 RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
     find /app/.venv -type d -path "*/tornado/test" -delete

README.md

@@ -484,6 +484,8 @@ make format-check   # Check formatting only
 
 For detailed contributing guidelines, see [CONTRIBUTING.md](CONTRIBUTING.md).
 
+> **📖 Contributing to documentation?** The LiteLLM docs have moved to a separate repository: [BerriAI/litellm-docs](https://github.com/BerriAI/litellm-docs). Please open doc PRs there. Docs are served at [docs.litellm.ai](https://docs.litellm.ai).
+
 ## Code Quality / Linting
 
 LiteLLM follows the [Google Python Style Guide](https://google.github.io/styleguide/pyguide.html).

ci_cd/run_migration.py

@@ -1,23 +1,234 @@
+import argparse
 import os
-import subprocess
-from pathlib import Path
-from datetime import datetime
-import testing.postgresql
+import re
 import shutil
+import subprocess
+import sys
+from datetime import datetime
+from pathlib import Path
+
+import testing.postgresql
 
 
-def create_migration(migration_name: str = None):
+DESTRUCTIVE_PATTERN = re.compile(r"\bDROP\s+(COLUMN|TABLE|INDEX)\b", re.IGNORECASE)
+DEFAULT_BASE_BRANCH = "litellm_internal_staging"
+
+
+def _find_destructive_statements(sql: str) -> list:
+    """Return SQL lines containing DROP COLUMN, DROP TABLE, or DROP INDEX."""
+    return [
+        line.strip() for line in sql.splitlines() if DESTRUCTIVE_PATTERN.search(line)
+    ]
+
+
+def _print_freshness_failure(
+    base_branch: str, reason: str, stderr_text: str = ""
+) -> None:
+    """Loudly refuse to run when the freshness check can't be completed."""
+    banner = "=" * 72
+    out = sys.stderr
+    print(banner, file=out)
+    print(f"  FRESHNESS CHECK FAILED — COULD NOT VERIFY origin/{base_branch}", file=out)
+    print(banner, file=out)
+    print("", file=out)
+    print(f"Reason: {reason}", file=out)
+    if stderr_text:
+        print("", file=out)
+        print("git stderr:", file=out)
+        for line in stderr_text.rstrip().splitlines():
+            print(f"    {line}", file=out)
+    print("", file=out)
+    print("Common causes:", file=out)
+    print("  - No network access (offline)", file=out)
+    print("  - 'origin' remote not configured, or base branch name is wrong", file=out)
+    print("  - Not a git repository", file=out)
+    print("", file=out)
+    print("Options:", file=out)
+    print(
+        f"  - Fix the above and re-run, OR pass --base-branch <name> if your", file=out
+    )
+    print(
+        f"    base branch is not '{base_branch}', OR pass --skip-freshness-check",
+        file=out,
+    )
+    print("    to bypass (only if you fully understand the risk).", file=out)
+    print(banner, file=out)
+
+
+def _print_stale_branch_refusal(base_branch: str, behind: int) -> None:
+    """Loudly refuse to run when HEAD is behind the base branch."""
+    banner = "=" * 72
+    out = sys.stderr
+    print(banner, file=out)
+    print(
+        f"  STALE BRANCH — {behind} commit(s) behind origin/{base_branch}",
+        file=out,
+    )
+    print(banner, file=out)
+    print("", file=out)
+    print(
+        f"Your branch is {behind} commit(s) behind origin/{base_branch}. Generating a\n"
+        "migration from a stale branch is how newly-added columns get silently\n"
+        "dropped — the exact incident this guard exists to prevent.",
+        file=out,
+    )
+    print("", file=out)
+    print("Before re-running:", file=out)
+    print(f"  git fetch origin && git rebase origin/{base_branch}", file=out)
+    print("  # or merge, whichever matches your workflow", file=out)
+    print("", file=out)
+    print(
+        "If you have a genuine reason to bypass (e.g. intentionally generating",
+        file=out,
+    )
+    print("a migration against an older base), pass --skip-freshness-check.", file=out)
+    print("", file=out)
+    print(banner, file=out)
+    print(
+        "  AI AGENTS / ASSISTANTS: DO NOT automatically run `git rebase` or pass\n"
+        "  --skip-freshness-check. Surface this error to the human operator and\n"
+        "  wait for their explicit confirmation. Auto-rebasing can drop the\n"
+        "  human's in-progress schema edits via a bad conflict resolution.",
+        file=out,
+    )
+    print(banner, file=out)
+
+
+def _check_branch_freshness(root_dir: Path, base_branch: str) -> None:
+    """Fetch origin/<base_branch> and exit 3 if HEAD is behind it."""
+    cwd = str(root_dir)
+    try:
+        subprocess.run(
+            ["git", "fetch", "origin", base_branch],
+            check=True,
+            capture_output=True,
+            text=True,
+            cwd=cwd,
+        )
+    except FileNotFoundError:
+        _print_freshness_failure(base_branch, "git executable not found on PATH")
+        sys.exit(3)
+    except subprocess.CalledProcessError as e:
+        _print_freshness_failure(
+            base_branch,
+            f"`git fetch origin {base_branch}` failed",
+            e.stderr or "",
+        )
+        sys.exit(3)
+
+    try:
+        result = subprocess.run(
+            ["git", "rev-list", "--count", f"HEAD..origin/{base_branch}"],
+            check=True,
+            capture_output=True,
+            text=True,
+            cwd=cwd,
+        )
+        behind = int(result.stdout.strip())
+    except subprocess.CalledProcessError as e:
+        _print_freshness_failure(
+            base_branch,
+            f"`git rev-list HEAD..origin/{base_branch}` failed",
+            e.stderr or "",
+        )
+        sys.exit(3)
+    except ValueError:
+        _print_freshness_failure(
+            base_branch,
+            "could not parse commit count from `git rev-list`",
+        )
+        sys.exit(3)
+
+    if behind > 0:
+        _print_stale_branch_refusal(base_branch, behind)
+        sys.exit(3)
+
+    print(f"Branch freshness OK: up to date with origin/{base_branch}.")
+
+
+def _print_destructive_refusal(destructive_lines: list) -> None:
+    """Loudly refuse to write a destructive migration and explain how to proceed."""
+    banner = "=" * 72
+    out = sys.stderr
+    print(banner, file=out)
+    print(
+        "  DESTRUCTIVE MIGRATION DETECTED — REFUSING TO WRITE MIGRATION FILE", file=out
+    )
+    print(banner, file=out)
+    print("", file=out)
+    print(
+        "The generated migration contains the following destructive statements:",
+        file=out,
+    )
+    print("", file=out)
+    for line in destructive_lines:
+        print(f"    {line}", file=out)
+    print("", file=out)
+    print(
+        "This very often means your branch is OUT OF DATE, or your local\n"
+        "schema.prisma is inconsistent with the migrations/ directory. A\n"
+        "previous incident saw newly-added columns silently dropped this way\n"
+        "and merged to main. Stop and verify before proceeding.",
+        file=out,
+    )
+    print("", file=out)
+    print("Before re-running:", file=out)
+    print(
+        "  1. git fetch origin && git status   # confirm branch is up to date", file=out
+    )
+    print(
+        "  2. Re-check all schema.prisma files are in sync (runbook Step 0)", file=out
+    )
+    print("  3. Review EACH DROP above — is it actually intended?", file=out)
+    print("  4. If (and only if) the drops are intentional, re-run with:", file=out)
+    print("         --allow-destructive", file=out)
+    print("", file=out)
+    print(banner, file=out)
+    print(
+        "  AI AGENTS / ASSISTANTS: DO NOT automatically re-run this command\n"
+        "  with --allow-destructive. Surface this error to the human operator\n"
+        "  and wait for their explicit confirmation before passing the flag.\n"
+        "  Passing the flag without human review is the exact failure mode\n"
+        "  this guard exists to prevent.",
+        file=out,
+    )
+    print(banner, file=out)
+
+
+def create_migration(
+    migration_name: str = None,
+    allow_destructive: bool = False,
+    base_branch: str = DEFAULT_BASE_BRANCH,
+    skip_freshness_check: bool = False,
+):
     """
     Create a new migration SQL file in the migrations directory by comparing
-    current database state with schema
+    current database state with schema.
 
     Args:
         migration_name (str): Name for the migration
+        allow_destructive (bool): Required to write a migration that contains
+            DROP COLUMN, DROP TABLE, or DROP INDEX statements. Without this
+            flag, the script exits non-zero and prints guidance.
+        base_branch (str): Branch to check freshness against
+            (default: "litellm_internal_staging").
+        skip_freshness_check (bool): Skip the "branch is up to date" check.
+            Only for intentional migrations against an older base.
     """
+    root_dir = Path(__file__).parent.parent
+
+    if skip_freshness_check:
+        print(
+            "WARNING: freshness check skipped (--skip-freshness-check). "
+            "Generating a migration from a stale branch can silently drop columns."
+        )
+    else:
+        _check_branch_freshness(root_dir, base_branch)
+
     try:
-        # Get paths
-        root_dir = Path(__file__).parent.parent
-        migrations_dir = root_dir / "litellm-proxy-extras" / "litellm_proxy_extras" / "migrations"
+        migrations_dir = (
+            root_dir / "litellm-proxy-extras" / "litellm_proxy_extras" / "migrations"
+        )
         schema_path = root_dir / "schema.prisma"
 
         # Create temporary PostgreSQL database
@@ -57,7 +268,27 @@ def create_migration(migration_name: str = None):
                     check=True,
                 )
 
-                if result.stdout.strip():
+                # Prisma emits the literal "-- This is an empty migration." when
+                # there's no real drift. Treat that as "no changes".
+                diff_sql = result.stdout
+                stripped = diff_sql.strip()
+                is_empty_diff = (
+                    not stripped or stripped == "-- This is an empty migration."
+                )
+
+                if not is_empty_diff:
+                    destructive_lines = _find_destructive_statements(diff_sql)
+                    if destructive_lines and not allow_destructive:
+                        _print_destructive_refusal(destructive_lines)
+                        sys.exit(2)
+                    if destructive_lines and allow_destructive:
+                        print(
+                            "WARNING: writing destructive migration "
+                            "(--allow-destructive passed). Statements:"
+                        )
+                        for line in destructive_lines:
+                            print(f"    {line}")
+
                     # Generate timestamp and create migration directory
                     timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
                     migration_name = migration_name or "unnamed_migration"
@@ -66,7 +297,7 @@ def create_migration(migration_name: str = None):
 
                     # Write the SQL to migration.sql
                     migration_file = migration_dir / "migration.sql"
-                    migration_file.write_text(result.stdout)
+                    migration_file.write_text(diff_sql)
 
                     print(f"Created migration in {migration_dir}")
                     return True
@@ -88,8 +319,48 @@ def create_migration(migration_name: str = None):
 
 
 if __name__ == "__main__":
-    # If running directly, can optionally pass migration name as argument
-    import sys
-
-    migration_name = sys.argv[1] if len(sys.argv) > 1 else None
-    create_migration(migration_name)
+    parser = argparse.ArgumentParser(
+        description=(
+            "Generate a Prisma migration by diffing the temp DB "
+            "(existing migrations applied) against schema.prisma."
+        )
+    )
+    parser.add_argument(
+        "migration_name",
+        nargs="?",
+        default=None,
+        help="Name for the migration (used in the generated directory name).",
+    )
+    parser.add_argument(
+        "--allow-destructive",
+        action="store_true",
+        help=(
+            "Required to write a migration that contains DROP COLUMN, "
+            "DROP TABLE, or DROP INDEX. Without this flag, destructive "
+            "diffs are refused."
+        ),
+    )
+    parser.add_argument(
+        "--base-branch",
+        default=DEFAULT_BASE_BRANCH,
+        help=(
+            f"Branch to check freshness against (default: {DEFAULT_BASE_BRANCH}). "
+            "The script fetches origin/<base-branch> and refuses to run if HEAD "
+            "is behind it."
+        ),
+    )
+    parser.add_argument(
+        "--skip-freshness-check",
+        action="store_true",
+        help=(
+            "Bypass the 'branch is up to date' check. Only for intentional "
+            "migrations against an older base. Pairs poorly with automation."
+        ),
+    )
+    args = parser.parse_args()
+    create_migration(
+        args.migration_name,
+        allow_destructive=args.allow_destructive,
+        base_branch=args.base_branch,
+        skip_freshness_check=args.skip_freshness_check,
+    )

cookbook/anthropic_agent_sdk/agent_with_mcp.py

@@ -24,24 +24,26 @@ async def interactive_chat_with_mcp():
     Interactive CLI chat with the agent and MCP server
     """
     config = Config()
-    
+
     # Configure Anthropic SDK to point to LiteLLM gateway
     litellm_base_url = setup_litellm_env(config)
-    
+
     # Fetch available models from proxy
-    available_models = await fetch_available_models(litellm_base_url, config.LITELLM_API_KEY)
-    
+    available_models = await fetch_available_models(
+        litellm_base_url, config.LITELLM_API_KEY
+    )
+
     current_model = config.LITELLM_MODEL
-    
+
     # MCP server configuration
     mcp_server_url = f"{litellm_base_url}/mcp/deepwiki2"
     use_mcp = os.getenv("USE_MCP", "true").lower() == "true"
-    
+
     if not use_mcp:
         print("⚠️  MCP disabled via USE_MCP=false")
-    
+
     print_header(litellm_base_url, current_model, has_mcp=use_mcp)
-    
+
     while True:
         # Configure agent options
         if use_mcp:
@@ -58,7 +60,7 @@ async def interactive_chat_with_mcp():
                             "url": mcp_server_url,
                             "headers": {
                                 "Authorization": f"Bearer {config.LITELLM_API_KEY}"
-                            }
+                            },
                         }
                     },
                 )
@@ -78,12 +80,12 @@ async def interactive_chat_with_mcp():
                 model=current_model,
                 max_turns=50,
             )
-        
+
         # Create agent client
         try:
             async with ClaudeSDKClient(options=options) as client:
                 conversation_active = True
-                
+
                 while conversation_active:
                     # Get user input
                     try:
@@ -91,34 +93,36 @@ async def interactive_chat_with_mcp():
                     except (EOFError, KeyboardInterrupt):
                         print("\n\n👋 Goodbye!")
                         return
-                    
+
                     # Handle commands
-                    if user_input.lower() in ['quit', 'exit']:
+                    if user_input.lower() in ["quit", "exit"]:
                         print("\n👋 Goodbye!")
                         return
-                    
-                    if user_input.lower() == 'clear':
+
+                    if user_input.lower() == "clear":
                         print("\n🔄 Starting new conversation...\n")
                         conversation_active = False
                         continue
-                    
-                    if user_input.lower() == 'models':
+
+                    if user_input.lower() == "models":
                         handle_model_list(available_models, current_model)
                         continue
-                    
-                    if user_input.lower() == 'model':
-                        new_model, should_restart = handle_model_switch(available_models, current_model)
+
+                    if user_input.lower() == "model":
+                        new_model, should_restart = handle_model_switch(
+                            available_models, current_model
+                        )
                         if should_restart:
                             current_model = new_model
                             conversation_active = False
                         continue
-                    
+
                     if not user_input:
                         continue
-                    
+
                     # Stream response from agent
                     await stream_response(client, user_input)
-        
+
         except Exception as e:
             print(f"\n❌ Error creating agent client: {e}")
             print("This might be an MCP configuration issue. Try running without MCP:")

cookbook/anthropic_agent_sdk/common.py

@@ -8,13 +8,13 @@ import httpx
 
 class Config:
     """Configuration for LiteLLM Gateway connection"""
-    
+
     # LiteLLM proxy URL (default to local instance)
     LITELLM_PROXY_URL = os.getenv("LITELLM_PROXY_URL", "http://localhost:4000")
-    
+
     # LiteLLM API key (master key or virtual key)
     LITELLM_API_KEY = os.getenv("LITELLM_API_KEY", "sk-1234")
-    
+
     # Model name as configured in LiteLLM (e.g., "bedrock-claude-sonnet-4", "gpt-4", etc.)
     LITELLM_MODEL = os.getenv("LITELLM_MODEL", "bedrock-claude-sonnet-4.5")
 
@@ -28,7 +28,7 @@ async def fetch_available_models(base_url: str, api_key: str) -> list[str]:
             response = await client.get(
                 f"{base_url}/models",
                 headers={"Authorization": f"Bearer {api_key}"},
-                timeout=10.0
+                timeout=10.0,
             )
             response.raise_for_status()
             data = response.json()
@@ -50,7 +50,7 @@ def setup_litellm_env(config: Config):
     """
     Configure environment variables to point Agent SDK to LiteLLM
     """
-    litellm_base_url = config.LITELLM_PROXY_URL.rstrip('/')
+    litellm_base_url = config.LITELLM_PROXY_URL.rstrip("/")
     os.environ["ANTHROPIC_BASE_URL"] = litellm_base_url
     os.environ["ANTHROPIC_API_KEY"] = config.LITELLM_API_KEY
     return litellm_base_url
@@ -87,10 +87,12 @@ def handle_model_list(available_models: list[str], current_model: str):
         print(f"  {marker} {i}. {model}")
 
 
-def handle_model_switch(available_models: list[str], current_model: str) -> tuple[str, bool]:
+def handle_model_switch(
+    available_models: list[str], current_model: str
+) -> tuple[str, bool]:
     """
     Handle model switching
-    
+
     Returns:
         tuple: (new_model, should_restart_conversation)
     """
@@ -98,7 +100,7 @@ def handle_model_switch(available_models: list[str], current_model: str) -> tupl
     for i, model in enumerate(available_models, 1):
         marker = "✓" if model == current_model else " "
         print(f"  {marker} {i}. {model}")
-    
+
     try:
         choice = input("\nEnter number (or press Enter to cancel): ").strip()
         if choice:
@@ -112,7 +114,7 @@ def handle_model_switch(available_models: list[str], current_model: str) -> tupl
                 print("❌ Invalid choice")
     except (ValueError, IndexError):
         print("❌ Invalid input")
-    
+
     return current_model, False
 
 
@@ -120,41 +122,43 @@ async def stream_response(client, user_input: str):
     """
     Stream response from the agent
     """
-    print("\n🤖 Assistant: ", end='', flush=True)
-    
+    print("\n🤖 Assistant: ", end="", flush=True)
+
     try:
         await client.query(user_input)
-        
+
         # Show loading indicator
-        print("⏳ thinking...", end='', flush=True)
-        
+        print("⏳ thinking...", end="", flush=True)
+
         # Stream the response
         first_chunk = True
         async for msg in client.receive_response():
             # Clear loading indicator on first message
             if first_chunk:
-                print("\r🤖 Assistant: ", end='', flush=True)
+                print("\r🤖 Assistant: ", end="", flush=True)
                 first_chunk = False
-            
+
             # Handle different message types
-            if hasattr(msg, 'type'):
-                if msg.type == 'content_block_delta':
+            if hasattr(msg, "type"):
+                if msg.type == "content_block_delta":
                     # Streaming text delta
-                    if hasattr(msg, 'delta') and hasattr(msg.delta, 'text'):
-                        print(msg.delta.text, end='', flush=True)
-                elif msg.type == 'content_block_start':
+                    if hasattr(msg, "delta") and hasattr(msg.delta, "text"):
+                        print(msg.delta.text, end="", flush=True)
+                elif msg.type == "content_block_start":
                     # Start of content block
-                    if hasattr(msg, 'content_block') and hasattr(msg.content_block, 'text'):
-                        print(msg.content_block.text, end='', flush=True)
-            
+                    if hasattr(msg, "content_block") and hasattr(
+                        msg.content_block, "text"
+                    ):
+                        print(msg.content_block.text, end="", flush=True)
+
             # Fallback to original content handling
-            if hasattr(msg, 'content'):
+            if hasattr(msg, "content"):
                 for content_block in msg.content:
-                    if hasattr(content_block, 'text'):
-                        print(content_block.text, end='', flush=True)
-        
+                    if hasattr(content_block, "text"):
+                        print(content_block.text, end="", flush=True)
+
         print()  # New line after response
-        
+
     except Exception as e:
         print(f"\r\n❌ Error: {e}")
         print("Please check your LiteLLM gateway is running and configured correctly.")

cookbook/anthropic_agent_sdk/main.py

@@ -24,17 +24,19 @@ async def interactive_chat():
     Interactive CLI chat with the agent
     """
     config = Config()
-    
+
     # Configure Anthropic SDK to point to LiteLLM gateway
     litellm_base_url = setup_litellm_env(config)
-    
+
     # Fetch available models from proxy
-    available_models = await fetch_available_models(litellm_base_url, config.LITELLM_API_KEY)
-    
+    available_models = await fetch_available_models(
+        litellm_base_url, config.LITELLM_API_KEY
+    )
+
     current_model = config.LITELLM_MODEL
-    
+
     print_header(litellm_base_url, current_model)
-    
+
     while True:
         # Configure agent options for each conversation
         options = ClaudeAgentOptions(
@@ -42,11 +44,11 @@ async def interactive_chat():
             model=current_model,
             max_turns=50,
         )
-        
+
         # Create agent client
         async with ClaudeSDKClient(options=options) as client:
             conversation_active = True
-            
+
             while conversation_active:
                 # Get user input
                 try:
@@ -54,31 +56,33 @@ async def interactive_chat():
                 except (EOFError, KeyboardInterrupt):
                     print("\n\n👋 Goodbye!")
                     return
-                
+
                 # Handle commands
-                if user_input.lower() in ['quit', 'exit']:
+                if user_input.lower() in ["quit", "exit"]:
                     print("\n👋 Goodbye!")
                     return
-                
-                if user_input.lower() == 'clear':
+
+                if user_input.lower() == "clear":
                     print("\n🔄 Starting new conversation...\n")
                     conversation_active = False
                     continue
-                
-                if user_input.lower() == 'models':
+
+                if user_input.lower() == "models":
                     handle_model_list(available_models, current_model)
                     continue
-                
-                if user_input.lower() == 'model':
-                    new_model, should_restart = handle_model_switch(available_models, current_model)
+
+                if user_input.lower() == "model":
+                    new_model, should_restart = handle_model_switch(
+                        available_models, current_model
+                    )
                     if should_restart:
                         current_model = new_model
                         conversation_active = False
                     continue
-                
+
                 if not user_input:
                     continue
-                
+
                 # Stream response from agent
                 await stream_response(client, user_input)
 

cookbook/litellm_proxy_server/batch_api/bedrock/bedrock.py

@@ -11,15 +11,15 @@ BEDROCK_BATCH_MODEL = "bedrock/batch-anthropic.claude-3-5-sonnet-20240620-v1:0"
 batch_input_file = client.files.create(
     file=open("./bedrock_batch_completions.jsonl", "rb"),
     purpose="batch",
-    extra_body={"target_model_names": BEDROCK_BATCH_MODEL}
+    extra_body={"target_model_names": BEDROCK_BATCH_MODEL},
 )
 print(batch_input_file)
 
 # Create batch
-batch = client.batches.create( 
+batch = client.batches.create(
     input_file_id=batch_input_file.id,
     endpoint="/v1/chat/completions",
     completion_window="24h",
     metadata={"description": "Test batch job"},
 )
-print(batch)
+print(batch)

cookbook/litellm_proxy_server/cli_token_usage.py

@@ -8,6 +8,7 @@ in your Python scripts after running `litellm-proxy login`.
 
 from textwrap import indent
 import litellm
+
 LITELLM_BASE_URL = "http://localhost:4000/"
 
 
@@ -15,38 +16,38 @@ def main():
     """Using CLI token with LiteLLM SDK"""
     print("🚀 Using CLI Token with LiteLLM SDK")
     print("=" * 40)
-    #litellm._turn_on_debug()
-    
+    # litellm._turn_on_debug()
+
     # Get the CLI token
     api_key = litellm.get_litellm_gateway_api_key()
-    
+
     if not api_key:
         print("❌ No CLI token found. Please run 'litellm-proxy login' first.")
         return
-    
+
     print("✅ Found CLI token.")
 
     available_models = litellm.get_valid_models(
         check_provider_endpoint=True,
         custom_llm_provider="litellm_proxy",
         api_key=api_key,
-        api_base=LITELLM_BASE_URL
+        api_base=LITELLM_BASE_URL,
     )
-    
+
     print("✅ Available models:")
     if available_models:
         for i, model in enumerate(available_models, 1):
             print(f"   {i:2d}. {model}")
     else:
         print("   No models available")
-    
+
     # Use with LiteLLM
     try:
         response = litellm.completion(
             model="litellm_proxy/gemini/gemini-2.5-flash",
             messages=[{"role": "user", "content": "Hello from CLI token!"}],
             api_key=api_key,
-            base_url=LITELLM_BASE_URL
+            base_url=LITELLM_BASE_URL,
         )
         print(f"✅ LLM Response: {response.model_dump_json(indent=4)}")
     except Exception as e:
@@ -55,7 +56,7 @@ def main():
 
 if __name__ == "__main__":
     main()
-    
+
     print("\n💡 Tips:")
     print("1. Run 'litellm-proxy login' to authenticate first")
     print("2. Replace 'https://your-proxy.com' with your actual proxy URL")

cookbook/litellm_proxy_server/mcp/mcp_with_litellm_proxy.py

@@ -3,11 +3,12 @@ Use LiteLLM Proxy MCP Gateway to call MCP tools.
 
 When using LiteLLM Proxy, you can use the same MCP tools across all your LLM providers.
 """
+
 import openai
 
 client = openai.OpenAI(
-    api_key="sk-1234", # paste your litellm proxy api key here
-    base_url="http://localhost:4000" # paste your litellm proxy base url here
+    api_key="sk-1234",  # paste your litellm proxy api key here
+    base_url="http://localhost:4000",  # paste your litellm proxy base url here
 )
 print("Making API request to Responses API with MCP tools")
 
@@ -17,7 +18,7 @@ response = client.responses.create(
         {
             "role": "user",
             "content": "give me TLDR of what BerriAI/litellm repo is about",
-            "type": "message"
+            "type": "message",
         }
     ],
     tools=[
@@ -25,11 +26,11 @@ response = client.responses.create(
             "type": "mcp",
             "server_label": "litellm",
             "server_url": "litellm_proxy",
-            "require_approval": "never"
+            "require_approval": "never",
         }
     ],
     stream=True,
-    tool_choice="required"
+    tool_choice="required",
 )
 
 for chunk in response:

cookbook/litellm_proxy_server/secret_manager/my_secret_manager.py

@@ -40,8 +40,10 @@ class InMemorySecretManager(CustomSecretManager):
     ) -> Optional[str]:
         """Read secret synchronously"""
         from litellm._logging import verbose_proxy_logger
-        
-        verbose_proxy_logger.info(f"CUSTOM SECRET MANAGER: LOOKING FOR SECRET: {secret_name}")
+
+        verbose_proxy_logger.info(
+            f"CUSTOM SECRET MANAGER: LOOKING FOR SECRET: {secret_name}"
+        )
         value = self.secrets.get(secret_name)
         verbose_proxy_logger.info(f"CUSTOM SECRET MANAGER: READ SECRET: {value}")
         return value
@@ -76,4 +78,3 @@ class InMemorySecretManager(CustomSecretManager):
             del self.secrets[secret_name]
             return {"status": "deleted", "secret_name": secret_name}
         return {"status": "not_found", "secret_name": secret_name}
-

cookbook/livekit_agent_sdk/main.py

@@ -5,6 +5,7 @@ This example shows how to use LiveKit's xAI realtime plugin through LiteLLM prox
 LiteLLM acts as a unified interface, allowing you to switch between xAI, OpenAI, 
 and Azure realtime APIs without changing your agent code.
 """
+
 import asyncio
 import json
 import os
@@ -23,71 +24,79 @@ async def run_voice_agent():
     2. Sends a user message
     3. Streams back the response
     """
-    
+
     url = f"ws://{PROXY_URL.replace('http://', '').replace('https://', '')}/v1/realtime?model={MODEL}"
     headers = {"Authorization": f"Bearer {API_KEY}"}
-    
+
     print(f"🎙️  Connecting to voice agent...")
     print(f"   Model: {MODEL}")
     print(f"   Proxy: {PROXY_URL}")
     print()
-    
+
     async with websockets.connect(url, additional_headers=headers) as ws:
         # Receive initial connection event
         initial = json.loads(await ws.recv())
         print(f"✅ Connected! Event: {initial['type']}\n")
-        
+
         # Get user input
         user_message = input("💬 Your message: ").strip()
         if not user_message:
             user_message = "Tell me a fun fact about AI!"
-        
+
         print(f"\n🤖 Sending to {MODEL}...\n")
-        
+
         # Send user message
-        await ws.send(json.dumps({
-            "type": "conversation.item.create",
-            "item": {
-                "type": "message",
-                "role": "user",
-                "content": [{"type": "input_text", "text": user_message}]
-            }
-        }))
-        
+        await ws.send(
+            json.dumps(
+                {
+                    "type": "conversation.item.create",
+                    "item": {
+                        "type": "message",
+                        "role": "user",
+                        "content": [{"type": "input_text", "text": user_message}],
+                    },
+                }
+            )
+        )
+
         # Request response
-        await ws.send(json.dumps({
-            "type": "response.create",
-            "response": {"modalities": ["text", "audio"]}
-        }))
-        
+        await ws.send(
+            json.dumps(
+                {
+                    "type": "response.create",
+                    "response": {"modalities": ["text", "audio"]},
+                }
+            )
+        )
+
         # Stream response
-        print("🎤 Response: ", end='', flush=True)
+        print("🎤 Response: ", end="", flush=True)
         transcript = []
-        
+
         try:
             while True:
                 msg = await asyncio.wait_for(ws.recv(), timeout=15.0)
                 event = json.loads(msg)
-                
+
                 # Capture transcript deltas
-                if event['type'] == 'response.output_audio_transcript.delta':
-                    delta = event.get('delta', '')
+                if event["type"] == "response.output_audio_transcript.delta":
+                    delta = event.get("delta", "")
                     if delta:
-                        print(delta, end='', flush=True)
+                        print(delta, end="", flush=True)
                         transcript.append(delta)
-                
+
                 # Done when response completes
-                elif event['type'] == 'response.done':
+                elif event["type"] == "response.done":
                     break
-        
+
         except asyncio.TimeoutError:
             pass
-        
+
         print("\n")
-        
+
         if transcript:
             print(f"✅ Complete response: {''.join(transcript)}")
-        
+
         await ws.close()
 
 
@@ -97,7 +106,7 @@ def main():
     print("LiveKit xAI Voice Agent via LiteLLM Proxy")
     print("=" * 70)
     print()
-    
+
     try:
         asyncio.run(run_voice_agent())
     except KeyboardInterrupt:

cookbook/misc/test_responses_api.py

@@ -1,10 +1,9 @@
 import base64
 from openai import OpenAI
 import time
-client = OpenAI(
-    base_url="http://0.0.0.0:4001",
-    api_key="sk-1234"
-)
+
+client = OpenAI(base_url="http://0.0.0.0:4001", api_key="sk-1234")
+
 
 # Function to encode the image
 def encode_image(image_path):
@@ -25,7 +24,7 @@ response = client.responses.create(
         {
             "role": "user",
             "content": [
-                { "type": "input_text", "text": "what color is the image"},
+                {"type": "input_text", "text": "what color is the image"},
                 {
                     "type": "input_image",
                     "image_url": f"data:image/jpeg;base64,{base64_image}",
@@ -36,7 +35,6 @@ response = client.responses.create(
 )
 
 
-
 print(response.output_text)
 print("response1 id===", response.id)
 print("sleeping for 20 seconds...")
@@ -45,9 +43,7 @@ print("making follow up request for existing id")
 response2 = client.responses.create(
     model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
     previous_response_id=response.id,
-    input="ok, and what objects are in the image?"
+    input="ok, and what objects are in the image?",
 )
 
 print(response2.output_text)
-
-

cookbook/nova_sonic_realtime.py

@@ -52,11 +52,11 @@ class RealtimeClient:
     async def connect(self):
         """Connect to LiteLLM proxy realtime endpoint."""
         print(f"Connecting to {self.url}...")
-        
+
         headers = {}
         if self.api_key:
             headers["Authorization"] = f"Bearer {self.api_key}"
-        
+
         self.ws = await websockets.connect(
             self.url,
             additional_headers=headers,
@@ -175,7 +175,9 @@ class RealtimeClient:
 
         try:
             while self.is_active:
-                audio_data = self.input_stream.read(CHUNK_SIZE, exception_on_overflow=False)
+                audio_data = self.input_stream.read(
+                    CHUNK_SIZE, exception_on_overflow=False
+                )
                 await self.send_audio_chunk(audio_data)
                 await asyncio.sleep(0.01)  # Small delay to prevent overwhelming
         except Exception as e:
@@ -270,6 +272,7 @@ async def main():
     except Exception as e:
         print(f"\n❌ Error: {e}")
         import traceback
+
         traceback.print_exc()
     finally:
         await client.close()
@@ -281,7 +284,7 @@ if __name__ == "__main__":
     print("2. Bedrock is configured in proxy_server_config.yaml")
     print("3. AWS credentials are set")
     print()
-    
+
     try:
         asyncio.run(main())
     except KeyboardInterrupt:

cookbook/veo_video_generation.py

@@ -21,49 +21,45 @@ from typing import Optional
 
 class VeoVideoGenerator:
     """Complete Veo video generation client using LiteLLM proxy."""
-    
-    def __init__(self, base_url: str = "http://localhost:4000/gemini/v1beta", 
-                 api_key: str = "sk-1234"):
+
+    def __init__(
+        self,
+        base_url: str = "http://localhost:4000/gemini/v1beta",
+        api_key: str = "sk-1234",
+    ):
         """
         Initialize the Veo video generator.
-        
+
         Args:
             base_url: Base URL for the LiteLLM proxy with Gemini pass-through
             api_key: API key for LiteLLM proxy authentication
         """
         self.base_url = base_url
         self.api_key = api_key
-        self.headers = {
-            "x-goog-api-key": api_key,
-            "Content-Type": "application/json"
-        }
-    
+        self.headers = {"x-goog-api-key": api_key, "Content-Type": "application/json"}
+
     def generate_video(self, prompt: str) -> Optional[str]:
         """
         Initiate video generation with Veo.
-        
+
         Args:
             prompt: Text description of the video to generate
-            
+
         Returns:
             Operation name if successful, None otherwise
         """
         print(f"🎬 Generating video with prompt: '{prompt}'")
-        
+
         url = f"{self.base_url}/models/veo-3.0-generate-preview:predictLongRunning"
-        payload = {
-            "instances": [{
-                "prompt": prompt
-            }]
-        }
-        
+        payload = {"instances": [{"prompt": prompt}]}
+
         try:
             response = requests.post(url, headers=self.headers, json=payload)
             response.raise_for_status()
-            
+
             data = response.json()
             operation_name = data.get("name")
-            
+
             if operation_name:
                 print(f"✅ Video generation started: {operation_name}")
                 return operation_name
@@ -71,58 +67,64 @@ class VeoVideoGenerator:
                 print("❌ No operation name returned")
                 print(f"Response: {json.dumps(data, indent=2)}")
                 return None
-                
+
         except requests.RequestException as e:
             print(f"❌ Failed to start video generation: {e}")
-            if hasattr(e, 'response') and e.response is not None:
+            if hasattr(e, "response") and e.response is not None:
                 try:
                     error_data = e.response.json()
                     print(f"Error details: {json.dumps(error_data, indent=2)}")
                 except:
                     print(f"Error response: {e.response.text}")
             return None
-    
-    def wait_for_completion(self, operation_name: str, max_wait_time: int = 600) -> Optional[str]:
+
+    def wait_for_completion(
+        self, operation_name: str, max_wait_time: int = 600
+    ) -> Optional[str]:
         """
         Poll operation status until video generation is complete.
-        
+
         Args:
             operation_name: Name of the operation to monitor
             max_wait_time: Maximum time to wait in seconds (default: 10 minutes)
-            
+
         Returns:
             Video URI if successful, None otherwise
         """
         print("⏳ Waiting for video generation to complete...")
-        
+
         operation_url = f"{self.base_url}/{operation_name}"
         start_time = time.time()
         poll_interval = 10  # Start with 10 seconds
-        
+
         while time.time() - start_time < max_wait_time:
             try:
-                print(f"🔍 Polling status... ({int(time.time() - start_time)}s elapsed)")
-                
+                print(
+                    f"🔍 Polling status... ({int(time.time() - start_time)}s elapsed)"
+                )
+
                 response = requests.get(operation_url, headers=self.headers)
                 response.raise_for_status()
-                
+
                 data = response.json()
-                
+
                 # Check for errors
                 if "error" in data:
                     print("❌ Error in video generation:")
                     print(json.dumps(data["error"], indent=2))
                     return None
-                
+
                 # Check if operation is complete
                 is_done = data.get("done", False)
-                
+
                 if is_done:
                     print("🎉 Video generation complete!")
-                    
+
                     try:
                         # Extract video URI from nested response
-                        video_uri = data["response"]["generateVideoResponse"]["generatedSamples"][0]["video"]["uri"]
+                        video_uri = data["response"]["generateVideoResponse"][
+                            "generatedSamples"
+                        ][0]["video"]["uri"]
                         print(f"📹 Video URI: {video_uri}")
                         return video_uri
                     except KeyError as e:
@@ -130,64 +132,68 @@ class VeoVideoGenerator:
                         print("Full response:")
                         print(json.dumps(data, indent=2))
                         return None
-                
+
                 # Wait before next poll, with exponential backoff
                 time.sleep(poll_interval)
                 poll_interval = min(poll_interval * 1.2, 30)  # Cap at 30 seconds
-                
+
             except requests.RequestException as e:
                 print(f"❌ Error polling operation status: {e}")
                 time.sleep(poll_interval)
-        
+
         print(f"⏰ Timeout after {max_wait_time} seconds")
         return None
-    
-    def download_video(self, video_uri: str, output_filename: str = "generated_video.mp4") -> bool:
+
+    def download_video(
+        self, video_uri: str, output_filename: str = "generated_video.mp4"
+    ) -> bool:
         """
         Download the generated video file.
-        
+
         Args:
             video_uri: URI of the video to download (from Google's response)
             output_filename: Local filename to save the video
-            
+
         Returns:
             True if download successful, False otherwise
         """
         print(f"⬇️  Downloading video...")
         print(f"Original URI: {video_uri}")
-        
+
         # Convert Google URI to LiteLLM proxy URI
         # Example: files/abc123 -> /gemini/v1beta/files/abc123:download?alt=media
         if video_uri.startswith("files/"):
             download_path = f"{video_uri}:download?alt=media"
         else:
             download_path = video_uri
-            
+
         litellm_download_url = f"{self.base_url}/{download_path}"
         print(f"Download URL: {litellm_download_url}")
-        
+
         try:
             # Download with streaming and redirect handling
             response = requests.get(
-                litellm_download_url, 
-                headers=self.headers, 
+                litellm_download_url,
+                headers=self.headers,
                 stream=True,
-                allow_redirects=True  # Handle redirects automatically
+                allow_redirects=True,  # Handle redirects automatically
             )
             response.raise_for_status()
-            
+
             # Save video file
-            with open(output_filename, 'wb') as f:
+            with open(output_filename, "wb") as f:
                 downloaded_size = 0
                 for chunk in response.iter_content(chunk_size=8192):
                     if chunk:
                         f.write(chunk)
                         downloaded_size += len(chunk)
-                        
+
                         # Progress indicator for large files
                         if downloaded_size % (1024 * 1024) == 0:  # Every MB
-                            print(f"📦 Downloaded {downloaded_size / (1024*1024):.1f} MB...")
-            
+                            print(
+                                f"📦 Downloaded {downloaded_size / (1024*1024):.1f} MB..."
+                            )
+
             # Verify file was created and has content
             if os.path.exists(output_filename):
                 file_size = os.path.getsize(output_filename)
@@ -203,48 +209,52 @@ class VeoVideoGenerator:
             else:
                 print("❌ File was not created")
                 return False
-                
+
         except requests.RequestException as e:
             print(f"❌ Download failed: {e}")
-            if hasattr(e, 'response') and e.response is not None:
+            if hasattr(e, "response") and e.response is not None:
                 print(f"Status code: {e.response.status_code}")
                 print(f"Response headers: {dict(e.response.headers)}")
             return False
-    
+
     def generate_and_download(self, prompt: str, output_filename: str = None) -> bool:
         """
         Complete workflow: generate video and download it.
-        
+
         Args:
             prompt: Text description for video generation
             output_filename: Output filename (auto-generated if None)
-            
+
         Returns:
             True if successful, False otherwise
         """
         # Auto-generate filename if not provided
         if output_filename is None:
             timestamp = int(time.time())
-            safe_prompt = "".join(c for c in prompt[:30] if c.isalnum() or c in (' ', '-', '_')).rstrip()
-            output_filename = f"veo_video_{safe_prompt.replace(' ', '_')}_{timestamp}.mp4"
-        
+            safe_prompt = "".join(
+                c for c in prompt[:30] if c.isalnum() or c in (" ", "-", "_")
+            ).rstrip()
+            output_filename = (
+                f"veo_video_{safe_prompt.replace(' ', '_')}_{timestamp}.mp4"
+            )
+
         print("=" * 60)
         print("🎬 VEO VIDEO GENERATION WORKFLOW")
         print("=" * 60)
-        
+
         # Step 1: Generate video
         operation_name = self.generate_video(prompt)
         if not operation_name:
             return False
-        
+
         # Step 2: Wait for completion
         video_uri = self.wait_for_completion(operation_name)
         if not video_uri:
             return False
-        
+
         # Step 3: Download video
         success = self.download_video(video_uri, output_filename)
-        
+
         if success:
             print("=" * 60)
             print("🎉 SUCCESS! Video generation complete!")
@@ -254,51 +264,51 @@ class VeoVideoGenerator:
             print("=" * 60)
             print("❌ FAILED! Video generation or download failed")
             print("=" * 60)
-        
+
         return success
 
 
 def main():
     """
     Example usage of the VeoVideoGenerator.
-    
+
     Configure these environment variables:
     - LITELLM_BASE_URL: Your LiteLLM proxy URL (default: http://localhost:4000/gemini/v1beta)
     - LITELLM_API_KEY: Your LiteLLM API key (default: sk-1234)
     """
-    
+
     # Configuration from environment or defaults
     base_url = os.getenv("LITELLM_BASE_URL", "http://localhost:4000/gemini/v1beta")
     api_key = os.getenv("LITELLM_API_KEY", "sk-1234")
-    
+
     print("🚀 Starting Veo Video Generation Example")
     print(f"📡 Using LiteLLM proxy at: {base_url}")
-    
+
     # Initialize generator
     generator = VeoVideoGenerator(base_url=base_url, api_key=api_key)
-    
+
     # Example prompts - try different ones!
     example_prompts = [
         "A cat playing with a ball of yarn in a sunny garden",
         "Ocean waves crashing against rocky cliffs at sunset",
         "A bustling city street with people walking and cars passing by",
-        "A peaceful forest with sunlight filtering through the trees"
+        "A peaceful forest with sunlight filtering through the trees",
     ]
-    
+
     # Use first example or get from user
     prompt = example_prompts[0]
     print(f"🎬 Using prompt: '{prompt}'")
-    
+
     # Generate and download video
     success = generator.generate_and_download(prompt)
-    
+
     if success:
         print("\n✅ Example completed successfully!")
         print("💡 Try modifying the prompt in the script for different videos!")
     else:
         print("\n❌ Example failed!")
         print("🔧 Check your LiteLLM proxy configuration and Google AI Studio API key")
-        
+
         # Troubleshooting tips
         print("\n🔍 Troubleshooting:")
         print("1. Ensure LiteLLM proxy is running with Google AI Studio pass-through")

deploy/charts/litellm-helm/templates/deployment.yaml

@@ -47,7 +47,7 @@ spec:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
       {{- with .Values.extraInitContainers }}
       initContainers:
-        {{- toYaml . | nindent 8 }}
+        {{- tpl (toYaml .) $ | nindent 8 }}
       {{- end }}
       containers:
         - name: {{ include "litellm.name" . }}
@@ -212,7 +212,7 @@ spec:
             {{- toYaml . | nindent 12 }}
           {{- end }}
       {{- with .Values.extraContainers }}
-        {{- toYaml . | nindent 8 }}
+        {{- tpl (toYaml .) $ | nindent 8 }}
       {{- end }}
       volumes:
         {{ if .Values.securityContext.readOnlyRootFilesystem }}

deploy/charts/litellm-helm/templates/migrations-job.yaml

@@ -37,7 +37,7 @@ spec:
       serviceAccountName: {{ include "litellm.migrationServiceAccountName" . }}
       {{- with .Values.migrationJob.extraInitContainers }}
       initContainers:
-        {{- toYaml . | nindent 8 }}
+        {{- tpl (toYaml .) $ | nindent 8 }}
       {{- end }} 
       containers:
         - name: prisma-migrations
@@ -96,7 +96,7 @@ spec:
             {{- toYaml . | nindent 12 }}
           {{- end }}
       {{- with .Values.migrationJob.extraContainers }}
-        {{- toYaml . | nindent 8 }}
+        {{- tpl (toYaml .) $ | nindent 8 }}
       {{- end }}
       {{- with .Values.volumes }}
       volumes:

deploy/charts/litellm-helm/tests/deployment_tests.yaml

@@ -319,3 +319,61 @@ tests:
     asserts:
       - notExists:
           path: spec.minReadySeconds
+  - it: should work with extraInitContainers
+    template: deployment.yaml
+    set:
+      extraInitContainers:
+        - name: init-test
+          image: busybox:latest
+          command: ["echo", "hello"]
+    asserts:
+      - contains:
+          path: spec.template.spec.initContainers
+          content:
+            name: init-test
+            image: busybox:latest
+            command: ["echo", "hello"]
+  - it: should support tpl in extraInitContainers
+    template: deployment.yaml
+    set:
+      image:
+        repository: ghcr.io/berriai/litellm-database
+        tag: test
+      extraInitContainers:
+        - name: init-tpl
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          command: ["echo", "hello"]
+    asserts:
+      - contains:
+          path: spec.template.spec.initContainers
+          content:
+            name: init-tpl
+            image: "ghcr.io/berriai/litellm-database:test"
+            command: ["echo", "hello"]
+  - it: should work with extraContainers
+    template: deployment.yaml
+    set:
+      extraContainers:
+        - name: sidecar
+          image: busybox:latest
+    asserts:
+      - contains:
+          path: spec.template.spec.containers
+          content:
+            name: sidecar
+            image: busybox:latest
+  - it: should support tpl in extraContainers
+    template: deployment.yaml
+    set:
+      image:
+        repository: ghcr.io/berriai/litellm-database
+        tag: test
+      extraContainers:
+        - name: sidecar-tpl
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+    asserts:
+      - contains:
+          path: spec.template.spec.containers
+          content:
+            name: sidecar-tpl
+            image: "ghcr.io/berriai/litellm-database:test"

deploy/charts/litellm-helm/tests/migrations-job_tests.yaml

@@ -188,3 +188,69 @@ tests:
       - equal:
           path: spec.template.spec.serviceAccountName
           value: pre-existing-sa
+  - it: should work with extraInitContainers
+    template: migrations-job.yaml
+    set:
+      migrationJob:
+        enabled: true
+        extraInitContainers:
+          - name: init-test
+            image: busybox:latest
+            command: ["echo", "hello"]
+    asserts:
+      - contains:
+          path: spec.template.spec.initContainers
+          content:
+            name: init-test
+            image: busybox:latest
+            command: ["echo", "hello"]
+  - it: should support tpl in extraInitContainers
+    template: migrations-job.yaml
+    set:
+      image:
+        repository: ghcr.io/berriai/litellm-database
+        tag: test
+      migrationJob:
+        enabled: true
+        extraInitContainers:
+          - name: init-tpl
+            image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+            command: ["echo", "hello"]
+    asserts:
+      - contains:
+          path: spec.template.spec.initContainers
+          content:
+            name: init-tpl
+            image: "ghcr.io/berriai/litellm-database:test"
+            command: ["echo", "hello"]
+  - it: should work with extraContainers
+    template: migrations-job.yaml
+    set:
+      migrationJob:
+        enabled: true
+        extraContainers:
+          - name: sidecar
+            image: busybox:latest
+    asserts:
+      - contains:
+          path: spec.template.spec.containers
+          content:
+            name: sidecar
+            image: busybox:latest
+  - it: should support tpl in extraContainers
+    template: migrations-job.yaml
+    set:
+      image:
+        repository: ghcr.io/berriai/litellm-database
+        tag: test
+      migrationJob:
+        enabled: true
+        extraContainers:
+          - name: sidecar-tpl
+            image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+    asserts:
+      - contains:
+          path: spec.template.spec.containers
+          content:
+            name: sidecar-tpl
+            image: "ghcr.io/berriai/litellm-database:test"

docker/Dockerfile.database

@@ -26,10 +26,8 @@ RUN apk add --no-cache \
     npm \
     libsndfile
 
-ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
-    UV_PROJECT_ENVIRONMENT=/app/.venv \
+ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
     UV_LINK_MODE=copy \
-    XDG_CACHE_HOME=/app/.cache \
     PATH="/app/.venv/bin:${PATH}"
 
 # Copy dependency metadata first for layer caching
@@ -92,11 +90,14 @@ RUN apk add --no-cache bash openssl tzdata nodejs npm python3 libsndfile supervi
     { apk del --no-cache npm 2>/dev/null || true; }
 
 WORKDIR /app
-ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
-    XDG_CACHE_HOME=/app/.cache \
-    PATH="/app/.venv/bin:${PATH}"
+ENV PATH="/app/.venv/bin:${PATH}"
 
 COPY --from=builder /app /app
+# Prisma binaries live in $HOME/.cache (default prisma-python location),
+# which is /root/.cache here. Copy them from the builder so they survive
+# deployments that volume-mount /app/.cache (e.g. readOnlyRootFilesystem
+# + emptyDir) — otherwise the mount would shadow the baked-in query engine.
+COPY --from=builder /root/.cache /root/.cache
 
 RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
     find /app/.venv -type d -path "*/tornado/test" -delete

docker/Dockerfile.non_root

@@ -15,29 +15,21 @@ COPY --from=uvbin /uv /usr/local/bin/uv
 COPY --from=uvbin /uvx /usr/local/bin/uvx
 
 RUN for i in 1 2 3; do \
-    apk add --no-cache \
-      python3 \
-      python3-dev \
-      clang \
-      llvm \
-      lld \
-      gcc \
-      linux-headers \
-      build-base \
-      bash \
-      coreutils \
-      curl \
-      openssl \
-      openssl-dev \
-      nodejs \
-      npm \
-      libsndfile && break || sleep 5; \
+      apk add --no-cache \
+        python3 \
+        python3-dev \
+        gcc \
+        bash \
+        coreutils \
+        curl \
+        openssl \
+        libsndfile \
+        nodejs && break || sleep 5; \
     done
 
 ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
     UV_LINK_MODE=copy \
-    NVM_DIR=/root/.nvm \
-    PATH="/root/.nvm/versions/node/v20.20.2/bin:/app/.venv/bin:${PATH}" \
+    PATH="/app/.venv/bin:${PATH}" \
     LITELLM_NON_ROOT=true \
     PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
     PRISMA_CLI_BINARY_TARGETS="debian-openssl-3.0.x" \
@@ -49,7 +41,8 @@ COPY enterprise/pyproject.toml enterprise/
 COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/
 
 # Install third-party dependencies (cached unless pyproject.toml/uv.lock change)
-RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
+RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
+    uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
     --extra proxy \
     --extra proxy-runtime \
     --extra extra_proxy \
@@ -62,38 +55,12 @@ COPY . .
 # Set non-root flag for build time consistency
 ENV LITELLM_NON_ROOT=true
 
-# Build Admin UI once and stage the static output for the runtime image.
-# NOTE: .npmrc files (which may set ignore-scripts=true and min-release-age=3d)
-# are temporarily renamed during npm install/ci so they don't block lifecycle
-# scripts needed by the build. This is safe because npm ci installs from
-# package-lock.json with pinned versions + integrity hashes.
+# Stage the pre-built Admin UI from the checked-in Next.js static export.
+# _experimental/out/ is regenerated as part of the release runbook.
+# Restructure extensionless routes (foo.html -> foo/index.html) to match the layout
+# proxy_server.py expects, and drop a readiness marker.
 RUN mkdir -p /var/lib/litellm/ui /var/lib/litellm/assets && \
-    ([ -f /app/.npmrc ] && mv /app/.npmrc /app/.npmrc.bak || true) && \
-    NVM_VERSION="v0.40.4" && \
-    NVM_CHECKSUM="4b7412c49960c7d31e8df72da90c1fb5b8cccb419ac99537b737028d497aba4f" && \
-    NODE_VERSION="v20.20.2" && \
-    NVM_SCRIPT="/tmp/install-nvm.sh" && \
-    curl -fsSL "https://raw.githubusercontent.com/nvm-sh/nvm/${NVM_VERSION}/install.sh" -o "$NVM_SCRIPT" && \
-    echo "${NVM_CHECKSUM}  ${NVM_SCRIPT}" | sha256sum -c - && \
-    bash "$NVM_SCRIPT" && \
-    export NVM_DIR="$HOME/.nvm" && \
-    . "$NVM_DIR/nvm.sh" && \
-    nvm install "${NODE_VERSION}" && \
-    nvm use "${NODE_VERSION}" && \
-    npm install -g npm@11.12.1 && \
-    npm install -g node-gyp@12.2.0 && \
-    ln -sf "$(npm root -g)/node-gyp" "$(npm root -g)/npm/node_modules/node-gyp" && \
-    npm cache clean --force && \
-    cd /app/ui/litellm-dashboard && \
-    if [ -f "/app/enterprise/enterprise_ui/enterprise_colors.json" ]; then \
-      cp /app/enterprise/enterprise_ui/enterprise_colors.json ./ui_colors.json; \
-    fi && \
-    ([ -f .npmrc ] && mv .npmrc .npmrc.bak || true) && \
-    npm ci --no-audit --no-fund && \
-    ([ -f .npmrc.bak ] && mv .npmrc.bak .npmrc || true) && \
-    ([ -f /app/.npmrc.bak ] && mv /app/.npmrc.bak /app/.npmrc || true) && \
-    npm run build && \
-    cp -r /app/ui/litellm-dashboard/out/* /var/lib/litellm/ui/ && \
+    cp -r /app/litellm/proxy/_experimental/out/. /var/lib/litellm/ui/ && \
     cp /app/litellm/proxy/logo.jpg /var/lib/litellm/assets/logo.jpg && \
     ( cd /var/lib/litellm/ui && \
       for html_file in *.html; do \
@@ -103,10 +70,10 @@ RUN mkdir -p /var/lib/litellm/ui /var/lib/litellm/assets && \
           mv "$html_file" "$folder_name/index.html"; \
         fi; \
       done && \
-      touch .litellm_ui_ready ) && \
-    cd /app/ui/litellm-dashboard && rm -rf ./out
+      touch .litellm_ui_ready )
 
-RUN if [ "$PROXY_EXTRAS_SOURCE" = "published" ]; then \
+RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
+    if [ "$PROXY_EXTRAS_SOURCE" = "published" ]; then \
       uv sync --frozen --no-default-groups --no-editable \
         --extra proxy \
         --extra proxy-runtime \
@@ -123,10 +90,7 @@ RUN if [ "$PROXY_EXTRAS_SOURCE" = "published" ]; then \
         --python python3; \
     fi
 
-RUN mkdir -p /app/.cache/npm && \
-    prisma generate --schema=./schema.prisma && \
-    prisma --version && \
-    prisma migrate diff --from-empty --to-schema-datamodel ./schema.prisma --script > /dev/null 2>&1 || true
+RUN prisma generate --schema=./schema.prisma
 
 RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
     sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
@@ -137,33 +101,11 @@ WORKDIR /app
 USER root
 
 RUN for i in 1 2 3; do \
-    apk upgrade --no-cache && break || sleep 5; \
+      apk upgrade --no-cache && break || sleep 5; \
     done && \
     for i in 1 2 3; do \
-      apk add --no-cache python3 bash openssl tzdata nodejs npm supervisor libsndfile && break || sleep 5; \
-    done && \
-    apk upgrade --no-cache nodejs && \
-    npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
-    GLOBAL="$(npm root -g)" && \
-    find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
-      rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
-    done && \
-    find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
-      rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
-    done && \
-    find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
-      rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
-    done && \
-    find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
-      rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
-    done && \
-    find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
-      rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
-    done && \
-    find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
-      sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null && \
-    npm cache clean --force && \
-    { apk del --no-cache npm 2>/dev/null || true; }
+      apk add --no-cache python3 bash openssl tzdata supervisor libsndfile nodejs && break || sleep 5; \
+    done
 
 COPY --from=builder /app /app
 COPY --from=builder /var/lib/litellm/ui /var/lib/litellm/ui
@@ -179,15 +121,10 @@ ENV PATH="/app/.venv/bin:${PATH}" \
     PRISMA_SKIP_POSTINSTALL_GENERATE=1 \
     PRISMA_HIDE_UPDATE_MESSAGE=1 \
     PRISMA_ENGINES_CHECKSUM_IGNORE_MISSING=1 \
-    NPM_CONFIG_CACHE=/app/.cache/npm \
-    NPM_CONFIG_PREFER_OFFLINE=true \
     PRISMA_OFFLINE_MODE=true
 
-RUN sed -i 's/\r$//' docker/entrypoint.sh && \
-    sed -i 's/\r$//' docker/prod_entrypoint.sh && \
-    chmod +x docker/entrypoint.sh docker/prod_entrypoint.sh && \
-    mkdir -p /nonexistent /.npm /var/lib/litellm/assets /var/lib/litellm/ui /tmp/.npm && \
-    chown -R nobody:nogroup /app /var/lib/litellm/ui /var/lib/litellm/assets /nonexistent /.npm /tmp/.npm && \
+RUN mkdir -p /nonexistent /var/lib/litellm/assets /var/lib/litellm/ui && \
+    chown -R nobody:nogroup /app /var/lib/litellm/ui /var/lib/litellm/assets /nonexistent && \
     PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \
     chown -R nobody:nogroup "$PRISMA_PATH" && \
     LITELLM_PKG_MIGRATIONS_PATH="$(python -c 'import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))' 2>/dev/null || echo '')/migrations" && \
@@ -201,7 +138,7 @@ RUN sed -i 's/\r$//' docker/entrypoint.sh && \
     [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g+w "$LITELLM_PROXY_EXTRAS_PATH" || true && \
     chmod -R g+rX "$PRISMA_PATH" /var/lib/litellm/ui /var/lib/litellm/assets /app/.cache
 
-USER nobody
+USER 65534
 
 RUN prisma generate --schema=./schema.prisma
 

docker/tests/nonroot.yaml

@@ -2,7 +2,7 @@ schemaVersion: 2.0.0
 
 metadataTest:
   entrypoint: ["docker/prod_entrypoint.sh"]
-  user: "nobody"
+  user: "65534"
   workdir: "/app"
 
 fileExistenceTests:

docs/my-website/.gitignore

@@ -1,23 +0,0 @@
-# Dependencies
-/node_modules
-
-# Production
-/build
-
-# Generated files
-.docusaurus
-.cache-loader
-
-# Misc
-.DS_Store
-.env
-.env.local
-.env.development.local
-.env.test.local
-.env.production.local
-
-npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
-yarn.lock
-pnpm-lock.yaml

docs/my-website/Dockerfile

@@ -1,32 +0,0 @@
-ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.10.9
-
-FROM $UV_IMAGE AS uvbin
-
-FROM python:3.14.0a3-slim
-
-COPY --from=uvbin /uv /usr/local/bin/uv
-COPY --from=uvbin /uvx /usr/local/bin/uvx
-COPY . /app
-WORKDIR /app
-
-ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
-    UV_LINK_MODE=copy \
-    PATH="/app/.venv/bin:${PATH}"
-
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    gcc \
-    python3-dev \
-    libssl-dev \
-    pkg-config \
-    && rm -rf /var/lib/apt/lists/*
-
-RUN uv sync --frozen --no-default-groups --no-editable \
-    --extra proxy \
-    --extra proxy-runtime \
-    --extra extra_proxy \
-    --extra semantic-router \
-    --python python
-
-EXPOSE $PORT 
-
-CMD ["sh", "-c", "litellm --host 0.0.0.0 --port $PORT --workers 10 --config config.yaml"]

docs/my-website/README.md

@@ -1,41 +0,0 @@
-# Website
-
-This website is built using [Docusaurus 2](https://docusaurus.io/), a modern static website generator.
-
-### Installation
-
-```
-$ yarn
-```
-
-### Local Development
-
-```
-$ yarn start
-```
-
-This command starts a local development server and opens up a browser window. Most changes are reflected live without having to restart the server.
-
-### Build
-
-```
-$ yarn build
-```
-
-This command generates static content into the `build` directory and can be served using any static contents hosting service.
-
-### Deployment
-
-Using SSH:
-
-```
-$ USE_SSH=true yarn deploy
-```
-
-Not using SSH:
-
-```
-$ GIT_USER=<Your GitHub username> yarn deploy
-```
-
-If you are using GitHub pages for hosting, this command is a convenient way to build the website and push to the `gh-pages` branch.

docs/my-website/babel.config.js

@@ -1,3 +0,0 @@
-module.exports = {
-  presets: [require.resolve('@docusaurus/core/lib/babel/preset')],
-};

docs/my-website/blog/anthropic_wildcard_model_access_incident/index.md

@@ -1,138 +0,0 @@
----
-slug: anthropic-wildcard-model-access-incident
-title: "Incident Report: Wildcard Blocking New Models After Cost Map Reload"
-date: 2026-02-23T10:00:00
-authors:
-  - sameer
-  - krrish
-  - ishaan-alt
-tags: [incident-report, proxy, auth, model-access]
-hide_table_of_contents: false
----
-
-**Date:** Feb 23, 2026  
-**Duration:** ~3 hours  
-**Severity:** High (for users with provider wildcard access rules)  
-**Status:** Resolved
-
-## Summary
-
-When a new Anthropic model (e.g. `claude-sonnet-4-6`) was added to the LiteLLM model cost map and a cost map reload was triggered, requests to the new model were rejected with:
-
-```
-key not allowed to access model. This key can only access models=['anthropic/*']. Tried to access claude-sonnet-4-6.
-```
-
-The reload updated `litellm.model_cost` correctly but never re-ran `add_known_models()`, so `litellm.anthropic_models` (the in-memory set used by the wildcard resolver) remained stale. The new model was invisible to the `anthropic/*` wildcard even though the cost map knew about it.
-
-- **LLM calls:** All requests to newly-added Anthropic models were blocked with a 401.
-- **Existing models:** Unaffected — only models missing from the stale provider set were impacted.
-- **Other providers:** Same bug class existed for any provider wildcard (e.g. `openai/*`, `gemini/*`).
-
-{/* truncate */}
-
----
-
-## Background
-
-LiteLLM supports provider-level wildcard access rules. When an admin configures a key or team with `models=['anthropic/*']`, any model whose provider resolves to `anthropic` should be allowed. The resolution happens in `_model_custom_llm_provider_matches_wildcard_pattern`:
-
-```mermaid
-flowchart TD
-    A["1. Request arrives for claude-sonnet-4-6"] --> B["2. Auth check: can this key call this model?
-    proxy/auth/auth_checks.py"]
-    B --> C["3. Key has models=['anthropic/*']
-    → wildcard match attempted"]
-    C --> D["4. get_llm_provider('claude-sonnet-4-6')
-    checks litellm.anthropic_models set"]
-    D -->|"model IN set"| E["5a. ✅ Provider = 'anthropic'
-    → 'anthropic/claude-sonnet-4-6' matches 'anthropic/*'"]
-    D -->|"model NOT IN set"| F["5b. ❌ Provider unknown
-    → exception raised → wildcard returns False"]
-    E --> G["6. Request allowed"]
-    F --> H["6. 401: key not allowed to access model"]
-
-    style E fill:#d4edda,stroke:#28a745
-    style F fill:#f8d7da,stroke:#dc3545
-    style H fill:#f8d7da,stroke:#dc3545
-    style D fill:#fff3cd,stroke:#ffc107
-```
-
-`litellm.anthropic_models` is a Python `set` populated at import time by `add_known_models()`. It is the source `get_llm_provider()` consults to map a bare model name like `claude-sonnet-4-6` to the provider string `"anthropic"`.
-
----
-
-## Root Cause
-
-`add_known_models()` is called **once** at module import time. Both reload paths in `proxy_server.py` updated `litellm.model_cost` with the fresh map but never called `add_known_models()` again:
-
-```python
-# Before the fix — both reload paths looked like this:
-new_model_cost_map = get_model_cost_map(url=model_cost_map_url)
-litellm.model_cost = new_model_cost_map          # ✅ cost map updated
-_invalidate_model_cost_lowercase_map()           # ✅ cache cleared
-# ❌ add_known_models() never called
-#    → litellm.anthropic_models still has the old set
-#    → new model not in the set
-#    → get_llm_provider() raises for the new model
-#    → wildcard match returns False
-#    → 401 for every request to the new model
-```
-
-The gap existed in two places:
-1. `_check_and_reload_model_cost_map` — the periodic automatic reload (every 10 s)
-2. The `/reload/model_cost_map` admin endpoint — the manual reload
-
-**Timeline:**
-
-1. New model (`claude-sonnet-4-6`) added to `model_prices_and_context_window.json`
-2. Admin triggers cost map reload via UI → `litellm.model_cost` updated
-3. Users with `anthropic/*` wildcard keys attempt requests to `claude-sonnet-4-6`
-4. `get_llm_provider('claude-sonnet-4-6')` raises → wildcard returns False → 401
-5. Admin reloads cost map again — same result (root cause not addressed)
-6. ~3 hours of investigation → root cause identified → fix deployed
-
----
-
-## The Fix
-
-After each reload, `add_known_models()` is called with the freshly fetched map passed explicitly. Passing the map directly (rather than relying on the module-level reference) removes any ambiguity about which dict is iterated:
-
-```python
-# After the fix — both reload paths now do:
-new_model_cost_map = get_model_cost_map(url=model_cost_map_url)
-litellm.model_cost = new_model_cost_map
-_invalidate_model_cost_lowercase_map()
-litellm.add_known_models(model_cost_map=new_model_cost_map)  # ✅ sets repopulated
-```
-
-`add_known_models()` was also updated to accept an optional explicit map so callers cannot accidentally iterate a stale module-level reference:
-
-```python
-# Before
-def add_known_models():
-    for key, value in model_cost.items():   # reads module global — ambiguous after reload
-        ...
-
-# After
-def add_known_models(model_cost_map: Optional[Dict] = None):
-    _map = model_cost_map if model_cost_map is not None else model_cost
-    for key, value in _map.items():         # always iterates the map you just fetched
-        ...
-```
-
-After the fix, the provider sets (`anthropic_models`, `open_ai_chat_completion_models`, etc.) are always consistent with `litellm.model_cost` immediately after every reload. New models become accessible via wildcard rules without any proxy restart.
-
----
-
-## Remediation
-
-| # | Action | Status | Code |
-|---|---|---|---|
-| 1 | Call `add_known_models(model_cost_map=...)` in the periodic reload path | ✅ Done | [`proxy_server.py#L4393`](https://github.com/BerriAI/litellm/blob/main/litellm/proxy/proxy_server.py#L4393) |
-| 2 | Call `add_known_models(model_cost_map=...)` in the `/reload/model_cost_map` endpoint | ✅ Done | [`proxy_server.py#L11904`](https://github.com/BerriAI/litellm/blob/main/litellm/proxy/proxy_server.py#L11904) |
-| 3 | Update `add_known_models()` to accept an explicit map parameter | ✅ Done | [`__init__.py#L617`](https://github.com/BerriAI/litellm/blob/main/litellm/__init__.py#L617) |
-| 4 | Regression test: `add_known_models(model_cost_map=...)` populates provider sets | ✅ Done | [`test_auth_checks.py`](https://github.com/BerriAI/litellm/blob/main/tests/proxy_unit_tests/test_auth_checks.py) |
-| 5 | Regression test: `anthropic/*` wildcard grants/denies access correctly after reload | ✅ Done | [`test_auth_checks.py`](https://github.com/BerriAI/litellm/blob/main/tests/proxy_unit_tests/test_auth_checks.py) |
-
----

docs/my-website/blog/april_townhall_announcement/index.md

@@ -1,40 +0,0 @@
----
-slug: april-townhall-announcement
-title: "April Townhall: Security + Product Roadmap"
-date: 2026-04-02T07:30:00
-authors:
-  - krrish
-  - ishaan-alt
-description: "Join the LiteLLM April townhall on Friday, 10 April at 7:30 AM to learn about LiteLLM's security and product roadmap."
-tags: [announcement, townhall]
-hide_table_of_contents: true
----
-
-import Image from '@theme/IdealImage';
-
-We are hosting our April townhall on **Friday, 10 April at 7:30 AM PST**.
-
-<Image
-  img={require('../../img/april_townhall_banner.png')}
-  style={{width: '900px', height: 'auto', display: 'block'}}
-/>
-
-{/* truncate */}
-
-## Agenda
-
-- Product updates and roadmap progress
-- Reliability and security updates
-- Open Q&A with the team
-
-## How to contribute
-
-Add your thoughts to this [ticket](https://github.com/BerriAI/litellm/issues/24825) to help us shape the agenda.
-
-## Register
-
-Register here: [LiteLLM April Townhall Form](https://forms.gle/hvyVXwbFjzJQE7dEA)
-
-We will hold the townhall from **7:30 AM to 8:30 AM PST on Zoom**.
-
-For security, attendance is restricted to corporate emails. If you register with a non-corporate email, we will share the townhall slides and accompanying blog post after the event.

docs/my-website/blog/april_townhall_updates/index.md

@@ -1,162 +0,0 @@
----
-slug: april-townhall-updates
-title: "April Townhall Updates: CI/CD v2, Stability, and Product Roadmap"
-date: 2026-04-10T12:00:00
-authors:
-  - krrish
-  - ishaan-alt
-description: "A recap of the April LiteLLM town hall covering CI/CD v2, product stability work, and the near-term roadmap."
-tags: [townhall, security, reliability, product]
-hide_table_of_contents: false
----
-
-import Image from '@theme/IdealImage';
-
-Thank you to everyone who joined our April town hall.
-
-We used the session to share our CI/CD v2 improvements, product stability work, and what we are prioritizing next across reliability and product roadmap.
-
-{/* truncate */}
-
-## CI/CD v2 improvements
-
-Our CI/CD v2 work is centered around four goals:
-
-1. **Limit** what each package can access
-2. **Reduce** the number of sensitive environment variables
-3. **Avoid** compromised packages
-4. **Reduce the risk of** release tampering
-
-#### New architecture: isolated environments
-
-We have begun moving to isolated environments for distinct CI/CD stages to reduce the chance that a single compromised step can inherit broad access across the entire pipeline.
-
-<Image
-  img={require('../../img/april_townhall_isolated_environments.png')}
-  style={{width: '900px', height: 'auto', display: 'block'}}
-/>
-
-#### Current rollout status
-
-These changes are deployed in our current release workflow. [See here](https://github.com/BerriAI/litellm/tags)
-
-#### Independently verify releases
-
-A key part of CI/CD v2 is supporting independent verification of release artifacts using our published verification process, while reducing reliance on any single credential or release path.
-
-[**Learn more about how to verify releases**](https://docs.litellm.ai/docs/proxy/docker_image_security)
-
-<Image
-  img={require('../../img/verify_releases.png')}
-  style={{width: '900px', height: 'auto', display: 'block'}}
-/>
-
-## Stability improvements
-
-### SDLC improvements
-
-This month, we're focusing on process stability improvements around:
-- Improving main-branch stability
-- Mapping UI QA to built Docker images for 1:1 environment parity
-- Consistent release tags across PyPI and Docker
-- Fixing release notes publication
-
-#### Improving main-branch stability
-
-We're introducing a staging-gated flow:
-
-<Image
-  img={require('../../img/stable_main.png')}
-  style={{width: '900px', height: 'auto', display: 'block'}}
-/>
-
-- Only an internal staging branch can push to `main`.
-- PRs to that staging branch must pass CircleCI LLM API testing.
-- Collision handling happens on staging, which is designed to reduce unstable changes reaching `main`.
-
-#### UI QA in Docker environment
-
-Moving forward, all UI QA will be performed in the built Docker image that users run.
-
-Previously, some UI QA paths were run in local environments that did not fully replicate Docker runtime conditions.
-
-That contributed to release-specific issues, including MCP registration problems in `v1.82.3`.
-
-#### Consistent release tags
-
-Today we publish releases for multiple scenarios:
-- Dev (Built of a PR for a customer-specific scenario)
-- Nightly (Passes all CI/CD checks)
-- Release Candidate (Passes all CI/CD checks + manual UI QA)
-- Stable (intended to pass all CI/CD checks + manual UI QA + 7 days of production testing)
-
-We are targeting a consistent naming convention across PyPI and Docker by the end of April.
-
-#### Release notes
-
-CI/CD v2 changes moved release notes to a manual path. This is a temporary solution while we investigate a better automated workflow. We are targeting a more consistent process by the end of April.
-
-### Product stability improvements
-
-#### Stable Prisma migrations
-
-Today, we have observed several migration failure classes:
-- Migration not applied
-- Migration marked applied but incomplete
-- Migration not applied due to non-root image issues
-
-We're prioritizing this work this month and have assigned an engineering owner to the effort. Our target is to resolve these error classes by the end of April.
-
-#### UI type safety
-
-Another area of focus is improving the stability of the UI. Today, one cause of errors is that the UI maintains its own assumptions about backend API types. This can lead to issues when backend responses differ from UI assumptions.
-
-We aim to move to having the UI and Backend be in sync with each other, and are exploring OpenAPI-driven mapping to achieve this.
-
-## Product roadmap
-
-### Our Assumptions
-
-Over the next few years, we expect:
-- Companies will give employees more AI tools.
-- More AI agents will move into production workflows across HR, finance, support, and operations.
-
-### Our Inferences
-#### Near-term
-
-- AI spend will increase.
-- Uptime and latency will become even more important.
-- More AI resources (skills, CLIs, and related assets) will require governance.
-- Agent and MCP usage patterns will require deeper controls.
-- Broader developer adoption will increase the need for simpler, more discoverable tooling.
-
-#### Long-term 
-
-- We expect many organizations to treat agent auditability (how decisions were made across LLM + MCP + sub-agent inputs/outputs) as a compliance expectation.
-- Permission management will get more complex as user-agent interaction chains deepen.
-
-Roadmap timelines in this post are targets and may evolve based on validation and user feedback.
-
-## April investments
-
-### Reliability
-
-- Increase uptime for 10k+ RPS scenarios.
-- Investigate latency overhead for long-running Claude Code requests.
-
-### Feature reliability
-
-- Polish MCP authentication.
-- Better understand how teams are using agents through LiteLLM.
-
-### Governance
-
-- Launch Skills as a first-class citizen in LiteLLM.
-
-## Q&A
-
-Thank you again for all the questions and direct feedback. We will keep sharing concrete progress updates as these efforts ship.
-
-## Hiring
-
-We are actively hiring across several roles, please apply [here](https://jobs.ashbyhq.com/litellm) if you're interested!

docs/my-website/blog/authors.yml

@@ -1,48 +0,0 @@
-litellm:
-  name: LiteLLM Team
-  title: LiteLLM Core Team
-  url: https://github.com/BerriAI/litellm
-  image_url: https://github.com/BerriAI.png
-
-sameer:
-  name: Sameer Kankute
-  title: SWE @ LiteLLM (LLM Translation)
-  url: https://www.linkedin.com/in/sameer-kankute/
-  image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
-
-krrish:
-  name: Krrish Dholakia
-  title: CEO, LiteLLM
-  url: https://www.linkedin.com/in/krish-d/
-  image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
-
-ishaan:
-  name: Ishaan Jaffer
-  title: CTO, LiteLLM
-  url: https://www.linkedin.com/in/reffajnaahsi/
-  image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
-
-# Alias for typo in name
-ishaan-alt:
-  name: Ishaan Jaffer
-  title: CTO, LiteLLM
-  url: https://www.linkedin.com/in/reffajnaahsi/
-  image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
-
-ryan:
-  name: Ryan Crabbe
-  title: Performance Engineer, LiteLLM
-  url: https://www.linkedin.com/in/ryan-crabbe-0b9687214
-  image_url: https://media.licdn.com/dms/image/v2/D5603AQHt1t9Z4BJ6Gw/profile-displayphoto-shrink_400_400/profile-displayphoto-shrink_400_400/0/1724453682340?e=1772064000&v=beta&t=VXdmr13rsNB05wyA2F1TENOB5UuDHUZ0FCHTolNyR5M
-
-alexsander:
-  name: Alexsander Hamir
-  title: Performance Engineer, LiteLLM
-  url: https://www.linkedin.com/in/alexsander-baptista/
-  image_url: https://github.com/AlexsanderHamir.png
-
-yuneng:
-  name: Yuneng Jiang
-  title: SWE @ LiteLLM (Full Stack)
-  url: https://www.linkedin.com/in/yuneng-david-jiang-455676139/
-  image_url: https://avatars.githubusercontent.com/u/171294688?v=4

docs/my-website/blog/ci_cd_v2_improvements/index.md

@@ -1,90 +0,0 @@
----
-slug: ci-cd-v2-improvements
-title: "Announcing CI/CD v2 for LiteLLM"
-date: 2026-03-30T21:30:00
-authors:
-  - krrish
-description: "CI/CD v2 introduces isolated environments, stronger security gates, and safer release separation for LiteLLM."
-tags: [engineering, ci-cd, security]
-hide_table_of_contents: false
----
-
-import Image from '@theme/IdealImage';
-
-The CI/CD v2 is now live for LiteLLM.
-
-<Image
-  img={require('../../img/ci_cd_architecture.png')}
-  style={{width: '700px', height: 'auto', display: 'block'}}
-/>
-
-<br/>
-Building on the roadmap from our [security incident](https://docs.litellm.ai/blog/security-townhall-updates#roadmap), CI/CD v2 introduces isolated environments, stronger security gates, and safer release separation for LiteLLM.
-
-## What changed
-
-- Security scans and unit tests run in isolated environments.
-- Validation and release are separated into different repositories, making it harder for an attacker to reach release credentials.
-- Trusted Publishing for PyPI releases - this means no long-lived credentials are used to publish releases.
-- Immutable Docker release tags - this means no tampering of Docker release tags after they are published [Learn more](https://docs.docker.com/docker-hub/repos/manage/hub-images/immutable-tags/). Note: work for GHCR docker releases is planned as well.
-- Docker image signing with [Cosign](https://github.com/sigstore/cosign) - all release images are signed so users can independently verify they came from us.
-
-## Verify Docker image signatures
-
-Starting from `v1.83.0-nightly`, all LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Every release is signed with the same key introduced in [commit `0112e53`](https://github.com/BerriAI/litellm/commit/0112e53046018d726492c814b3644b7d376029d0).
-
-**Verify using the pinned commit hash (recommended):**
-
-A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the original signing key:
-
-```bash
-cosign verify \
-  --key https://raw.githubusercontent.com/BerriAI/litellm/0112e53046018d726492c814b3644b7d376029d0/cosign.pub \
-  ghcr.io/berriai/litellm:<release-tag>
-```
-
-**Verify using a release tag (convenience):**
-
-Tags are protected in this repository and resolve to the same key. This option is easier to read but relies on tag protection rules:
-
-```bash
-cosign verify \
-  --key https://raw.githubusercontent.com/BerriAI/litellm/<release-tag>/cosign.pub \
-  ghcr.io/berriai/litellm:<release-tag>
-```
-
-Replace `<release-tag>` with the version you are deploying (e.g. `v1.83.0-stable`).
-
-Expected output:
-
-```
-The following checks were performed on each of these signatures:
-  - The cosign claims were validated
-  - The signatures were verified against the specified public key
-```
-
-## What's next
-
-Moving forward, we plan on:
-- Adopting OpenSSF (this is a set of security criteria that projects should meet to demonstrate a strong security posture - [Learn more](https://baseline.openssf.org/versions/2026-02-19.html))
-  - We've added Scorecard and Allstar to our Github
-
-- Adding SLSA Build Provenance to our CI/CD pipeline - this means we allow users to independently verify that a release came from us and prevent silent modifications of releases after they are published.
-
-
-We hope that this will mean you can be confident that the releases you are using are safe and from us.
-
-
-## The principle
-
-The new CI/CD pipeline reflects the principles, outlined below, and is designed to be more secure and reliable:
-
-- **Limit** what each package can access
-- **Reduce** the number of sensitive environment variables
-- **Avoid** compromised packages
-- **Prevent** release tampering
-
-
-## How to help: 
-
-Help us plan April's stability sprint - https://github.com/BerriAI/litellm/issues/24825

docs/my-website/blog/claude_code_beta_headers/index.md

@@ -1,168 +0,0 @@
----
-slug: claude-code-beta-headers-incident
-title: "Incident Report: Invalid beta headers with Claude Code"
-date: 2026-02-16T10:00:00
-authors:
-  - sameer
-  - ishaan-alt
-  - krrish
-tags: [incident-report, anthropic, stability]
-hide_table_of_contents: false
----
-
-**Date:** February 13, 2026
-**Duration:** ~3 hours
-**Severity:** High
-**Status:** Resolved
-
-> **Note:** This fix will be available starting from `v1.81.13-nightly` or higher of LiteLLM.
-
-## Summary
-
-Claude Code began sending unsupported Anthropic beta headers to non-Anthropic providers (Bedrock, Azure AI, Vertex AI), causing `invalid beta flag` errors. LiteLLM was forwarding all beta headers without provider-specific validation. Users experienced request failures when routing Claude Code requests through LiteLLM to these providers.
-
-- **LLM calls to Anthropic:** No impact.
-- **LLM calls to Bedrock/Azure/Vertex:** Failed with `invalid beta flag` errors when unsupported headers were present.
-- **Cost tracking and routing:** No impact.
-
-{/* truncate */}
-
----
-
-## Background
-
-Anthropic uses beta headers to enable experimental features in Claude. When Claude Code makes API requests, it includes headers like `anthropic-beta: prompt-caching-scope-2026-01-05,advanced-tool-use-2025-11-20`. However, not all providers support all Anthropic beta features.
-
-Before this incident, LiteLLM forwarded all beta headers to all providers without validation:
-
-```mermaid
-sequenceDiagram
-    participant CC as Claude Code
-    participant LP as LiteLLM (old behavior)
-    participant Provider as Provider (Bedrock/Azure/Vertex)
-
-    CC->>LP: Request with beta headers
-    Note over CC,LP: anthropic-beta: header1,header2,header3
-
-    LP->>Provider: Forward ALL headers (no validation)
-    Note over LP,Provider: anthropic-beta: header1,header2,header3
-
-    Provider-->>LP: ❌ Error: invalid beta flag
-    LP-->>CC: Request fails
-```
-
-Requests succeeded for Anthropic (native support) but failed for other providers when Claude Code sent headers those providers didn't support.
-
----
-
-## Root cause
-
-LiteLLM lacked provider-specific beta header validation. When Claude Code introduced new beta features or sent headers that specific providers didn't support, those headers were blindly forwarded, causing provider API errors.
-
----
-
-## Remediation
-
-| # | Action | Status | Code |
-|---|---|---|---|
-| 1 | Create `anthropic_beta_headers_config.json` with provider-specific mappings | ✅ Done | [`anthropic_beta_headers_config.json`](https://github.com/BerriAI/litellm/blob/main/litellm/anthropic_beta_headers_config.json) |
-| 2 | Implement strict validation: headers must be explicitly mapped to be forwarded | ✅ Done | [`litellm_logging.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm_core_utils/litellm_logging.py) |
-| 3 | Add `/reload/anthropic_beta_headers` endpoint for dynamic config updates | ✅ Done | Proxy management endpoints |
-| 4 | Add `/schedule/anthropic_beta_headers_reload` for automatic periodic updates | ✅ Done | Proxy management endpoints |
-| 5 | Support `LITELLM_ANTHROPIC_BETA_HEADERS_URL` for custom config sources | ✅ Done | Environment configuration |
-| 6 | Support `LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS` for air-gapped deployments | ✅ Done | Environment configuration |
-
-Now LiteLLM validates and transforms headers per-provider:
-
-```mermaid
-sequenceDiagram
-    participant CC as Claude Code
-    participant LP as LiteLLM (new behavior)
-    participant Config as Beta Headers Config
-    participant Provider as Provider (Bedrock/Azure/Vertex)
-
-    CC->>LP: Request with beta headers
-    Note over CC,LP: anthropic-beta: header1,header2,header3
-
-    LP->>Config: Load header mapping for provider
-    Config-->>LP: Returns mapping (header→value or null)
-
-    Note over LP: Validate & Transform:<br/>1. Check if header exists in mapping<br/>2. Filter out null values<br/>3. Map to provider-specific names
-
-    LP->>Provider: Request with filtered & mapped headers
-    Note over LP,Provider: anthropic-beta: mapped-header2<br/>(header1, header3 filtered out)
-
-    Provider-->>LP: ✅ Success response
-    LP-->>CC: Response
-```
-
----
-
-## Dynamic configuration updates
-
-A key improvement is zero-downtime configuration updates. When Anthropic releases new beta features, users can update their configuration without restarting:
-
-```bash
-# Manually trigger reload (no restart needed)
-curl -X POST "https://your-proxy-url/reload/anthropic_beta_headers" \
-  -H "Authorization: Bearer YOUR_ADMIN_TOKEN"
-
-# Or schedule automatic reloads every 24 hours
-curl -X POST "https://your-proxy-url/schedule/anthropic_beta_headers_reload?hours=24" \
-  -H "Authorization: Bearer YOUR_ADMIN_TOKEN"
-```
-
-This prevents future incidents where Claude Code introduces new headers before LiteLLM configuration is updated.
-
----
-
-## Configuration format
-
-The `anthropic_beta_headers_config.json` file maps input headers to provider-specific output headers:
-
-```json
-{
-  "description": "Mapping of Anthropic beta headers for each provider.",
-  "anthropic": {
-    "advanced-tool-use-2025-11-20": "advanced-tool-use-2025-11-20",
-    "computer-use-2025-01-24": "computer-use-2025-01-24"
-  },
-  "bedrock_converse": {
-    "advanced-tool-use-2025-11-20": null,
-    "computer-use-2025-01-24": "computer-use-2025-01-24"
-  },
-  "azure_ai": {
-    "advanced-tool-use-2025-11-20": "advanced-tool-use-2025-11-20",
-    "computer-use-2025-01-24": "computer-use-2025-01-24"
-  }
-}
-```
-
-**Validation rules:**
-1. Headers must exist in the mapping for the target provider
-2. Headers with `null` values are filtered out (unsupported)
-3. Header names can be transformed per-provider (e.g., Bedrock uses different names for some features)
-
----
-
-## Resolution steps for users
-
-For users still experiencing issues, update to the latest LiteLLM version if < v1.81.11-nightly:
-
-```bash
-pip install --upgrade litellm
-```
-
-Or manually reload the configuration without restarting:
-
-```bash
-curl -X POST "https://your-proxy-url/reload/anthropic_beta_headers" \
-  -H "Authorization: Bearer YOUR_ADMIN_TOKEN"
-```
-
----
-
-## Related documentation
-
-- [Managing Anthropic Beta Headers](../../docs/proxy/sync_anthropic_beta_headers) - Complete configuration guide
-- [`anthropic_beta_headers_config.json`](https://github.com/BerriAI/litellm/blob/main/litellm/anthropic_beta_headers_config.json) - Current configuration file

docs/my-website/blog/claude_opus_4_6/index.md

@@ -1,723 +0,0 @@
----
-slug: claude_opus_4_6
-title: "Day 0 Support: Claude Opus 4.6"
-date: 2026-02-05T10:00:00
-authors:
-  - sameer
-  - ishaan-alt
-  - krrish
-description: "Day 0 support for Claude Opus 4.6 on LiteLLM AI Gateway - use across Anthropic, Azure, Vertex AI, and Bedrock."
-tags: [anthropic, claude, opus 4.6]
-hide_table_of_contents: false
----
-
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-LiteLLM now supports Claude Opus 4.6 on Day 0. Use it across Anthropic, Azure, Vertex AI, and Bedrock through the LiteLLM AI Gateway.
-
-{/* truncate */}
-
-## Docker Image
-
-```bash
-docker pull ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.80.0-stable.opus-4-6
-```
-
-## Usage - Anthropic
-
-<Tabs>
-<TabItem value="proxy" label="LiteLLM Proxy">
-
-**1. Setup config.yaml**
-
-```yaml
-model_list:
-  - model_name: claude-opus-4-6
-    litellm_params:
-      model: anthropic/claude-opus-4-6
-      api_key: os.environ/ANTHROPIC_API_KEY
-```
-
-**2. Start the proxy**
-
-```bash
-docker run -d \
-  -p 4000:4000 \
-  -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
-  -v $(pwd)/config.yaml:/app/config.yaml \
-  ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.80.0-stable.opus-4-6 \
-  --config /app/config.yaml
-```
-
-**3. Test it!**
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
---header 'Content-Type: application/json' \
---header 'Authorization: Bearer $LITELLM_KEY' \
---data '{
-  "model": "claude-opus-4-6",
-  "messages": [
-    {
-      "role": "user",
-      "content": "what llm are you"
-    }
-  ]
-}'
-```
-
-</TabItem>
-</Tabs>
-
-## Usage - Azure
-
-<Tabs>
-<TabItem value="proxy" label="LiteLLM Proxy">
-
-**1. Setup config.yaml**
-
-```yaml
-model_list:
-  - model_name: claude-opus-4-6
-    litellm_params:
-      model: azure_ai/claude-opus-4-6
-      api_key: os.environ/AZURE_AI_API_KEY
-      api_base: os.environ/AZURE_AI_API_BASE  # https://<resource>.services.ai.azure.com
-```
-
-**2. Start the proxy**
-
-```bash
-docker run -d \
-  -p 4000:4000 \
-  -e AZURE_AI_API_KEY=$AZURE_AI_API_KEY \
-  -e AZURE_AI_API_BASE=$AZURE_AI_API_BASE \
-  -v $(pwd)/config.yaml:/app/config.yaml \
-  ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.80.0-stable.opus-4-6 \
-  --config /app/config.yaml
-```
-
-**3. Test it!**
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
---header 'Content-Type: application/json' \
---header 'Authorization: Bearer $LITELLM_KEY' \
---data '{
-  "model": "claude-opus-4-6",
-  "messages": [
-    {
-      "role": "user",
-      "content": "what llm are you"
-    }
-  ]
-}'
-```
-
-</TabItem>
-</Tabs>
-
-## Usage - Vertex AI
-
-<Tabs>
-<TabItem value="proxy" label="LiteLLM Proxy">
-
-**1. Setup config.yaml**
-
-```yaml
-model_list:
-  - model_name: claude-opus-4-6
-    litellm_params:
-      model: vertex_ai/claude-opus-4-6
-      vertex_project: os.environ/VERTEX_PROJECT
-      vertex_location: us-east5
-```
-
-**2. Start the proxy**
-
-```bash
-docker run -d \
-  -p 4000:4000 \
-  -e VERTEX_PROJECT=$VERTEX_PROJECT \
-  -e GOOGLE_APPLICATION_CREDENTIALS=/app/credentials.json \
-  -v $(pwd)/config.yaml:/app/config.yaml \
-  -v $(pwd)/credentials.json:/app/credentials.json \
-  ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.80.0-stable.opus-4-6 \
-  --config /app/config.yaml
-```
-
-**3. Test it!**
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
---header 'Content-Type: application/json' \
---header 'Authorization: Bearer $LITELLM_KEY' \
---data '{
-  "model": "claude-opus-4-6",
-  "messages": [
-    {
-      "role": "user",
-      "content": "what llm are you"
-    }
-  ]
-}'
-```
-
-</TabItem>
-</Tabs>
-
-## Usage - Bedrock
-
-<Tabs>
-<TabItem value="proxy" label="LiteLLM Proxy">
-
-**1. Setup config.yaml**
-
-```yaml
-model_list:
-  - model_name: claude-opus-4-6
-    litellm_params:
-      model: bedrock/anthropic.claude-opus-4-6-v1
-      aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
-      aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
-      aws_region_name: us-east-1
-```
-
-**2. Start the proxy**
-
-```bash
-docker run -d \
-  -p 4000:4000 \
-  -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
-  -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
-  -v $(pwd)/config.yaml:/app/config.yaml \
-  ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.80.0-stable.opus-4-6 \
-  --config /app/config.yaml
-```
-
-**3. Test it!**
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
---header 'Content-Type: application/json' \
---header 'Authorization: Bearer $LITELLM_KEY' \
---data '{
-  "model": "claude-opus-4-6",
-  "messages": [
-    {
-      "role": "user",
-      "content": "what llm are you"
-    }
-  ]
-}'
-```
-
-</TabItem>
-</Tabs>
-
-## Advanced Features
-
-### Compaction
-
-<Tabs>
-<TabItem value="completions" label="/chat/completions">
-
-Litellm supports enabling compaction for the new claude-opus-4-6.
-
-**Enabling Compaction**
-
-To enable compaction, add the `context_management` parameter with the `compact_20260112` edit type:
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
---header 'Content-Type: application/json' \
---header 'Authorization: Bearer $LITELLM_KEY' \
---data '{
-  "model": "claude-opus-4-6",
-  "messages": [
-    {
-      "role": "user",
-      "content": "What is the weather in San Francisco?"
-    }
-  ],
-  "context_management": {
-    "edits": [
-      {
-        "type": "compact_20260112"
-      }
-    ]
-  },
-  "max_tokens": 100
-}'
-```
-All the parameters supported for context_management by anthropic are supported and can be directly added. Litellm automatically adds the `compact-2026-01-12` beta header in the request.
-
-</TabItem>
-<TabItem value="messages" label="/v1/messages">
-
-Enable compaction to reduce context size while preserving key information. LiteLLM automatically adds the `compact-2026-01-12` beta header when compaction is enabled.
-
-:::info
-**Provider Support:** Compaction is supported on Anthropic, Azure AI, and Vertex AI. It is **not supported** on Bedrock (Invoke or Converse APIs).
-:::
-
-```bash
-curl --location 'http://0.0.0.0:4000/v1/messages' \
---header 'x-api-key: sk-12345' \
---header 'content-type: application/json' \
---data '{
-    "model": "claude-opus-4-6",
-    "max_tokens": 4096,
-    "messages": [
-        {
-            "role": "user",
-            "content": "Hi"
-        }
-    ],
-    "context_management": {
-        "edits": [
-            {
-                "type": "compact_20260112"
-            }
-        ]
-    }
-}'
-```
-
-</TabItem>
-</Tabs>
-
-
-**Response with Compaction Block**
-
-The response will include the compaction summary in `provider_specific_fields.compaction_blocks`:
-
-```json
-{
-  "id": "chatcmpl-a6c105a3-4b25-419e-9551-c800633b6cb2",
-  "created": 1770357619,
-  "model": "claude-opus-4-6",
-  "object": "chat.completion",
-  "choices": [
-    {
-      "finish_reason": "length",
-      "index": 0,
-      "message": {
-        "content": "I don't have access to real-time data, so I can't provide the current weather in San Francisco. To get up-to-date weather information, I'd recommend checking:\n\n- **Weather websites** like weather.com, accuweather.com, or wunderground.com\n- **Search engines** – just Google \"San Francisco weather\"\n- **Weather apps** on your phone (e.g., Apple Weather, Google Weather)\n- **National",
-        "role": "assistant",
-        "provider_specific_fields": {
-          "compaction_blocks": [
-            {
-              "type": "compaction",
-              "content": "Summary of the conversation: The user requested help building a web scraper..."
-            }
-          ]
-        }
-      }
-    }
-  ],
-  "usage": {
-    "completion_tokens": 100,
-    "prompt_tokens": 86,
-    "total_tokens": 186
-  }
-}
-```
-
-**Using Compaction Blocks in Follow-up Requests**
-
-To continue the conversation with compaction, include the compaction block in the assistant message's `provider_specific_fields`:
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
---header 'Content-Type: application/json' \
---header 'Authorization: Bearer $LITELLM_KEY' \
---data '{
-  "model": "claude-opus-4-6",
-  "messages": [
-    {
-      "role": "user",
-      "content": "How can I build a web scraper?"
-    },
-    {
-      "role": "assistant",
-      "content": [
-        {
-          "type": "text",
-          "text": "Certainly! To build a basic web scraper, you'll typically use a programming language like Python along with libraries such as `requests` (for fetching web pages) and `BeautifulSoup` (for parsing HTML). Here's a basic example:\n\n```python\nimport requests\nfrom bs4 import BeautifulSoup\n\nurl = 'https://example.com'\nresponse = requests.get(url)\nsoup = BeautifulSoup(response.text, 'html.parser')\n\n# Extract and print all text\ntext = soup.get_text()\nprint(text)\n```\n\nLet me know what you're interested in scraping or if you need help with a specific website!"
-        }
-      ],
-      "provider_specific_fields": {
-        "compaction_blocks": [
-          {
-            "type": "compaction",
-            "content": "Summary of the conversation: The user asked how to build a web scraper, and the assistant gave an overview using Python with requests and BeautifulSoup."
-          }
-        ]
-      }
-    },
-    {
-      "role": "user",
-      "content": "How do I use it to scrape product prices?"
-    }
-  ],
-  "context_management": {
-    "edits": [
-      {
-        "type": "compact_20260112"
-      }
-    ]
-  },
-  "max_tokens": 100
-}'
-```
-
-**Streaming Support**
-
-Compaction blocks are also supported in streaming mode. You'll receive:
-- `compaction_start` event when a compaction block begins
-- `compaction_delta` events with the compaction content
-- The accumulated `compaction_blocks` in `provider_specific_fields`
-
-### Adaptive Thinking
-
-:::note
-When using `reasoning_effort` with Claude Opus 4.6, all values (`low`, `medium`, `high`) are mapped to `thinking: {type: "adaptive"}`. To use explicit thinking budgets with `type: "enabled"`, pass the native `thinking` parameter directly (see "Native thinking param" tab below).
-:::
-
-<Tabs>
-<TabItem value="completions" label="/chat/completions">
-
-LiteLLM supports adaptive thinking through the `reasoning_effort` parameter:
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
---header 'Content-Type: application/json' \
---header 'Authorization: Bearer $LITELLM_KEY' \
---data '{
-  "model": "claude-opus-4-6",
-  "messages": [
-    {
-      "role": "user",
-      "content": "Solve this complex problem: What is the optimal strategy for..."
-    }
-  ],
-  "reasoning_effort": "high"
-}'
-```
-
-</TabItem>
-<TabItem value="messages" label="/v1/messages">
-
-Use the `thinking` parameter with `type: "adaptive"` to enable adaptive thinking mode:
-
-```bash
-curl --location 'http://0.0.0.0:4000/v1/messages' \
---header 'x-api-key: sk-12345' \
---header 'content-type: application/json' \
---data '{
-    "model": "claude-opus-4-6",
-    "max_tokens": 16000,
-    "thinking": {
-        "type": "adaptive"
-    },
-    "messages": [
-        {
-            "role": "user",
-            "content": "Explain why the sum of two even numbers is always even."
-        }
-    ]
-}'
-```
-
-</TabItem>
-<TabItem value="native" label="Native thinking param">
-
-Use the `thinking` parameter directly for adaptive thinking via the SDK:
-
-```python
-import litellm
-
-response = litellm.completion(
-  model="anthropic/claude-opus-4-6",
-  messages=[{"role": "user", "content": "Solve this complex problem: What is the optimal strategy for..."}],
-  thinking={"type": "adaptive"},
-)
-```
-
-</TabItem>
-</Tabs>
-
-### Effort Levels
-
-<Tabs>
-<TabItem value="completions" label="/chat/completions">
-
-Four effort levels available: `low`, `medium`, `high` (default), and `max`. Pass directly via the `output_config` parameter:
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
---header 'Content-Type: application/json' \
---header 'Authorization: Bearer $LITELLM_KEY' \
---data '{
-  "model": "claude-opus-4-6",
-  "messages": [
-    {
-      "role": "user",
-      "content": "Explain quantum computing"
-    }
-  ],
-  "output_config": {
-        "effort": "medium"
-    }
-}'
-```
-
-You can use reasoning effort plus output_config to have more control on the model.
-
-</TabItem>
-<TabItem value="messages" label="/v1/messages">
-
-Four effort levels available: `low`, `medium`, `high` (default), and `max`. Pass directly via the `output_config` parameter:
-
-```bash
-curl --location 'http://0.0.0.0:4000/v1/messages' \
---header 'x-api-key: sk-12345' \
---header 'content-type: application/json' \
---data '{
-    "model": "claude-opus-4-6",
-    "max_tokens": 4096,
-    "messages": [
-        {
-            "role": "user",
-            "content": "Explain quantum computing"
-        }
-    ],
-    "output_config": {
-        "effort": "medium"
-    }
-}'
-```
-
-</TabItem>
-</Tabs>
-
-### 1M Token Context (Beta)
-
-Opus 4.6 supports 1M token context. Premium pricing applies for prompts exceeding 200k tokens ($10/$37.50 per million input/output tokens). LiteLLM supports cost calculations for 1M token contexts.
-
-<Tabs>
-<TabItem value="completions" label="/chat/completions">
-
-To use the 1M token context window, you need to forward the `anthropic-beta` header from your client to the LLM provider.
-
-**Step 1: Enable header forwarding in your config**
-
-```yaml
-general_settings:
-  forward_client_headers_to_llm_api: true
-```
-
-**Step 2: Send requests with the beta header**
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
---header 'Content-Type: application/json' \
---header 'Authorization: Bearer $LITELLM_KEY' \
---header 'anthropic-beta: context-1m-2025-08-07' \
---data '{
-  "model": "claude-opus-4-6",
-  "messages": [
-    {
-      "role": "user",
-      "content": "Analyze this large document..."
-    }
-  ]
-}'
-```
-
-</TabItem>
-<TabItem value="messages" label="/v1/messages">
-
-To use the 1M token context window, you need to forward the `anthropic-beta` header from your client to the LLM provider.
-
-**Step 1: Enable header forwarding in your config**
-
-```yaml
-general_settings:
-  forward_client_headers_to_llm_api: true
-```
-
-**Step 2: Send requests with the beta header**
-
-```bash
-curl --location 'http://0.0.0.0:4000/v1/messages' \
---header 'x-api-key: sk-12345' \
---header 'anthropic-beta: context-1m-2025-08-07' \
---header 'content-type: application/json' \
---data '{
-    "model": "claude-opus-4-6",
-    "max_tokens": 16000,
-    "messages": [
-        {
-            "role": "user",
-            "content": "Analyze this large document..."
-        }
-    ]
-}'
-```
-
-:::tip
-You can combine multiple beta headers by separating them with commas:
-```bash
---header 'anthropic-beta: context-1m-2025-08-07,compact-2026-01-12'
-```
-:::
-
-</TabItem>
-</Tabs>
-
-### US-Only Inference
-
-Available at 1.1× token pricing. LiteLLM automatically tracks costs for US-only inference.
-
-<Tabs>
-<TabItem value="completions" label="/chat/completions">
-
-Use the `inference_geo` parameter to specify US-only inference:
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
---header 'Content-Type: application/json' \
---header 'Authorization: Bearer $LITELLM_KEY' \
---data '{
-  "model": "claude-opus-4-6",
-  "messages": [
-    {
-      "role": "user",
-      "content": "What is the capital of France?"
-    }
-  ],
-  "inference_geo": "us"
-}'
-```
-
-LiteLLM will automatically apply the 1.1× pricing multiplier for US-only inference in cost tracking.
-
-</TabItem>
-<TabItem value="messages" label="/v1/messages">
-
-Use the `inference_geo` parameter to specify US-only inference:
-
-```bash
-curl --location 'http://0.0.0.0:4000/v1/messages' \
---header 'x-api-key: sk-12345' \
---header 'content-type: application/json' \
---data '{
-    "model": "claude-opus-4-6",
-    "max_tokens": 4096,
-    "messages": [
-        {
-            "role": "user",
-            "content": "What is the capital of France?"
-        }
-    ],
-    "inference_geo": "us"
-}'
-```
-
-LiteLLM will automatically apply the 1.1× pricing multiplier for US-only inference in cost tracking.
-
-</TabItem>
-</Tabs>
-
-### Fast Mode
-
-:::info
-Fast mode is **only supported on the Anthropic provider** (`anthropic/claude-opus-4-6`). It is not available on Azure AI, Vertex AI, or Bedrock.
-:::
-
-**Pricing:**
-- Standard: $5 input / $25 output per MTok
-- Fast: $30 input / $150 output per MTok (6× premium)
-
-<Tabs>
-<TabItem value="completions" label="/chat/completions">
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
---header 'Content-Type: application/json' \
---header 'Authorization: Bearer $LITELLM_KEY' \
---data '{
-  "model": "claude-opus-4-6",
-  "messages": [
-    {
-      "role": "user",
-      "content": "Refactor this module..."
-    }
-  ],
-  "max_tokens": 4096,
-  "speed": "fast"
-}'
-```
-
-**Using OpenAI SDK:**
-
-```python
-import openai
-
-client = openai.OpenAI(
-    api_key="your-litellm-key",
-    base_url="http://0.0.0.0:4000"
-)
-
-response = client.chat.completions.create(
-    model="claude-opus-4-6",
-    messages=[{"role": "user", "content": "Refactor this module..."}],
-    max_tokens=4096,
-    extra_body={"speed": "fast"}
-)
-```
-
-**Using LiteLLM SDK:**
-
-```python
-from litellm import completion
-
-response = completion(
-    model="anthropic/claude-opus-4-6",
-    messages=[{"role": "user", "content": "Refactor this module..."}],
-    max_tokens=4096,
-    speed="fast"
-)
-```
-
-LiteLLM automatically tracks the higher costs for fast mode in usage and cost calculations.
-
-</TabItem>
-<TabItem value="messages" label="/v1/messages">
-
-```bash
-curl --location 'http://0.0.0.0:4000/v1/messages' \
---header 'x-api-key: sk-12345' \
---header 'content-type: application/json' \
---data '{
-    "model": "claude-opus-4-6",
-    "max_tokens": 4096,
-    "speed": "fast",
-    "messages": [
-        {
-            "role": "user",
-            "content": "Refactor this module..."
-        }
-    ]
-}'
-```
-
-LiteLLM automatically:
-- Adds the `fast-mode-2026-02-01` beta header
-- Tracks the 6× premium pricing in cost calculations
-
-</TabItem>
-</Tabs>

docs/my-website/blog/claude_sonnet_4_6/index.md

@@ -1,279 +0,0 @@
----
-slug: claude_sonnet_4_6
-title: "Day 0 Support: Claude Sonnet 4.6"
-date: 2026-02-17T10:00:00
-authors:
-  - ishaan-alt
-  - krrish
-description: "Day 0 support for Claude Sonnet 4.6 on LiteLLM AI Gateway - use across Anthropic, Azure, Vertex AI, and Bedrock."
-tags: [anthropic, claude, sonnet 4.6]
-hide_table_of_contents: false
----
-
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-LiteLLM now supports Claude Sonnet 4.6 on Day 0. Use it across Anthropic, Azure, Vertex AI, and Bedrock through the LiteLLM AI Gateway.
-
-{/* truncate */}
-
-## Docker Image
-
-```bash
-docker pull ghcr.io/berriai/litellm:v1.81.3-stable.sonnet-4-6
-```
-
-## Usage - Anthropic
-
-<Tabs>
-<TabItem value="proxy" label="LiteLLM Proxy">
-
-**1. Setup config.yaml**
-
-```yaml
-model_list:
-  - model_name: claude-sonnet-4-6
-    litellm_params:
-      model: anthropic/claude-sonnet-4-6
-      api_key: os.environ/ANTHROPIC_API_KEY
-```
-
-**2. Start the proxy**
-
-```bash
-docker run -d \
-  -p 4000:4000 \
-  -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
-  -v $(pwd)/config.yaml:/app/config.yaml \
-  ghcr.io/berriai/litellm:v1.81.3-stable.sonnet-4-6 \
-  --config /app/config.yaml
-```
-
-**3. Test it!**
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
---header 'Content-Type: application/json' \
---header 'Authorization: Bearer $LITELLM_KEY' \
---data '{
-  "model": "claude-sonnet-4-6",
-  "messages": [
-    {
-      "role": "user",
-      "content": "what llm are you"
-    }
-  ]
-}'
-```
-
-</TabItem>
-
-<TabItem value="sdk" label="LiteLLM SDK">
-
-```python
-from litellm import completion
-
-response = completion(
-    model="anthropic/claude-sonnet-4-6",
-    messages=[{"role": "user", "content": "what llm are you"}]
-)
-print(response.choices[0].message.content)
-```
-
-</TabItem>
-</Tabs>
-
-## Usage - Azure
-
-<Tabs>
-<TabItem value="proxy" label="LiteLLM Proxy">
-
-**1. Setup config.yaml**
-
-```yaml
-model_list:
-  - model_name: claude-sonnet-4-6
-    litellm_params:
-      model: azure_ai/claude-sonnet-4-6
-      api_key: os.environ/AZURE_AI_API_KEY
-      api_base: os.environ/AZURE_AI_API_BASE  # https://<resource>.services.ai.azure.com
-```
-
-**2. Start the proxy**
-
-```bash
-docker run -d \
-  -p 4000:4000 \
-  -e AZURE_AI_API_KEY=$AZURE_AI_API_KEY \
-  -e AZURE_AI_API_BASE=$AZURE_AI_API_BASE \
-  -v $(pwd)/config.yaml:/app/config.yaml \
-  ghcr.io/berriai/litellm:v1.81.3-stable.sonnet-4-6 \
-  --config /app/config.yaml
-```
-
-**3. Test it!**
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
---header 'Content-Type: application/json' \
---header 'Authorization: Bearer $LITELLM_KEY' \
---data '{
-  "model": "claude-sonnet-4-6",
-  "messages": [
-    {
-      "role": "user",
-      "content": "what llm are you"
-    }
-  ]
-}'
-```
-
-</TabItem>
-
-<TabItem value="sdk" label="LiteLLM SDK">
-
-```python
-from litellm import completion
-
-response = completion(
-    model="azure_ai/claude-sonnet-4-6",
-    api_key="your-azure-api-key",
-    api_base="https://<resource>.services.ai.azure.com",
-    messages=[{"role": "user", "content": "what llm are you"}]
-)
-print(response.choices[0].message.content)
-```
-
-</TabItem>
-</Tabs>
-
-## Usage - Vertex AI
-
-<Tabs>
-<TabItem value="proxy" label="LiteLLM Proxy">
-
-**1. Setup config.yaml**
-
-```yaml
-model_list:
-  - model_name: claude-sonnet-4-6
-    litellm_params:
-      model: vertex_ai/claude-sonnet-4-6
-      vertex_project: os.environ/VERTEX_PROJECT
-      vertex_location: us-east5
-```
-
-**2. Start the proxy**
-
-```bash
-docker run -d \
-  -p 4000:4000 \
-  -e VERTEX_PROJECT=$VERTEX_PROJECT \
-  -e GOOGLE_APPLICATION_CREDENTIALS=/app/credentials.json \
-  -v $(pwd)/config.yaml:/app/config.yaml \
-  -v $(pwd)/credentials.json:/app/credentials.json \
-  ghcr.io/berriai/litellm:v1.81.3-stable.sonnet-4-6 \
-  --config /app/config.yaml
-```
-
-**3. Test it!**
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
---header 'Content-Type: application/json' \
---header 'Authorization: Bearer $LITELLM_KEY' \
---data '{
-  "model": "claude-sonnet-4-6",
-  "messages": [
-    {
-      "role": "user",
-      "content": "what llm are you"
-    }
-  ]
-}'
-```
-
-</TabItem>
-
-<TabItem value="sdk" label="LiteLLM SDK">
-
-```python
-from litellm import completion
-
-response = completion(
-    model="vertex_ai/claude-sonnet-4-6",
-    vertex_project="your-project-id",
-    vertex_location="us-east5",
-    messages=[{"role": "user", "content": "what llm are you"}]
-)
-print(response.choices[0].message.content)
-```
-
-</TabItem>
-</Tabs>
-
-## Usage - Bedrock
-
-<Tabs>
-<TabItem value="proxy" label="LiteLLM Proxy">
-
-**1. Setup config.yaml**
-
-```yaml
-model_list:
-  - model_name: claude-sonnet-4-6
-    litellm_params:
-      model: bedrock/anthropic.claude-sonnet-4-6-v1
-      aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
-      aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
-      aws_region_name: us-east-1
-```
-
-**2. Start the proxy**
-
-```bash
-docker run -d \
-  -p 4000:4000 \
-  -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
-  -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
-  -v $(pwd)/config.yaml:/app/config.yaml \
-  ghcr.io/berriai/litellm:v1.81.3-stable.sonnet-4-6 \
-  --config /app/config.yaml
-```
-
-**3. Test it!**
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
---header 'Content-Type: application/json' \
---header 'Authorization: Bearer $LITELLM_KEY' \
---data '{
-  "model": "claude-sonnet-4-6",
-  "messages": [
-    {
-      "role": "user",
-      "content": "what llm are you"
-    }
-  ]
-}'
-```
-
-</TabItem>
-
-<TabItem value="sdk" label="LiteLLM SDK">
-
-```python
-from litellm import completion
-
-response = completion(
-    model="bedrock/anthropic.claude-sonnet-4-6-v1",
-    aws_access_key_id="your-access-key",
-    aws_secret_access_key="your-secret-key",
-    aws_region_name="us-east-1",
-    messages=[{"role": "user", "content": "what llm are you"}]
-)
-print(response.choices[0].message.content)
-```
-
-</TabItem>
-</Tabs>

docs/my-website/blog/fastapi_middleware_performance/index.mdx

@@ -1,211 +0,0 @@
----
-slug: fastapi-middleware-performance
-title: "Your Middleware Could Be a Bottleneck"
-date: 2026-02-07T10:00:00
-authors:
-  - krrish
-  - ishaan-alt
-  - ryan
-description: "How we improved LiteLLM proxy latency and throughput by replacing a single middleware base class"
-tags: [performance, fastapi, middleware]
-hide_table_of_contents: false
----
-
-import { BaseHTTPMiddlewareAnimation, PureASGIAnimation, BenchmarkVisualization } from '@site/src/components/MiddlewareDiagrams';
-
-> How we improved LiteLLM proxy latency and throughput by replacing a single, simple middleware base class
-
----
-
-## Our Setup
-
-The LiteLLM proxy server has two middleware layers. The first is Starlette's `CORSMiddleware` (re-exported by FastAPI), which is a pure ASGI middleware. Then we have a simple BaseHTTPMiddleware called PrometheusAuthMiddleware.
-
-The job of `PrometheusAuthMiddleware` is to authenticate requests to the `/metrics` endpoint. It's not on by default, you enable it with a flag in your proxy config:
-
-<details>
-<summary>Proxy config flag</summary>
-
-```yaml
-litellm_settings:
-    require_auth_for_metrics_endpoint: true
-```
-
-</details>
-
-The middleware checks two things: is the request hitting `/metrics`, and is auth even enabled? If both checks fail, which they do for the vast majority of requests, it just passes the request through unchanged.
-
-<details>
-<summary>PrometheusAuthMiddleware source</summary>
-
-```python
-class PrometheusAuthMiddleware(BaseHTTPMiddleware):
-    async def dispatch(self, request: Request, call_next):
-        if self._is_prometheus_metrics_endpoint(request):
-            if self._should_run_auth_on_metrics_endpoint() is True:
-                try:
-                    await user_api_key_auth(request=request, api_key=...)
-                except Exception as e:
-                    return JSONResponse(status_code=401, content=...)
-        response = await call_next(request)
-        return response
-
-    @staticmethod
-    def _is_prometheus_metrics_endpoint(request: Request):
-        if "/metrics" in request.url.path:
-            return True
-        return False
-```
-
-</details>
-
-Looks harmless. Subclass `BaseHTTPMiddleware`, implement `dispatch()`, done. This is what you will see in Starlette's documentation<sup>[1](#footnote-1)</sup>.
-
-{/* truncate */}
-
----
-
-## What BaseHTTPMiddleware Actually Does
-
-When you write a `dispatch()` method, you'd expect the request to flow straight through your function and out the other side. What actually happens is much more involved. 
-
-On every request, even a pure passthrough (meaning nothing happens), `BaseHTTPMiddleware` creates **7 intermediate objects and tasks**:
-
-<BaseHTTPMiddlewareAnimation />
-
-It wraps the request in a new object to track body state, creates a synchronization event, allocates an in-memory channel to pass messages between your middleware and the inner app, sets up a task group to manage the lifecycle, and then runs your actual route handler in a *separate background task* when you call `call_next()`. The response body then flows back through that in-memory channel, gets re-wrapped in a streaming response object, and finally reaches the caller. That's a lot.
-
-For a middleware that for us, does nothing on 99.9% of requests, paying this cost doesn't make sense.
-
-Compare that to a pure ASGI middleware, which we can have just check the request path and continue along. 
-
-<PureASGIAnimation />
-
-Our middleware is doing something really simple. For the vast majority of requests it doesn't need to do anything at all but just let the request pass through. It doesn't need task groups, memory streams, or cancel scopes. It needs a function call.
-
----
-
-## Comparing Both
-
-We replaced the `BaseHTTPMiddleware` subclass with a pure ASGI middleware. To benchmark the difference, we used Apache Bench<sup>[2](#footnote-2)</sup> to compare both configurations of LiteLLM's middleware stack: the old setup (1 pure ASGI + 1 `BaseHTTPMiddleware`) against the new setup (2 pure ASGI).
-
-A minimal FastAPI app serves `GET /health` → `PlainTextResponse("ok")`. The endpoint does zero work to isolate the middleware overhead: any difference between configs is purely the cost of the middleware plumbing itself. Both middlewares are just calling the next layer. Same work, different base class.
-
-Apache Bench (`ab`) fires requests at the server with 1,000 concurrent connections and a single uvicorn worker. One worker means one event loop, so the benchmark directly measures how each middleware design handles concurrent load on a single thread.
-
-<BenchmarkVisualization />
-
-<details>
-<summary>Try it yourself</summary>
-
-Save the script below as `benchmark_middleware.py`, then run:
-
-```bash
-# Terminal 1 — start the "before" server (1 ASGI + 1 BaseHTTPMiddleware)
-python benchmark_middleware.py --middleware mixed
-
-# Terminal 2 — benchmark it
-ab -n 50000 -c 1000 http://localhost:8000/health
-
-# Stop the server, then start the "after" server (2x pure ASGI)
-python benchmark_middleware.py --middleware asgi
-
-# Terminal 2 — benchmark again
-ab -n 50000 -c 1000 http://localhost:8000/health
-```
-
-```python
-import argparse
-import uvicorn
-from fastapi import FastAPI
-from fastapi.responses import PlainTextResponse
-from starlette.middleware.base import BaseHTTPMiddleware
-from starlette.requests import Request
-from starlette.types import ASGIApp, Receive, Scope, Send
-
-
-class NoOpBaseHTTPMiddleware(BaseHTTPMiddleware):
-    async def dispatch(self, request: Request, call_next):
-        return await call_next(request)
-
-
-class NoOpPureASGIMiddleware:
-    def __init__(self, app: ASGIApp) -> None:
-        self.app = app
-
-    async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
-        await self.app(scope, receive, send)
-
-
-def create_app(middleware_type: str | None = None, layers: int = 2) -> FastAPI:
-    app = FastAPI()
-
-    @app.get("/health")
-    async def health():
-        return PlainTextResponse("ok")
-
-    if middleware_type == "mixed":
-        app.add_middleware(NoOpBaseHTTPMiddleware)
-        app.add_middleware(NoOpPureASGIMiddleware)
-    elif middleware_type == "asgi":
-        for _ in range(layers):
-            app.add_middleware(NoOpPureASGIMiddleware)
-
-    return app
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--middleware", choices=["asgi", "mixed"], default=None)
-    parser.add_argument("--layers", type=int, default=2)
-    parser.add_argument("--port", type=int, default=8000)
-    args = parser.parse_args()
-
-    app = create_app(middleware_type=args.middleware, layers=args.layers)
-    uvicorn.run(app, host="0.0.0.0", port=args.port, workers=1, log_level="warning")
-```
-
-</details>
-
----
-
-## Our Change
-
-Here's what we replaced it with:
-
-```python
-class PrometheusAuthMiddleware:
-    def __init__(self, app: ASGIApp) -> None:
-        self.app = app
-
-    async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
-        if scope["type"] != "http" or "/metrics" not in scope.get("path", ""):
-            await self.app(scope, receive, send)
-            return
-
-        if litellm.require_auth_for_metrics_endpoint is True:
-            request = Request(scope, receive)
-            api_key = request.headers.get("Authorization") or ""
-            try:
-                await user_api_key_auth(request=request, api_key=api_key)
-            except Exception as e:
-                # send 401 directly via ASGI protocol
-                ...
-                return
-
-        await self.app(scope, receive, send)
-```
-
-For the 99.9% of requests that aren't hitting `/metrics`, the middleware is now one dict lookup, one string check, and one function call. No objects allocated, no tasks spawned. 
-
-It's important to evaluate if the tools you're using are the right fit for the job as your software grows and handles more responsiblity. We're now putting in a static analysis check to prevent this from happening again with any newly introduced middlewares. If we find the use case is necessary then that's okay and we'll reevalute but for everything LiteLLM needs to do at the moment it's not. 
-
-This middleware change was one part of a broader optimization effort on the LiteLLM proxy. Across all optimizations combined, we've measured about a **30% reduction in proxy overhead** over the past two weeks. 
-
----
-
-<a id="footnote-1"></a>
-<sup>1</sup> [Starlette Middleware — BaseHTTPMiddleware](https://starlette.dev/middleware/#basehttpmiddleware)
-
-<a id="footnote-2"></a>
-<sup>2</sup> [Apache HTTP server benchmarking tool (`ab`)](https://httpd.apache.org/docs/2.4/programs/ab.html)

docs/my-website/blog/gemin_3.1/index.md

@@ -1,142 +0,0 @@
----
-slug: gemini_3_1_pro
-title: "DAY 0 Support: Gemini 3.1 Pro on LiteLLM"
-date: 2026-02-19T10:00:00
-authors:
-  - sameer
-  - krrish
-  - ishaan-alt
-description: "Guide to using Gemini 3.1 Pro on LiteLLM Proxy and SDK with day 0 support."
-tags: [gemini, day 0 support, llms]
-hide_table_of_contents: false
----
-
-
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-# Gemini 3.1 Pro Day 0 Support 
-
-LiteLLM now supports `gemini-3.1-pro-preview` and all the new API changes along with it.
-
-{/* truncate */}
-
-## Deploy this version
-
-<Tabs>
-<TabItem value="docker" label="Docker">
-
-``` showLineNumbers title="docker run litellm"
-docker run \
--e STORE_MODEL_IN_DB=True \
--p 4000:4000 \
-ghcr.io/berriai/litellm:main-v1.81.9-stable.gemini.3.1-pro
-```
-
-</TabItem>
-
-<TabItem value="pip" label="Pip">
-
-``` showLineNumbers title="pip install litellm"
-pip install litellm==v1.81.9-stable.gemini.3.1-pro
-```
-
-</TabItem>
-</Tabs>
-
-## What's New
-
-### 1. New Thinking Levels: `thinkingLevel` with MINIMAL & MEDIUM
-
-Gemini 3.1 Pro introduces support for **medium** thinking level
-
-LiteLLM automatically maps the OpenAI `reasoning_effort` parameter to Gemini's `thinkingLevel`, so you can use familiar `reasoning_effort` values (`minimal`, `low`, `medium`, `high`) without changing your code!
-
----
-## Supported Endpoints
-
-LiteLLM provides **full end-to-end support** for Gemini 3.1 Pro on:
-
-- ✅ `/v1/chat/completions` - OpenAI-compatible chat completions endpoint
-- ✅ `/v1/responses` - OpenAI Responses API endpoint (streaming and non-streaming)
-- ✅ [`/v1/messages`](../../docs/anthropic_unified) - Anthropic-compatible messages endpoint
-- ✅ `/v1/generateContent` – [Google Gemini API](../../docs/generateContent) compatible endpoint 
-
-All endpoints support:
-- Streaming and non-streaming responses
-- Function calling with thought signatures
-- Multi-turn conversations
-- All Gemini 3-specific features
-- Conversion of provider specific thinking related param to thinkingLevel
-
-## Quick Start
-
-<Tabs>
-<TabItem value="sdk" label="SDK">
-
-**Basic Usage with MEDIUM thinking (NEW)**
-
-```python
-from litellm import completion
-
-# No need to make any changes to your code as we map openai reasoning param to thinkingLevel
-response = completion(
-    model="gemini/gemini-3.1-pro-preview",
-    messages=[{"role": "user", "content": "Solve this complex math problem: 25 * 4 + 10"}],
-    reasoning_effort="medium",  # NEW: MEDIUM thinking level
-)
-
-print(response.choices[0].message.content)
-```
-
-</TabItem>
-
-<TabItem value="proxy" label="PROXY">
-
-**1. Setup config.yaml**
-
-```yaml
-model_list:
-  - model_name: gemini-3.1-pro-preview
-    litellm_params:
-      model: gemini/gemini-3.1-pro-preview
-      api_key: os.environ/GEMINI_API_KEY
-  - model_name: vertex-gemini-3.1-pro-preview
-    litellm_params:
-      model: vertex_ai/gemini-3.1-pro-preview
-```
-
-**2. Start proxy**
-
-```bash
-litellm --config /path/to/config.yaml
-```
-
-**3. Call with MEDIUM thinking**
-
-```bash
-curl -X POST http://localhost:4000/v1/chat/completions \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer <YOUR-LITELLM-KEY>" \
-  -d '{
-    "model": "gemini-3.1-pro-preview",
-    "messages": [{"role": "user", "content": "Complex reasoning task"}],
-    "reasoning_effort": "medium"
-  }'
-```
-
-</TabItem>
-</Tabs>
-
----
-
-## `reasoning_effort` Mapping for Gemini 3+
-
-| reasoning_effort | thinking_level | 
-|------------------|----------------|
-| `minimal` | `minimal` |
-| `low` | `low` |
-| `medium` | `medium` |
-| `high` | `high` |
-| `disable` | `minimal` |
-| `none` | `minimal` |

docs/my-website/blog/gemini_3/index.md

@@ -1,975 +0,0 @@
----
-slug: gemini_3
-title: "DAY 0 Support: Gemini 3 on LiteLLM"
-date: 2025-11-19T10:00:00
-authors:
-  - sameer
-  - krrish
-  - ishaan-alt
-description: "Common questions and best practices for using gemini-3-pro-preview with LiteLLM Proxy and SDK."
-tags: [gemini, day 0 support, llms]
-hide_table_of_contents: false
----
-
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-:::info
-
-This guide covers common questions and best practices for using `gemini-3-pro-preview` with LiteLLM Proxy and SDK.
-
-:::
-
-{/* truncate */}
-
-## Quick Start
-
-<Tabs>
-<TabItem value="sdk" label="Python SDK">
-
-```python
-from litellm import completion
-import os
-
-os.environ["GEMINI_API_KEY"] = "your-api-key"
-
-response = completion(
-    model="gemini/gemini-3-pro-preview",
-    messages=[{"role": "user", "content": "Hello!"}],
-    reasoning_effort="low"
-)
-
-print(response.choices[0].message.content)
-```
-
-</TabItem>
-<TabItem value="proxy" label="LiteLLM Proxy">
-
-**1. Add to config.yaml:**
-
-```yaml
-model_list:
-  - model_name: gemini-3-pro-preview
-    litellm_params:
-      model: gemini/gemini-3-pro-preview
-      api_key: os.environ/GEMINI_API_KEY
-```
-
-**2. Start proxy:**
-
-```bash
-litellm --config /path/to/config.yaml
-```
-
-**3. Make request:**
-
-```bash
-curl http://0.0.0.0:4000/v1/chat/completions \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer sk-1234" \
-  -d '{
-    "model": "gemini-3-pro-preview",
-    "messages": [{"role": "user", "content": "Hello!"}],
-    "reasoning_effort": "low"
-  }'
-```
-
-</TabItem>
-</Tabs>
-
-## Supported Endpoints
-
-LiteLLM provides **full end-to-end support** for Gemini 3 Pro Preview on:
-
-- ✅ `/v1/chat/completions` - OpenAI-compatible chat completions endpoint
-- ✅ `/v1/responses` - OpenAI Responses API endpoint (streaming and non-streaming)
-- ✅ [`/v1/messages`](../../docs/anthropic_unified) - Anthropic-compatible messages endpoint
-- ✅ `/v1/generateContent` – [Google Gemini API](https://cloud.google.com/vertex-ai/docs/generative-ai/model-reference/gemini#rest) compatible endpoint (for code, see: `client.models.generate_content(...)`)
-
-All endpoints support:
-- Streaming and non-streaming responses
-- Function calling with thought signatures
-- Multi-turn conversations
-- All Gemini 3-specific features
-
-## Thought Signatures
-
-#### What are Thought Signatures?
-
-Thought signatures are encrypted representations of the model's internal reasoning process. They're essential for maintaining context across multi-turn conversations, especially with function calling.
-
-#### How Thought Signatures Work
-
-1. **Automatic Extraction**: When Gemini 3 returns a function call, LiteLLM automatically extracts the `thought_signature` from the response
-2. **Storage**: Thought signatures are stored in `provider_specific_fields.thought_signature` of tool calls
-3. **Automatic Preservation**: When you include the assistant's message in conversation history, LiteLLM automatically preserves and returns thought signatures to Gemini
-
-## Example: Multi-Turn Function Calling
-
-#### Streaming with Thought Signatures
-
-When using streaming mode with `stream_chunk_builder()`, thought signatures are now automatically preserved:
-
-<Tabs>
-<TabItem value="streaming" label="Streaming SDK">
-
-```python
-import os
-import litellm
-from litellm import completion
-
-os.environ["GEMINI_API_KEY"] = "your-api-key"
-
-MODEL = "gemini/gemini-3-pro-preview"
-
-messages = [
-    {"role": "system", "content": "You are a helpful assistant. Use the calculate tool."},
-    {"role": "user", "content": "What is 2+2?"},
-]
-
-tools = [{
-    "type": "function",
-    "function": {
-        "name": "calculate",
-        "description": "Calculate a mathematical expression",
-        "parameters": {
-            "type": "object",
-            "properties": {"expression": {"type": "string"}},
-            "required": ["expression"],
-        },
-    },
-}]
-
-print("Step 1: Sending request with stream=True...")
-response = completion(
-    model=MODEL,
-    messages=messages,
-    stream=True,
-    tools=tools,
-    reasoning_effort="low"
-)
-
-# Collect all chunks
-chunks = []
-for part in response:
-    chunks.append(part)
-
-# Reconstruct message using stream_chunk_builder
-# Thought signatures are now preserved automatically!
-full_response = litellm.stream_chunk_builder(chunks, messages=messages)
-print(f"Full response: {full_response}")
-
-assistant_msg = full_response.choices[0].message
-
-# ✅ Thought signature is now preserved in provider_specific_fields
-if assistant_msg.tool_calls and assistant_msg.tool_calls[0].provider_specific_fields:
-    thought_sig = assistant_msg.tool_calls[0].provider_specific_fields.get("thought_signature")
-    print(f"Thought signature preserved: {thought_sig is not None}")
-
-# Append assistant message (includes thought signatures automatically)
-messages.append(assistant_msg)
-
-# Mock tool execution
-messages.append({
-    "role": "tool",
-    "content": "4",
-    "tool_call_id": assistant_msg.tool_calls[0].id
-})
-
-print("\nStep 2: Sending tool result back to model...")
-response_2 = completion(
-    model=MODEL,
-    messages=messages,
-    stream=True,
-    tools=tools,
-    reasoning_effort="low"
-)
-
-for part in response_2:
-    if part.choices[0].delta.content:
-        print(part.choices[0].delta.content, end="")
-print()  # New line
-```
-
-**Key Points:**
-- ✅ `stream_chunk_builder()` now preserves `provider_specific_fields` including thought signatures
-- ✅ Thought signatures are automatically included when appending `assistant_msg` to conversation history
-- ✅ Multi-turn conversations work seamlessly with streaming
-
-</TabItem>
-<TabItem value="sdk" label="Non-Streaming SDK">
-
-```python
-from openai import OpenAI
-import json
-
-client = OpenAI(api_key="sk-1234", base_url="http://localhost:4000")
-
-# Define tools
-tools = [
-    {
-        "type": "function",
-        "function": {
-            "name": "get_weather",
-            "description": "Get the current weather",
-            "parameters": {
-                "type": "object",
-                "properties": {
-                    "location": {"type": "string"}
-                },
-                "required": ["location"]
-            }
-        }
-    }
-]
-
-# Step 1: Initial request
-messages = [{"role": "user", "content": "What's the weather in Tokyo?"}]
-
-response = client.chat.completions.create(
-    model="gemini-3-pro-preview",
-    messages=messages,
-    tools=tools,
-    reasoning_effort="low"
-)
-
-# Step 2: Append assistant message (thought signatures automatically preserved)
-messages.append(response.choices[0].message)
-
-# Step 3: Execute tool and append result
-for tool_call in response.choices[0].message.tool_calls:
-    if tool_call.function.name == "get_weather":
-        result = {"temperature": 30, "unit": "celsius"}
-        messages.append({
-            "role": "tool",
-            "content": json.dumps(result),
-            "tool_call_id": tool_call.id
-        })
-
-# Step 4: Follow-up request (thought signatures automatically included)
-response2 = client.chat.completions.create(
-    model="gemini-3-pro-preview",
-    messages=messages,
-    tools=tools,
-    reasoning_effort="low"
-)
-
-print(response2.choices[0].message.content)
-```
-
-**Key Points:**
-- ✅ Thought signatures are automatically extracted from `response.choices[0].message.tool_calls[].provider_specific_fields.thought_signature`
-- ✅ When you append `response.choices[0].message` to your conversation history, thought signatures are automatically preserved
-- ✅ You don't need to manually extract or manage thought signatures
-
-</TabItem>
-<TabItem value="proxy" label="cURL">
-
-```bash
-# Step 1: Initial request
-curl http://localhost:4000/v1/chat/completions \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer sk-1234" \
-  -d '{
-    "model": "gemini-3-pro-preview",
-    "messages": [
-      {"role": "user", "content": "What'\''s the weather in Tokyo?"}
-    ],
-    "tools": [
-      {
-        "type": "function",
-        "function": {
-          "name": "get_weather",
-          "description": "Get the current weather",
-          "parameters": {
-            "type": "object",
-            "properties": {
-              "location": {"type": "string"}
-            },
-            "required": ["location"]
-          }
-        }
-      }
-    ],
-    "reasoning_effort": "low"
-  }'
-```
-
-**Response includes thought signature:**
-
-```json
-{
-  "choices": [{
-    "message": {
-      "role": "assistant",
-      "tool_calls": [{
-        "id": "call_abc123",
-        "type": "function",
-        "function": {
-          "name": "get_weather",
-          "arguments": "{\"location\": \"Tokyo\"}"
-        },
-        "provider_specific_fields": {
-          "thought_signature": "CpcHAdHtim9+q4rstcbvQC0ic4x1/vqQlCJWgE+UZ6dTLYGHMMBkF/AxqL5UmP6SY46uYC8t4BTFiXG5zkw6EMJ..."
-        }
-      }]
-    }
-  }]
-}
-```
-
-```bash
-# Step 2: Follow-up request (include assistant message with thought signature)
-curl http://localhost:4000/v1/chat/completions \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer sk-1234" \
-  -d '{
-    "model": "gemini-3-pro-preview",
-    "messages": [
-      {"role": "user", "content": "What'\''s the weather in Tokyo?"},
-      {
-        "role": "assistant",
-        "content": null,
-        "tool_calls": [{
-          "id": "call_abc123",
-          "type": "function",
-          "function": {
-            "name": "get_weather",
-            "arguments": "{\"location\": \"Tokyo\"}"
-          },
-          "provider_specific_fields": {
-            "thought_signature": "CpcHAdHtim9+q4rstcbvQC0ic4x1/vqQlCJWgE+UZ6dTLYGHMMBkF/AxqL5UmP6SY46uYC8t4BTFiXG5zkw6EMJ..."
-          }
-        }]
-      },
-      {
-        "role": "tool",
-        "content": "{\"temperature\": 30, \"unit\": \"celsius\"}",
-        "tool_call_id": "call_abc123"
-      }
-    ],
-    "tools": [...],
-    "reasoning_effort": "low"
-  }'
-```
-
-</TabItem>
-</Tabs>
-
-#### Important Notes on Thought Signatures
-
-1. **Automatic Handling**: LiteLLM automatically extracts and preserves thought signatures. You don't need to manually manage them.
-
-2. **Parallel Function Calls**: When the model makes parallel function calls, only the **first function call** has a thought signature.
-
-3. **Sequential Function Calls**: In multi-step function calling, each step's first function call has its own thought signature that must be preserved.
-
-4. **Required for Context**: Thought signatures are essential for maintaining reasoning context. Without them, the model may lose context of its previous reasoning.
-
-## Conversation History: Switching from Non-Gemini-3 Models
-
-#### Common Question: Will switching from a non-Gemini-3 model to Gemini-3 break conversation history?
-
-**Answer: No!** LiteLLM automatically handles this by adding dummy thought signatures when needed.
-
-#### How It Works
-
-When you switch from a model that doesn't use thought signatures (e.g., `gemini-2.5-flash`) to Gemini 3, LiteLLM:
-
-1. **Detects missing signatures**: Identifies assistant messages with tool calls that lack thought signatures
-2. **Adds dummy signature**: Automatically injects a dummy thought signature (`skip_thought_signature_validator`) for compatibility
-3. **Maintains conversation flow**: Your conversation history continues to work seamlessly
-
-#### Example: Switching Models Mid-Conversation
-
-<Tabs>
-<TabItem value="sdk" label="Python SDK">
-
-```python
-from openai import OpenAI
-
-client = OpenAI(api_key="sk-1234", base_url="http://localhost:4000")
-
-# Step 1: Start with gemini-2.5-flash (no thought signatures)
-messages = [{"role": "user", "content": "What's the weather?"}]
-
-response1 = client.chat.completions.create(
-    model="gemini-2.5-flash",
-    messages=messages,
-    tools=[...],
-    reasoning_effort="low"
-)
-
-# Append assistant message (no tool call thought signature from gemini-2.5-flash)
-messages.append(response1.choices[0].message)
-
-# Step 2: Switch to gemini-3-pro-preview
-# LiteLLM automatically adds dummy thought signature to the previous assistant message
-response2 = client.chat.completions.create(
-    model="gemini-3-pro-preview",  # 👈 Switched model
-    messages=messages,  # 👈 Same conversation history
-    tools=[...],
-    reasoning_effort="low"
-)
-
-# ✅ Works seamlessly! No errors, no breaking changes
-print(response2.choices[0].message.content)
-```
-
-</TabItem>
-<TabItem value="proxy" label="cURL">
-
-```bash
-# Step 1: Start with gemini-2.5-flash
-curl http://localhost:4000/v1/chat/completions \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer sk-1234" \
-  -d '{
-    "model": "gemini-2.5-flash",
-    "messages": [{"role": "user", "content": "What'\''s the weather?"}],
-    "tools": [...],
-    "reasoning_effort": "low"
-  }'
-
-# Step 2: Switch to gemini-3-pro-preview with same conversation history
-# LiteLLM automatically handles the missing thought signature
-curl http://localhost:4000/v1/chat/completions \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer sk-1234" \
-  -d '{
-    "model": "gemini-3-pro-preview",  # 👈 Switched model
-    "messages": [
-      {"role": "user", "content": "What'\''s the weather?"},
-      {
-        "role": "assistant",
-        "tool_calls": [...]  # 👈 No thought_signature from gemini-2.5-flash
-      }
-    ],
-    "tools": [...],
-    "reasoning_effort": "low"
-  }'
-# ✅ Works! LiteLLM adds dummy signature automatically
-```
-
-</TabItem>
-</Tabs>
-
-#### Dummy Signature Details
-
-The dummy signature used is: `base64("skip_thought_signature_validator")`
-
-This is the recommended approach by Google for handling conversation history from models that don't support thought signatures. It allows Gemini 3 to:
-- Accept the conversation history without validation errors
-- Continue the conversation seamlessly
-- Maintain context across model switches
-
-## Thinking Level Parameter
-
-#### How `reasoning_effort` Maps to `thinking_level`
-
-For Gemini 3 Pro Preview, LiteLLM automatically maps `reasoning_effort` to the new `thinking_level` parameter:
-
-| `reasoning_effort` | `thinking_level` | Notes |
-|-------------------|------------------|-------|
-| `"minimal"` | `"low"` | Maps to low thinking level |
-| `"low"` | `"low"` | Default for most use cases |
-| `"medium"` | `"high"` | Medium not available yet, maps to high |
-| `"high"` | `"high"` | Maximum reasoning depth |
-| `"disable"` | `"low"` | Gemini 3 cannot fully disable thinking |
-| `"none"` | `"low"` | Gemini 3 cannot fully disable thinking |
-
-#### Default Behavior
-
-If you don't specify `reasoning_effort`, LiteLLM automatically sets `thinking_level="low"` for Gemini 3 models, to avoid high costs. 
-
-### Example Usage
-
-<Tabs>
-<TabItem value="sdk" label="Python SDK">
-
-```python
-from litellm import completion
-
-# Low thinking level (faster, lower cost)
-response = completion(
-    model="gemini/gemini-3-pro-preview",
-    messages=[{"role": "user", "content": "What's the weather?"}],
-    reasoning_effort="low"  # Maps to thinking_level="low"
-)
-
-# High thinking level (deeper reasoning, higher cost)
-response = completion(
-    model="gemini/gemini-3-pro-preview",
-    messages=[{"role": "user", "content": "Solve this complex math problem step by step."}],
-    reasoning_effort="high"  # Maps to thinking_level="high"
-)
-```
-
-</TabItem>
-<TabItem value="proxy" label="LiteLLM Proxy">
-
-```bash
-# Low thinking level
-curl http://localhost:4000/v1/chat/completions \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer sk-1234" \
-  -d '{
-    "model": "gemini-3-pro-preview",
-    "messages": [{"role": "user", "content": "What'\''s the weather?"}],
-    "reasoning_effort": "low"
-  }'
-
-# High thinking level
-curl http://localhost:4000/v1/chat/completions \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer sk-1234" \
-  -d '{
-    "model": "gemini-3-pro-preview",
-    "messages": [{"role": "user", "content": "Solve this complex problem."}],
-    "reasoning_effort": "high"
-  }'
-```
-
-</TabItem>
-</Tabs>
-
-## Important Notes
-
-1. **Gemini 3 Cannot Disable Thinking**: Unlike Gemini 2.5 models, Gemini 3 cannot fully disable thinking. Even when you set `reasoning_effort="none"` or `"disable"`, it maps to `thinking_level="low"`.
-
-2. **Temperature Recommendation**: For Gemini 3 models, LiteLLM defaults `temperature` to `1.0` and strongly recommends keeping it at this default. Setting `temperature < 1.0` can cause:
-   - Infinite loops
-   - Degraded reasoning performance
-   - Failure on complex tasks
-
-3. **Automatic Defaults**: If you don't specify `reasoning_effort`, LiteLLM automatically sets `thinking_level="low"` for optimal performance.
-
-## Cost Tracking: Prompt Caching & Context Window
-
-LiteLLM provides comprehensive cost tracking for Gemini 3 Pro Preview, including support for prompt caching and tiered pricing based on context window size.
-
-### Prompt Caching Cost Tracking
-
-Gemini 3 supports prompt caching, which allows you to cache frequently used prompt prefixes to reduce costs. LiteLLM automatically tracks and calculates costs for:
-
-- **Cache Hit Tokens**: Tokens that are read from cache (charged at a lower rate)
-- **Cache Creation Tokens**: Tokens that are written to cache (one-time cost)
-- **Text Tokens**: Regular prompt tokens that are processed normally
-
-#### How It Works
-
-LiteLLM extracts caching information from the `prompt_tokens_details` field in the usage object:
-
-```python
-{
-  "usage": {
-    "prompt_tokens": 50000,
-    "completion_tokens": 1000,
-    "total_tokens": 51000,
-    "prompt_tokens_details": {
-      "cached_tokens": 30000,  # Cache hit tokens
-      "cache_creation_tokens": 5000,  # Tokens written to cache
-      "text_tokens": 15000  # Regular processed tokens
-    }
-  }
-}
-```
-
-### Context Window Tiered Pricing
-
-Gemini 3 Pro Preview supports up to 1M tokens of context, with tiered pricing that automatically applies when your prompt exceeds 200k tokens.
-
-#### Automatic Tier Detection
-
-LiteLLM automatically detects when your prompt exceeds the 200k token threshold and applies the appropriate tiered pricing:
-
-```python
-from litellm import completion_cost
-
-# Example: Small prompt (< 200k tokens)
-response_small = completion(
-    model="gemini/gemini-3-pro-preview",
-    messages=[{"role": "user", "content": "Hello!"}]
-)
-# Uses base pricing: $0.000002/input token, $0.000012/output token
-
-# Example: Large prompt (> 200k tokens)
-response_large = completion(
-    model="gemini/gemini-3-pro-preview",
-    messages=[{"role": "user", "content": "..." * 250000}]  # 250k tokens
-)
-# Automatically uses tiered pricing: $0.000004/input token, $0.000018/output token
-```
-
-#### Cost Breakdown
-
-The cost calculation includes:
-
-1. **Text Processing Cost**: Regular tokens processed at base or tiered rate
-2. **Cache Read Cost**: Cached tokens read at discounted rate
-3. **Cache Creation Cost**: One-time cost for writing tokens to cache (applies tiered rate if above 200k)
-4. **Output Cost**: Generated tokens at base or tiered rate
-
-### Example: Viewing Cost Breakdown
-
-You can view the detailed cost breakdown using LiteLLM's cost tracking:
-
-```python
-from litellm import completion, completion_cost
-
-response = completion(
-    model="gemini/gemini-3-pro-preview",
-    messages=[{"role": "user", "content": "Explain prompt caching"}],
-    caching=True  # Enable prompt caching
-)
-
-# Get total cost
-total_cost = completion_cost(completion_response=response)
-print(f"Total cost: ${total_cost:.6f}")
-
-# Access usage details
-usage = response.usage
-print(f"Prompt tokens: {usage.prompt_tokens}")
-print(f"Completion tokens: {usage.completion_tokens}")
-
-# Access caching details
-if usage.prompt_tokens_details:
-    print(f"Cache hit tokens: {usage.prompt_tokens_details.cached_tokens}")
-    print(f"Cache creation tokens: {usage.prompt_tokens_details.cache_creation_tokens}")
-    print(f"Text tokens: {usage.prompt_tokens_details.text_tokens}")
-```
-
-### Cost Optimization Tips
-
-1. **Use Prompt Caching**: For repeated prompt prefixes, enable caching to reduce costs by up to 90% for cached portions
-2. **Monitor Context Size**: Be aware that prompts above 200k tokens use tiered pricing (2x for input, 1.5x for output)
-3. **Cache Management**: Cache creation tokens are charged once when writing to cache, then subsequent reads are much cheaper
-4. **Track Usage**: Use LiteLLM's built-in cost tracking to monitor spending across different token types
-
-### Integration with LiteLLM Proxy
-
-When using LiteLLM Proxy, all cost tracking is automatically logged and available through:
-
-- **Usage Logs**: Detailed token and cost breakdowns in proxy logs
-- **Budget Management**: Set budgets and alerts based on actual usage
-- **Analytics Dashboard**: View cost trends and breakdowns by token type
-
-```yaml
-# config.yaml
-model_list:
-  - model_name: gemini-3-pro-preview
-    litellm_params:
-      model: gemini/gemini-3-pro-preview
-      api_key: os.environ/GEMINI_API_KEY
-
-litellm_settings:
-  # Enable detailed cost tracking
-  success_callback: ["langfuse"]  # or your preferred logging service
-```
-
-## Using with Claude Code CLI
-
-You can use `gemini-3-pro-preview` with **Claude Code CLI** - Anthropic's command-line interface. This allows you to use Gemini 3 Pro Preview with Claude Code's native syntax and workflows.
-
-### Setup
-
-**1. Add Gemini 3 Pro Preview to your `config.yaml`:**
-
-```yaml
-model_list:
-  - model_name: gemini-3-pro-preview
-    litellm_params:
-      model: gemini/gemini-3-pro-preview
-      api_key: os.environ/GEMINI_API_KEY
-
-litellm_settings:
-  master_key: os.environ/LITELLM_MASTER_KEY
-```
-
-**2. Set environment variables:**
-
-```bash
-export GEMINI_API_KEY="your-gemini-api-key"
-export LITELLM_MASTER_KEY="sk-1234567890"  # Generate a secure key
-```
-
-**3. Start LiteLLM Proxy:**
-
-```bash
-litellm --config /path/to/config.yaml
-
-# RUNNING on http://0.0.0.0:4000
-```
-
-**4. Configure Claude Code to use LiteLLM Proxy:**
-
-```bash
-export ANTHROPIC_BASE_URL="http://0.0.0.0:4000"
-export ANTHROPIC_AUTH_TOKEN="$LITELLM_MASTER_KEY"
-```
-
-**5. Use Gemini 3 Pro Preview with Claude Code:**
-
-```bash
-# Claude Code will use gemini-3-pro-preview from your LiteLLM proxy
-claude --model gemini-3-pro-preview
-
-```
-
-### Example Usage
-
-Once configured, you can interact with Gemini 3 Pro Preview using Claude Code's native interface:
-
-```bash
-$ claude --model gemini-3-pro-preview
-> Explain how thought signatures work in multi-turn conversations.
-
-# Gemini 3 Pro Preview responds through Claude Code interface
-```
-
-### Benefits
-
-- ✅ **Native Claude Code Experience**: Use Gemini 3 Pro Preview with Claude Code's familiar CLI interface
-- ✅ **Unified Authentication**: Single API key for all models through LiteLLM proxy
-- ✅ **Cost Tracking**: All usage tracked through LiteLLM's centralized logging
-- ✅ **Seamless Model Switching**: Easily switch between Claude and Gemini models
-- ✅ **Full Feature Support**: All Gemini 3 features (thought signatures, function calling, etc.) work through Claude Code
-
-### Troubleshooting
-
-**Claude Code not finding the model:**
-- Ensure the model name in Claude Code matches exactly: `gemini-3-pro-preview`
-- Verify your proxy is running: `curl http://0.0.0.0:4000/health`
-- Check that `ANTHROPIC_BASE_URL` points to your LiteLLM proxy
-
-**Authentication errors:**
-- Verify `ANTHROPIC_AUTH_TOKEN` matches your LiteLLM master key
-- Ensure `GEMINI_API_KEY` is set correctly
-- Check LiteLLM proxy logs for detailed error messages
-
-## Responses API Support
-
-LiteLLM fully supports the OpenAI Responses API for Gemini 3 Pro Preview, including both streaming and non-streaming modes. The Responses API provides a structured way to handle multi-turn conversations with function calling, and LiteLLM automatically preserves thought signatures throughout the conversation.
-
-### Example: Using Responses API with Gemini 3
-
-<Tabs>
-<TabItem value="sdk" label="Non-Streaming">
-
-```python
-from openai import OpenAI
-import json
-
-client = OpenAI()
-
-# 1. Define a list of callable tools for the model
-tools = [
-    {
-        "type": "function",
-        "name": "get_horoscope",
-        "description": "Get today's horoscope for an astrological sign.",
-        "parameters": {
-            "type": "object",
-            "properties": {
-                "sign": {
-                    "type": "string",
-                    "description": "An astrological sign like Taurus or Aquarius",
-                },
-            },
-            "required": ["sign"],
-        },
-    },
-]
-
-def get_horoscope(sign):
-    return f"{sign}: Next Tuesday you will befriend a baby otter."
-
-# Create a running input list we will add to over time
-input_list = [
-    {"role": "user", "content": "What is my horoscope? I am an Aquarius."}
-]
-
-# 2. Prompt the model with tools defined
-response = client.responses.create(
-    model="gemini-3-pro-preview",
-    tools=tools,
-    input=input_list,
-)
-
-# Save function call outputs for subsequent requests
-input_list += response.output
-
-for item in response.output:
-    if item.type == "function_call":
-        if item.name == "get_horoscope":
-            # 3. Execute the function logic for get_horoscope
-            horoscope = get_horoscope(json.loads(item.arguments))
-            
-            # 4. Provide function call results to the model
-            input_list.append({
-                "type": "function_call_output",
-                "call_id": item.call_id,
-                "output": json.dumps({
-                  "horoscope": horoscope
-                })
-            })
-
-print("Final input:")
-print(input_list)
-
-response = client.responses.create(
-    model="gemini-3-pro-preview",
-    instructions="Respond only with a horoscope generated by a tool.",
-    tools=tools,
-    input=input_list,
-)
-
-# 5. The model should be able to give a response!
-print("Final output:")
-print(response.model_dump_json(indent=2))
-print("\n" + response.output_text)
-```
-
-**Key Points:**
-- ✅ Thought signatures are automatically preserved in function calls
-- ✅ Works seamlessly with multi-turn conversations
-- ✅ All Gemini 3-specific features are fully supported
-
-</TabItem>
-<TabItem value="streaming" label="Streaming">
-
-```python
-from openai import OpenAI
-import json
-
-client = OpenAI()
-
-tools = [
-    {
-        "type": "function",
-        "name": "get_horoscope",
-        "description": "Get today's horoscope for an astrological sign.",
-        "parameters": {
-            "type": "object",
-            "properties": {
-                "sign": {
-                    "type": "string",
-                    "description": "An astrological sign like Taurus or Aquarius",
-                },
-            },
-            "required": ["sign"],
-        },
-    },
-]
-
-def get_horoscope(sign):
-    return f"{sign}: Next Tuesday you will befriend a baby otter."
-
-input_list = [
-    {"role": "user", "content": "What is my horoscope? I am an Aquarius."}
-]
-
-# Streaming mode
-response = client.responses.create(
-    model="gemini-3-pro-preview",
-    tools=tools,
-    input=input_list,
-    stream=True,
-)
-
-# Collect all chunks
-chunks = []
-for chunk in response:
-    chunks.append(chunk)
-    # Process streaming chunks as they arrive
-    print(chunk)
-
-# Thought signatures are automatically preserved in streaming mode
-```
-
-**Key Points:**
-- ✅ Streaming mode fully supported
-- ✅ Thought signatures preserved across streaming chunks
-- ✅ Real-time processing of function calls and responses
-
-</TabItem>
-</Tabs>
-
-### Responses API Benefits
-
-- ✅ **Structured Output**: Responses API provides a clear structure for handling function calls and multi-turn conversations
-- ✅ **Thought Signature Preservation**: LiteLLM automatically preserves thought signatures in both streaming and non-streaming modes
-- ✅ **Seamless Integration**: Works with existing OpenAI SDK patterns
-- ✅ **Full Feature Support**: All Gemini 3 features (thought signatures, function calling, reasoning) are fully supported
-
-
-## Best Practices
-
-#### 1. Always Include Thought Signatures in Conversation History
-
-When building multi-turn conversations with function calling:
-
-✅ **Do:**
-```python
-# Append the full assistant message (includes thought signatures)
-messages.append(response.choices[0].message)
-```
-
-❌ **Don't:**
-```python
-# Don't manually construct assistant messages without thought signatures
-messages.append({
-    "role": "assistant",
-    "tool_calls": [...]  # Missing thought signatures!
-})
-```
-
-#### 2. Use Appropriate Thinking Levels
-
-- **`reasoning_effort="low"`**: For simple queries, quick responses, cost optimization
-- **`reasoning_effort="high"`**: For complex problems requiring deep reasoning
-
-#### 3. Keep Temperature at Default
-
-For Gemini 3 models, always use `temperature=1.0` (default). Lower temperatures can cause issues.
-
-#### 4. Handle Model Switches Gracefully
-
-When switching from non-Gemini-3 to Gemini-3:
-- ✅ LiteLLM automatically handles missing thought signatures
-- ✅ No manual intervention needed
-- ✅ Conversation history continues seamlessly
-
-
-## Troubleshooting
-
-#### Issue: Missing Thought Signatures
-
-**Symptom**: Error when including assistant messages in conversation history
-
-**Solution**: Ensure you're appending the full assistant message from the response:
-```python
-messages.append(response.choices[0].message)  # ✅ Includes thought signatures
-```
-
-#### Issue: Conversation Breaks When Switching Models
-
-**Symptom**: Errors when switching from gemini-2.5-flash to gemini-3-pro-preview
-
-**Solution**: This should work automatically! LiteLLM adds dummy signatures. If you see errors, ensure you're using the latest LiteLLM version.
-
-#### Issue: Infinite Loops or Poor Performance
-
-**Symptom**: Model gets stuck or produces poor results
-
-**Solution**: 
-- Ensure `temperature=1.0` (default for Gemini 3)
-- Check that `reasoning_effort` is set appropriately
-- Verify you're using the correct model name: `gemini/gemini-3-pro-preview`
-
-## Additional Resources
-
-- [Gemini Provider Documentation](../../docs/providers/gemini)
-- [Thought Signatures Guide](../../docs/providers/gemini#thought-signatures)
-- [Reasoning Content Documentation](../../docs/reasoning_content)
-- [Function Calling Guide](../../docs/completion/function_call)

docs/my-website/blog/gemini_3_1_flash_lite/index.md

@@ -1,168 +0,0 @@
----
-slug: gemini_3_1_flash_lite_preview
-title: "DAY 0 Support: Gemini 3.1 Flash Lite Preview on LiteLLM"
-date: 2026-03-03T08:00:00
-authors:
-  - sameer
-  - krrish
-  - ishaan-alt
-description: "Guide to using Gemini 3.1 Flash Lite Preview on LiteLLM Proxy and SDK with day 0 support."
-tags: [gemini, day 0 support, llms, supernova]
-hide_table_of_contents: false
----
-
-
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-# Gemini 3.1 Flash Lite Preview Day 0 Support 
-
-LiteLLM now supports `gemini-3.1-flash-lite-preview` with full day 0 support!
-
-:::note
-If you only want cost tracking, you need no change in your current Litellm version. But if you want the support for new features introduced along with it like thinking levels, you will need to use v1.80.8-stable.1 or above.
-:::
-
-{/* truncate */}
-
-## Deploy this version
-
-<Tabs>
-<TabItem value="docker" label="Docker">
-
-``` showLineNumbers title="docker run litellm"
-docker run \
--e STORE_MODEL_IN_DB=True \
--p 4000:4000 \
-ghcr.io/berriai/litellm:main-v1.80.8-stable.1
-```
-
-</TabItem>
-
-<TabItem value="pip" label="Pip">
-
-``` showLineNumbers title="pip install litellm"
-pip install litellm==v1.80.8-stable.1
-```
-
-</TabItem>
-</Tabs>
-
-## What's New
-
-Supports all four thinking levels:
-- **MINIMAL**: Ultra-fast responses with minimal reasoning
-- **LOW**: Simple instruction following
-- **MEDIUM**: Balanced reasoning for complex tasks
-- **HIGH**: Maximum reasoning depth (dynamic)
-
----
-
-## Quick Start
-
-<Tabs>
-<TabItem value="sdk" label="SDK">
-
-**Basic Usage**
-
-```python
-from litellm import completion
-
-response = completion(
-    model="gemini/gemini-3.1-flash-lite-preview",
-    messages=[{"role": "user", "content": "Extract key entities from this text: ..."}],
-)
-
-print(response.choices[0].message.content)
-```
-
-**With Thinking Levels**
-
-```python
-from litellm import completion
-
-# Use MEDIUM thinking for complex reasoning tasks
-response = completion(
-    model="gemini/gemini-3.1-flash-lite-preview",
-    messages=[{"role": "user", "content": "Analyze this dataset and identify patterns"}],
-    reasoning_effort="medium",  # low, medium , high
-)
-
-print(response.choices[0].message.content)
-```
-
-</TabItem>
-
-<TabItem value="proxy" label="PROXY">
-
-**1. Setup config.yaml**
-
-```yaml
-model_list:
-  - model_name: gemini-3.1-flash-lite
-    litellm_params:
-      model: gemini/gemini-3.1-flash-lite-preview
-      api_key: os.environ/GEMINI_API_KEY
-  
-  # Or use Vertex AI
-  - model_name: vertex-gemini-3.1-flash-lite
-    litellm_params:
-      model: vertex_ai/gemini-3.1-flash-lite-preview
-      vertex_project: your-project-id
-      vertex_location: us-central1
-```
-
-**2. Start proxy**
-
-```bash
-litellm --config /path/to/config.yaml
-```
-
-**3. Make requests**
-
-```bash
-curl -X POST http://localhost:4000/v1/chat/completions \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer <YOUR-LITELLM-KEY>" \
-  -d '{
-    "model": "gemini-3.1-flash-lite",
-    "messages": [{"role": "user", "content": "Extract structured data from this text"}],
-    "reasoning_effort": "low"
-  }'
-```
-
-</TabItem>
-</Tabs>
-
----
-
-## Supported Endpoints
-
-LiteLLM provides **full end-to-end support** for Gemini 3.1 Flash Lite Preview on:
-
-- ✅ `/v1/chat/completions` - OpenAI-compatible chat completions endpoint
-- ✅ `/v1/responses` - OpenAI Responses API endpoint (streaming and non-streaming)
-- ✅ [`/v1/messages`](../../docs/anthropic_unified) - Anthropic-compatible messages endpoint
-- ✅ `/v1/generateContent` – [Google Gemini API](../../docs/generateContent) compatible endpoint 
-
-All endpoints support:
-- Streaming and non-streaming responses
-- Function calling with thought signatures
-- Multi-turn conversations
-- All Gemini 3-specific features (thinking levels, thought signatures)
-- Full multimodal support (text, image, audio, video)
-
----
-
-## `reasoning_effort` Mapping for Gemini 3.1
-
-LiteLLM automatically maps OpenAI's `reasoning_effort` parameter to Gemini's `thinkingLevel`:
-
-| reasoning_effort | thinking_level | Use Case |
-|------------------|----------------|----------|
-| `minimal` | `minimal` | Ultra-fast responses, simple queries |
-| `low` | `low` | Basic instruction following |
-| `medium` | `medium` | Balanced reasoning for moderate complexity |
-| `high` | `high` | Maximum reasoning depth, complex problems |
-| `disable` | `minimal` | Disable extended reasoning |
-| `none` | `minimal` | No extended reasoning |

docs/my-website/blog/gemini_3_flash/index.md

@@ -1,247 +0,0 @@
----
-slug: gemini_3_flash
-title: "DAY 0 Support: Gemini 3 Flash on LiteLLM"
-date: 2025-12-17T10:00:00
-authors:
-  - sameer
-  - krrish
-  - ishaan-alt
-description: "Guide to using Gemini 3 Flash on LiteLLM Proxy and SDK with day 0 support."
-tags: [gemini, day 0 support, llms]
-hide_table_of_contents: false
----
-
-
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-# Gemini 3 Flash Day 0 Support 
-
-LiteLLM now supports `gemini-3-flash-preview` and all the new API changes along with it.
-
-:::note
-If you only want cost tracking, you need no change in your current Litellm version. But if you want the support for new features introduced along with it like thinking levels, you will need to use v1.80.8-stable.1 or above.
-:::
-
-{/* truncate */}
-
-## Deploy this version
-
-<Tabs>
-<TabItem value="docker" label="Docker">
-
-``` showLineNumbers title="docker run litellm"
-docker run \
--e STORE_MODEL_IN_DB=True \
--p 4000:4000 \
-ghcr.io/berriai/litellm:main-v1.80.8-stable.1
-```
-
-</TabItem>
-
-<TabItem value="pip" label="Pip">
-
-``` showLineNumbers title="pip install litellm"
-pip install litellm==1.80.8.post1
-```
-
-</TabItem>
-</Tabs>
-
-## What's New
-
-### 1. New Thinking Levels: `thinkingLevel` with MINIMAL & MEDIUM
-
-Gemini 3 Flash introduces granular thinking control with `thinkingLevel` instead of `thinkingBudget`.
-- **MINIMAL**: Ultra-lightweight thinking for fast responses
-- **MEDIUM**: Balanced thinking for complex reasoning  
-- **HIGH**: Maximum reasoning depth
-
-LiteLLM automatically maps the OpenAI `reasoning_effort` parameter to Gemini's `thinkingLevel`, so you can use familiar `reasoning_effort` values (`minimal`, `low`, `medium`, `high`) without changing your code!
-
-### 2. Thought Signatures
-
-Like `gemini-3-pro`, this model also includes thought signatures for tool calls. LiteLLM handles signature extraction and embedding internally. [Learn more about thought signatures](../gemini_3/index.md#thought-signatures).
-
-**Edge Case Handling**: If thought signatures are missing in the request, LiteLLM adds a dummy signature ensuring the API call doesn't break
-
----
-## Supported Endpoints
-
-LiteLLM provides **full end-to-end support** for Gemini 3 Flash on:
-
-- ✅ `/v1/chat/completions` - OpenAI-compatible chat completions endpoint
-- ✅ `/v1/responses` - OpenAI Responses API endpoint (streaming and non-streaming)
-- ✅ [`/v1/messages`](../../docs/anthropic_unified) - Anthropic-compatible messages endpoint
-- ✅ `/v1/generateContent` – [Google Gemini API](../../docs/generateContent) compatible endpoint 
-All endpoints support:
-- Streaming and non-streaming responses
-- Function calling with thought signatures
-- Multi-turn conversations
-- All Gemini 3-specific features
-- Converstion of provider specific thinking related param to thinkingLevel
-
-## Quick Start
-
-<Tabs>
-<TabItem value="sdk" label="SDK">
-
-**Basic Usage with MEDIUM thinking (NEW)**
-
-```python
-from litellm import completion
-
-# No need to make any changes to your code as we map openai reasoning param to thinkingLevel
-response = completion(
-    model="gemini/gemini-3-flash-preview",
-    messages=[{"role": "user", "content": "Solve this complex math problem: 25 * 4 + 10"}],
-    reasoning_effort="medium",  # NEW: MEDIUM thinking level
-)
-
-print(response.choices[0].message.content)
-```
-
-</TabItem>
-
-<TabItem value="proxy" label="PROXY">
-
-**1. Setup config.yaml**
-
-```yaml
-model_list:
-  - model_name: gemini-3-flash
-    litellm_params:
-      model: gemini/gemini-3-flash-preview
-      api_key: os.environ/GEMINI_API_KEY
-```
-
-**2. Start proxy**
-
-```bash
-litellm --config /path/to/config.yaml
-```
-
-**3. Call with MEDIUM thinking**
-
-```bash
-curl -X POST http://localhost:4000/v1/chat/completions \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer <YOUR-LITELLM-KEY>" \
-  -d '{
-    "model": "gemini-3-flash",
-    "messages": [{"role": "user", "content": "Complex reasoning task"}],
-    "reasoning_effort": "medium"
-  }'
-``'
-
-</TabItem>
-</Tabs>
-
----
-
-## All `reasoning_effort` Levels
-
-<Tabs>
-<TabItem value="minimal" label="MINIMAL">
-
-**Ultra-fast, minimal reasoning**
-
-```python
-from litellm import completion
-
-response = completion(
-    model="gemini/gemini-3-flash-preview",
-    messages=[{"role": "user", "content": "What's 2+2?"}],
-    reasoning_effort="minimal",
-)
-```
-
-</TabItem>
-
-<TabItem value="low" label="LOW">
-
-**Simple instruction following**
-
-```python
-response = completion(
-    model="gemini/gemini-3-flash-preview",
-    messages=[{"role": "user", "content": "Write a haiku about coding"}],
-    reasoning_effort="low",
-)
-```
-
-</TabItem>
-
-<TabItem value="medium" label="MEDIUM (NEW)">
-
-**Balanced reasoning for complex tasks** ✨
-
-```python
-response = completion(
-    model="gemini/gemini-3-flash-preview",
-    messages=[{"role": "user", "content": "Analyze this dataset and find patterns"}],
-    reasoning_effort="medium",  # NEW!
-)
-```
-
-</TabItem>
-
-<TabItem value="high" label="HIGH">
-
-**Maximum reasoning depth**
-
-```python
-response = completion(
-    model="gemini/gemini-3-flash-preview",
-    messages=[{"role": "user", "content": "Prove this mathematical theorem"}],
-    reasoning_effort="high",
-)
-```
-
-</TabItem>
-</Tabs>
-
----
-
-## Key Features
-
-✅ **Thinking Levels**: MINIMAL, LOW, MEDIUM, HIGH  
-✅ **Thought Signatures**: Track reasoning with unique identifiers  
-✅ **Seamless Integration**: Works with existing OpenAI-compatible client  
-✅ **Backward Compatible**: Gemini 2.5 models continue using `thinkingBudget`  
-
----
-
-## Installation
-
-```bash
-pip install litellm --upgrade
-```
-
-```python
-import litellm
-from litellm import completion
-
-response = completion(
-    model="gemini/gemini-3-flash-preview",
-    messages=[{"role": "user", "content": "Your question here"}],
-    reasoning_effort="medium",  # Use MEDIUM thinking
-)
-print(response)
-```
-
-:::note
-If using this model via vertex_ai, keep the location as global as this is the only supported location as of now.
-:::
-
-
-## `reasoning_effort` Mapping for Gemini 3+
-
-| reasoning_effort | thinking_level | 
-|------------------|----------------|
-| `minimal` | `minimal` |
-| `low` | `low` |
-| `medium` | `medium` |
-| `high` | `high` |
-| `disable` | `minimal` |
-| `none` | `minimal` |

docs/my-website/blog/gemini_embedding_2_multimodal/index.md

@@ -1,168 +0,0 @@
----
-slug: gemini_embedding_2_multimodal
-title: "Gemini Embedding 2 Preview: Multimodal Embeddings on LiteLLM"
-date: 2025-03-11T10:00:00
-authors:
-  - sameer
-description: "Generate embeddings from text, images, audio, video, and PDFs with gemini-embedding-2-preview on LiteLLM via Gemini API and Vertex AI."
-tags: [gemini, embeddings, multimodal, vertex ai]
-hide_table_of_contents: false
----
-
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-# Gemini Embedding 2 Preview: Multimodal Embeddings
-
-LiteLLM now supports **multimodal embeddings** with `gemini-embedding-2-preview`—generating a single embedding from a mix of text, images, audio, video, and PDF content. Available via both the **Gemini API** (API key) and **Vertex AI** (GCP credentials).
-
-{/* truncate */}
-
-## Supported Input Types
-
-| Modality | Supported Formats | 
-|----------|-------------------|
-| **Text** | Plain text |
-| **Image** | PNG, JPEG | 
-| **Audio** | MP3, WAV | 
-| **Video** | MP4, MOV | 
-| **Documents** | PDF | 
-
-## Input Formats
-
-LiteLLM accepts three input formats for multimodal content:
-
-1. **Data URIs** – Base64-encoded inline: `data:image/png;base64,<encoded_data>`
-2. **GCS URLs** – Cloud Storage paths (Vertex AI): `gs://bucket/path/to/file.png`
-3. **Gemini File References** – Pre-uploaded files (Gemini API): `files/abc123`
-
-## Quick Start
-
-<Tabs>
-<TabItem value="gemini" label="Gemini API">
-
-```python
-from litellm import embedding
-import os
-
-os.environ["GEMINI_API_KEY"] = "your-api-key"
-
-# Text + Image (base64)
-response = embedding(
-    model="gemini/gemini-embedding-2-preview",
-    input=[
-        "The food was delicious and the waiter...",
-        "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAIAQMAAAD+wSzIAAAABlBMVEX///+/v7+jQ3Y5AAAADklEQVQI12P4AIX8EAgALgAD/aNpbtEAAAAASUVORK5CYII"
-    ],
-)
-print(response)
-```
-
-</TabItem>
-
-<TabItem value="vertex" label="Vertex AI">
-
-```python
-import litellm
-from litellm import embedding
-
-litellm.vertex_project = "your-project-id"
-litellm.vertex_location = "us-central1"
-
-# Text + Image (GCS URL)
-response = embedding(
-    model="vertex_ai/gemini-embedding-2-preview",
-    input=[
-        "Describe this image",
-        "gs://my-bucket/images/photo.png"
-    ],
-)
-print(response)
-```
-
-</TabItem>
-
-<TabItem value="proxy" label="LiteLLM Proxy">
-
-**1. Config (config.yaml)**
-
-```yaml
-model_list:
-  - model_name: gemini-embedding-2-preview
-    litellm_params:
-      model: gemini/gemini-embedding-2-preview
-      api_key: os.environ/GEMINI_API_KEY
-  - model_name: vertex-gemini-embedding-2-preview
-    litellm_params:
-      model: vertex_ai/gemini-embedding-2-preview
-      vertex_project: os.environ/VERTEXAI_PROJECT
-      vertex_location: os.environ/VERTEXAI_LOCATION
-
-general_settings:
-  master_key: sk-1234
-```
-
-**2. Start proxy**
-
-```bash
-litellm --config config.yaml
-```
-
-**3. Call embeddings**
-
-```bash
-curl -X POST http://localhost:4000/embeddings \
-  -H "Authorization: Bearer sk-1234" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "model": "gemini-embedding-2-preview",
-    "input": [
-      "The food was delicious and the waiter...",
-      "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAIAQMAAAD+wSzIAAAABlBMVEX///+/v7+jQ3Y5AAAADklEQVQI12P4AIX8EAgALgAD/aNpbtEAAAAASUVORK5CYII"
-    ]
-  }'
-```
-
-</TabItem>
-</Tabs>
-
-## Input Format Examples
-
-| Format | Example | Provider |
-|--------|---------|----------|
-| **Data URI** | `data:image/png;base64,...` | Gemini, Vertex AI |
-| **GCS URL** | `gs://bucket/path/image.png` | Vertex AI |
-| **File reference** | `files/abc123` | Gemini API only |
-
-### Supported MIME Types for Data URIs
-
-- **Images:** `image/png`, `image/jpeg`
-- **Audio:** `audio/mpeg`, `audio/wav`
-- **Video:** `video/mp4`, `video/quicktime`
-- **Documents:** `application/pdf`
-
-### GCS URL MIME Inference
-
-For Vertex AI, MIME types are inferred from file extensions:
-
-- `.png` → `image/png`
-- `.jpg` / `.jpeg` → `image/jpeg`
-- `.mp3` → `audio/mpeg`
-- `.wav` → `audio/wav`
-- `.mp4` → `video/mp4`
-- `.mov` → `video/quicktime`
-- `.pdf` → `application/pdf`
-
-## Optional Parameters
-
-| Parameter | Description | Maps to |
-|-----------|-------------|---------|
-| `dimensions` | Output embedding size | `outputDimensionality` |
-
-```python
-response = embedding(
-    model="gemini/gemini-embedding-2-preview",
-    input=["text to embed"],
-    dimensions=768,  # Optional: control output vector size
-)
-```

docs/my-website/blog/gpt_5_3_codex/index.md

@@ -1,138 +0,0 @@
----
-slug: gpt_5_3_codex
-title: "Day 0 Support: GPT-5.3-Codex"
-date: 2026-02-24T10:00:00
-authors:
-  - sameer
-  - krrish
-  - ishaan-alt
-description: "Day 0 support for GPT-5.3-Codex on LiteLLM, including phase parameter handling for Responses API."
-tags: [openai, gpt-5.3-codex, codex, day 0 support]
-hide_table_of_contents: false
----
-
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-LiteLLM now supports GPT-5.3-Codex on Day 0, including support for the new assistant `phase` metadata on Responses API output items.
-
-{/* truncate */}
-
-## Why `phase` matters for GPT-5.3-Codex
-
-`phase` appears on assistant output items and helps distinguish preamble/commentary turns from final closeout responses.
-
-Reference: [Phase parameter docs](https://developers.openai.com/api/reference/overview)
-
-Supported values:
-- `null`
-- `"commentary"`
-- `"final_answer"`
-
-Important:
-- Persist assistant output items with `phase` exactly as returned.
-- Send those assistant items back on the next turn.
-- Do **not** add `phase` to user messages.
-
-## Docker Image
-
-```bash
-docker pull ghcr.io/berriai/litellm:v1.81.12-stable.gpt-5.3
-```
-
-## Usage 
-
-<Tabs>
-<TabItem value="proxy" label="LiteLLM Proxy">
-
-**1. Setup config.yaml**
-
-```yaml
-model_list:
-  - model_name: gpt-5.3-codex
-    litellm_params:
-      model: openai/gpt-5.3-codex
-```
-
-**2. Start the proxy**
-
-```bash
-docker run -d \
-  -p 4000:4000 \
-  -e ANTHROPIC_API_KEY=$OPENAI_API_KEY \
-  -v $(pwd)/config.yaml:/app/config.yaml \
-  ghcr.io/berriai/litellm:v1.81.12-stable.gpt-5.3 \
-  --config /app/config.yaml
-```
-
-
-**3. Test it**
-
-```bash
-curl -X POST "http://0.0.0.0:4000/v1/responses" \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer $LITELLM_KEY" \
-  -d '{
-    "model": "gpt-5.3-codex",
-    "input": "Write a Python script that checks if a number is prime."
-  }'
-```
-
-</TabItem>
-</Tabs>
-
-## Python Example: Persist `phase` with OpenAI Client + LiteLLM Base URL
-
-```python
-from openai import OpenAI
-
-client = OpenAI(
-    base_url="http://0.0.0.0:4000/v1",  # LiteLLM Proxy
-    api_key="your-litellm-api-key",
-)
-
-items = []  # Persist this per conversation/thread
-
-
-def _item_get(item, key, default=None):
-    if isinstance(item, dict):
-        return item.get(key, default)
-    return getattr(item, key, default)
-
-
-def run_turn(user_text: str):
-    global items
-
-    # User message: no phase field
-    items.append(
-        {
-            "type": "message",
-            "role": "user",
-            "content": [{"type": "input_text", "text": user_text}],
-        }
-    )
-
-    resp = client.responses.create(
-        model="gpt-5.3-codex",
-        input=items,
-    )
-
-    # Persist assistant output items verbatim, including phase
-    for out_item in (resp.output or []):
-        items.append(out_item)
-
-    # Optional: inspect latest phase for UI/telemetry routing
-    latest_phase = None
-    for out_item in reversed(resp.output or []):
-        if _item_get(out_item, "type") == "output_item.done" and _item_get(out_item, "phase") is not None:
-            latest_phase = _item_get(out_item, "phase")
-            break
-
-    return resp, latest_phase
-```
-
-## Notes
-
-- Use `/v1/responses` for GPT Codex models.
-- Preserve full assistant output history for best multi-turn behavior.
-- If `phase` metadata is dropped during history reconstruction, output quality can degrade on long-running tasks.

docs/my-website/blog/gpt_5_4/index.md

@@ -1,90 +0,0 @@
----
-slug: gpt_5_4
-title: "Day 0 Support: GPT-5.4"
-date: 2026-03-05T10:00:00
-authors:
-  - sameer
-  - krrish
-  - ishaan-alt
-description: "GPT-5.4 model support in LiteLLM"
-tags: [openai, gpt-5.4, completion]
-hide_table_of_contents: false
----
-
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-LiteLLM now supports fully GPT-5.4!
-
-{/* truncate */}
-
-## Docker Image
-
-```bash
-docker pull ghcr.io/berriai/litellm:v1.81.14-stable.gpt-5.4_patch
-```
-
-## Usage
-
-<Tabs>
-<TabItem value="proxy" label="LiteLLM Proxy">
-
-**1. Setup config.yaml**
-
-```yaml
-model_list:
-  - model_name: gpt-5.4
-    litellm_params:
-      model: openai/gpt-5.4
-      api_key: os.environ/OPENAI_API_KEY
-```
-
-**2. Start the proxy**
-
-```bash
-docker run -d \
-  -p 4000:4000 \
-  -e OPENAI_API_KEY=$OPENAI_API_KEY \
-  -v $(pwd)/config.yaml:/app/config.yaml \
-  ghcr.io/berriai/litellm:v1.81.14-stable.gpt-5.4_patch \
-  --config /app/config.yaml
-```
-
-**3. Test it**
-
-```bash
-curl -X POST "http://0.0.0.0:4000/chat/completions" \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer $LITELLM_KEY" \
-  -d '{
-    "model": "gpt-5.4",
-    "messages": [
-      {"role": "user", "content": "Write a Python function to check if a number is prime."}
-    ]
-  }'
-```
-
-</TabItem>
-<TabItem value="sdk" label="LiteLLM SDK">
-
-```python
-from litellm import completion
-
-response = completion(
-    model="openai/gpt-5.4",
-    messages=[
-        {"role": "user", "content": "Write a Python function to check if a number is prime."}
-    ],
-)
-
-print(response.choices[0].message.content)
-```
-
-</TabItem>
-</Tabs>
-
-## Notes
-
-- Restart your container to get the cost tracking for this model.
-- Use `/responses` for better model performance.
-- GPT-5.4 supports reasoning, function calling, vision, and tool-use — see the [OpenAI provider docs](../../docs/providers/openai) for advanced usage.

docs/my-website/blog/gpt_5_4_mini_nano/index.md

@@ -1,106 +0,0 @@
----
-slug: gpt_5_4_mini_nano
-title: "Day 0 Support: GPT-5.4-mini and GPT-5.4-nano"
-date: 2026-03-17T10:00:00
-authors:
-  - name: Sameer Kankute
-    title: SWE @ LiteLLM (LLM Translation)
-    url: https://www.linkedin.com/in/sameer-kankute/
-    image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
-  - name: Krrish Dholakia
-    title: "CEO, LiteLLM"
-    url: https://www.linkedin.com/in/krish-d/
-    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
-  - name: Ishaan Jaff
-    title: "CTO, LiteLLM"
-    url: https://www.linkedin.com/in/reffajnaahsi/
-    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
-description: "GPT-5.4-mini and GPT-5.4-nano model support in LiteLLM"
-tags: [openai, gpt-5.4-mini, gpt-5.4-nano, completion]
-hide_table_of_contents: false
----
-
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-LiteLLM now supports GPT-5.4-mini and GPT-5.4-nano — cost-effective models for simple completions and high-throughput workloads.
-
-:::note
-If you're on **v1.82.3-stable** or above, you don't need any update to use these models.
-:::
-
-## Usage
-
-<Tabs>
-<TabItem value="proxy" label="LiteLLM Proxy">
-
-**1. Setup config.yaml**
-
-```yaml
-model_list:
-  - model_name: gpt-5.4-mini
-    litellm_params:
-      model: openai/gpt-5.4-mini
-      api_key: os.environ/OPENAI_API_KEY
-  - model_name: gpt-5.4-nano
-    litellm_params:
-      model: openai/gpt-5.4-nano
-      api_key: os.environ/OPENAI_API_KEY
-```
-
-**2. Start the proxy**
-
-```bash
-litellm --config /path/to/config.yaml
-```
-
-**3. Test it**
-
-```bash
-# GPT-5.4-mini
-curl -X POST "http://localhost:4000/v1/chat/completions" \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer $LITELLM_KEY" \
-  -d '{
-    "model": "gpt-5.4-mini",
-    "messages": [{"role": "user", "content": "What is the capital of France?"}]
-  }'
-
-# GPT-5.4-nano
-curl -X POST "http://localhost:4000/v1/chat/completions" \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer $LITELLM_KEY" \
-  -d '{
-    "model": "gpt-5.4-nano",
-    "messages": [{"role": "user", "content": "What is 2 + 2?"}]
-  }'
-```
-
-</TabItem>
-<TabItem value="sdk" label="LiteLLM SDK">
-
-```python
-from litellm import completion
-
-# GPT-5.4-mini
-response = completion(
-    model="openai/gpt-5.4-mini",
-    messages=[{"role": "user", "content": "What is the capital of France?"}],
-)
-print(response.choices[0].message.content)
-
-# GPT-5.4-nano
-response = completion(
-    model="openai/gpt-5.4-nano",
-    messages=[{"role": "user", "content": "What is 2 + 2?"}],
-)
-print(response.choices[0].message.content)
-```
-
-</TabItem>
-</Tabs>
-
-## Notes
-
-- Both models support function calling, vision, and tool-use — see the [OpenAI provider docs](../../docs/providers/openai) for advanced usage.
-- GPT-5.4-nano is the most cost-effective option for simple tasks; GPT-5.4-mini offers a balance of speed and capability.

docs/my-website/blog/guardrail_logging_secret_exposure_incident/index.md

@@ -1,78 +0,0 @@
----
-slug: guardrail-logging-secret-exposure-incident
-title: "Incident Report: Guardrail logging exposed secret headers in spend logs and traces"
-date: 2026-03-18T10:00:00
-authors:
-  - litellm
-tags: [incident-report, security, guardrails]
-hide_table_of_contents: false
----
-
-**Date:** March 18, 2026
-**Duration:** Unknown
-**Severity:** High
-**Status:** Resolved
-
-## Summary
-
-When a custom guardrail returned the full LiteLLM request/data dictionary, the guardrail response logged by LiteLLM could include `secret_fields.raw_headers`, including plaintext `Authorization` headers containing API keys or other credentials.
-
-This information could then propagate to logging and observability surfaces that consume guardrail metadata, including:
-
-- **Spend logs in the LiteLLM UI:** visible to admins with access to spend-log data
-- **OpenTelemetry traces:** visible to anyone with access to the relevant telemetry backend
-
-LLM calls, proxy routing, and provider execution were not blocked by this bug. The impact was exposure of sensitive request headers in observability and logging paths.
-
-{/* truncate */}
-
----
-
-## Background
-
-LiteLLM keeps internal request data (including request headers) for use during the call. That data is not meant to be written to logs or telemetry.
-
-When custom guardrails run, their outcomes are logged so they can appear in spend logs, OpenTelemetry traces, and other observability backends. If a guardrail returned the full request payload instead of a minimal result, that internal request data could be included in what was logged. Before the fix, the guardrail logging path did not strip that data before sending it to those systems.
-
-```mermaid
-flowchart TD
-    inboundRequest["1. Incoming proxy request"] --> storeSecrets["2. Store internal request data"]
-    storeSecrets --> guardrailRuns["3. Custom guardrail runs"]
-    guardrailRuns --> fullDataReturn["4. Guardrail returns full request payload"]
-    fullDataReturn --> loggingBuild["5. Build guardrail log payload"]
-    loggingBuild --> spendLogs["6a. Persist to spend logs / UI"]
-    loggingBuild --> otelTraces["6b. Attach to OTEL guardrail spans"]
-```
-
----
-
-## Root Cause
-
-The root cause was incomplete sanitization in the guardrail logging path. When building the payload that gets sent to spend logs and traces, LiteLLM prepared guardrail responses for logging but did not strip internal request data (such as headers) from them. If a guardrail returned a response that included that data, it was passed through to the logging and observability systems unchanged.
-
----
-
-## Impact
-
-This issue required all of the following:
-
-1. A custom guardrail returned the full LiteLLM request/data dictionary, or another response object containing `secret_fields`.
-2. LiteLLM logged that guardrail response through the standard guardrail logging path.
-3. An operator, admin, or telemetry consumer had access to the resulting logs or traces.
-
-When those conditions were met, sensitive values could become visible through:
-
-- **Spend logs / UI responses:** guardrail metadata could be included in spend-log payloads rendered in the admin UI.
-- **OpenTelemetry traces:** `guardrail_response` could be written as a span attribute on guardrail spans.
-- **Other downstream observability backends:** any integration consuming the same guardrail metadata could receive the leaked values.
-
-This was a logging and telemetry exposure bug. It did not let callers bypass auth, access other tenants directly, or change model behavior, but it could expose plaintext credentials to people with access to those observability systems.
-
----
-
-## Guidance For Users
-
-- Upgrade to LiteLLM 1.82.3+.
-- If you operated custom guardrails that return the full request/data dict, review whether spend logs or telemetry traces were retained during the affected period.
-- Rotate any credentials that may have appeared in `Authorization` or other forwarded request headers in those systems.
-- Apply least-privilege access controls to spend-log views and telemetry backends that may contain request-derived metadata.

docs/my-website/blog/httpx_cache_eviction_incident/index.md

@@ -1,126 +0,0 @@
----
-slug: httpx-cache-eviction-incident
-title: "Incident Report: Cache Eviction Closes In-Use httpx Clients"
-date: 2026-02-27T10:00:00
-authors:
-  - ryan
-  - ishaan-alt
-  - krrish
-tags: [incident-report, caching, stability]
-hide_table_of_contents: false
----
-
-**Date:** February 27, 2026
-**Duration:** ~6 days (Feb 21 merge -> Feb 27 fix)
-**Severity:** High
-**Status:** Resolved
-
-> **Note:** This fix is available starting from LiteLLM `v1.81.14.rc.2` or higher.
-
-## Summary
-
-A change to improve Redis connection pool cleanup introduced a regression that closed **httpx clients** that were still actively being used by the proxy. The `LLMClientCache` (an in-memory TTL cache) stores both Redis clients *and* httpx clients under the same eviction policy. When a cache entry expired or was evicted, the new cleanup code called `aclose()`/`close()` on the evicted value which worked correctly for Redis clients, but destroyed httpx clients that other parts of the system still held references to and were actively using for LLM API calls.
-
-**Impact:** Any proxy instance that hit the cache TTL (default 10 minutes) or capacity limit (200 entries) would have its httpx clients closed out from under it, causing requests to LLM providers to fail with connection errors.
-
-{/* truncate */}
-
----
-
-## Background
-
-`LLMClientCache` extends `InMemoryCache` and is used to cache SDK clients (OpenAI, Anthropic, etc.) to avoid re-creating them on every request. These clients are keyed by configuration + event loop ID. The cache has:
-
-- **Max size:** 200 entries
-- **Default TTL:** 10 minutes
-
-When the cache is full or entries expire, `InMemoryCache.evict_cache()` calls `_remove_key()` to drop entries.
-
-The cached values are a mix of:
-- **Redis/async Redis clients** — owned exclusively by the cache, safe to close on eviction
-- **httpx-backed SDK clients** (OpenAI, Anthropic, etc.) — shared references, still in use by router/model instances
-
----
-
-## Root Cause
-
-[PR #21717](https://github.com/BerriAI/litellm/pull/21717) overrode `_remove_key()` in `LLMClientCache` to close async clients on eviction:
-
-<details>
-<summary>Problematic code added in PR #21717</summary>
-
-```python
-class LLMClientCache(InMemoryCache):
-    def _remove_key(self, key: str) -> None:
-        value = self.cache_dict.get(key)
-        super()._remove_key(key)
-        if value is not None:
-            close_fn = getattr(value, "aclose", None) or getattr(value, "close", None)
-            if close_fn and asyncio.iscoroutinefunction(close_fn):
-                try:
-                    asyncio.get_running_loop().create_task(close_fn())
-                except RuntimeError:
-                    pass
-            elif close_fn and callable(close_fn):
-                try:
-                    close_fn()
-                except Exception:
-                    pass
-```
-
-</details>
-
-The intent was correct for Redis clients — prevent connection pool leaks when cached Redis clients expire. But `LLMClientCache` also stores httpx-backed SDK clients (e.g., `AsyncOpenAI`, `AsyncAnthropic`). These clients:
-
-1. Have an `aclose()` method (inherited from httpx)
-2. Are still held by references elsewhere in the codebase (router, model instances)
-3. Were being closed without any check on whether they were still in use
-
-So when the cache evicted an entry, it would call `aclose()` on an httpx client that was still being used for active LLM requests → closed transport → connection errors.
-
----
-
-## The Fix
-
-[PR #22247](https://github.com/BerriAI/litellm/pull/22247) removed the `_remove_key` override entirely:
-
-<details>
-<summary>The fix (PR #22247)</summary>
-
-```diff
- class LLMClientCache(InMemoryCache):
--    def _remove_key(self, key: str) -> None:
--        """Close async clients before evicting them to prevent connection pool leaks."""
--        value = self.cache_dict.get(key)
--        super()._remove_key(key)
--        if value is not None:
--            close_fn = getattr(value, "aclose", None) or getattr(
--                value, "close", None
--            )
--            ...
--
-     def update_cache_key_with_event_loop(self, key):
-```
-
-</details>
-
-The eviction now simply drops the reference and lets Python's GC handle cleanup, which is safe because:
-- httpx clients that are still referenced elsewhere stay alive
-- Unreferenced clients get cleaned up by GC naturally
-
-The other improvements from PR #21717 were kept:
-- **`max_connections` respected for URL-based Redis configs**, previously silently dropped
-- **`disconnect()` now closes both sync and async Redis clients**, sync client was previously leaked
-- **Connection pool passthrough**, when a pool is provided with a URL config, it's used directly instead of creating a duplicate
-
----
-
-## Remediation
-
-| Action | Status | Code |
-|--------|--------|------|
-| Remove `_remove_key` override that closes shared clients on eviction | ✅ Done | [PR #22247](https://github.com/BerriAI/litellm/pull/22247) |
-| Add e2e test: evicted client still usable (capacity) | ✅ Done | [PR #22313](https://github.com/BerriAI/litellm/pull/22313) |
-| Add e2e test: expired client still usable (TTL) | ✅ Done | [PR #22313](https://github.com/BerriAI/litellm/pull/22313) |
-
-The e2e tests go through `get_async_httpx_client()` the same code path the proxy uses in production and assert the client is still functional after eviction. These run in CI on every PR against `main`. If anyone modifies `LLMClientCache` eviction behavior, overrides `_remove_key`, or adds any form of client cleanup on eviction, these tests will fail regardless of the implementation approach.

docs/my-website/blog/litellm_observatory/index.md

@@ -1,128 +0,0 @@
----
-slug: litellm-observatory
-title: "Improve release stability with 24 hour load tests"
-date: 2026-02-06T10:00:00
-authors:
-  - alexsander
-  - krrish
-  - ishaan-alt
-description: "How we built a long-running, release-validation system to catch regressions before they reach users."
-tags: [testing, observability, reliability, releases]
-hide_table_of_contents: false
----
-
-![LiteLLM Observatory](https://raw.githubusercontent.com/AlexsanderHamir/assets/main/Screenshot%202026-01-31%20175355.png)
-
-# Improve release stability with 24 hour load tests
-
-As LiteLLM adoption has grown, so have expectations around reliability, performance, and operational safety. Meeting those expectations requires more than correctness-focused tests, it requires validating how the system behaves over time, under real-world conditions.
-
-This post introduces **LiteLLM Observatory**, a long-running release-validation system we built to catch regressions before they reach users.
-
-{/* truncate */}
-
----
-
-## Why We Built the Observatory
-
-LiteLLM operates at the intersection of external providers, long-lived network connections, and high-throughput workloads. While our unit and integration tests do an excellent job validating correctness, they are not designed to surface issues that only appear after extended operation.
-
-A subtle lifecycle edge case discovered in v1.81.3 reinforced the need for stronger release validation in this area.
-
----
-
-## A Real-World Lifecycle Edge Case
-
-In v1.81.3, we shipped a fix for an HTTP client memory leak. The change passed unit and integration tests and behaved correctly in short-lived runs.
-
-The issue that surfaced was not caused by a single incorrect line of logic, but by how multiple components interacted over time:
-
-- A cached `httpx` client was configured with a 1-hour TTL
-- When the cache expired, the underlying HTTP connection was closed as expected
-- A higher-level client continued to hold a reference to that connection
-- Subsequent requests failed with:
-
-```
-Cannot send a request, as the client has been closed
-```
-
-**Before (with bug):**
-
-| Provider | Requests | Success | Failures | Fail % |
-|----------|----------|---------|----------|--------|
-| OpenAI   | 720,000  | 432,000 | 288,000  | 40%    |
-| Azure    | 692,000  | 415,200 | 276,800  | 40%    |
-
-**After (fixed):**
-
-| Provider | Requests   | Success   | Failures | Fail %  |
-|----------|------------|-----------|----------|---------|
-| OpenAI   | 1,200,000  | 1,199,988 | 12       | 0.001%  |
-| Azure    | 1,150,000  | 1,149,982 | 18       | 0.002%  |
-
-Our focus moving forward is on being the first to detect issues, even when they aren’t covered by unit tests. LiteLLM Observatory is designed to surface latency regressions, OOMs, and failure modes that only appear under real traffic patterns in **our own production deployments** during release validation.
-
-
----
-
-### How the Observatory Works
-
-[LiteLLM Observatory](https://github.com/BerriAI/litellm-observatory) is a testing service that runs long-running tests against our LiteLLM deployments. We trigger tests by sending API requests, and results are automatically sent to Slack when tests complete.
-
-#### How Tests Run
-
-1. **Start a Test**: We send a request to the Observatory API with:
-   - Which LiteLLM deployment to test (URL and API key)
-   - Which test to run (e.g., `TestOAIAzureRelease`)
-   - Test settings (which models to test, how long to run, failure thresholds)
-
-2. **Smart Queueing**:
-   - The system checks whether we are attempting to run the exact same test more than once
-   - If a duplicate test is already running or queued, we receive an error to avoid wasting resources
-   - Otherwise, the test is added to a queue and runs when capacity is available (up to 5 tests can run concurrently by default)
-
-3. **Instant Response**: The API responds immediately—we do not wait for the test to finish. Tests may run for hours, but the request itself completes in milliseconds.
-
-4. **Background Execution**:
-   - The test runs in the background, issuing requests against our LiteLLM deployment
-   - It tracks request success and failure rates over time
-   - When the test completes, results are automatically posted to our Slack channel
-
-#### Example: The OpenAI / Azure Reliability Test
-
-The `TestOAIAzureRelease` test is designed to catch a class of bugs that only surface after sustained runtime:
-
-- **Duration**: Runs continuously for 3 hours
-- **Behavior**: Cycles through specified models (such as `gpt-4` and `gpt-3.5-turbo`), issuing requests continuously
-- **Why 3 Hours**: This helps catch issues where HTTP clients degrade or fail after extended use (for example, a bug observed in LiteLLM v1.81.3)
-- **Pass / Fail Criteria**: The test passes if fewer than 1% of requests fail. If the failure rate exceeds 1%, the test fails and we are notified in Slack
-- **Key Detail**: The same HTTP client is reused for the entire run, allowing us to detect lifecycle-related bugs that only appear under prolonged reuse
-
-#### When We Use It
-
-- **Before Deployments**: Run tests before promoting a new LiteLLM version to production
-- **Routine Validation**: Schedule regular runs (daily or weekly) to catch regressions early
-- **Issue Investigation**: Run tests on demand when we suspect a deployment issue
-- **Long-Running Failure Detection**: Identify bugs that only appear under sustained load, beyond what short smoke tests can reveal
-
-
-### Complementing Unit Tests
-
-Unit tests remain a foundational part of our development process. They are fast and precise, but they don’t cover:
-
-- Real provider behavior
-- Long-lived network interactions
-- Resource lifecycle edge cases
-- Time-dependent regressions
-
-LiteLLM Observatory complements unit tests by validating the system as it actually runs in production-like environments.
-
----
-
-### Looking Ahead
-
-Reliability is an ongoing investment.
-
-LiteLLM Observatory is one of several systems we’re building to continuously raise the bar on release quality and operational safety. As LiteLLM evolves, so will our validation tooling, informed by real-world usage and lessons learned.
-
-We’ll continue to share those improvements openly as we go.

docs/my-website/blog/minimax_m2_5/index.md

@@ -1,387 +0,0 @@
----
-slug: minimax_m2_5
-title: "Day 0 Support: MiniMax-M2.5"
-date: 2026-02-12T10:00:00
-authors:
-  - sameer
-  - krrish
-  - ishaan-alt
-description: "Day 0 support for MiniMax-M2.5 on LiteLLM"
-tags: [minimax, M2.5, llm]
-hide_table_of_contents: false
----
-
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-LiteLLM now supports MiniMax-M2.5 on Day 0. Use it across OpenAI-compatible and Anthropic-compatible APIs through the LiteLLM AI Gateway.
-
-{/* truncate */}
-
-## Supported Models
-
-LiteLLM supports the following MiniMax models:
-
-| Model | Description | Input Cost | Output Cost | Context Window |
-|-------|-------------|------------|-------------|----------------|
-| **MiniMax-M2.5** | Advanced reasoning, Agentic capabilities | $0.3/M tokens | $1.2/M tokens | 1M tokens |
-| **MiniMax-M2.5-lightning** | Faster and More Agile (~100 tps) | $0.3/M tokens | $2.4/M tokens | 1M tokens |
-
-## Features Supported
-
-- **Prompt Caching**: Reduce costs with cached prompts ($0.03/M tokens for cache read, $0.375/M tokens for cache write)
-- **Function Calling**: Built-in tool calling support
-- **Reasoning**: Advanced reasoning capabilities with thinking support
-- **System Messages**: Full system message support
-- **Cost Tracking**: Automatic cost calculation for all requests
-
-## Docker Image
-
-```bash
-docker pull litellm/litellm:v1.81.3-stable
-```
-
-## Usage - OpenAI Compatible API (/v1/chat/completions)
-
-<Tabs>
-<TabItem value="proxy" label="LiteLLM Proxy">
-
-**1. Setup config.yaml**
-
-```yaml
-model_list:
-  - model_name: minimax-m2-5
-    litellm_params:
-      model: minimax/MiniMax-M2.5
-      api_key: os.environ/MINIMAX_API_KEY
-      api_base: https://api.minimax.io/v1
-```
-
-**2. Start the proxy**
-
-```bash
-docker run -d \
-  -p 4000:4000 \
-  -e MINIMAX_API_KEY=$MINIMAX_API_KEY \
-  -v $(pwd)/config.yaml:/app/config.yaml \
-  ghcr.io/berriai/litellm:v1.81.3-stable \
-  --config /app/config.yaml
-```
-
-**3. Test it!**
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
---header 'Content-Type: application/json' \
---header 'Authorization: Bearer $LITELLM_KEY' \
---data '{
-  "model": "minimax-m2-5",
-  "messages": [
-    {
-      "role": "user",
-      "content": "what llm are you"
-    }
-  ]
-}'
-```
-
-</TabItem>
-</Tabs>
-
-### With Reasoning Split
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
---header 'Content-Type: application/json' \
---header 'Authorization: Bearer $LITELLM_KEY' \
---data '{
-  "model": "minimax-m2-5",
-  "messages": [
-    {
-      "role": "user",
-      "content": "Solve: 2+2=?"
-    }
-  ],
-  "extra_body": {
-    "reasoning_split": true
-  }
-}'
-```
-
-## Usage - Anthropic Compatible API (/v1/messages)
-
-<Tabs>
-<TabItem value="proxy" label="LiteLLM Proxy">
-
-**1. Setup config.yaml**
-
-```yaml
-model_list:
-  - model_name: minimax-m2-5
-    litellm_params:
-      model: minimax/MiniMax-M2.5
-      api_key: os.environ/MINIMAX_API_KEY
-      api_base: https://api.minimax.io/anthropic/v1/messages
-```
-
-**2. Start the proxy**
-
-```bash
-docker run -d \
-  -p 4000:4000 \
-  -e MINIMAX_API_KEY=$MINIMAX_API_KEY \
-  -v $(pwd)/config.yaml:/app/config.yaml \
-  ghcr.io/berriai/litellm:v1.81.3-stable \
-  --config /app/config.yaml
-```
-
-**3. Test it!**
-
-```bash
-curl --location 'http://0.0.0.0:4000/v1/messages' \
---header 'Content-Type: application/json' \
---header 'Authorization: Bearer $LITELLM_KEY' \
---data '{
-  "model": "minimax-m2-5",
-  "max_tokens": 1000,
-  "messages": [
-    {
-      "role": "user",
-      "content": "what llm are you"
-    }
-  ]
-}'
-```
-
-</TabItem>
-</Tabs>
-
-### With Thinking
-
-```bash
-curl --location 'http://0.0.0.0:4000/v1/messages' \
---header 'Content-Type: application/json' \
---header 'Authorization: Bearer $LITELLM_KEY' \
---data '{
-  "model": "minimax-m2-5",
-  "max_tokens": 1000,
-  "thinking": {
-    "type": "enabled",
-    "budget_tokens": 1000
-  },
-  "messages": [
-    {
-      "role": "user",
-      "content": "Solve: 2+2=?"
-    }
-  ]
-}'
-```
-
-## Usage - LiteLLM SDK
-
-### OpenAI-compatible API
-
-```python
-import litellm
-
-response = litellm.completion(
-    model="minimax/MiniMax-M2.5",
-    messages=[
-        {"role": "user", "content": "Hello, how are you?"}
-    ],
-    api_key="your-minimax-api-key",
-    api_base="https://api.minimax.io/v1"
-)
-
-print(response.choices[0].message.content)
-```
-
-### Anthropic-compatible API
-
-```python
-import litellm
-
-response = litellm.anthropic.messages.acreate(
-    model="minimax/MiniMax-M2.5",
-    messages=[{"role": "user", "content": "Hello, how are you?"}],
-    api_key="your-minimax-api-key",
-    api_base="https://api.minimax.io/anthropic/v1/messages",
-    max_tokens=1000
-)
-
-print(response.choices[0].message.content)
-```
-
-### With Thinking
-
-```python
-response = litellm.anthropic.messages.acreate(
-    model="minimax/MiniMax-M2.5",
-    messages=[{"role": "user", "content": "Solve: 2+2=?"}],
-    thinking={"type": "enabled", "budget_tokens": 1000},
-    api_key="your-minimax-api-key"
-)
-
-# Access thinking content
-for block in response.choices[0].message.content:
-    if hasattr(block, 'type') and block.type == 'thinking':
-        print(f"Thinking: {block.thinking}")
-```
-
-### With Reasoning Split (OpenAI API)
-
-```python
-response = litellm.completion(
-    model="minimax/MiniMax-M2.5",
-    messages=[
-        {"role": "user", "content": "Solve: 2+2=?"}
-    ],
-    extra_body={"reasoning_split": True},
-    api_key="your-minimax-api-key",
-    api_base="https://api.minimax.io/v1"
-)
-
-# Access thinking and response
-if hasattr(response.choices[0].message, 'reasoning_details'):
-    print(f"Thinking: {response.choices[0].message.reasoning_details}")
-print(f"Response: {response.choices[0].message.content}")
-```
-
-## Cost Tracking
-
-LiteLLM automatically tracks costs for MiniMax-M2.5 requests. The pricing is:
-
-- **Input**: $0.3 per 1M tokens
-- **Output**: $1.2 per 1M tokens
-- **Cache Read**: $0.03 per 1M tokens
-- **Cache Write**: $0.375 per 1M tokens
-
-### Accessing Cost Information
-
-```python
-response = litellm.completion(
-    model="minimax/MiniMax-M2.5",
-    messages=[{"role": "user", "content": "Hello!"}],
-    api_key="your-minimax-api-key"
-)
-
-# Access cost information
-print(f"Cost: ${response._hidden_params.get('response_cost', 0)}")
-```
-
-## Streaming Support
-
-### OpenAI API
-
-```python
-response = litellm.completion(
-    model="minimax/MiniMax-M2.5",
-    messages=[{"role": "user", "content": "Tell me a story"}],
-    stream=True,
-    api_key="your-minimax-api-key",
-    api_base="https://api.minimax.io/v1"
-)
-
-for chunk in response:
-    if chunk.choices[0].delta.content:
-        print(chunk.choices[0].delta.content, end="")
-```
-
-### Streaming with Reasoning Split
-
-```python
-stream = litellm.completion(
-    model="minimax/MiniMax-M2.5",
-    messages=[
-        {"role": "user", "content": "Tell me a story"},
-    ],
-    extra_body={"reasoning_split": True},
-    stream=True,
-    api_key="your-minimax-api-key",
-    api_base="https://api.minimax.io/v1"
-)
-
-reasoning_buffer = ""
-text_buffer = ""
-
-for chunk in stream:
-    if hasattr(chunk.choices[0].delta, "reasoning_details") and chunk.choices[0].delta.reasoning_details:
-        for detail in chunk.choices[0].delta.reasoning_details:
-            if "text" in detail:
-                reasoning_text = detail["text"]
-                new_reasoning = reasoning_text[len(reasoning_buffer):]
-                if new_reasoning:
-                    print(new_reasoning, end="", flush=True)
-                    reasoning_buffer = reasoning_text
-
-    if chunk.choices[0].delta.content:
-        content_text = chunk.choices[0].delta.content
-        new_text = content_text[len(text_buffer):] if text_buffer else content_text
-        if new_text:
-            print(new_text, end="", flush=True)
-            text_buffer = content_text
-```
-
-## Using with Native SDKs
-
-### Anthropic SDK via LiteLLM Proxy
-
-```python
-import os
-os.environ["ANTHROPIC_BASE_URL"] = "http://localhost:4000"
-os.environ["ANTHROPIC_API_KEY"] = "sk-1234"  # Your LiteLLM proxy key
-
-import anthropic
-
-client = anthropic.Anthropic()
-
-message = client.messages.create(
-    model="minimax-m2-5",
-    max_tokens=1000,
-    system="You are a helpful assistant.",
-    messages=[
-        {
-            "role": "user",
-            "content": [
-                {
-                    "type": "text",
-                    "text": "Hi, how are you?"
-                }
-            ]
-        }
-    ]
-)
-
-for block in message.content:
-    if block.type == "thinking":
-        print(f"Thinking:\n{block.thinking}\n")
-    elif block.type == "text":
-        print(f"Text:\n{block.text}\n")
-```
-
-### OpenAI SDK via LiteLLM Proxy
-
-```python
-import os
-os.environ["OPENAI_BASE_URL"] = "http://localhost:4000"
-os.environ["OPENAI_API_KEY"] = "sk-1234"  # Your LiteLLM proxy key
-
-from openai import OpenAI
-
-client = OpenAI()
-
-response = client.chat.completions.create(
-    model="minimax-m2-5",
-    messages=[
-        {"role": "system", "content": "You are a helpful assistant."},
-        {"role": "user", "content": "Hi, how are you?"},
-    ],
-    extra_body={"reasoning_split": True},
-)
-
-# Access thinking and response
-if hasattr(response.choices[0].message, 'reasoning_details'):
-    print(f"Thinking:\n{response.choices[0].message.reasoning_details[0]['text']}\n")
-print(f"Text:\n{response.choices[0].message.content}\n")
-```

docs/my-website/blog/model_cost_map_incident/index.md

@@ -1,92 +0,0 @@
----
-slug: model-cost-map-incident
-title: "Incident Report: Invalid model cost map on main"
-date: 2026-02-10T10:00:00
-authors:
-  - ishaan
-tags: [incident-report, stability]
-hide_table_of_contents: false
----
-
-**Date:** January 27, 2026
-**Duration:** ~20 minutes
-**Severity:** Low
-**Status:** Resolved
-
-## Summary
-
-A malformed JSON entry in `model_prices_and_context_window.json` was merged to `main` ([`562f0a0`](https://github.com/BerriAI/litellm/commit/562f0a028251750e3d75386bee0e630d9796d0df)). This caused LiteLLM to silently fall back to a stale local copy of the model cost map. Users on older package versions lost cost tracking for newer models only (e.g. `azure/gpt-5.2`). No LLM calls were blocked.
-
-- **LLM calls and proxy routing:** No impact.
-- **Cost tracking:** Impacted for newer models not present in the local backup. Older models were unaffected. The incident lasted ~20 minutes until the commit was reverted.
-
-{/* truncate */}
-
----
-
-## Background
-
-The model cost map is not in the request path. It is used after the LLM response comes back, inside a try/catch, to calculate spend. A missing entry never blocks a call.
-
-```mermaid
-flowchart TD
-    A["1. litellm.completion() receives request
-    litellm/main.py"] --> B["2. Route to provider
-    litellm/litellm_core_utils/get_llm_provider_logic.py"]
-    B --> C["3. LLM returns response
-    litellm/main.py"]
-    C --> D["4. Post-call: look up model in cost map
-    litellm/cost_calculator.py"]
-    D -->|"found"| E["5a. Attach cost to response"]
-    D -->|"not found (try/catch)"| F["5b. Log warning, set cost=0"]
-    E --> G["6. Return response to caller"]
-    F --> G
-
-    style D fill:#fff3cd,stroke:#ffc107
-    style F fill:#fff3cd,stroke:#ffc107
-    style E fill:#d4edda,stroke:#28a745
-    style G fill:#d4edda,stroke:#28a745
-```
-
-Both paths return a response to the caller. When the cost map lookup fails, the only difference is `cost=0` on that request.
-
----
-
-## Root cause
-
-LiteLLM fetches the model cost map from GitHub `main` at import time. If the fetch fails, it falls back to a local backup bundled with the package. Before this incident, the fallback was completely silent -- no warning was logged.
-
-A contributor PR introduced an extra `{` bracket, producing invalid JSON. The remote fetch failed with `JSONDecodeError`, triggering the silent fallback. Users on older package versions had backup files missing newer models.
-
-**Timeline:**
-
-1. Malformed JSON merged to `main`
-2. LiteLLM installations fall back to local backup on next import
-3. Users report `"This model isn't mapped yet"` for newer models
-4. Bad commit identified and reverted (~20 minutes)
-
----
-
-## Remediation
-
-| # | Action | Status | Code |
-|---|---|---|---|
-| 1 | CI validation on `model_prices_and_context_window.json` | ✅ Done | [`test-model-map.yaml`](https://github.com/BerriAI/litellm/blob/main/.github/workflows/test-model-map.yaml) |
-| 2 | Warning log on fallback to local backup | ✅ Done | [`get_model_cost_map.py#L57-L68`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm_core_utils/get_model_cost_map.py#L57-L68) |
-| 3 | `GetModelCostMap` class with integrity validation helpers | ✅ Done | [`get_model_cost_map.py#L24-L149`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm_core_utils/get_model_cost_map.py#L24-L149) |
-| 4 | Resilience test suite (bad hosted map, fallback, completion) | ✅ Done | [`test_model_cost_map_resilience.py#L150-L291`](https://github.com/BerriAI/litellm/blob/main/tests/llm_translation/test_model_cost_map_resilience.py#L150-L291) |
-| 5 | Test that backup model cost map always exists and contains common models | ✅ Done | [`test_model_cost_map_resilience.py#L213-L228`](https://github.com/BerriAI/litellm/blob/main/tests/llm_translation/test_model_cost_map_resilience.py#L213-L228) |
-
-Enterprises that require zero external dependencies at import time can set `LITELLM_LOCAL_MODEL_COST_MAP=True` to skip the GitHub fetch entirely.
-
----
-
-## Other dependencies on external resources
-
-| Dependency | Impact if unavailable | Fallback |
-|---|---|---|
-| Model cost map (GitHub) | Cost tracking for newer models | Local backup (now with warning) |
-| JWT public keys (IDP/SSO) | Auth fails | None |
-| OIDC UserInfo (IDP/SSO) | Auth fails | None |
-| HuggingFace model API | HF provider calls fail | None |
-| Ollama tags (localhost) | Ollama model list stale | Static list |

docs/my-website/blog/realtime_webrtc_http_endpoints/index.md

@@ -1,111 +0,0 @@
----
-slug: realtime_webrtc_http_endpoints
-title: "Realtime WebRTC HTTP Endpoints"
-date: 2026-03-12T10:00:00
-authors:
-  - sameer
-  - krrish
-  - ishaan-alt
-description: "Use the LiteLLM proxy to route OpenAI-style WebRTC realtime via HTTP: client_secrets and SDP exchange."
-tags: [realtime, webrtc, proxy, openai]
-hide_table_of_contents: false
----
-
-import WebRTCTester from '@site/src/components/WebRTCTester';
-
-Connect to the Realtime API via WebRTC from browser/mobile clients. LiteLLM handles auth and key management.
-
-{/* truncate */}
-
-## How it works
-
-![WebRTC flow: Browser, LiteLLM Proxy, and OpenAI/Azure](../../img/webrtc_flow.png)
-
-**Flow of generating ephemeral token**
-
-![Ephemeral token flow: Browser requests token, LiteLLM gets real token from OpenAI, returns encrypted token](../../img/ephemeral_token.png)
-
-
-## Proxy Setup
-
-```yaml
-model_list:
-  - model_name: gpt-4o-realtime
-    litellm_params:
-      model: openai/gpt-4o-realtime-preview-2024-12-17
-      api_key: os.environ/OPENAI_API_KEY
-    model_info:
-      mode: realtime
-```
-
-**Azure:** use `model: azure/gpt-4o-realtime-preview`, `api_key`, `api_base`.
-
-```bash
-litellm --config /path/to/config.yaml
-```
-
-## Try it live
-
-<WebRTCTester />
-
-## Client Usage
-
-**1. Get token** - `POST /v1/realtime/client_secrets` with LiteLLM API key and `{ model }`.
-
-**2. WebRTC handshake** - Create `RTCPeerConnection`, add mic track, create data channel `oai-events`, send SDP offer to `POST /v1/realtime/calls` with `Authorization: Bearer <encrypted_token>` and `Content-Type: application/sdp`.
-
-**3. Events** - Use the data channel for `session.update` and other events.
-
-<details>
-<summary>Full code example</summary>
-
-```javascript
-// 1. Token
-const r = await fetch("http://proxy:4000/v1/realtime/client_secrets", {
-  method: "POST",
-  headers: { "Authorization": "Bearer sk-litellm-key", "Content-Type": "application/json" },
-  body: JSON.stringify({ model: "gpt-4o-realtime" }),
-});
-const { client_secret } = await r.json();
-const token = client_secret.value;
-
-// 2. WebRTC
-const pc = new RTCPeerConnection();
-const audio = document.createElement("audio");
-audio.autoplay = true;
-pc.ontrack = (e) => (audio.srcObject = e.streams[0]);
-const ms = await navigator.mediaDevices.getUserMedia({ audio: true });
-pc.addTrack(ms.getTracks()[0]);
-const dc = pc.createDataChannel("oai-events");
-const offer = await pc.createOffer();
-await pc.setLocalDescription(offer);
-
-const sdpRes = await fetch("http://proxy:4000/v1/realtime/calls", {
-  method: "POST",
-  headers: { "Authorization": `Bearer ${token}`, "Content-Type": "application/sdp" },
-  body: offer.sdp,
-});
-await pc.setRemoteDescription({ type: "answer", sdp: await sdpRes.text() });
-
-// 3. Events
-dc.send(JSON.stringify({ type: "session.update", session: { instructions: "..." } }));
-```
-
-</details>
-
-## FAQ
-
-**Q: What do I do if I get a 401 Token expired error?**  
-A: Tokens are short-lived. Get a fresh token right before creating the WebRTC offer.
-
-**Q: Which key should I use for `/v1/realtime/calls`?**  
-A: Use the **encrypted token** from `client_secrets`, not your raw API key.
-
-**Q: Should I pass the `model` parameter when making the call?**  
-A: No, the encrypted token already encodes all routing information including model.
-
-**Q: How do I resolve Azure `api-version` errors?**  
-A: Set the correct `api_version` in `litellm_params` (or via the `AZURE_API_VERSION` environment variable), along with the right `api_base` and deployment values.
-
-**Q: What if I get no audio?**  
-A: Make sure you grant microphone permission, ensure `pc.ontrack` assigns the audio element with `autoplay` enabled, check your network/firewall for WebRTC traffic, and inspect the browser console for ICE or SDP errors.

docs/my-website/blog/redis_circuit_breaker/diagrams.js

@@ -1,159 +0,0 @@
-import React from 'react';
-
-const s = {
-  fig: {margin: '2.5rem 0', fontFamily: 'inherit'},
-  box: {borderRadius: 12, border: '1px solid #e5e7eb', background: '#fff', padding: '2rem 2.5rem'},
-  label: {fontSize: 11, fontWeight: 700, textTransform: 'uppercase', letterSpacing: '0.12em', color: '#9ca3af', textAlign: 'center', marginBottom: '1.5rem'},
-  caption: {textAlign: 'center', fontSize: 12, color: '#9ca3af', marginTop: 12},
-  node: (border='#d1d5db', bg='#f9fafb') => ({
-    border: `1px solid ${border}`, borderRadius: 6, padding: '8px 20px',
-    fontSize: 13, background: bg, display: 'inline-block',
-  }),
-  arrow: {display: 'flex', flexDirection: 'column', alignItems: 'center'},
-};
-
-const SmallArrow = ({color='#9ca3af'}) => (
-  <svg width="2" height="28" style={{display:'block'}}>
-    <line x1="1" y1="0" x2="1" y2="22" stroke={color} strokeWidth="1.5"/>
-    <polygon points="1,28 -2,21 4,21" fill={color}/>
-  </svg>
-);
-
-export function CascadeFailure() {
-  return (
-    <figure style={s.fig}>
-      <div style={s.box}>
-        <p style={s.label}>Without circuit breaker — cascade failure</p>
-        <div style={{display:'flex', flexDirection:'column', alignItems:'center', gap:0}}>
-          <div style={s.node()}>LiteLLM Pod (×100)</div>
-          <SmallArrow />
-          <div style={s.node()}>Rate limit / cache check</div>
-          <div style={{position:'relative', display:'flex', flexDirection:'column', alignItems:'center'}}>
-            <SmallArrow color="#f87171"/>
-            <span style={{position:'absolute', left:8, top:4, fontSize:11, color:'#f87171', whiteSpace:'nowrap'}}>hangs 30s per request</span>
-          </div>
-          <div style={s.node('#fca5a5','#fef2f2')}><span style={{color:'#b91c1c', fontWeight:600}}>Redis — degraded, timing out</span></div>
-          <SmallArrow color="#fb923c"/>
-          <div style={s.node('#fdba74','#fff7ed')}><span style={{color:'#c2410c', fontWeight:600}}>Postgres — 100× normal read load</span></div>
-          <SmallArrow />
-          <div style={{...s.node('#111827','#111827'), color:'#fff', fontWeight:600}}>Total outage — gateway down</div>
-        </div>
-      </div>
-      <figcaption style={s.caption}>Slow Redis → every auth check times out → database overwhelmed → full cascade</figcaption>
-    </figure>
-  );
-}
-
-export function CircuitBreakerStates() {
-  const circle = (border, color, label, sub) => (
-    <div style={{display:'flex', flexDirection:'column', alignItems:'center', width: 140}}>
-      <div style={{width:88, height:88, borderRadius:'50%', border:`2px solid ${border}`, background:'#fff', display:'flex', flexDirection:'column', alignItems:'center', justifyContent:'center'}}>
-        <span style={{fontSize:11, fontWeight:700, color, letterSpacing:'0.06em'}}>{label}</span>
-        <span style={{fontSize:10, color:'#9ca3af', marginTop:2}}>{sub}</span>
-      </div>
-      <p style={{fontSize:11, color:'#6b7280', textAlign:'center', marginTop:10, lineHeight:1.5}}>{'\u00a0'}</p>
-    </div>
-  );
-  const arrow = (label) => (
-    <div style={{display:'flex', flexDirection:'column', alignItems:'center', marginTop:36, marginLeft:4, marginRight:4}}>
-      <span style={{fontSize:10, color:'#6b7280', marginBottom:4}}>{label}</span>
-      <div style={{display:'flex', alignItems:'center'}}>
-        <div style={{height:1, width:48, background:'#9ca3af'}}/>
-        <svg width="8" height="8" style={{marginLeft:-1}}><polygon points="0,0 8,4 0,8" fill="#6b7280"/></svg>
-      </div>
-    </div>
-  );
-  return (
-    <figure style={s.fig}>
-      <div style={s.box}>
-        <p style={s.label}>Circuit breaker state machine</p>
-        <div style={{display:'flex', justifyContent:'center', alignItems:'flex-start'}}>
-          {circle('#1f2937','#111827','CLOSED','normal')}
-          {arrow('5 failures')}
-          {circle('#f87171','#dc2626','OPEN','fast-fail')}
-          {arrow('60s timeout')}
-          {circle('#fbbf24','#b45309','HALF-OPEN','probing')}
-        </div>
-        <div style={{display:'flex', justifyContent:'center', gap:32, marginTop:24}}>
-          <div style={{display:'flex', flexDirection:'column', alignItems:'center', gap:4}}>
-            <div style={{display:'flex', alignItems:'center', gap:4}}>
-              <svg width="8" height="8"><polygon points="8,0 0,4 8,8" fill="#16a34a"/></svg>
-              <div style={{height:1, width:100, background:'#16a34a'}}/>
-            </div>
-            <span style={{fontSize:10, color:'#16a34a'}}>probe success → CLOSED</span>
-          </div>
-          <div style={{display:'flex', flexDirection:'column', alignItems:'center', gap:4}}>
-            <div style={{display:'flex', alignItems:'center', gap:4}}>
-              <svg width="8" height="8"><polygon points="8,0 0,4 8,8" fill="#ef4444"/></svg>
-              <div style={{height:1, width:100, borderTop:'2px dashed #f87171'}}/>
-            </div>
-            <span style={{fontSize:10, color:'#ef4444'}}>probe failure → OPEN again</span>
-          </div>
-        </div>
-      </div>
-    </figure>
-  );
-}
-
-export function CircuitBreakerFlow() {
-  return (
-    <figure style={s.fig}>
-      <div style={s.box}>
-        <p style={s.label}>With circuit breaker — graceful degradation</p>
-        <div style={{display:'flex', flexDirection:'column', alignItems:'center'}}>
-          <div style={s.node()}>Incoming request</div>
-          <SmallArrow />
-          <div style={{...s.node('#111827'), border:'2px solid #111827', fontWeight:600}}>Circuit Breaker</div>
-          <div style={{display:'flex', gap:80, marginTop:20, alignItems:'flex-start'}}>
-            <div style={{display:'flex', flexDirection:'column', alignItems:'center', gap:8}}>
-              <SmallArrow />
-              <span style={{fontSize:10, fontWeight:700, textTransform:'uppercase', letterSpacing:'0.06em', color:'#6b7280', border:'1px solid #e5e7eb', borderRadius:4, padding:'2px 8px'}}>Closed</span>
-              <div style={{...s.node(), textAlign:'center', fontSize:13}}>Redis call<br/><span style={{fontSize:11, color:'#9ca3af'}}>normal latency</span></div>
-            </div>
-            <div style={{display:'flex', flexDirection:'column', alignItems:'center', gap:8}}>
-              <SmallArrow />
-              <span style={{fontSize:10, fontWeight:700, textTransform:'uppercase', letterSpacing:'0.06em', color:'#ef4444', border:'1px solid #fca5a5', borderRadius:4, padding:'2px 8px'}}>Open</span>
-              <div style={{...s.node('#fca5a5'), textAlign:'center', fontSize:13}}>Fast-fail — 0ms<br/><span style={{fontSize:11, color:'#9ca3af'}}>no network call</span></div>
-              <SmallArrow />
-              <div style={{...s.node(), textAlign:'center', fontSize:13}}>DB fallback<br/><span style={{fontSize:11, color:'#9ca3af'}}>bounded load</span></div>
-            </div>
-          </div>
-          <div style={{...s.node('#111827','#111827'), color:'#fff', fontWeight:600, marginTop:24}}>Request completes — gateway stays up</div>
-        </div>
-      </div>
-      <figcaption style={s.caption}>Redis down → circuit opens → 0ms rejection → DB absorbs bounded fallback traffic</figcaption>
-    </figure>
-  );
-}
-
-export function IncidentTimeline() {
-  const row = (color, text) => (
-    <div style={{display:'flex', alignItems:'flex-start', gap:10, marginBottom:12}}>
-      <div style={{marginTop:5, width:6, height:6, borderRadius:'50%', background:color, flexShrink:0}}/>
-      <p style={{fontSize:13, color:'#4b5563', margin:0, lineHeight:1.5}}>{text}</p>
-    </div>
-  );
-  return (
-    <figure style={s.fig}>
-      <div style={s.box}>
-        <p style={s.label}>Redis degrades — before vs. after</p>
-        <div style={{display:'grid', gridTemplateColumns:'1fr 1fr', gap:20}}>
-          <div style={{border:'1px solid #e5e7eb', borderRadius:8, padding:20}}>
-            <p style={{fontSize:10, fontWeight:700, textTransform:'uppercase', letterSpacing:'0.1em', color:'#9ca3af', marginBottom:16}}>Without circuit breaker</p>
-            {row('#f87171','All 100 pods hang for 30s on each auth check')}
-            {row('#f87171','Threadpools fill up, requests queue')}
-            {row('#f87171','100× simultaneous DB fallbacks overwhelm Postgres')}
-            {row('#f87171','Requires manual intervention to recover')}
-          </div>
-          <div style={{border:'1px solid #111827', borderRadius:8, padding:20}}>
-            <p style={{fontSize:10, fontWeight:700, textTransform:'uppercase', letterSpacing:'0.1em', color:'#9ca3af', marginBottom:16}}>With circuit breaker</p>
-            {row('#111827','Circuit opens after 5 failures — 0ms fast-fail')}
-            {row('#111827','Auth falls back to DB — bounded, not 100× load')}
-            {row('#111827','Cache miss rate temporarily elevated — gateway stays up')}
-            {row('#111827','Auto-recovers when Redis comes back — no intervention needed')}
-          </div>
-        </div>
-      </div>
-    </figure>
-  );
-}

docs/my-website/blog/redis_circuit_breaker/index.md

@@ -1,141 +0,0 @@
----
-slug: redis-circuit-breaker
-title: "Making the AI Gateway Resilient to Redis Failures"
-date: 2026-04-11T09:00:00
-authors:
-  - ishaan
-description: "How LiteLLM's production AI Gateway handles Redis degradation at scale without cascading failures — circuit breaker pattern, 0ms fast-fail, automatic recovery."
-tags: [reliability, redis, infrastructure, engineering, ai-gateway]
-hide_table_of_contents: true
----
-
-import { CascadeFailure, CircuitBreakerStates, CircuitBreakerFlow, IncidentTimeline } from './diagrams';
-
-*Last Updated: April 2026*
-
-Enterprise AI Gateway deployments put Redis in the hot path for nearly every request: rate limiting, cache lookups, spend tracking. When Redis is healthy, the latency contribution is single-digit milliseconds — invisible to end users. When it degrades, a production AI Gateway needs to stay up regardless.
-
-Running LiteLLM at scale across 100+ pods means designing for failure modes before they appear. The easy case is Redis going fully down: fail fast, fall through to the database, continue serving requests. The hard case — the one that takes down gateways — is a *slow* Redis: still accepting connections, still responding, but timing out after 20-30 seconds per operation.
-
-{/* truncate */}
-
-## Why slow Redis is harder than a full outage
-
-<CascadeFailure />
-
-With 100 pods each hanging 30 seconds on every auth check, threadpools fill up and requests queue. By the time Redis times out and falls through to Postgres, the database receives 100× its normal load from simultaneous fallbacks. A slow Redis becomes a database outage becomes a full gateway outage. A production-grade AI Gateway cannot allow one degraded dependency to cascade into total failure.
-
-## The fix: circuit breaker pattern
-
-The circuit breaker pattern tracks consecutive failures and cuts off the unhealthy dependency before it cascades. Instead of hanging 30 seconds on each Redis call, the circuit opens after 5 consecutive failures and fast-fails at 0ms — no network call, no wait.
-
-<CircuitBreakerStates />
-
-Three states:
-
-- **CLOSED** — normal. All Redis calls pass through.
-- **OPEN** — Redis is unhealthy. Every call fast-fails instantly. Requests continue with degraded-but-functional behavior: auth and rate limiting fall back to the database.
-- **HALF-OPEN** — after 60 seconds, one probe request tests recovery. Success closes the circuit; failure resets the timer.
-
-This is how a reliable AI Gateway handles infrastructure degradation: stay up, degrade gracefully, recover automatically.
-
-## How requests flow through the AI Gateway
-
-<CircuitBreakerFlow />
-
-When the circuit is open, the gateway does not stall. Auth checks fall back to Postgres — slower, but bounded. The database absorbs the load because it receives *some* requests via DB fallback, not *all* 100 pods simultaneously dumping their queued requests after a 30-second timeout.
-
-The difference between a resilient AI Gateway and a fragile one: controlled degradation vs. uncontrolled cascade.
-
-## The implementation
-
-```python
-class RedisCircuitBreaker:
-    def __init__(self, failure_threshold: int, recovery_timeout: int):
-        self.failure_threshold = failure_threshold  # default: 5
-        self.recovery_timeout = recovery_timeout    # default: 60s
-        self._failure_count = 0
-        self._state = self.CLOSED
-
-    def is_open(self) -> bool:
-        if self._state == self.OPEN:
-            if time.time() - self._opened_at > self.recovery_timeout:
-                self._state = self.HALF_OPEN
-                return False  # this caller is the recovery probe
-            return True       # fast-fail
-        return False
-
-    def record_failure(self):
-        self._failure_count += 1
-        self._opened_at = time.time()
-        if self._failure_count >= self.failure_threshold:
-            self._state = self.OPEN  # open the circuit
-
-    def record_success(self):
-        self._failure_count = 0
-        self._state = self.CLOSED   # Redis recovered
-```
-
-Every async Redis operation goes through a decorator that checks the breaker before touching the network. When open, it raises immediately:
-
-```python
-@_redis_circuit_breaker_guard
-async def async_get_cache(self, key: str):
-    ...
-```
-
-The decorator handles all bookkeeping — success resets nothing, failures increment the counter, exceptions trigger `record_failure()`. The caller sees a clean exception and falls through to its normal non-Redis path. No changes required in calling code.
-
-## AI Gateway resilience in production
-
-<IncidentTimeline />
-
-Redis degradation events no longer cascade in production. The observable symptom during a Redis slowdown is a temporary bump in cache miss rate — the right failure mode for a resilient AI Gateway. Auth still works. Rate limiting still works. Spend tracking still works, at slightly higher DB cost. Recovery is fully automatic when Redis comes back.
-
-```bash
-# configure via environment variables
-REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD=5   # failures before opening
-REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT=60  # seconds before probe
-```
-
-The circuit breaker ships on by default in all LiteLLM versions since `v1.82.0`. No configuration needed for most deployments.
-
-## Key Takeaways
-
-- A slow Redis is more dangerous than a downed one: 30-second timeouts across 100+ pods overwhelm Postgres at 100× normal load
-- LiteLLM's AI Gateway uses a circuit breaker that fast-fails Redis calls at 0ms after 5 consecutive failures
-- Three states: CLOSED (normal), OPEN (fast-fail + DB fallback), HALF-OPEN (probe recovery)
-- Auth, rate limiting, and spend tracking continue working during Redis outages
-- Resilient, production-grade behavior — enabled by default since `v1.82.0`, no configuration required
-
----
-
-### Frequently Asked Questions
-
-### Does the circuit breaker affect normal Redis performance?
-
-No. When Redis is healthy (circuit CLOSED), every call passes through with zero overhead. The breaker only activates after 5 consecutive failures — transparent under normal conditions.
-
-### What happens to rate limiting when the circuit is open?
-
-Rate limiting falls back to Postgres with bounded load. Limits remain enforced at slightly higher DB cost until Redis recovers and the circuit closes automatically.
-
-### How is this different from basic Redis retry logic?
-
-Retry logic still waits for each timeout (30s × retries). The circuit breaker cuts the connection immediately at 0ms after the failure threshold, preventing threadpool exhaustion across all pods simultaneously. Retries make slow-Redis worse; the circuit breaker contains it.
-
-### Is this available in LiteLLM OSS?
-
-Yes. The circuit breaker ships in LiteLLM OSS (Apache 2.0) by default since `v1.82.0`. [LiteLLM Enterprise](https://litellm.ai/enterprise) adds SSO/SCIM, air-gapped deployment, 24/7 SLA support, and advanced guardrails on top of the OSS foundation.
-
----
-
-## Conclusion
-
-Redis resilience is one layer of what makes LiteLLM a production-grade, reliable AI Gateway at scale. The circuit breaker pattern ensures infrastructure degradation stays contained — the right failure mode is a temporary cache miss rate bump, not a full outage. This is how AI Gateway infrastructure should behave under pressure: degrade gracefully, recover automatically, keep serving traffic. For teams with strict uptime and compliance requirements, [LiteLLM Enterprise](https://litellm.ai/enterprise) provides the additional controls needed for regulated production environments.
-
-## Recommended Reading
-
-- [LiteLLM AI Gateway — full feature overview](https://docs.litellm.ai/docs/simple_proxy)
-- [Load balancing and routing across 100+ LLM providers](https://docs.litellm.ai/docs/routing)
-- [Spend tracking and budget controls](https://docs.litellm.ai/docs/proxy/cost_tracking)

docs/my-website/blog/responses_api_encrypted_content_incident/index.md

@@ -1,312 +0,0 @@
----
-slug: responses-api-encrypted-content-incident
-title: "Incident Report: Encrypted Content Failures in Multi-Region Responses API Load Balancing"
-date: 2026-02-24T10:00:00
-authors:
-  - sameer
-  - krrish
-  - ishaan-alt
-tags: [incident-report, proxy, responses-api, load-balancing]
-hide_table_of_contents: false
----
-
-**Date:** Feb 24, 2026  
-**Duration:** Ongoing (until fix deployed)  
-**Severity:** High (for users load balancing Responses API across different API keys)  
-**Status:** Resolved
-
-## Summary
-
-When load balancing OpenAI's Responses API across deployments with **different API keys** (e.g., different Azure regions or OpenAI organizations), follow-up requests containing encrypted content items (like `rs_...` reasoning items) would fail with:
-
-```json
-{
-  "error": {
-    "message": "The encrypted content for item rs_0d09d6e56879e76500699d6feee41c8197bd268aae76141f87 could not be verified. Reason: Encrypted content organization_id did not match the target organization.",
-    "type": "invalid_request_error",
-    "code": "invalid_encrypted_content"
-  }
-}
-```
-
-Encrypted content items are cryptographically tied to the API key's organization that created them. When the router load balanced a follow-up request to a deployment with a different API key, decryption failed.
-
-- **Responses API calls with encrypted content:** Complete failure when routed to wrong deployment
-- **Initial requests:** Unaffected — only follow-up requests containing encrypted items failed
-- **Other API endpoints:** No impact — chat completions, embeddings, etc. functioned normally
-
-{/* truncate */}
-
----
-
-## Background
-
-OpenAI's Responses API can return encrypted "reasoning items" (with IDs like `rs_...`) that contain intermediate reasoning steps. These items are encrypted with the organization's key and can only be decrypted by the same organization's API key.
-
-When load balancing across deployments with different API keys, the existing affinity mechanisms were insufficient:
-
-- **`responses_api_deployment_check`**: Requires `previous_response_id` which some clients (like Codex) don't provide
-- **`deployment_affinity`**: Too broad — pins *all* requests from a user to one deployment, reducing effective quota by the number of users
-- **`session_affinity`**: Requires explicit session IDs and still reduces quota
-
-```mermaid
-flowchart TD
-    A["1. Initial request to Responses API
-    router.aresponses()"] --> B["2. Router load balances to Deployment A
-    (API Key 1, Azure East US)"]
-    B --> C["3. Response contains encrypted item
-    rs_abc123 (encrypted with Org 1 key)"]
-    C --> D["4. Follow-up request includes rs_abc123 in input"]
-    D --> E["5. Router load balances to Deployment B
-    (API Key 2, Azure West Europe)"]
-    E -->|"Different API key"| F["6. ❌ Deployment B cannot decrypt rs_abc123
-    Error: invalid_encrypted_content"]
-    
-    D -.->|"With encrypted_content_affinity"| G["5b. Router detects rs_abc123 was created by Deployment A"]
-    G --> H["6b. ✅ Routes to Deployment A (bypasses rate limits)
-    Request succeeds"]
-
-    style F fill:#f8d7da,stroke:#dc3545
-    style H fill:#d4edda,stroke:#28a745
-    style E fill:#fff3cd,stroke:#ffc107
-    style G fill:#d4edda,stroke:#28a745
-```
-
----
-
-## Root Cause
-
-LiteLLM's router had no mechanism to track which deployment created specific encrypted content items and route follow-up requests accordingly. The router treated all deployments as interchangeable, leading to decryption failures when encrypted content crossed organizational boundaries.
-
-**The Problem Flow:**
-
-1. User calls `router.aresponses()` with model `gpt-5.1-codex`
-2. Router load balances to Deployment A (Azure East US, API Key 1)
-3. Response contains encrypted reasoning item `rs_abc123` (encrypted with Org 1's key)
-4. User makes follow-up request with `rs_abc123` in the input
-5. Router load balances to Deployment B (Azure West Europe, API Key 2)
-6. Deployment B tries to decrypt `rs_abc123` with Org 2's key → **fails**
-
-**Why Existing Solutions Didn't Work:**
-
-- **`previous_response_id`**: Not provided by all clients (e.g., Codex)
-- **`deployment_affinity`**: Pins *all* user requests to one deployment → reduces quota to 1/N where N = number of deployments
-- **`session_affinity`**: Requires explicit session management and still reduces quota
-
-**Timeline:**
-
-1. Users configured multi-region Responses API load balancing with different API keys
-2. Initial requests succeeded, but follow-up requests with encrypted content failed intermittently
-3. Error rate correlated with number of deployments (more deployments = higher chance of routing to wrong one)
-4. Investigation revealed encrypted content was organization-bound
-5. Existing affinity mechanisms deemed unsuitable (quota reduction, missing `previous_response_id`)
-6. New solution designed and implemented: `encrypted_content_affinity`
-
----
-
-## The Fix
-
-Implemented a new `encrypted_content_affinity` pre-call check that intelligently tracks encrypted content and routes follow-up requests **only when necessary**.
-
-### Implementation
-
-**1. Encoding `model_id` into output items** ([`responses/utils.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/responses/utils.py))
-
-The same approach used for `previous_response_id` affinity — no cache needed. When a response contains output items with `encrypted_content`, LiteLLM encodes the originating deployment's `model_id` in **two places** for redundancy:
-
-1. **Into the item ID** (if present): `rs_abc123` → `encitem_{base64("litellm:model_id:{model_id};item_id:rs_abc123")}`
-2. **Into the encrypted_content itself**: Wraps the content with `litellm_enc:{base64("model_id:{model_id}")};{original_encrypted_content}`
-
-```python
-# Encoding item IDs (when present)
-def _build_encrypted_item_id(model_id: str, item_id: str) -> str:
-    assembled = f"litellm:model_id:{model_id};item_id:{item_id}"
-    encoded = base64.b64encode(assembled.encode("utf-8")).decode("utf-8")
-    return f"encitem_{encoded}"
-
-# Wrapping encrypted_content (always, for redundancy)
-def _wrap_encrypted_content_with_model_id(encrypted_content: str, model_id: str) -> str:
-    metadata = f"model_id:{model_id}"
-    encoded_metadata = base64.b64encode(metadata.encode("utf-8")).decode("utf-8")
-    return f"litellm_enc:{encoded_metadata};{encrypted_content}"
-```
-
-**Why wrap encrypted_content directly?** Some clients (like Codex) don't consistently send item IDs in follow-up requests, but they always send the `encrypted_content` itself. By embedding `model_id` into the content, affinity works even when IDs are missing.
-
-**Streaming responses:** The wrapping logic is applied to both:
-- Final response objects (non-streaming)
-- Individual streaming events (`response.output_item.added`, `response.output_item.done`)
-
-This ensures clients receiving streaming responses get wrapped content they can send back.
-
-Before forwarding to the upstream provider, LiteLLM restores the original item IDs and unwraps encrypted_content so the provider never sees the encoded form:
-
-```python
-# In responses/main.py — before calling the handler
-input = ResponsesAPIRequestUtils._restore_encrypted_content_item_ids_in_input(input)
-```
-
-**2. `EncryptedContentAffinityCheck` — routing only** ([`encrypted_content_affinity_check.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py))
-
-No `async_log_success_event` or cache lookups — the `model_id` is decoded directly from the item ID or encrypted_content:
-
-```python
-class EncryptedContentAffinityCheck(CustomLogger):
-    async def async_filter_deployments(self, model, healthy_deployments, ...):
-        """Extract model_id from input items (ID or encrypted_content) and pin to that deployment."""
-        for item in request_kwargs.get("input", []):
-            # Try to extract model_id from two sources:
-            model_id = self._extract_model_id_from_input(item)
-            
-            if model_id:
-                deployment = self._find_deployment_by_model_id(
-                    healthy_deployments, model_id
-                )
-                if deployment:
-                    request_kwargs["_encrypted_content_affinity_pinned"] = True
-                    return [deployment]
-        return healthy_deployments
-    
-    def _extract_model_id_from_input(self, item: dict) -> Optional[str]:
-        """Extract model_id from either encoded ID or wrapped encrypted_content."""
-        # 1. Try decoding from item ID (if present)
-        item_id = item.get("id", "")
-        if item_id:
-            decoded = ResponsesAPIRequestUtils._decode_encrypted_item_id(item_id)
-            if decoded:
-                return decoded["model_id"]
-        
-        # 2. Try unwrapping from encrypted_content (fallback for clients that omit IDs)
-        encrypted_content = item.get("encrypted_content", "")
-        if encrypted_content and encrypted_content.startswith("litellm_enc:"):
-            model_id, _ = ResponsesAPIRequestUtils._unwrap_encrypted_content_with_model_id(
-                encrypted_content
-            )
-            return model_id
-        
-        return None
-```
-
-**3. Rate Limit Bypass** ([`router.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/router.py))
-
-When encrypted content requires a specific deployment, RPM/TPM limits are bypassed (the request would fail on any other deployment anyway):
-
-```python
-# In async_get_available_deployment, after filtering healthy deployments:
-if (
-    request_kwargs.get("_encrypted_content_affinity_pinned")
-    and len(healthy_deployments) == 1
-):
-    return healthy_deployments[0]  # Bypass routing strategy (RPM/TPM checks)
-```
-
-**3. Configuration**
-
-```yaml
-router_settings:
-  routing_strategy: usage-based-routing-v2
-  enable_pre_call_checks: true
-  optional_pre_call_checks:
-    - encrypted_content_affinity
-  deployment_affinity_ttl_seconds: 86400  # 24 hours
-```
-
-### Key Benefits
-
-✅ **No quota reduction**: Only pins requests containing encrypted items  
-✅ **Bypasses rate limits**: When encrypted content requires a specific deployment, RPM/TPM limits don't block it  
-✅ **No `previous_response_id` required**: Works by encoding `model_id` directly into the item ID  
-✅ **No cache required**: `model_id` is decoded on-the-fly from the item ID — no Redis, no TTL  
-✅ **Globally safe**: Can be enabled for all models; non-Responses-API calls are unaffected  
-✅ **Surgical precision**: Normal requests continue to load balance freely
-
----
-
-## Remediation
-
-| # | Action | Status | Code |
-|---|---|---|---|
-| 1 | Encode `model_id` into encrypted-content item IDs on response | ✅ Done | [`responses/utils.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/responses/utils.py) |
-| 2 | Restore original item IDs before forwarding to upstream provider | ✅ Done | [`responses/main.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/responses/main.py) |
-| 3 | `EncryptedContentAffinityCheck`: decode item IDs to route (no cache) | ✅ Done | [`encrypted_content_affinity_check.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py) |
-| 4 | Add `encrypted_content_affinity` to `OptionalPreCallChecks` type | ✅ Done | [`types/router.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/types/router.py) |
-| 5 | Implement rate limit bypass for affinity-pinned requests | ✅ Done | [`router.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/router.py) |
-| 6 | Unit tests: encoding/decoding utilities, routing, RPM bypass | ✅ Done | [`test_encrypted_content_affinity_check.py`](https://github.com/BerriAI/litellm/blob/main/litellm/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py) |
-| 7 | Documentation: Responses API guide, load balancing guide, config reference | ✅ Done | [Docs](https://docs.litellm.ai/docs/response_api#encrypted-content-affinity-multi-region-load-balancing) |
-| 8 | **[Mar 3]** Fix streaming events to wrap encrypted_content | ✅ Done | [`responses/streaming_iterator.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/responses/streaming_iterator.py) |
-
----
-
-## Follow-up Fix: Streaming Responses (Mar 3, 2026)
-
-### The Issue
-
-After the initial fix was deployed, users reported that the `invalid_encrypted_content` error **still occurred** when using streaming responses with clients like Codex. Investigation revealed:
-
-- ✅ Non-streaming responses: `encrypted_content` was correctly wrapped with `litellm_enc:` prefix
-- ❌ Streaming responses: Individual `response.output_item.added` and `response.output_item.done` events contained **raw, unwrapped** `encrypted_content`
-
-Since Codex and other clients consume responses as streams, they received unwrapped content in these events and sent it back in follow-up requests, causing the affinity check to fail.
-
-### The Root Cause
-
-The `_update_encrypted_content_item_ids_in_response` function only modified the **final** response object, which is used for non-streaming responses. For streaming responses, individual chunks are processed by `ResponsesAPIStreamingIterator._process_chunk`, which was **not** applying the wrapping logic to streaming events.
-
-### The Fix
-
-Modified `litellm/litellm/responses/streaming_iterator.py` to wrap `encrypted_content` in streaming events:
-
-```python
-# In ResponsesAPIStreamingIterator._process_chunk
-if (
-    self.litellm_metadata
-    and self.litellm_metadata.get("encrypted_content_affinity_enabled")
-):
-    event_type = getattr(openai_responses_api_chunk, "type", None)
-    if event_type in (
-        ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED,
-        ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
-    ):
-        item = getattr(openai_responses_api_chunk, "item", None)
-        if item:
-            encrypted_content = getattr(item, "encrypted_content", None)
-            if encrypted_content and isinstance(encrypted_content, str):
-                model_id = (
-                    self.litellm_metadata.get("model_info", {}).get("id")
-                    if self.litellm_metadata
-                    else None
-                )
-                if model_id:
-                    wrapped_content = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id(
-                        encrypted_content, model_id
-                    )
-                    setattr(item, "encrypted_content", wrapped_content)
-```
-
-This ensures that **all** `encrypted_content` sent to clients (streaming or non-streaming) is wrapped with `model_id` metadata, enabling consistent affinity routing.
-
----
-
-## Migration Guide
-
-### Before (Using `deployment_affinity`)
-
-```yaml
-router_settings:
-  optional_pre_call_checks:
-    - deployment_affinity  # ❌ Reduces quota by number of users
-```
-
-**Problem:** All requests from a user pin to one deployment, reducing effective quota to 1/N.
-
-### After (Using `encrypted_content_affinity`)
-
-```yaml
-router_settings:
-  optional_pre_call_checks:
-    - encrypted_content_affinity  # ✅ Only pins requests with encrypted content
-```
-
-**Benefit:** Normal requests load balance freely, only encrypted content requests pin when necessary.
-
----

docs/my-website/blog/security_hardening_april_2026/index.md

@@ -1,66 +0,0 @@
----
-slug: security-hardening-april-2026
-title: "Security Update: Vulnerability Disclosures and Ongoing Hardening"
-date: 2026-04-03T12:00:00
-authors:
-  - krrish
-  - ishaan-alt
-description: "Disclosure of security vulnerabilities fixed in LiteLLM v1.83.0, and the launch of our bug bounty program."
-tags: [security]
-hide_table_of_contents: false
----
-
-After the [supply chain incident](https://docs.litellm.ai/blog/security-update-march-2026) in March, we brought in [Veria Labs](https://verialabs.com/) to audit the LiteLLM proxy and fixed a number of vulnerability reports from independent researchers. All issues below are fixed in v1.83.0. If you are affected, particularly if you have JWT auth enabled, we recommend upgrading.
-
-We've also launched a [bug bounty program](#bug-bounty-program) and Veria Labs is continuing to audit the proxy. More fixes will ship in upcoming versions.
-
-The two high-severity issues ([CVE-2026-35029](https://github.com/BerriAI/litellm/security/advisories/GHSA-53mr-6c8q-9789) and [GHSA-69x8-hrgq-fjj8](https://github.com/BerriAI/litellm/security/advisories/GHSA-69x8-hrgq-fjj8)) **both require the attacker to already have a valid API key for the proxy**. These are not exploitable by unauthenticated users. 
-
-The critical-severity issue ([CVE-2026-35030](https://github.com/BerriAI/litellm/security/advisories/GHSA-jjhc-v7c2-5hh6)) is an authentication bypass, but only affects deployments with `enable_jwt_auth` explicitly enabled, which is off by default. **The default LiteLLM configuration is not affected, and no LiteLLM Cloud customers had this feature enabled.**
-
-{/* truncate */}
-
-## Vulnerabilities
-
-### CVE-2026-35030: Authentication bypass via OIDC cache collision (Critical)
-
-Found by Veria Labs.
-
-When `enable_jwt_auth` is enabled, LiteLLM cached OIDC userinfo using `token[:20]` as the cache key. JWTs from the same signing algorithm share the same header prefix, so an attacker could forge a token that hits another user's cache entry and inherit their session. We fixed this by keying the cache on `sha256(token)` instead.
-
-**Most deployments are not affected.** This requires `enable_jwt_auth: true`, which is off by default. If you can't upgrade, disable JWT auth as a workaround.
-
-Full advisory: [GHSA-jjhc-v7c2-5hh6](https://github.com/BerriAI/litellm/security/advisories/GHSA-jjhc-v7c2-5hh6)
-
-### CVE-2026-35029: Privilege escalation via `/config/update` (High)
-
-Found by Lakera.
-
-`/config/update` didn't check the caller's role. Any authenticated user could modify the proxy's runtime configuration, which could lead to arbitrary file read, admin account takeover, or remote code execution. We now require the `proxy_admin` role on this endpoint.
-
-Full advisory: [GHSA-53mr-6c8q-9789](https://github.com/BerriAI/litellm/security/advisories/GHSA-53mr-6c8q-9789)
-
-### Password hash exposure and pass-the-hash login (High)
-
-Weak hashing originally reported by GitHub user [hamzayevmaqsud](https://github.com/hamzayevmaqsud) ([#15484](https://github.com/BerriAI/litellm/issues/15484)). The full chain was identified by Luca Vandenweghe and Maarten De Rammelaere of [iO Digital](https://www.iodigital.com/).
-
-Passwords were stored as unsalted SHA-256 hashes, and in some cases plaintext. Several API endpoints returned the hash to any authenticated user, and `/v2/login` accepted the raw hash as a credential without re-hashing it, so a stolen hash was as good as the password itself. We've moved to scrypt with random salts and stripped hashes from all API responses.
-
-Full advisory: [GHSA-69x8-hrgq-fjj8](https://github.com/BerriAI/litellm/security/advisories/GHSA-69x8-hrgq-fjj8)
-
-## Bug bounty program
-
-After the supply chain incident and these disclosures it was clear we needed more external eyes on the project. We've set up a bug bounty program so researchers have a way to report issues. 
-
-Bounties are currently paid for P0 (supply chain) and P1 (unauthenticated proxy access) vulnerabilities:
-
-| Severity | Bounty | Example |
-|----------|--------|---------|
-| Critical | $1,500 – $3,000 | Supply chain compromise |
-| High | $500 – $1,500 | Unauthenticated access to protected data |
-
-We plan on expanding the program further in the coming months. More info about the bug bounty program is available [here](https://github.com/BerriAI/litellm/security).
-
-## What's next
-
-Veria Labs is continuing to work with us on a broader audit of the proxy. Security advisories sent through Github will be responded to within five business days. We'll publish advisories as issues are confirmed and fixed.

docs/my-website/blog/security_townhall_updates/index.md

@@ -1,223 +0,0 @@
----
-slug: security-townhall-updates
-title: "Security Townhall Updates"
-date: 2026-03-27T12:00:00
-authors:
-  - krrish
-  - ishaan-alt
-description: "What happened, what we've done, and what comes next for LiteLLM's release and security processes."
-tags: [security, incident-report]
-hide_table_of_contents: false
----
-
-import Image from '@theme/IdealImage';
-
-Thank you to everyone who joined our town hall.
-
-We wanted to use that time to walk through what we know, what we've done so far, and how we're improving LiteLLM's release and security processes going forward. This post is a written version of that update. [Slides available here](https://drive.google.com/file/d/17hsSG7nk-OYL7VRCTbTa7McrWREtS9OO/view?usp=sharing)
-
-{/* truncate */}
-
-## What happened
-
-On March 24, 2026 at 10:39 UTC, LiteLLM v1.82.7 was pushed to PyPI. Version v1.82.8 was published soon after. Those packages were live for about 40 minutes before being quarantined by PyPI. By 16:00 UTC, the LiteLLM team had worked with PyPI to delete the affected packages.
-
-At this point, our understanding is that this was a supply-chain incident affecting those two published versions.
-
-## How did this happen?
-
-Our understanding is that the issue came from the [compromised Trivy security scanner](https://www.aquasec.com/blog/trivy-supply-chain-attack-what-you-need-to-know/) dependency in our CI/CD pipeline.
-
-<Image 
-  img={require('../../img/shared_ci_cd_environment.png')}
-  style={{width: '500px', height: '400px', display: 'block'}}
-/>
-
-There were three major contributing factors:
-
-### 1. Shared CI/CD environment
-
-At the time, everything was running on CircleCI, and all steps shared a common environment. That increased blast radius: if one component was compromised, it could potentially access credentials or context intended for other parts of the pipeline.
-
-### 2. Static credentials in environment variables
-
-Release credentials, including credentials for PyPI, GHCR, and Docker publishing, were available as static secrets in the environment. That meant a compromised step could access long-lived release credentials.
-
-### 3. Unpinned Trivy dependency
-
-In our security scanning component, we had an unpinned Trivy dependency. Our present understanding is that a compromised Trivy package ran during the scan, had access to environment variables, and enabled attackers to obtain those credentials.
-
-**In summary:** a compromised package in CI had access to secrets it should not have had, and those secrets were then used in the release path.
-
-## What we've already done
-
-
-In the last 3 days, we've taken the following steps:
-
-### 1. Minimize Scope of Impact
-
-#### Prevented further key abuse
-
-We deleted or rotated all impacted or adjacent secret keys, including PyPI, GitHub, Docker, and related credentials. Out of an abundance of caution, we've also rotated LiteLLM maintainer accounts. 
-
-#### Prevent branch attacks
-
-We removed roughly 6,000 open branches and added an auto-deletion policy for branches merged into `main`. This reduces the surface area for branch-based abuse.
-
-#### Pinned CI/CD dependencies
-
-We've pinned all Github Actions, and are working on pinning all CircleCI dependencies as well.
-
-#### Paused releases
-
-We've paused new releases until we've confirmed codebase security and put stronger release controls in place.
-
-### 2. Secured LiteLLM
-
-#### Forensic analysis
-
-We are working with Google's Mandiant cybersecurity team to confirm the source of the attack and verify the security of the codebase. We also confirmed that no malicious code was pushed to `main`.
-
-#### Confirm Application Security
-
-In parallel, we are working with whitehat hackers at [Veria Labs](https://verialabs.com/) to verify application security and review improvements to our CI/CD process.
-
-We have also confirmed that the last 20 LiteLLM releases contain no indicators of compromise, and that no unauthenticated attacks can be made against LiteLLM Proxy based on our current investigation. [Check Security Blog for release verification.](https://docs.litellm.ai/blog/security-update-march-2026#verified-safe-versions)
-
-#### Created a security working group
-
-We created a new security working group inside LiteLLM focused on:
-
-- Building threat models
-- Auditing the build process and dependencies
-
-If you're interested in joining the security working group, please file an issue [here](https://github.com/BerriAI/litellm-security-wg).
-
-### 3. Improved CI/CD
-
-We've already begun making structural changes to how releases are built and published. These align with our goals (covered in the next section) around isolated environments, ephemeral credentials, and release auditing.
-
-## Roadmap
-
-We plan on following 4 guiding principles for our new CI/CD pipeline:
-
-1. **Limit** what each package can access
-2. **Reduce** the number of sensitive environment variables
-3. **Avoid** compromised packages
-4. **Prevent** release tampering
-
-
-### Isolated environments
-
-<Image 
-  img={require('../../img/isolated_ci_cd_environments.png')}
-  style={{width: '400px', height: 'auto'}}
-/>
-
-We are breaking our CI/CD into 4 semantic concepts:
-
-1. Unit tests
-2. Integration tests
-3. Security scans
-4. Release publishing
-
-And will be running each of these in isolated environments.
-
-This will limit the damage that any single compromised component can cause.
-
-### Ephemeral credentials
-
-We plan to move to ephemeral credentials for PyPI (Trusted Publisher) and GHCR (Token-based authentication) releases. This will reduce the risk of credentials being leaked or compromised.
-
-We have already begun doing this: 
-
-- PyPI Trusted Publisher on GitHub Actions [PR](https://github.com/BerriAI/litellm/pull/24654)
-- GHCR Token-based authentication on GitHub Actions [PR](https://github.com/BerriAI/litellm/pull/24683)
-
-### Release auditing
-
-Our goal is to allow users to independently verify that a release came from us and prevent silent modifications of releases after they are published.
-
-This will ensure, your releases are safe, even when: 
-- Stolen PyPI/GHCR credentials are used to publish malicious releases
-- Tampered registry artifacts are published
-- Tag mutations are made after the release is published
-
-We believe that [Cosign](https://github.com/sigstore/cosign) is a good fit for this, and have shipped it in [PR #24683](https://github.com/BerriAI/litellm/pull/24683).
-
-#### How to verify a Docker image with Cosign
-
-Starting from `v1.83.0-nightly`, all LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Every release is signed with the same key that was introduced in [commit `0112e53`](https://github.com/BerriAI/litellm/commit/0112e53046018d726492c814b3644b7d376029d0).
-
-**Verify using the pinned commit hash (recommended):**
-
-A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the original signing key:
-
-```bash
-cosign verify \
-  --key https://raw.githubusercontent.com/BerriAI/litellm/0112e53046018d726492c814b3644b7d376029d0/cosign.pub \
-  ghcr.io/berriai/litellm:<release-tag>
-```
-
-**Verify using a release tag (convenience):**
-
-Tags are protected in this repository and resolve to the same key. This option is easier to read but relies on tag protection rules:
-
-```bash
-cosign verify \
-  --key https://raw.githubusercontent.com/BerriAI/litellm/<release-tag>/cosign.pub \
-  ghcr.io/berriai/litellm:<release-tag>
-```
-
-Replace `<release-tag>` with the version you are deploying (e.g. `v1.83.0-stable`).
-
-Expected output:
-
-```
-The following checks were performed on each of these signatures:
-  - The cosign claims were validated
-  - The signatures were verified against the specified public key
-```
-
-### Avoid Compromised Packages
-
-- Move to pinned, verified SHAs for packages and actions used in CI/CD, avoiding `latest` wherever possible. 
-- Add a cooldown period before upgrading to a new version of a package - allows more time to investigate and verify the new version. 
-
-We've added zizmor to help us catch issues such as unpinned dependencies and credential leakage. [commit](https://github.com/BerriAI/litellm/commit/a671275f5c5b0e1fb1adacdf3b6ef779aaa5d56c).
-
-
-## Frequently Asked Questions
-
-**Q: Did you observe any lateral movement into your corporate environment during this incident?**
-
-A: No. Our investigation to date, conducted in coordination with external security experts, has found no evidence of lateral movement into our internal corporate systems. The incident was isolated to the CI/CD pipeline and the release path for specific versions (v1.82.7 and v1.82.8). As a proactive measure, we have rotated all potentially impacted or adjacent secrets—including PyPI, GitHub, and Docker credentials—and updated maintainer account security to ensure continued isolation.
-
-**Q: Do you expect delays in future product releases due to these new security measures?**
-
-A: We are committed to balancing security with speed. While we have temporarily paused releases to implement stronger controls, we are moving quickly to automate our new security protocols. We are currently implementing isolated CI/CD environments, ephemeral credentials (via Trusted Publishers), and release auditing with Cosign. These improvements are designed to be integrated into our automated pipeline, allowing us to maintain a fast release cadence while ensuring every package is verified and secure.
-
-**Q: Were older packages impacted?**
-
-Our current findings show no indicators of compromise in the last 20 versions of LiteLLM. This was manually verified by our team and independently reviewed by Veria Labs.
-
-We have also published the verified versions for users to use. [Check Security Blog for release verification.](https://docs.litellm.ai/blog/security-update-march-2026#verified-safe-versions)
-
-
-
-## Questions & Support 
-
-If you believe your systems may be affected, contact us immediately:
-
-- **Security:** security@berri.ai
-- **Support:** support@berri.ai
-- **Slack:** Reach out to the LiteLLM team directly [here](https://join.slack.com/t/litellmossslack/shared_invite/zt-3o7nkuyfr-p_kbNJj8taRfXGgQI1~YyA)
-
-## Hiring 
-
-We are currently hiring for: 
-
-- DevOps Engineer - to keep ci/cd secure and running smoothly
-- Security Engineer - to keep the application secure
-
-If you're interest in joining, please apply [here](https://jobs.ashbyhq.com/litellm)

docs/my-website/blog/security_update_march_2026/index.md

@@ -1,820 +0,0 @@
----
-slug: security-update-march-2026
-title: "Security Update: Suspected Supply Chain Incident"
-date: 2026-03-24T14:00:00
-authors:
-  - krrish
-  - ishaan-alt
-description: "As of 2:00 PM ET on March 24, 2026"
-tags: [security, incident-report]
-hide_table_of_contents: false
----
-
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-import VersionVerificationTable from '@site/src/components/VersionVerificationTable';
-
-> **Status:** Active investigation
-> **Last updated:** March 27, 2026
-
-> **Update (March 30):** A new **clean** version of LiteLLM is now available (v1.83.0). This was released by our new [CI/CD v2](https://docs.litellm.ai/blog/ci-cd-v2-improvements) pipeline which added isolated environments, stronger security gates, and safer release separation for LiteLLM.
-
-> **Update (March 27):** Review Townhall updates, including explanation of the incident, what we've done, and what comes next. [Learn more](https://docs.litellm.ai/blog/security-townhall-updates)
-
-> **Update (March 27):** Added [Verified safe versions](#verified-safe-versions) section with SHA-256 checksums for all audited PyPI and Docker releases.
-
-> **Update (March 26):** Added `checkmarx[.]zone` to [Indicators of compromise](#indicators-of-compromise-iocs)
-
-> **Update (March 25):** Added community-contributed scripts for scanning GitHub Actions and GitLab CI pipelines for the compromised versions. See [How to check if you are affected](#how-to-check-if-you-are-affected). s/o [@Zach Fury](https://www.linkedin.com/in/fryware/) for these scripts.
-
-
-## TLDR; 
-- The compromised PyPI packages were **litellm==1.82.7** and **litellm==1.82.8**. Those packages were live on March 24, 2026 from 10:39 UTC for about 40 minutes before being quarantined by PyPI.
-- We believe that the compromise originated from the [Trivy dependency](https://www.aquasec.com/blog/trivy-supply-chain-attack-what-you-need-to-know/) used in our CI/CD security scanning workflow.
-- Customers running the official LiteLLM Proxy Docker image were not impacted. That deployment path pins dependencies in requirements.txt and does not rely on the compromised PyPI packages.
-- ~~We have paused all new LiteLLM releases until we complete a broader supply-chain review and confirm the release path is safe.~~ **Updated:** We have now released a new **safe** version of LiteLLM (v1.83.0) by our new [CI/CD v2](https://docs.litellm.ai/blog/ci-cd-v2-improvements) pipeline which added isolated environments, stronger security gates, and safer release separation for LiteLLM. We have also verified the codebase is safe and no malicious code was pushed to `main`.
-
-
-## Overview
-
-LiteLLM AI Gateway is investigating a suspected supply chain attack involving unauthorized PyPI package publishes. Current evidence suggests a maintainer's PyPI account may have been compromised and used to distribute malicious code.
-
-At this time, we believe this incident may be linked to the broader [Trivy security compromise](https://www.aquasec.com/blog/trivy-supply-chain-attack-what-you-need-to-know/), in which stolen credentials were reportedly used to gain unauthorized access to the LiteLLM publishing pipeline.
-
-This investigation is ongoing. Details below may change as we confirm additional findings.
-
-## Confirmed affected versions
-
-The following LiteLLM versions published to PyPI were impacted:
-
-- **v1.82.7**: contained a malicious payload in the LiteLLM AI Gateway `proxy_server.py`
-- **v1.82.8**: contained `litellm_init.pth` and a malicious payload in the LiteLLM AI Gateway `proxy_server.py`
-
-If you installed or ran either of these versions, review the recommendations below immediately.
-
-Note: These versions have already been removed from PyPI.
-
-## What happened
-
-Initial evidence suggests the attacker bypassed official CI/CD workflows and uploaded malicious packages directly to PyPI.
-
-These compromised versions appear to have included a credential stealer designed to:
-
-- Harvest secrets by scanning for:
-  - environment variables
-  - SSH keys
-  - cloud provider credentials (AWS, GCP, Azure)
-  - Kubernetes tokens
-  - database passwords
-- Encrypt and exfiltrate data via a `POST` request to `models.litellm.cloud`, which is **not** an official BerriAI / LiteLLM domain
-
-## Who is affected
-
-You may be affected if **any** of the following are true:
-
-- You installed or upgraded LiteLLM via `pip` on **March 24, 2026**, between **10:39 UTC and 16:00 UTC**
-- You ran `pip install litellm` without pinning a version and received **v1.82.7** or **v1.82.8**
-- You built a Docker image during this window that included `pip install litellm` without a pinned version
-- A dependency in your project pulled in LiteLLM as a transitive, unpinned dependency
-  (for example through AI agent frameworks, MCP servers, or LLM orchestration tools)
-
-You are **not** affected if any of the following are true:
-
-**LiteLLM AI Gateway/Proxy users:** Customers running the official LiteLLM Proxy Docker image were not impacted. That deployment path pins dependencies in requirements.txt and does not rely on the compromised PyPI packages.
-
-- You are using **LiteLLM Cloud**
-- You are using the official LiteLLM AI Gateway Docker image: `ghcr.io/berriai/litellm`
-- You are on **v1.82.6 or earlier** and did not upgrade during the affected window
-- You installed LiteLLM from source via the GitHub repository, which was **not** compromised
-
-
-### How to check if you are affected
-
-<Tabs>
-<TabItem value="sdk" label="SDK">
-
-```bash
-pip show litellm
-```
-</TabItem>
-<TabItem value="proxy" label="PROXY">
-
-Go to the proxy base url, and check the version of the installed LiteLLM.
-
-![Proxy version check](../../img/security_update_march_2026/proxy_version.png)
-</TabItem>
-<TabItem value="github" label="GitHub Actions">
-
-Scans all repositories in a GitHub organization for workflow jobs that installed the compromised versions.
-
-**Requirements:** Python 3 and `requests` (`pip install requests`).
-
-**Setup:**
-
-```bash
-export GITHUB_TOKEN="your-github-pat"
-```
-
-**Run:**
-
-```bash
-python find_litellm_github.py
-```
-
-Set the `ORG` variable in the script to your GitHub organization name.
-
-Both scripts default to scanning jobs from **today**. Adjust the `WINDOW_START` and `WINDOW_END` constants to cover **March 24, 2026** (the incident date) if running on a different day.
-
-<details>
-<summary>View full script (find_litellm_github.py)</summary>
-
-```python
-#!/usr/bin/env python3
-"""
-Scan all GitHub Actions jobs in a GitHub org that ran between
-0800-1244 UTC today and identify any that installed litellm 1.82.7 or 1.82.8.
-
-Adjust WINDOW_START / WINDOW_END to cover March 24, 2026 if running later.
-"""
-
-import io
-import os
-import re
-import sys
-import zipfile
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from datetime import datetime, timezone
-
-import requests
-
-GITHUB_URL   = "https://api.github.com"
-ORG          = "your-org"  # <-- set to your GitHub organization
-TOKEN        = os.environ.get("GITHUB_TOKEN", "")
-
-TODAY        = datetime.now(timezone.utc).date()
-WINDOW_START = datetime(TODAY.year, TODAY.month, TODAY.day,  8,  0, 0, tzinfo=timezone.utc)
-WINDOW_END   = datetime(TODAY.year, TODAY.month, TODAY.day, 12, 44, 0, tzinfo=timezone.utc)
-
-TARGET_VERSIONS = {"1.82.7", "1.82.8"}
-VERSION_PATTERN = re.compile(r"litellm[=\-](\d+\.\d+\.\d+)", re.IGNORECASE)
-
-SESSION = requests.Session()
-SESSION.headers.update({
-    "Authorization": f"Bearer {TOKEN}",
-    "Accept": "application/vnd.github+json",
-    "X-GitHub-Api-Version": "2022-11-28",
-})
-
-
-def get_paginated(url, params=None):
-    params = dict(params or {})
-    params.setdefault("per_page", 100)
-    page = 1
-    while True:
-        params["page"] = page
-        resp = SESSION.get(url, params=params, timeout=30)
-        if resp.status_code == 404:
-            return
-        resp.raise_for_status()
-        data = resp.json()
-        if isinstance(data, dict):
-            items = next((v for v in data.values() if isinstance(v, list)), [])
-        else:
-            items = data
-        if not items:
-            break
-        yield from items
-        if len(items) < params["per_page"]:
-            break
-        page += 1
-
-
-def parse_ts(ts_str):
-    if not ts_str:
-        return None
-    return datetime.fromisoformat(ts_str.replace("Z", "+00:00"))
-
-
-def get_repos():
-    repos = []
-    for r in get_paginated(f"{GITHUB_URL}/orgs/{ORG}/repos", {"type": "all"}):
-        repos.append({"id": r["id"], "name": r["name"], "full_name": r["full_name"]})
-    return repos
-
-
-def get_runs_in_window(repo_full_name):
-    created_filter = (
-        f"{WINDOW_START.strftime('%Y-%m-%dT%H:%M:%SZ')}"
-        f"..{WINDOW_END.strftime('%Y-%m-%dT%H:%M:%SZ')}"
-    )
-    url = f"{GITHUB_URL}/repos/{repo_full_name}/actions/runs"
-    runs = []
-    for run in get_paginated(url, {"created": created_filter, "per_page": 100}):
-        ts = parse_ts(run.get("run_started_at") or run.get("created_at"))
-        if ts and WINDOW_START <= ts <= WINDOW_END:
-            runs.append(run)
-    return runs
-
-
-def get_jobs_for_run(repo_full_name, run_id):
-    url = f"{GITHUB_URL}/repos/{repo_full_name}/actions/runs/{run_id}/jobs"
-    jobs = []
-    for job in get_paginated(url, {"filter": "all"}):
-        ts = parse_ts(job.get("started_at"))
-        if ts and WINDOW_START <= ts <= WINDOW_END:
-            jobs.append(job)
-    return jobs
-
-
-def fetch_job_log(repo_full_name, job_id):
-    url = f"{GITHUB_URL}/repos/{repo_full_name}/actions/jobs/{job_id}/logs"
-    resp = SESSION.get(url, timeout=60, allow_redirects=True)
-    if resp.status_code in (403, 404, 410):
-        return ""
-    resp.raise_for_status()
-
-    content_type = resp.headers.get("Content-Type", "")
-    if "zip" in content_type or resp.content[:2] == b"PK":
-        try:
-            with zipfile.ZipFile(io.BytesIO(resp.content)) as zf:
-                parts = []
-                for name in sorted(zf.namelist()):
-                    with zf.open(name) as f:
-                        parts.append(f.read().decode("utf-8", errors="replace"))
-                return "\n".join(parts)
-        except zipfile.BadZipFile:
-            pass
-    return resp.text
-
-
-def check_job(repo_full_name, job):
-    job_id   = job["id"]
-    job_name = job["name"]
-    run_id   = job["run_id"]
-    started  = job.get("started_at", "")
-
-    log_text = fetch_job_log(repo_full_name, job_id)
-    if not log_text:
-        return None
-
-    found_versions = set()
-    context_lines  = []
-    for line in log_text.splitlines():
-        m = VERSION_PATTERN.search(line)
-        if m:
-            ver = m.group(1)
-            if ver in TARGET_VERSIONS:
-                found_versions.add(ver)
-                context_lines.append(line.strip())
-
-    if not found_versions:
-        return None
-
-    return {
-        "repo":       repo_full_name,
-        "run_id":     run_id,
-        "job_id":     job_id,
-        "job_name":   job_name,
-        "started_at": started,
-        "versions":   sorted(found_versions),
-        "context":    context_lines[:10],
-        "job_url":    job.get("html_url", f"https://github.com/{repo_full_name}/actions/runs/{run_id}"),
-    }
-
-
-def main():
-    if not TOKEN:
-        print("ERROR: Set GITHUB_TOKEN environment variable.", file=sys.stderr)
-        sys.exit(1)
-
-    print(f"Time window : {WINDOW_START.isoformat()} -> {WINDOW_END.isoformat()}")
-    print(f"Hunting for : litellm {', '.join(sorted(TARGET_VERSIONS))}")
-    print()
-
-    print(f"Fetching repositories for org '{ORG}'...")
-    repos = get_repos()
-    print(f"  Found {len(repos)} repositories")
-    print()
-
-    jobs_to_check = []
-
-    print("Scanning workflow runs for time window...")
-    for repo in repos:
-        full_name = repo["full_name"]
-        try:
-            runs = get_runs_in_window(full_name)
-        except requests.HTTPError as e:
-            print(f"  WARN: {full_name} - {e}", file=sys.stderr)
-            continue
-        if not runs:
-            continue
-        print(f"  {full_name}: {len(runs)} run(s) in window")
-        for run in runs:
-            try:
-                jobs = get_jobs_for_run(full_name, run["id"])
-            except requests.HTTPError as e:
-                print(f"    WARN: run {run['id']} - {e}", file=sys.stderr)
-                continue
-            for job in jobs:
-                jobs_to_check.append((full_name, job))
-
-    total = len(jobs_to_check)
-    print(f"\nFetching logs for {total} job(s)...")
-    print()
-
-    hits = []
-    with ThreadPoolExecutor(max_workers=8) as pool:
-        futures = {
-            pool.submit(check_job, full_name, job): (full_name, job["id"])
-            for full_name, job in jobs_to_check
-        }
-        done = 0
-        for future in as_completed(futures):
-            done += 1
-            full_name, jid = futures[future]
-            try:
-                result = future.result()
-            except Exception as e:
-                print(f"  ERROR {full_name} job {jid}: {e}", file=sys.stderr)
-                continue
-            if result:
-                hits.append(result)
-            print(
-                f"  [{done}/{total}] {full_name} job {jid}" +
-                (f"  *** HIT: litellm {result['versions']} ***" if result else ""),
-                flush=True,
-            )
-
-    print()
-    print("=" * 72)
-    print(f"RESULTS: {len(hits)} job(s) installed litellm {' or '.join(sorted(TARGET_VERSIONS))}")
-    print("=" * 72)
-
-    if not hits:
-        print("No matches found.")
-        return
-
-    for h in sorted(hits, key=lambda x: x["started_at"]):
-        print()
-        print(f"  Repo      : {h['repo']}")
-        print(f"  Job       : {h['job_name']} (#{h['job_id']})")
-        print(f"  Run ID    : {h['run_id']}")
-        print(f"  Started   : {h['started_at']}")
-        print(f"  Versions  : litellm {', '.join(h['versions'])}")
-        print(f"  URL       : {h['job_url']}")
-        print(f"  Log lines :")
-        for line in h["context"]:
-            print(f"    {line}")
-
-
-if __name__ == "__main__":
-    main()
-```
-
-</details>
-
-</TabItem>
-<TabItem value="gitlab" label="GitLab CI">
-
-Scans all projects in a GitLab group (including subgroups) for CI/CD jobs that installed the compromised versions.
-
-**Requirements:** Python 3 and `requests` (`pip install requests`).
-
-**Setup:**
-
-```bash
-export GITLAB_TOKEN="your-gitlab-pat"
-```
-
-**Run:**
-
-```bash
-python find_litellm_jobs.py
-```
-
-Set the `GROUP_NAME` variable in the script to your GitLab group name.
-
-Both scripts default to scanning jobs from **today**. Adjust the `WINDOW_START` and `WINDOW_END` constants to cover **March 24, 2026** (the incident date) if running on a different day.
-
-<details>
-<summary>View full script (find_litellm_jobs.py)</summary>
-
-```python
-#!/usr/bin/env python3
-"""
-Scan all GitLab CI/CD jobs in a GitLab group that ran between
-0800-1244 UTC today and identify any that installed litellm 1.82.7 or 1.82.8.
-
-Adjust WINDOW_START / WINDOW_END to cover March 24, 2026 if running later.
-"""
-
-import os
-import re
-import sys
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from datetime import datetime, timezone
-
-import requests
-
-GITLAB_URL = "https://gitlab.com"
-GROUP_NAME = "YourGroup"  # <-- set to your GitLab group name
-TOKEN = os.environ.get("GITLAB_TOKEN", "")
-
-TODAY = datetime.now(timezone.utc).date()
-WINDOW_START = datetime(TODAY.year, TODAY.month, TODAY.day, 8, 0, 0, tzinfo=timezone.utc)
-WINDOW_END   = datetime(TODAY.year, TODAY.month, TODAY.day, 12, 44, 0, tzinfo=timezone.utc)
-
-TARGET_VERSIONS = {"1.82.7", "1.82.8"}
-VERSION_PATTERN = re.compile(r"litellm[=\-](\d+\.\d+\.\d+)", re.IGNORECASE)
-
-HEADERS = {"PRIVATE-TOKEN": TOKEN}
-SESSION = requests.Session()
-SESSION.headers.update(HEADERS)
-
-
-def get_paginated(url, params=None):
-    params = dict(params or {})
-    params.setdefault("per_page", 100)
-    page = 1
-    while True:
-        params["page"] = page
-        resp = SESSION.get(url, params=params, timeout=30)
-        resp.raise_for_status()
-        data = resp.json()
-        if not data:
-            break
-        yield from data
-        if len(data) < params["per_page"]:
-            break
-        page += 1
-
-
-def get_group_id(group_name):
-    resp = SESSION.get(f"{GITLAB_URL}/api/v4/groups/{group_name}", timeout=30)
-    resp.raise_for_status()
-    return resp.json()["id"]
-
-
-def get_all_projects(group_id):
-    projects = []
-    for p in get_paginated(
-        f"{GITLAB_URL}/api/v4/groups/{group_id}/projects",
-        {"include_subgroups": "true", "archived": "false"},
-    ):
-        projects.append({"id": p["id"], "name": p["path_with_namespace"]})
-    return projects
-
-
-def parse_ts(ts_str):
-    if not ts_str:
-        return None
-    ts_str = ts_str.replace("Z", "+00:00")
-    return datetime.fromisoformat(ts_str)
-
-
-def jobs_in_window(project_id):
-    matching = []
-    url = f"{GITLAB_URL}/api/v4/projects/{project_id}/jobs"
-    params = {"per_page": 100, "scope[]": ["success", "failed", "canceled", "running"]}
-
-    page = 1
-    while True:
-        params["page"] = page
-        resp = SESSION.get(url, params=params, timeout=30)
-        if resp.status_code == 403:
-            return matching
-        resp.raise_for_status()
-        jobs = resp.json()
-        if not jobs:
-            break
-
-        stop_early = False
-        for job in jobs:
-            ts = parse_ts(job.get("started_at") or job.get("created_at"))
-            if ts is None:
-                continue
-            if ts > WINDOW_END:
-                continue
-            if ts < WINDOW_START:
-                stop_early = True
-                continue
-            matching.append(job)
-
-        if stop_early or len(jobs) < 100:
-            break
-        page += 1
-
-    return matching
-
-
-def fetch_trace(project_id, job_id):
-    url = f"{GITLAB_URL}/api/v4/projects/{project_id}/jobs/{job_id}/trace"
-    resp = SESSION.get(url, timeout=60)
-    if resp.status_code in (403, 404):
-        return ""
-    resp.raise_for_status()
-    return resp.text
-
-
-def check_job(project_name, project_id, job):
-    job_id   = job["id"]
-    job_name = job["name"]
-    ref      = job.get("ref", "")
-    started  = job.get("started_at", job.get("created_at", ""))
-
-    trace = fetch_trace(project_id, job_id)
-    if not trace:
-        return None
-
-    found_versions = set()
-    for match in VERSION_PATTERN.finditer(trace):
-        ver = match.group(1)
-        if ver in TARGET_VERSIONS:
-            found_versions.add(ver)
-
-    if not found_versions:
-        return None
-
-    context_lines = []
-    for line in trace.splitlines():
-        if VERSION_PATTERN.search(line):
-            ver_match = VERSION_PATTERN.search(line)
-            if ver_match and ver_match.group(1) in TARGET_VERSIONS:
-                context_lines.append(line.strip())
-
-    return {
-        "project":    project_name,
-        "project_id": project_id,
-        "job_id":     job_id,
-        "job_name":   job_name,
-        "ref":        ref,
-        "started_at": started,
-        "versions":   sorted(found_versions),
-        "context":    context_lines[:10],
-        "job_url":    f"{GITLAB_URL}/{project_name}/-/jobs/{job_id}",
-    }
-
-
-def main():
-    if not TOKEN:
-        print("ERROR: Set GITLAB_TOKEN environment variable.", file=sys.stderr)
-        sys.exit(1)
-
-    print(f"Time window : {WINDOW_START.isoformat()} -> {WINDOW_END.isoformat()}")
-    print(f"Hunting for : litellm {', '.join(sorted(TARGET_VERSIONS))}")
-    print()
-
-    print(f"Resolving group '{GROUP_NAME}'...")
-    group_id = get_group_id(GROUP_NAME)
-
-    print("Fetching projects...")
-    projects = get_all_projects(group_id)
-    print(f"  Found {len(projects)} projects")
-    print()
-
-    all_jobs_to_check = []
-
-    print("Scanning job listings for time window...")
-    for proj in projects:
-        try:
-            jobs = jobs_in_window(proj["id"])
-        except requests.HTTPError as e:
-            print(f"  WARN: {proj['name']} - {e}", file=sys.stderr)
-            continue
-        if jobs:
-            print(f"  {proj['name']}: {len(jobs)} job(s) in window")
-        for j in jobs:
-            all_jobs_to_check.append((proj["name"], proj["id"], j))
-
-    total = len(all_jobs_to_check)
-    print(f"\nFetching traces for {total} job(s)...")
-    print()
-
-    hits = []
-    with ThreadPoolExecutor(max_workers=10) as pool:
-        futures = {
-            pool.submit(check_job, pname, pid, job): (pname, job["id"])
-            for pname, pid, job in all_jobs_to_check
-        }
-        done = 0
-        for future in as_completed(futures):
-            done += 1
-            pname, jid = futures[future]
-            try:
-                result = future.result()
-            except Exception as e:
-                print(f"  ERROR checking {pname} job {jid}: {e}", file=sys.stderr)
-                continue
-            if result:
-                hits.append(result)
-            print(f"  [{done}/{total}] checked {pname} job {jid}" +
-                  (f"  *** HIT: litellm {result['versions']} ***" if result else ""),
-                  flush=True)
-
-    print()
-    print("=" * 72)
-    print(f"RESULTS: {len(hits)} job(s) installed litellm {' or '.join(sorted(TARGET_VERSIONS))}")
-    print("=" * 72)
-
-    if not hits:
-        print("No matches found.")
-        return
-
-    for h in sorted(hits, key=lambda x: x["started_at"]):
-        print()
-        print(f"  Project   : {h['project']}")
-        print(f"  Job       : {h['job_name']} (#{h['job_id']})")
-        print(f"  Branch/tag: {h['ref']}")
-        print(f"  Started   : {h['started_at']}")
-        print(f"  Versions  : litellm {', '.join(h['versions'])}")
-        print(f"  URL       : {h['job_url']}")
-        print(f"  Log lines :")
-        for line in h["context"]:
-            print(f"    {line}")
-
-
-if __name__ == "__main__":
-    main()
-```
-
-</details>
-
-</TabItem>
-</Tabs>
-
-*CI/CD scripts contributed by the community ([original gist](https://gist.github.com/fryz/93ec8d4898ffe5b5ac5706a208823ef3)). Review before running.*
-
-
-## Indicators of compromise (IoCs)
-
-Review affected systems for the following indicators:
-
-- `litellm_init.pth` present in your `site-packages`
-- Outbound traffic or requests to `models.litellm[.]cloud`
-  This domain is **not** affiliated with LiteLLM
-- Outbound traffic or requests to `checkmarx[.]zone`
-  This domain is **not** affiliated with LiteLLM
-
-
-## Immediate actions for affected users
-
-If you installed or ran **v1.82.7** or **v1.82.8**, take the following actions immediately.
-
-### 1. Rotate all secrets
-
-Treat any credentials present on the affected systems as compromised, including:
-
-- API keys
-- Cloud access keys
-- Database passwords
-- SSH keys
-- Kubernetes tokens
-- Any secrets stored in environment variables or configuration files
-
-### 2. Inspect your filesystem
-
-Check your `site-packages` directory for a file named `litellm_init.pth`:
-
-```bash
-find /usr/lib/python3.13/site-packages/ -name "litellm_init.pth"
-```
-
-If present:
-
-- remove it immediately
-- investigate the host for further compromise
-- preserve relevant artifacts if your security team is performing forensics
-
-### 3. Audit version history
-
-Review your:
-
-- Local environments
-- CI/CD pipelines
-- Docker builds
-- Deployment logs
-
-Confirm whether **v1.82.7** or **v1.82.8** was installed anywhere.
-
-Pin LiteLLM to a known safe version such as **v1.82.6 or earlier**, or to a later verified release once announced.
-
-
-## Response and remediation
-
-The LiteLLM AI Gateway team has already taken the following steps:
-
-- Removed compromised packages from PyPI
-- Rotated maintainer credentials and established new authorized maintainers
-- Engaged Google's Mandiant security team to assist with forensic analysis of the build and publishing chain
-
-
-## Verify Docker image signatures
-
-Starting from `v1.83.0-nightly`, all LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Every release is signed with the same key introduced in [commit `0112e53`](https://github.com/BerriAI/litellm/commit/0112e53046018d726492c814b3644b7d376029d0).
-
-**Verify using the pinned commit hash (recommended):**
-
-A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the original signing key:
-
-```bash
-cosign verify \
-  --key https://raw.githubusercontent.com/BerriAI/litellm/0112e53046018d726492c814b3644b7d376029d0/cosign.pub \
-  ghcr.io/berriai/litellm:<release-tag>
-```
-
-**Verify using a release tag (convenience):**
-
-Tags are protected in this repository and resolve to the same key. This option is easier to read but relies on tag protection rules:
-
-```bash
-cosign verify \
-  --key https://raw.githubusercontent.com/BerriAI/litellm/<release-tag>/cosign.pub \
-  ghcr.io/berriai/litellm:<release-tag>
-```
-
-Replace `<release-tag>` with the version you are deploying (e.g. `v1.83.0-stable`).
-
-Expected output:
-
-```
-The following checks were performed on each of these signatures:
-  - The cosign claims were validated
-  - The signatures were verified against the specified public key
-```
-
-## Verified safe versions
-
-We have audited every LiteLLM release published between v1.78.0 and v1.82.6 across both PyPI and Docker. Each artifact was verified by:
-
-1. Downloading the published artifact and computing its SHA-256 digest
-2. Scanning for the known [indicators of compromise](#indicators-of-compromise-iocs) (IOCs)
-3. Comparing the artifact contents against the corresponding Git commit in the BerriAI/litellm repository
-
-**All versions listed below are confirmed clean.**
-
-<Tabs>
-<TabItem value="pypi" label="PyPI Releases">
-
-<VersionVerificationTable entries={[
-  { version: "1.82.6", sha256: "164a3ef3e19f309e3cabc199bef3d2045212712fefdfa25fc7f75884a5b5b205", gitCommit: "38d477507dad" },
-  { version: "1.82.5", sha256: "e1012ab816352215c4e00776dd48b0c68058b537888a8ff82cca62af19e6fb11", gitCommit: "1998c4f3703f" },
-  { version: "1.82.4", sha256: "d37c34a847e7952a146ed0e2888a24d3edec7787955c6826337395e755ad5c4b", gitCommit: "cfeafbe38811" },
-  { version: "1.82.3", sha256: "609901f6c5a5cf8c24386e4e3f50738bb8a9db719709fd76b208c8ee6d00f7a7", gitCommit: "61409275c8d8" },
-  { version: "1.82.2", sha256: "641ed024774fa3d5b4dd9347f0efb1e31fa422fba2a6500aabedee085d1194cb", gitCommit: "f351bbdb3683" },
-  { version: "1.82.1", sha256: "a9ec3fe42eccb1611883caaf8b1bf33c9f4e12163f94c7d1004095b14c379eb2", gitCommit: "94b002066e3a" },
-  { version: "1.82.0", sha256: "5496b5d4532cccdc7a095c21cbac4042f7662021c57bc1d17be4e39838929e80", gitCommit: "6c6585af568e" },
-  { version: "1.81.16", sha256: "d6bcc13acbd26719e07bfa6b9923740e88409cbf1f9d626d85fc9ae0e0eec88c", gitCommit: "678200ee4887" },
-  { version: "1.81.15", sha256: "2fa253658702509ce09fe0e172e5a47baaadf697fb0f784c7fd4ff665ae76ae1", gitCommit: "2e819656cee9" },
-  { version: "1.81.14", sha256: "6394e61bbdef7121e5e3800349f6b01e9369e7cf611e034f1832750c481abfed", gitCommit: "96bcee0b0af7" },
-  { version: "1.81.13", sha256: "ae4aea2a55e85993f5f6dd36d036519422d24812a1a3e8540d9e987f2d7a4304", gitCommit: "cc957a19a560" },
-  { version: "1.81.12", sha256: "219cf9729e5ea30c6d3f75aa43fef3c56a717369939a6d717cbad0fd78e3c146", gitCommit: "ba0d541b1982" },
-  { version: "1.81.11", sha256: "06a66c24742e082ddd2813c87f40f5c12fe7baa73ce1f9457eaf453dc44a0f65", gitCommit: "231aedeeff7e" },
-  { version: "1.81.10", sha256: "9efa1cbe61ac051f6500c267b173d988ff2d511c2eecf1c8f2ee546c0870747c", gitCommit: "7488abece8e7" },
-  { version: "1.81.9", sha256: "24ee273bc8a62299fbb754035f83fb7d8d44329c383701a2bd034f4fd1c19084", gitCommit: "a09d3e9162eb" },
-  { version: "1.81.8", sha256: "78cca92f36bc6c267c191d1fe1e2630c812bff6daec32c58cade75748c2692f6", gitCommit: "4fea649f519b" },
-  { version: "1.81.7", sha256: "58466c88c3289c6a3830d88768cf8f307581d9e6c87861de874d1128bb2de90d", gitCommit: "3f6a281d0f7a" },
-  { version: "1.81.6", sha256: "573206ba194d49a1691370ba33f781671609ac77c35347f8a0411d852cf6341a", gitCommit: "8da3a93e6e63" },
-  { version: "1.81.5", sha256: "206505c5a0c6503e465154b9c979772be3ede3f5bf746d15b37dca5ae54d239f", gitCommit: "2cc3778761d4" },
-  { version: "1.81.3", sha256: "3f60fd8b727587952ad3dd18b68f5fed538d6f43d15bb0356f4c3a11bccb2b92", gitCommit: "f30742fe6e8e" },
-]} />
-
-</TabItem>
-<TabItem value="docker" label="Docker Images">
-
-<VersionVerificationTable entries={[
-  { version: "1.82.3", sha256: "0a571da849db5f9c3cf3fead2ffbf1df982eebff7e7b38b46dbec3f640dafdbb", gitCommit: "61409275c8d8" },
-  { version: "1.82.3-stable", sha256: "0c2b2a0ad3e50af1702fc493ecd07f22a5180b6d1cfb169440b429b40e340e29", gitCommit: "61409275c8d8" },
-  { version: "1.82.0-stable", sha256: "71bf7283767ca436edcfa9f1f26c1743487b5fa29736c61c3eb6977776007c42", gitCommit: "97947c254252" },
-  { version: "1.81.15", sha256: "303c31af87e7915e7b34d6c4d55a6ac753ef947a5deaa899e9ccfd3d1d58f7c2", gitCommit: "20bf3aa8070a" },
-  { version: "1.81.14-stable", sha256: "a34f9758048231817d799b703fb998e40e2a5cbabb89ab95039fc30798f01b3c", gitCommit: "0435375b1271" },
-  { version: "1.81.13", sha256: "a876f3f22f9b6fd481c9091c44a8a893d81c172d66dc2749298dcd3dc4a3d6f0", gitCommit: "cc957a19a560" },
-  { version: "1.81.12-stable", sha256: "e24022878ccc87f57d808ac9304f18b87b8359e6556746d81cc20a5dc85f423a", gitCommit: "ba0d541b1982" },
-  { version: "1.81.9-stable", sha256: "262e53d7702ed82579717faff0b08f7c0b7e9973a6406cfcc0e4af7826327627", gitCommit: "a09d3e9162eb" },
-  { version: "1.81.3-stable", sha256: "dff82ccc32fb648927c090607887401c7e8ec814fe7c951beb95fe51073ca02b", gitCommit: "61ed8f9e0355" },
-  { version: "1.81.0-stable", sha256: "f4913297d1bb3dc373eb8911a5ac816b597be9b5e08a91636b6c2786dd572aa8", gitCommit: "790a5ce0b323" },
-  { version: "1.80.15-stable", sha256: "0b4ec3861e978b4aa254f4070f292cd345496a5fb59c72e1ee21cd6db94b670b", gitCommit: "17c8d8d109b5" },
-  { version: "1.80.11-stable", sha256: "4068108d9101cd2affba3924310fd7f34f23d14e36dd4853733898b9e04d81ca", gitCommit: "57e07bddd341" },
-  { version: "1.80.8-stable", sha256: "0304c2eb1f3cf54262d1b4e0629487232bab459e95b99a21e5810231d2b27021", gitCommit: "3381d63152f8" },
-  { version: "1.80.5-stable", sha256: "a89e173135fff96af4b5b91ea31845164eadcf6497c82adeb64c36a23c8a3d11", gitCommit: "6c49b95a4ab7" },
-  { version: "1.80.0-stable", sha256: "a3416f4cd0c896c94a1f526d872ff6c19bee22ff4afcdcc6f9ff690707900176", gitCommit: "98365205acd0" },
-  { version: "1.79.3-stable", sha256: "27aae83d6ab6cb0b63bf8179e375ce0e11f5cfef51f2675b0c1e60c6f546dbc1", gitCommit: "c0548542d4a9" },
-  { version: "1.79.1-stable", sha256: "7780d29a9543c4ce762430db7dfb0640105f7357fc38e35bf3fb7bbb1e6ba63f", gitCommit: "c217bddb59ba" },
-  { version: "1.79.0-stable", sha256: "32bf6ac059a56641e11e4712f63b8467c295f988b6c160dc7229660417ee44bd", gitCommit: "8d495f56a9cc" },
-  { version: "1.78.5-stable", sha256: "d5e607648eafa15edc63b0b1a5ed01f8b31a1fa0c80f7d25b252ae18a593ee29", gitCommit: "c471bf1f16c2" },
-  { version: "1.78.0-stable", sha256: "7a56b32dc7153763d31c0a056123dc878a598959935d8c7daacb1fca5272c205", gitCommit: "5fde83d9f154" },
-]} />
-
-</TabItem>
-</Tabs>
-
-
-## Questions and support
-
-If you believe your systems may be affected, contact us immediately:
-
-- **Security:** `security@berri.ai`
-- **Support:** `support@berri.ai`
-- **Slack:** Reach out to the LiteLLM team directly
-
-For real-time updates, follow [LiteLLM (YC W23) on X](https://x.com/LiteLLM).
-

docs/my-website/blog/server_root_path/index.md

@@ -1,146 +0,0 @@
----
-slug: server-root-path-incident
-title: "Incident Report: SERVER_ROOT_PATH regression broke UI routing"
-date: 2026-02-21T10:00:00
-authors:
-  - yuneng
-  - ishaan-alt
-  - krrish
-tags: [incident-report, ui, stability]
-hide_table_of_contents: false
----
-
-**Date:** January 22, 2026
-**Duration:** ~4 days (until fix merged January 26, 2026)
-**Severity:** High
-**Status:** Resolved
-
-> **Note:** This fix is available starting from LiteLLM `v1.81.3.rc.6` or higher.
-
-## Summary
-
-A PR ([`#19467`](https://github.com/BerriAI/litellm/pull/19467)) accidentally removed the `root_path=server_root_path` parameter from the FastAPI app initialization in `proxy_server.py`. This caused the proxy to ignore the `SERVER_ROOT_PATH` environment variable when serving the UI. Users who deploy LiteLLM behind a reverse proxy with a path prefix (e.g., `/api/v1` or `/llmproxy`) found that all UI pages returned 404 Not Found.
-
-- **LLM API calls:** No impact. API routing was unaffected.
-- **UI pages:** All UI pages returned 404 for deployments using `SERVER_ROOT_PATH`.
-- **Swagger/OpenAPI docs:** Broken when accessed through the configured root path.
-
-{/* truncate */}
-
----
-
-## Background
-
-Many LiteLLM deployments run behind a reverse proxy (e.g., Nginx, Traefik, AWS ALB) that routes traffic to LiteLLM under a path prefix. FastAPI's `root_path` parameter tells the application about this prefix so it can correctly serve static files, generate URLs, and handle routing.
-
-```mermaid
-sequenceDiagram
-    participant User as User Browser
-    participant RP as Reverse Proxy
-    participant LP as LiteLLM Proxy
-
-    User->>RP: GET /llmproxy/ui/
-    RP->>LP: GET /ui/ (X-Forwarded-Prefix: /llmproxy)
-
-    Note over LP: Before regression:<br/>FastAPI root_path="/llmproxy"<br/>→ Serves UI correctly
-
-    Note over LP: After regression:<br/>FastAPI root_path=""<br/>→ UI assets resolve to wrong paths<br/>→ 404 Not Found
-```
-
-The `root_path` parameter was present in `proxy_server.py` since early versions of LiteLLM. It was removed as a side effect of PR [#19467](https://github.com/BerriAI/litellm/pull/19467), which was intended to fix a different UI 404 issue.
-
----
-
-## Root cause
-
-PR [#19467](https://github.com/BerriAI/litellm/pull/19467) (`73d49f8`) removed the `root_path=server_root_path` line from the `FastAPI()` constructor in `proxy_server.py`:
-
-```diff
- app = FastAPI(
-     docs_url=_get_docs_url(),
-     redoc_url=_get_redoc_url(),
-     title=_title,
-     description=_description,
-     version=version,
--    root_path=server_root_path,
-     lifespan=proxy_startup_event,
- )
-```
-
-Without `root_path`, FastAPI treated all requests as if the application was mounted at `/`, causing path mismatches for any deployment using `SERVER_ROOT_PATH`.
-
-The regression went undetected because:
-
-1. **No automated test** verified that `root_path` was set on the FastAPI app.
-2. **No manual test procedure** existed for `SERVER_ROOT_PATH` functionality.
-3. **Default deployments** (without `SERVER_ROOT_PATH`) were unaffected, so most CI tests passed.
-
----
-
-## Remediation
-
-| #   | Action                                                                                            | Status  | Code                                                                                                                       |
-| --- | ------------------------------------------------------------------------------------------------- | ------- | -------------------------------------------------------------------------------------------------------------------------- |
-| 1   | Restore `root_path=server_root_path` in FastAPI app initialization                                | ✅ Done | [`#19790`](https://github.com/BerriAI/litellm/pull/19790) (`5426b3c`)                                                      |
-| 2   | Add unit tests for `get_server_root_path()` and FastAPI app initialization                        | ✅ Done | [`test_server_root_path.py`](https://github.com/BerriAI/litellm/blob/main/tests/proxy_unit_tests/test_server_root_path.py) |
-| 3   | Add CI workflow that builds Docker image and tests UI routing with `SERVER_ROOT_PATH` on every PR | ✅ Done | [`test_server_root_path.yml`](https://github.com/BerriAI/litellm/blob/main/.github/workflows/test_server_root_path.yml)    |
-| 4   | Document manual test procedure for `SERVER_ROOT_PATH`                                             | ✅ Done | [Discussion #8495](https://github.com/BerriAI/litellm/discussions/8495)                                                    |
-
----
-
-## CI workflow details
-
-The new [`test_server_root_path.yml`](https://github.com/BerriAI/litellm/blob/main/.github/workflows/test_server_root_path.yml) workflow runs on every PR against `main`. It:
-
-1. Builds the LiteLLM Docker image
-2. Starts a container with `SERVER_ROOT_PATH` set (tests both `/api/v1` and `/llmproxy`)
-3. Verifies the UI returns valid HTML at `{ROOT_PATH}/ui/`
-4. Fails the workflow if the UI is unreachable
-
-```mermaid
-flowchart TD
-    A["PR opened/updated"] --> B["Build Docker image"]
-    B --> C["Start container with SERVER_ROOT_PATH=/api/v1"]
-    B --> D["Start container with SERVER_ROOT_PATH=/llmproxy"]
-    C --> E["curl {ROOT_PATH}/ui/ → expect HTML"]
-    D --> F["curl {ROOT_PATH}/ui/ → expect HTML"]
-    E -->|"HTML found"| G["✅ Pass"]
-    E -->|"404 or no HTML"| H["❌ Fail Workflow"]
-    F -->|"HTML found"| G
-    F -->|"404 or no HTML"| H
-
-    style G fill:#d4edda,stroke:#28a745
-    style H fill:#f8d7da,stroke:#dc3545
-```
-
-This prevents future regressions where changes to `proxy_server.py` accidentally break `SERVER_ROOT_PATH` support.
-
----
-
-## Timeline
-
-| Time (UTC)         | Event                                                                                                                                                        |
-| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------ |
-| Jan 22, 2026 04:20 | PR [#19467](https://github.com/BerriAI/litellm/pull/19467) merged, removing `root_path=server_root_path`                                                     |
-| Jan 22–26          | Users on nightly builds report UI 404 errors when using `SERVER_ROOT_PATH`                                                                                   |
-| Jan 26, 2026 17:48 | Fix PR [#19790](https://github.com/BerriAI/litellm/pull/19790) merged, restoring `root_path=server_root_path`                                                |
-| Feb 18, 2026       | CI workflow [`test_server_root_path.yml`](https://github.com/BerriAI/litellm/blob/main/.github/workflows/test_server_root_path.yml) added to run on every PR |
-
----
-
-## Resolution steps for users
-
-For users still experiencing issues, update to the latest LiteLLM version:
-
-```bash
-pip install --upgrade litellm
-```
-
-Verify your `SERVER_ROOT_PATH` is correctly set:
-
-```bash
-# In your environment or docker-compose.yml
-SERVER_ROOT_PATH="/your-prefix"
-```
-
-Then confirm the UI is accessible at `http://your-host:4000/your-prefix/ui/`.

docs/my-website/blog/sub_millisecond_proxy_overhead/index.md

@@ -1,85 +0,0 @@
----
-slug: sub-millisecond-proxy-overhead
-title: "Achieving Sub-Millisecond Proxy Overhead"
-date: 2026-02-02T10:00:00
-authors:
-  - alexsander
-  - krrish
-  - ishaan-alt
-description: "Our Q1 performance target and architectural direction for achieving sub-millisecond proxy overhead on modest hardware."
-tags: [performance, architecture]
-hide_table_of_contents: false
----
-
-![Sidecar architecture: Python control plane vs. sidecar hot path](https://raw.githubusercontent.com/AlexsanderHamir/assets/main/Screenshot%202026-02-02%20172554.png)
-
-# Achieving Sub-Millisecond Proxy Overhead
-
-## Introduction
-
-Our Q1 performance target is to aggressively move toward sub-millisecond proxy overhead on a single instance with 4 CPUs and 8 GB of RAM, and to continue pushing that boundary over time. Our broader goal is to make LiteLLM inexpensive to deploy, lightweight, and fast. This post outlines the architectural direction behind that effort.
-
-Proxy overhead refers to the latency introduced by LiteLLM itself, independent of the upstream provider.
-
-To measure it, we run the same workload directly against the provider and through LiteLLM at identical QPS (for example, 1,000 QPS) and compare the latency delta. To reduce noise, the load generator, LiteLLM, and a mock LLM endpoint all run on the same machine, ensuring the difference reflects proxy overhead rather than network latency.
-
-{/* truncate */}
-
----
-
-## Where We're Coming From
-
-Under the same benchmark originally conducted by [TensorZero](https://www.tensorzero.com/docs/gateway/benchmarks), LiteLLM previously failed at around 1,000 QPS.
-
-That is no longer the case. Today, LiteLLM can be stress-tested at 1,000 QPS with no failures and can scale up to 5,000 QPS without failures on a 4-CPU, 8-GB RAM single instance setup.
-
-This establishes a more up to date baseline and provides useful context as we continue working on proxy overhead and overall performance.
-
----
-
-## Design Choice
-
-Achieving sub-millisecond proxy overhead with a Python-based system requires being deliberate about where work happens.
-
-Python is a strong fit for flexibility and extensibility: provider abstraction, configuration-driven routing, and a rich callback ecosystem. These are areas where development velocity and correctness matter more than raw throughput.
-
-At higher request rates, however, certain classes of work become expensive when executed inside the Python process on every request. Rather than rewriting LiteLLM or introducing complex deployment requirements, we adopt an optional **sidecar architecture**.
-
-This architectural change is how we intend to make LiteLLM **permanently fast**. While it supports our near-term performance targets, it is a long-term investment.
-
-Python continues to own:
-
-- Request validation and normalization
-- Model and provider selection
-- Callbacks and integrations
-
-The sidecar owns **performance-critical execution**, such as:
-
-- Efficient request forwarding
-- Connection reuse and pooling
-- Enforcing timeouts and limits
-- Aggregating high-frequency metrics
-
-This separation allows each component to focus on what it does best: Python acts as the control plane, while the sidecar handles the hot path.
-
----
-
-### Why the Sidecar Is Optional
-
-The sidecar is intentionally **optional**.
-
-This allows us to ship it incrementally, validate it under real-world workloads, and avoid making it a hard dependency before it is fully battle-tested across all LiteLLM features.
-
-Just as importantly, this ensures that self-hosting LiteLLM remains simple. The sidecar is bundled and started automatically, requires no additional infrastructure, and can be disabled entirely. From a user's perspective, LiteLLM continues to behave like a single service.
-
-As of today, the sidecar is an optimization, not a requirement.
-
----
-
-## Conclusion
-
-Sub-millisecond proxy overhead is not achieved through a single optimization, but through architectural changes.
-
-By keeping Python focused on orchestration and extensibility, and offloading performance-critical execution to a sidecar, we establish a foundation for making LiteLLM **permanently fast over time**—even on modest hardware such as a 1-CPU, 2-GB RAM instance, while keeping deployment and self-hosting simple.
-
-This work extends beyond Q1, and we will continue sharing benchmarks and updates as the architecture evolves.

docs/my-website/blog/vanta_compliance_recertification/index.md

@@ -1,18 +0,0 @@
----
-slug: vanta-compliance-recertification
-title: "LiteLLM + Vanta: SOC 2 Type 2 and ISO 27001 Recertification"
-date: 2026-03-30T10:00:00
-authors:
-  - krrish
-description: "LiteLLM is partnering with Vanta on SOC 2 Type 2 and ISO 27001 recertification and engaging independent auditors for verification."
-tags: [security, compliance]
-hide_table_of_contents: true
----
-
-![LiteLLM x Vanta SOC-2 Recertification](/img/blog/vanta_soc2_recertification.png)
-
-We are partnering with [Vanta](https://www.vanta.com/) to recertify LiteLLM's compliance for SOC 2 Type 2 and ISO 27001.
-
-As part of this process, we are also identifying independent auditors to validate and verify our compliance posture.
-
-This is part of our commitment to being the most secure and transparent AI Gateway possible.

docs/my-website/blog/video_characters_litellm/index.md

@@ -1,121 +0,0 @@
----
-slug: video_characters_api
-title: "New Video Characters, Edit and Extension API support"
-date: 2026-03-16T10:00:00
-authors:
-  - sameer
-  - krrish
-  - ishaan-alt
-description: "LiteLLM now supports creating, retrieving, and managing reusable video characters across multiple video generations."
-tags: [videos, characters, proxy, routing]
-hide_table_of_contents: false
----
-
-LiteLLM now supoports videos character, edit and extension apis.
-
-{/* truncate */}
-
-## What's New
-
-Four new endpoints for video character operations:
-- **Create character** - Upload a video to create a reusable asset
-- **Get character** - Retrieve character metadata
-- **Edit video** - Modify generated videos
-- **Extend video** - Continue clips with character consistency
-
-**Available from:** LiteLLM v1.83.0+
-
-## Quick Example
-
-```python
-import litellm
-
-# Create character from video
-character = litellm.avideo_create_character(
-    name="Luna",
-    video=open("luna.mp4", "rb"),
-    custom_llm_provider="openai",
-    model="sora-2"
-)
-print(f"Character: {character.id}")
-
-# Use in generation
-video = litellm.avideo(
-    model="sora-2",
-    prompt="Luna dances through a magical forest.",
-    characters=[{"id": character.id}],
-    seconds="8"
-)
-
-# Get character info
-fetched = litellm.avideo_get_character(
-    character_id=character.id,
-    custom_llm_provider="openai"
-)
-
-# Edit with character preserved
-edited = litellm.avideo_edit(
-    video_id=video.id,
-    prompt="Add warm golden lighting"
-)
-
-# Extend sequence
-extended = litellm.avideo_extension(
-    video_id=video.id,
-    prompt="Luna waves goodbye",
-    seconds="5"
-)
-```
-
-## Via Proxy
-
-```bash
-# Create character
-curl -X POST "http://localhost:4000/v1/videos/characters" \
-  -H "Authorization: Bearer sk-litellm-key" \
-  -F "video=@luna.mp4" \
-  -F "name=Luna"
-
-# Get character
-curl -X GET "http://localhost:4000/v1/videos/characters/char_abc123def456" \
-  -H "Authorization: Bearer sk-litellm-key"
-
-# Edit video
-curl -X POST "http://localhost:4000/v1/videos/edits" \
-  -H "Authorization: Bearer sk-litellm-key" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "video": {"id": "video_xyz789"},
-    "prompt": "Add warm golden lighting and enhance colors"
-  }'
-
-# Extend video
-curl -X POST "http://localhost:4000/v1/videos/extensions" \
-  -H "Authorization: Bearer sk-litellm-key" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "video": {"id": "video_xyz789"},
-    "prompt": "Luna waves goodbye and walks into the sunset",
-    "seconds": "5"
-  }'
-```
-
-## Managed Character IDs
-
-LiteLLM automatically encodes provider and model metadata into character IDs:
-
-**What happens:**
-```
-Upload character "Luna" with model "sora-2" on OpenAI
-  ↓
-LiteLLM creates: char_abc123def456 (contains provider + model_id)
-  ↓
-When you reference it later, LiteLLM decodes automatically
-  ↓
-Router knows exactly which deployment to use
-```
-
-**Behind the scenes:**
-- Character ID format: `character_<base64_encoded_metadata>`
-- Metadata includes: provider, model_id, original_character_id
-- Transparent to you - just use the ID, LiteLLM handles routing

docs/my-website/blog/vllm_embeddings_incident/index.md

@@ -1,108 +0,0 @@
----
-slug: vllm-embeddings-incident
-title: "Incident Report: vLLM Embeddings Broken by encoding_format Parameter"
-date: 2026-02-18T10:00:00
-authors:
-  - sameer
-  - krrish
-  - ishaan-alt
-tags: [incident-report, embeddings, vllm]
-hide_table_of_contents: false
----
-
-**Date:** Feb 16, 2026
-**Duration:** ~3 hours
-**Severity:** High (for vLLM embedding users)
-**Status:** Resolved
-
-## Summary
-
-A commit ([`dbcae4a`](https://github.com/BerriAI/litellm/commit/dbcae4aca5836770d0e9cd43abab0333c3d61ab2)) intended to fix OpenAI SDK behavior broke vLLM embeddings by explicitly passing `encoding_format=None` in API requests. vLLM rejects this with error: `"unknown variant \`\`, expected float or base64"`.
-
-- **vLLM embedding calls:** Complete failure - all requests rejected
-- **Other providers:** No impact - OpenAI and other providers functioned normally
-- **Other vLLM functionality:** No impact - only embeddings were affected
-
-{/* truncate */}
-
----
-
-## Background
-
-The `encoding_format` parameter for embeddings specifies whether vectors should be returned as `float` arrays or `base64` encoded strings. Different providers have different expectations:
-
-- **OpenAI SDK:** If `encoding_format` is omitted, the SDK adds a default value of `"float"`
-- **vLLM:** Strictly validates `encoding_format` - only accepts `"float"`, `"base64"`, or complete omission. Rejects `None` or empty string values.
-
-```mermaid
-flowchart TD
-    A["1. User calls litellm.embedding()
-    litellm/main.py"] --> B["2. Transform request for provider
-    litellm/llms/openai_like/embedding/handler.py"]
-    B --> C["3. Send request to vLLM endpoint"]
-    C -->|"encoding_format omitted"| D["4a. ✅ vLLM processes request"]
-    C -->|"encoding_format='float' or 'base64'"| D
-    C -->|"encoding_format=None or ''"| E["4b. ❌ vLLM rejects with error:
-    'unknown variant, expected float or base64'"]
-
-    style D fill:#d4edda,stroke:#28a745
-    style E fill:#f8d7da,stroke:#dc3545
-    style B fill:#fff3cd,stroke:#ffc107
-```
-
----
-
-## Root cause
-
-A well-intentioned fix for OpenAI SDK behavior inadvertently broke vLLM embeddings:
-
-**The Breaking Change ([`dbcae4a`](https://github.com/BerriAI/litellm/commit/dbcae4aca5836770d0e9cd43abab0333c3d61ab2)):**
-
-In `litellm/main.py`, the code was changed to explicitly set `encoding_format=None` instead of omitting it:
-
-```python
-# Added in dbcae4a
-if encoding_format is not None:
-    optional_params["encoding_format"] = encoding_format
-else:
-    # Omitting causes openai sdk to add default value of "float"
-    optional_params["encoding_format"] = None
-```
-
-This fix worked correctly for OpenAI - explicitly passing `None` prevented the SDK from adding its default value. However, vLLM's strict parameter validation rejected `None` values, causing all embedding requests to fail.
-
----
-
-## The Fix
-
-Fix deployed ([`55348dd`](https://github.com/BerriAI/litellm/commit/55348dd9c51b5b028f676d25ad023b8f052fc071)). The solution filters out `None` and empty string values from `optional_params` before sending requests to OpenAI-like providers (including vLLM).
-
-**In `litellm/llms/openai_like/embedding/handler.py`:**
-
-```python
-# Before (broken)
-data = {"model": model, "input": input, **optional_params}
-
-# After (fixed)
-filtered_optional_params = {k: v for k, v in optional_params.items() if v not in (None, '')}
-data = {"model": model, "input": input, **filtered_optional_params}
-```
-
-This ensures:
-- Valid values (`"float"`, `"base64"`) are preserved and sent
-- `None` and empty string values are filtered out (parameter omitted entirely)
-- OpenAI SDK no longer adds defaults because liteLLM handles the parameter upstream
-
----
-
-## Remediation
-
-| # | Action | Status | Code |
-|---|---|---|---|
-| 1 | Filter `None` and empty string values in OpenAI-like embedding handler | ✅ Done | [`handler.py#L108`](https://github.com/BerriAI/litellm/blob/main/litellm/llms/openai_like/embedding/handler.py#L108) |
-| 2 | Unit tests for parameter filtering (None, empty string, valid values) | ✅ Done | [`test_openai_like_embedding.py`](https://github.com/BerriAI/litellm/blob/main/tests/test_litellm/llms/openai_like/embedding/test_openai_like_embedding.py) |
-| 3 | Transformation tests for hosted_vllm embedding config | ✅ Done | [`test_hosted_vllm_embedding_transformation.py`](https://github.com/BerriAI/litellm/blob/main/tests/test_litellm/llms/hosted_vllm/embedding/test_hosted_vllm_embedding_transformation.py) |
-| 4 | E2E tests with actual vLLM endpoint | ✅ Done | [`test_hosted_vllm_embedding_e2e.py`](https://github.com/BerriAI/litellm/blob/main/tests/test_litellm/llms/hosted_vllm/embedding/test_hosted_vllm_embedding_e2e.py) |
-| 5 | Validate JSON payload structure matches vLLM expectations | ✅ Done | Tests verify exact JSON sent to endpoint |
-
----

docs/my-website/docs/a2a.md

@@ -1,265 +0,0 @@
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-import Image from '@theme/IdealImage';
-
-# Agent Gateway (A2A Protocol) - Overview
-
-Add A2A Agents on LiteLLM AI Gateway, Invoke agents in A2A Protocol, track request/response logs in LiteLLM Logs. Manage which Teams, Keys can access which Agents onboarded.
-
-<Image 
-  img={require('../img/a2a_gateway.png')}
-  style={{width: '80%', display: 'block', margin: '0', borderRadius: '8px'}}
-/>
-
-<br />
-<br />
-
-| Feature | Supported | 
-|---------|-----------|
-| Supported Agent Providers | A2A, Vertex AI Agent Engine, LangGraph, Azure AI Foundry, Bedrock AgentCore, Pydantic AI |
-| Logging | ✅ |
-| Load Balancing | ✅ |
-| Streaming | ✅ |
-| [Iteration Budgets](a2a_iteration_budgets) | ✅ |
-
-
-:::tip
-
-LiteLLM follows the [A2A (Agent-to-Agent) Protocol](https://github.com/google/A2A) for invoking agents.
-
-:::
-
-## Adding your Agent
-
-### Add A2A Agents
-
-You can add A2A-compatible agents through the LiteLLM Admin UI.
-
-1. Navigate to the **Agents** tab
-2. Click **Add Agent**
-3. Enter the agent name (e.g., `ij-local`) and the URL of your A2A agent
-
-<Image 
-  img={require('../img/add_agent_1.png')}
-  style={{width: '80%', display: 'block', margin: '0'}}
-/>
-
-The URL should be the invocation URL for your A2A agent (e.g., `http://localhost:10001`).
-
-
-### Add Azure AI Foundry Agents
-
-Follow [this guide, to add your azure ai foundry agent to LiteLLM Agent Gateway](./providers/azure_ai_agents#litellm-a2a-gateway)
-
-### Add Vertex AI Agent Engine
-
-Follow [this guide, to add your Vertex AI Agent Engine to LiteLLM Agent Gateway](./providers/vertex_ai_agent_engine)
-
-### Add Bedrock AgentCore Agents
-
-Follow [this guide, to add your bedrock agentcore agent to LiteLLM Agent Gateway](./providers/bedrock_agentcore#litellm-a2a-gateway)
-
-### Add LangGraph Agents
-
-Follow [this guide, to add your langgraph agent to LiteLLM Agent Gateway](./providers/langgraph#litellm-a2a-gateway)
-
-### Add Pydantic AI Agents
-
-Follow [this guide, to add your pydantic ai agent to LiteLLM Agent Gateway](./providers/pydantic_ai_agent#litellm-a2a-gateway)
-
-## Invoking your Agents
-
-See the [Invoking A2A Agents](./a2a_invoking_agents) guide to learn how to call your agents using:
-- **A2A SDK** - Native A2A protocol with full support for tasks and artifacts
-- **OpenAI SDK** - Familiar `/chat/completions` interface with `a2a/` model prefix
-
-## Tracking Agent Logs
-
-After invoking an agent, you can view the request logs in the LiteLLM **Logs** tab.
-
-The logs show:
-- **Request/Response content** sent to and received from the agent
-- **User, Key, Team** information for tracking who made the request
-- **Latency and cost** metrics
-
-<Image 
-  img={require('../img/agent2.png')}
-  style={{width: '100%', display: 'block', margin: '2rem auto'}}
-/>
-
-
-## Forwarding LiteLLM Context Headers
-
-When LiteLLM invokes your A2A agent, it sends special headers that enable:
-- **Trace Grouping**: All LLM calls from the same agent execution appear under one trace
-- **Agent Spend Tracking**: Costs are attributed to the specific agent
-
-| Header | Purpose |
-|--------|---------|
-| `X-LiteLLM-Trace-Id` | Links all LLM calls to the same execution flow |
-| `X-LiteLLM-Agent-Id` | Attributes spend to the correct agent |
-
-
-To enable these features, your A2A server must **forward these headers** to any LLM calls it makes back to LiteLLM.
-
-### Implementation Steps
-
-**Step 1: Extract headers from incoming A2A request**
-```python def get_litellm_headers(request) -> dict:
-    """Extract X-LiteLLM-* headers from incoming A2A request."""
-    all_headers = request.call_context.state.get('headers', {})
-    return {
-        k: v for k, v in all_headers.items() 
-        if k.lower().startswith('x-litellm-')
-    }
-```
-
-**Step 2: Forward headers to your LLM calls**
-Pass the extracted headers when making calls back to LiteLLM:
-<Tabs>
-<TabItem value="openai" label="OpenAI SDK" default>
-
-```python from openai import OpenAI
-
-headers = get_litellm_headers(request)
-
-client = OpenAI(
-    api_key="sk-your-litellm-key",
-    base_url="http://localhost:4000",
-    default_headers=headers,  # Forward headers
-)
-
-response = client.chat.completions.create(
-    model="gpt-4o",
-    messages=[{"role": "user", "content": "Hello"}]
-)
-```
-</TabItem>
-
-<TabItem value="langchain" label="LangChain">
-
-```python
-from langchain_openai import ChatOpenAI
-
-headers = get_litellm_headers(request)
-
-llm = ChatOpenAI(
-    model="gpt-4o",
-    openai_api_key="sk-your-litellm-key",
-    base_url="http://localhost:4000",
-    default_headers=headers,  # Forward headers
-)
-```
-</TabItem>
-<TabItem value="litellm" label="LiteLLM SDK">
-
-```python
-import litellm
-
-headers = get_litellm_headers(request)
-
-response = litellm.completion(
-    model="gpt-4o",
-    messages=[{"role": "user", "content": "Hello"}],
-    api_base="http://localhost:4000",
-    extra_headers=headers,  # Forward headers
-)
-```
-</TabItem>
-<TabItem value="requests" label="HTTP (requests/httpx)">
-
-```python
-import httpx
-
-headers = get_litellm_headers(request)
-headers["Authorization"] = "Bearer sk-your-litellm-key"
-
-response = httpx.post(
-    "http://localhost:4000/v1/chat/completions",
-    headers=headers,
-    json={"model": "gpt-4o", "messages": [{"role": "user", "content": "Hello"}]}
-)
-```
-</TabItem>
-</Tabs>
-
-### Result
-
-With header forwarding enabled, you'll see:
-
-**Trace Grouping in Langfuse:**
-
-<Image
-  img={require('../img/a2a_trace_grouping.png')}
-  style={{width: '80%', display: 'block', margin: '0', borderRadius: '8px'}}
-/>
-
-**Agent Spend Attribution:**
-
-<Image
-  img={require('../img/a2a_agent_spend.png')}
-  style={{width: '80%', display: 'block', margin: '0', borderRadius: '8px'}}
-/>
-
-## API Reference
-
-### Endpoint
-
-```
-POST /a2a/{agent_name}/message/send
-```
-
-### Authentication
-
-Include your LiteLLM Virtual Key in the `Authorization` header:
-
-```
-Authorization: Bearer sk-your-litellm-key
-```
-
-### Request Format
-
-LiteLLM follows the [A2A JSON-RPC 2.0 specification](https://github.com/google/A2A):
-
-```json title="Request Body"
-{
-  "jsonrpc": "2.0",
-  "id": "unique-request-id",
-  "method": "message/send",
-  "params": {
-    "message": {
-      "role": "user",
-      "parts": [{"kind": "text", "text": "Your message here"}],
-      "messageId": "unique-message-id"
-    }
-  }
-}
-```
-
-### Response Format
-
-```json title="Response"
-{
-  "jsonrpc": "2.0",
-  "id": "unique-request-id",
-  "result": {
-    "kind": "task",
-    "id": "task-id",
-    "contextId": "context-id",
-    "status": {"state": "completed", "timestamp": "2025-01-01T00:00:00Z"},
-    "artifacts": [
-      {
-        "artifactId": "artifact-id",
-        "name": "response",
-        "parts": [{"kind": "text", "text": "Agent response here"}]
-      }
-    ]
-  }
-}
-```
-
-## Agent Registry
-
-Want to create a central registry so your team can discover what agents are available within your company?
-
-Use the [AI Hub](./proxy/ai_hub) to make agents public and discoverable across your organization. This allows developers to browse available agents without needing to rebuild them.

docs/my-website/docs/a2a_agent_headers.md

@@ -1,252 +0,0 @@
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-# A2A Agent Authentication Headers
-
-Forward authentication credentials (Bearer tokens, API keys, etc.) from clients to backend A2A agents.
-
-## Overview
-
-When LiteLLM proxies a request to a backend A2A agent, the agent may require its own authentication headers. There are three ways to supply them:
-
-| Method | Who configures | How it works |
-|---|---|---|
-| **Static headers** | Admin (UI / API) | Always sent, regardless of client request |
-| **Forward client headers** | Admin (UI / API) | Header names to extract from client request and forward |
-| **Convention-based** | Client (no admin config) | Client sends `x-a2a-{agent_name}-{header}` — automatically routed |
-
-All three methods can be combined. **Static headers always win** on key conflicts.
-
----
-
-## Method 1 — Static Headers
-
-Admin-configured headers that are always sent to the backend agent. Use this for server-to-server tokens or internal credentials that clients should never see or override.
-
-<Tabs>
-<TabItem value="ui" label="UI">
-
-1. Go to **Agents** in the LiteLLM dashboard.
-2. Create or edit an agent.
-3. Open the **Authentication Headers** panel.
-4. Under **Static Headers**, click **Add Static Header** and fill in the header name and value.
-
-</TabItem>
-<TabItem value="api" label="REST API">
-
-```bash
-curl -X POST http://localhost:4000/v1/agents \
-  -H "Authorization: Bearer sk-admin" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "agent_name": "my-agent",
-    "agent_card_params": { ... },
-    "static_headers": {
-      "Authorization": "Bearer internal-server-token",
-      "X-Internal-Service": "litellm-proxy"
-    }
-  }'
-```
-
-To update an existing agent:
-
-```bash
-curl -X PATCH http://localhost:4000/v1/agents/{agent_id} \
-  -H "Authorization: Bearer sk-admin" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "static_headers": {
-      "Authorization": "Bearer new-token"
-    }
-  }'
-```
-
-</TabItem>
-</Tabs>
-
-**Client call — no special headers needed:**
-
-```bash
-curl -X POST http://localhost:4000/a2a/my-agent \
-  -H "Authorization: Bearer sk-client-key" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "jsonrpc": "2.0", "id": "1", "method": "message/send",
-    "params": { "message": { "role": "user", "parts": [{"kind": "text", "text": "Hello"}], "messageId": "msg-1" } }
-  }'
-```
-
-The backend agent receives `Authorization: Bearer internal-server-token` without the client ever knowing the value.
-
----
-
-## Method 2 — Forward Client Headers
-
-Admin specifies a list of header **names**. When the client sends a request that includes those headers, LiteLLM extracts their values and forwards them to the backend agent. The client controls the values; the admin controls which headers are eligible to be forwarded.
-
-<Tabs>
-<TabItem value="ui" label="UI">
-
-1. Go to **Agents** in the LiteLLM dashboard.
-2. Create or edit an agent.
-3. Open the **Authentication Headers** panel.
-4. Under **Forward Client Headers**, type header names and press **Enter** (e.g. `x-api-key`, `Authorization`).
-
-</TabItem>
-<TabItem value="api" label="REST API">
-
-```bash
-curl -X POST http://localhost:4000/v1/agents \
-  -H "Authorization: Bearer sk-admin" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "agent_name": "my-agent",
-    "agent_card_params": { ... },
-    "extra_headers": ["x-api-key", "x-user-token"]
-  }'
-```
-
-</TabItem>
-</Tabs>
-
-**Client call — include the forwarded headers:**
-
-```bash
-curl -X POST http://localhost:4000/a2a/my-agent \
-  -H "Authorization: Bearer sk-client-key" \
-  -H "x-api-key: user-secret-value" \
-  -H "Content-Type: application/json" \
-  -d '{ ... }'
-```
-
-The backend agent receives `x-api-key: user-secret-value`.
-
-:::note
-Header name matching is **case-insensitive**. If the client sends `X-API-Key` and `extra_headers` lists `x-api-key`, they match.
-:::
-
----
-
-## Method 3 — Convention-Based Forwarding
-
-Clients can forward headers to a specific agent without any admin pre-configuration by using the naming convention:
-
-```
-x-a2a-{agent_name_or_id}-{header_name}: value
-```
-
-LiteLLM parses these headers automatically and routes them to the matching agent only.
-
-**Examples:**
-
-| Client header sent | Agent name/ID | Forwarded as |
-|---|---|---|
-| `x-a2a-my-agent-authorization: Bearer tok` | `my-agent` | `authorization: Bearer tok` |
-| `x-a2a-my-agent-x-api-key: secret` | `my-agent` | `x-api-key: secret` |
-| `x-a2a-abc123-authorization: Bearer tok` | agent ID `abc123` | `authorization: Bearer tok` |
-
-```bash
-curl -X POST http://localhost:4000/a2a/my-agent \
-  -H "Authorization: Bearer sk-client-key" \
-  -H "x-a2a-my-agent-authorization: Bearer agent-specific-token" \
-  -H "Content-Type: application/json" \
-  -d '{ ... }'
-```
-
-The `x-a2a-other-agent-authorization` header sent in the same request is **not** forwarded to `my-agent` — it is silently ignored.
-
-:::tip Matches both agent name and agent ID
-Both the human-readable name (e.g. `my-agent`) and the UUID (e.g. `abc123-...`) are valid. Use whichever is convenient for the client.
-:::
-
----
-
-## Merge Precedence
-
-When multiple methods supply the same header name, **static headers win**:
-
-```
-dynamic (forwarded/convention)  →  merged  ←  static (overlays, wins)
-```
-
-Example:
-
-| Source | `Authorization` value |
-|---|---|
-| Client sends (via `extra_headers` or convention) | `Bearer client-token` |
-| Admin-configured `static_headers` | `Bearer server-token` |
-| **What the backend agent receives** | **`Bearer server-token`** |
-
-This ensures admin-controlled credentials cannot be overridden by client requests.
-
----
-
-## Combining All Three Methods
-
-```bash
-# Register agent with static + forwarded headers
-curl -X POST http://localhost:4000/v1/agents \
-  -H "Authorization: Bearer sk-admin" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "agent_name": "my-agent",
-    "agent_card_params": { ... },
-    "static_headers": {
-      "X-Internal-Token": "secret123"
-    },
-    "extra_headers": ["x-user-id"]
-  }'
-
-# Client call using all three mechanisms
-curl -X POST http://localhost:4000/a2a/my-agent \
-  -H "Authorization: Bearer sk-client-key" \
-  -H "x-user-id: user-42" \
-  -H "x-a2a-my-agent-x-request-id: req-abc" \
-  -H "Content-Type: application/json" \
-  -d '{ ... }'
-```
-
-The backend agent receives:
-
-```
-X-Internal-Token: secret123          ← static header (always)
-x-user-id: user-42                   ← forwarded (in extra_headers)
-x-request-id: req-abc                ← convention-based (x-a2a-my-agent-*)
-X-LiteLLM-Trace-Id: <uuid>           ← LiteLLM internal
-X-LiteLLM-Agent-Id: <agent-id>       ← LiteLLM internal
-```
-
----
-
-## Header Isolation
-
-Each agent invocation uses an isolated HTTP connection. Headers configured for agent A are **never** sent to agent B, even if both agents are running and receiving requests simultaneously.
-
----
-
-## API Reference
-
-### `POST /v1/agents` / `PATCH /v1/agents/{agent_id}`
-
-| Field | Type | Description |
-|---|---|---|
-| `static_headers` | `object` | `{"Header-Name": "value"}` — always forwarded |
-| `extra_headers` | `string[]` | Header names to extract from client request and forward |
-
-### Agent Response
-
-Both fields are returned in `GET /v1/agents` and `GET /v1/agents/{agent_id}`:
-
-```json
-{
-  "agent_id": "...",
-  "agent_name": "my-agent",
-  "static_headers": { "X-Internal-Token": "secret123" },
-  "extra_headers": ["x-user-id"],
-  ...
-}
-```
-
-:::caution
-`static_headers` values are stored in the database and returned by the API. Treat them as you would any credential — do not store sensitive long-lived tokens here if your API is publicly accessible. Consider using short-lived tokens or environment-injected secrets instead.
-:::

docs/my-website/docs/a2a_agent_permissions.md

@@ -1,259 +0,0 @@
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-import Image from '@theme/IdealImage';
-
-# Agent Permission Management
-
-Control which A2A agents can be accessed by specific keys or teams in LiteLLM.
-
-## Overview
-
-Agent Permission Management lets you restrict which agents a LiteLLM Virtual Key or Team can access. This is useful for:
-
-- **Multi-tenant environments**: Give different teams access to different agents
-- **Security**: Prevent keys from invoking agents they shouldn't have access to
-- **Compliance**: Enforce access policies for sensitive agent workflows
-
-When permissions are configured:
-- `GET /v1/agents` only returns agents the key/team can access
-- `POST /a2a/{agent_id}` (Invoking an agent) returns `403 Forbidden` if access is denied
-
-## Setting Permissions on a Key
-
-This example shows how to create a key with agent permissions and test access.
-
-### 1. Get Your Agent ID
-
-<Tabs>
-<TabItem value="ui" label="UI">
-
-1. Go to **Agents** in the sidebar
-2. Click into the agent you want
-3. Copy the **Agent ID**
-
-<Image 
-  img={require('../img/agent_id.png')}
-  style={{width: '80%', display: 'block', margin: '0', borderRadius: '8px'}}
-/>
-
-</TabItem>
-<TabItem value="api" label="API">
-
-```bash title="List all agents" showLineNumbers
-curl "http://localhost:4000/v1/agents" \
-  -H "Authorization: Bearer sk-master-key"
-```
-
-Response:
-```json title="Response" showLineNumbers
-{
-  "agents": [
-    {"agent_id": "agent-123", "name": "Support Agent"},
-    {"agent_id": "agent-456", "name": "Sales Agent"}
-  ]
-}
-```
-
-</TabItem>
-</Tabs>
-
-### 2. Create a Key with Agent Permissions
-
-<Tabs>
-<TabItem value="ui" label="UI">
-
-1. Go to **Keys** → **Create Key**
-2. Expand **Agent Settings**
-3. Select the agents you want to allow
-
-<Image 
-  img={require('../img/agent_key.png')}
-  style={{width: '80%', display: 'block', margin: '0', borderRadius: '8px'}}
-/>
-
-</TabItem>
-<TabItem value="api" label="API">
-
-```bash title="Create key with agent permissions" showLineNumbers
-curl -X POST "http://localhost:4000/key/generate" \
-  -H "Authorization: Bearer sk-master-key" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "object_permission": {
-      "agents": ["agent-123"]
-    }
-  }'
-```
-
-</TabItem>
-</Tabs>
-
-### 3. Test Access
-
-**Allowed agent (succeeds):**
-```bash title="Invoke allowed agent" showLineNumbers
-curl -X POST "http://localhost:4000/a2a/agent-123" \
-  -H "Authorization: Bearer sk-your-new-key" \
-  -H "Content-Type: application/json" \
-  -d '{"message": {"role": "user", "parts": [{"type": "text", "text": "Hello"}]}}'
-```
-
-**Blocked agent (fails with 403):**
-```bash title="Invoke blocked agent" showLineNumbers
-curl -X POST "http://localhost:4000/a2a/agent-456" \
-  -H "Authorization: Bearer sk-your-new-key" \
-  -H "Content-Type: application/json" \
-  -d '{"message": {"role": "user", "parts": [{"type": "text", "text": "Hello"}]}}'
-```
-
-Response:
-```json title="403 Forbidden Response" showLineNumbers
-{
-  "error": {
-    "message": "Access denied to agent: agent-456",
-    "code": 403
-  }
-}
-```
-
-## Setting Permissions on a Team
-
-Restrict all keys belonging to a team to only access specific agents.
-
-### 1. Create a Team with Agent Permissions
-
-<Tabs>
-<TabItem value="ui" label="UI">
-
-1. Go to **Teams** → **Create Team**
-2. Expand **Agent Settings**
-3. Select the agents you want to allow for this team
-
-<Image 
-  img={require('../img/agent_key.png')}
-  style={{width: '80%', display: 'block', margin: '0', borderRadius: '8px'}}
-/>
-
-</TabItem>
-<TabItem value="api" label="API">
-
-```bash title="Create team with agent permissions" showLineNumbers
-curl -X POST "http://localhost:4000/team/new" \
-  -H "Authorization: Bearer sk-master-key" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "team_alias": "support-team",
-    "object_permission": {
-      "agents": ["agent-123"]
-    }
-  }'
-```
-
-Response:
-```json title="Response" showLineNumbers
-{
-  "team_id": "team-abc-123",
-  "team_alias": "support-team"
-}
-```
-
-</TabItem>
-</Tabs>
-
-### 2. Create a Key for the Team
-
-<Tabs>
-<TabItem value="ui" label="UI">
-
-1. Go to **Keys** → **Create Key**
-2. Select the **Team** from the dropdown
-
-<Image 
-  img={require('../img/agent_team.png')}
-  style={{width: '80%', display: 'block', margin: '0', borderRadius: '8px'}}
-/>
-
-</TabItem>
-<TabItem value="api" label="API">
-
-```bash title="Create key for team" showLineNumbers
-curl -X POST "http://localhost:4000/key/generate" \
-  -H "Authorization: Bearer sk-master-key" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "team_id": "team-abc-123"
-  }'
-```
-
-</TabItem>
-</Tabs>
-
-### 3. Test Access
-
-The key inherits agent permissions from the team.
-
-**Allowed agent (succeeds):**
-```bash title="Invoke allowed agent" showLineNumbers
-curl -X POST "http://localhost:4000/a2a/agent-123" \
-  -H "Authorization: Bearer sk-team-key" \
-  -H "Content-Type: application/json" \
-  -d '{"message": {"role": "user", "parts": [{"type": "text", "text": "Hello"}]}}'
-```
-
-**Blocked agent (fails with 403):**
-```bash title="Invoke blocked agent" showLineNumbers
-curl -X POST "http://localhost:4000/a2a/agent-456" \
-  -H "Authorization: Bearer sk-team-key" \
-  -H "Content-Type: application/json" \
-  -d '{"message": {"role": "user", "parts": [{"type": "text", "text": "Hello"}]}}'
-```
-
-## How It Works
-
-```mermaid
-flowchart TD
-    A[Request to invoke agent] --> B{LiteLLM Virtual Key has agent restrictions?}
-    B -->|Yes| C{LiteLLM Team has agent restrictions?}
-    B -->|No| D{LiteLLM Team has agent restrictions?}
-    
-    C -->|Yes| E[Use intersection of key + team permissions]
-    C -->|No| F[Use key permissions only]
-    
-    D -->|Yes| G[Inherit team permissions]
-    D -->|No| H[Allow ALL agents]
-    
-    E --> I{Agent in allowed list?}
-    F --> I
-    G --> I
-    H --> J[Allow request]
-    
-    I -->|Yes| J
-    I -->|No| K[Return 403 Forbidden]
-```
-
-| Key Permissions | Team Permissions | Result | Notes |
-|-----------------|------------------|--------|-------|
-| None | None | Key can access **all** agents | Open access by default when no restrictions are set |
-| `["agent-1", "agent-2"]` | None | Key can access `agent-1` and `agent-2` | Key uses its own permissions |
-| None | `["agent-1", "agent-3"]` | Key can access `agent-1` and `agent-3` | Key inherits team's permissions |
-| `["agent-1", "agent-2"]` | `["agent-1", "agent-3"]` | Key can access `agent-1` only | Intersection of both lists (most restrictive wins) |
-
-## Viewing Permissions
-
-<Tabs>
-<TabItem value="ui" label="UI">
-
-1. Go to **Keys** or **Teams**
-2. Click into the key/team you want to view
-3. Agent permissions are displayed in the info view
-
-</TabItem>
-<TabItem value="api" label="API">
-
-```bash title="Get key info" showLineNumbers
-curl "http://localhost:4000/key/info?key=sk-your-key" \
-  -H "Authorization: Bearer sk-master-key"
-```
-
-</TabItem>
-</Tabs>