Merge branch 'main' into fix/litellm-usa-under

.circleci/requirements.txt

@@ -17,4 +17,5 @@ mcp==1.25.0    # for MCP server
 semantic_router==0.1.10 # for auto-routing with litellm
 fastuuid==0.12.0
 responses==0.25.7  # for proxy client tests
-pytest-retry==1.6.3  # for automatic test retries
+pytest-retry==1.6.3  # for automatic test retries
+litellm-proxy-extras  # for prisma migrations

.claude/settings.json

@@ -0,0 +1,36 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(git show:*)",
+      "Bash(git worktree add:*)",
+      "Read(//Users/krrishdholakia/Documents/litellm/**)",
+      "Read(//Users/krrishdholakia/Documents/litellm-claude-code-guardrails/litellm/types/**)",
+      "Read(//Users/krrishdholakia/Documents/litellm-claude-code-guardrails/**)",
+      "Read(//Users/krrishdholakia/Documents/litellm-claude-code-guardrails/litellm/**)",
+      "Bash(python:*)",
+      "Bash(python -c \"\nimport sys; sys.path.insert\\(0, ''.''\\)\nfrom litellm.proxy.guardrails.guardrail_hooks.claude_code.guardrail import ClaudeCodeGuardrail, HOSTED_TOOL_PREFIXES\nprint\\(''HOSTED_TOOL_PREFIXES:'', HOSTED_TOOL_PREFIXES\\)\nprint\\(''ClaudeCodeGuardrail imported OK''\\)\n\")",
+      "Read(//Users/krrishdholakia/Documents/litellm-mcp-jwt-groups/litellm/proxy/**)",
+      "Read(//Users/krrishdholakia/Documents/litellm-mcp-jwt-groups/**)",
+      "Bash(poetry run pytest:*)",
+      "Bash(git add:*)",
+      "Bash(git commit:*)",
+      "Bash(poetry run python:*)",
+      "Bash(poetry run pip:*)",
+      "Bash(git reset:*)",
+      "Bash(git cherry-pick:*)",
+      "Bash(git checkout:*)",
+      "Read(//Users/krrishdholakia/Documents/litellm/litellm/proxy/guardrails/guardrail_hooks/**)",
+      "Read(//Users/krrishdholakia/Documents/**)",
+      "Bash(git -C /Users/krrishdholakia/Documents/litellm-mcp-user-permissions worktree list)",
+      "Bash(ls:*)"
+    ],
+    "additionalDirectories": [
+      "/Users/krrishdholakia/Documents/litellm-mcp-group-plan/plan",
+      "/Users/krrishdholakia/Documents/litellm-claude-code-guardrails/litellm/proxy/guardrails/guardrail_hooks/claude_code",
+      "/Users/krrishdholakia/Documents/litellm-claude-code-guardrails/litellm/types",
+      "/Users/krrishdholakia/Documents/litellm-claude-code-guardrails",
+      "/Users/krrishdholakia/Documents/litellm-mcp-jwt-groups/litellm/proxy",
+      "/Users/krrishdholakia/Documents/litellm-mcp-jwt-groups/tests/test_litellm/proxy/auth"
+    ]
+  }
+}

.github/ISSUE_TEMPLATE/config.yml

@@ -1,7 +1,7 @@
 blank_issues_enabled: true
 contact_links:
   - name: Schedule Demo
-    url: https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat
+    url: https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions
     about: Speak directly with Krrish and Ishaan, the founders, to discuss issues, share feedback, or explore improvements for LiteLLM
   - name: Discord
     url: https://discord.com/invite/wuPM9dRgDw

.github/codeql/codeql-config.yml

@@ -0,0 +1,22 @@
+name: "LiteLLM CodeQL config"
+
+# Use security-extended suite instead of security-and-quality to avoid
+# result sets > 2 GiB on this codebase that cause fatal OOM failures.
+queries:
+  - uses: security-extended
+
+# These two queries are security queries included in security-extended that
+# individually produce result sets > 2 GiB on this codebase, causing fatal
+# OOM failures. Exclude them as a safety net until CI confirms they no longer
+# OOM; drop these exclusions in a follow-up once verified.
+query-filters:
+  - exclude:
+      id: py/clear-text-logging-sensitive-data  # CWE-312 — > 2 GiB result set
+  - exclude:
+      id: py/polynomial-redos                   # CWE-730 — > 2 GiB result set
+
+paths-ignore:
+  - tests
+  - docs
+  - "**/*.md"
+  - litellm/proxy/_experimental/out

.github/observatory/litellm_config.yaml

@@ -0,0 +1,19 @@
+# LiteLLM Observatory Test Configuration
+# This config is used by CI to spin up a temporary LiteLLM instance
+# for running observatory tests against RC/stable releases.
+#
+# Add model definitions for the providers you want to test.
+# Provider API keys are injected via environment variables in CI.
+
+model_list:
+  - model_name: gpt-4o
+    litellm_params:
+      model: azure/gpt-4o
+      api_key: os.environ/AZURE_API_KEY
+      api_base: os.environ/AZURE_API_BASE
+
+  - model_name: gpt-4o-mini
+    litellm_params:
+      model: azure/gpt-4o-mini
+      api_key: os.environ/AZURE_API_KEY
+      api_base: os.environ/AZURE_API_BASE

.github/pull_request_template.md

@@ -6,11 +6,15 @@
 
 **Please complete all items before asking a LiteLLM maintainer to review your PR**
 
-- [ ] I have Added testing in the [`tests/litellm/`](https://github.com/BerriAI/litellm/tree/main/tests/litellm) directory, **Adding at least 1 test is a hard requirement** - [see details](https://docs.litellm.ai/docs/extras/contributing_code)
+- [ ] I have Added testing in the [`tests/test_litellm/`](https://github.com/BerriAI/litellm/tree/main/tests/test_litellm) directory, **Adding at least 1 test is a hard requirement** - [see details](https://docs.litellm.ai/docs/extras/contributing_code)
 - [ ] My PR passes all unit tests on [`make test-unit`](https://docs.litellm.ai/docs/extras/contributing_code)
 - [ ] My PR's scope is as isolated as possible, it only solves 1 specific problem
 - [ ] I have requested a Greptile review by commenting `@greptileai` and received a **Confidence Score of at least 4/5** before requesting a maintainer review
 
+## Delays in PR merge?
+
+If you're seeing a delay in your PR being merged, ping the LiteLLM Team on [Slack (#pr-review)](https://join.slack.com/t/litellmossslack/shared_invite/zt-3o7nkuyfr-p_kbNJj8taRfXGgQI1~YyA).
+
 ## CI (LiteLLM team)
 
 > **CI status guideline:**

.github/scripts/close_duplicate_issues.py

@@ -0,0 +1,208 @@
+#!/usr/bin/env python3
+"""
+Detect and close duplicate GitHub issues using title similarity.
+
+Modes:
+  --scan           Compare all open issues against each other (batch)
+  --issue-number N Check a single issue against older open issues
+
+Requires the `gh` CLI to be authenticated.
+"""
+
+import argparse
+import difflib
+import json
+import re
+import subprocess
+import sys
+
+
+def normalize_title(title: str) -> str:
+    """Strip common prefixes, lowercase, and collapse whitespace."""
+    title = re.sub(
+        r"^\[?(bug|feature request|enhancement|question|docs)[:\]]?\s*",
+        "",
+        title,
+        flags=re.IGNORECASE,
+    )
+    return " ".join(title.lower().split())
+
+
+def gh(*args: str) -> str:
+    """Run a gh CLI command and return stdout."""
+    result = subprocess.run(
+        ["gh", *args],
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    return result.stdout
+
+
+def fetch_open_issues(repo: str | None) -> list[dict]:
+    """Fetch all open issues (excluding PRs) via gh api --paginate."""
+    if repo:
+        endpoint = f"repos/{repo}/issues?state=open&per_page=100&sort=created&direction=asc"
+    else:
+        endpoint = "repos/{owner}/{repo}/issues?state=open&per_page=100&sort=created&direction=asc"
+    cmd = ["api", "--paginate", endpoint]
+
+    raw = gh(*cmd)
+    # gh --paginate concatenates JSON arrays, so we may get multiple arrays
+    issues = []
+    for line in raw.strip().splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        parsed = json.loads(line)
+        if isinstance(parsed, list):
+            issues.extend(parsed)
+        else:
+            issues.append(parsed)
+
+    # Filter out pull requests (they also appear in the issues endpoint)
+    return [i for i in issues if "pull_request" not in i]
+
+
+def close_as_duplicate(
+    issue_number: int, duplicate_of: int, repo: str | None, dry_run: bool
+) -> None:
+    """Close an issue as duplicate of another, adding a comment and label."""
+    repo_args = ["--repo", repo] if repo else []
+
+    if dry_run:
+        print(f"  [DRY RUN] Would close #{issue_number} as duplicate of #{duplicate_of}")
+        return
+
+    # Add comment
+    comment_body = (
+        f"Closing as duplicate of #{duplicate_of}.\n\n"
+        "If you believe this is not a duplicate, please reopen and add context "
+        "explaining how this differs."
+    )
+    gh("issue", "comment", str(issue_number), "--body", comment_body, *repo_args)
+
+    # Add label
+    gh("issue", "edit", str(issue_number), "--add-label", "duplicate", *repo_args)
+
+    # Close with not_planned reason
+    gh(
+        "api",
+        f"repos/{repo or '{owner}/{repo}'}/issues/{issue_number}",
+        "-X",
+        "PATCH",
+        "-f",
+        "state=closed",
+        "-f",
+        "state_reason=not_planned",
+    )
+
+    print(f"  Closed #{issue_number} as duplicate of #{duplicate_of}")
+
+
+def find_duplicate(
+    issue: dict, candidates: list[dict], threshold: float
+) -> dict | None:
+    """Return the first candidate whose normalized title is above threshold."""
+    norm = normalize_title(issue["title"])
+    for candidate in candidates:
+        if candidate["number"] == issue["number"]:
+            continue
+        cand_norm = normalize_title(candidate["title"])
+        ratio = difflib.SequenceMatcher(None, norm, cand_norm).ratio()
+        if ratio >= threshold:
+            return candidate
+    return None
+
+
+def scan_all(issues: list[dict], threshold: float, repo: str | None, dry_run: bool) -> int:
+    """Compare every issue against all older issues. Returns count of duplicates found."""
+    # Sort oldest first
+    issues.sort(key=lambda i: i["number"])
+    closed_count = 0
+
+    for idx, issue in enumerate(issues):
+        older = issues[:idx]
+        if not older:
+            continue
+        dup = find_duplicate(issue, older, threshold)
+        if dup:
+            ratio = difflib.SequenceMatcher(
+                None,
+                normalize_title(issue["title"]),
+                normalize_title(dup["title"]),
+            ).ratio()
+            print(
+                f"#{issue['number']}: \"{issue['title']}\"\n"
+                f"  -> duplicate of #{dup['number']}: \"{dup['title']}\" "
+                f"({ratio:.0%} similar)"
+            )
+            close_as_duplicate(issue["number"], dup["number"], repo, dry_run)
+            closed_count += 1
+
+    return closed_count
+
+
+def check_single(
+    issue_number: int, issues: list[dict], threshold: float, repo: str | None, dry_run: bool
+) -> bool:
+    """Check a single issue against all older open issues. Returns True if duplicate found."""
+    target = None
+    for i in issues:
+        if i["number"] == issue_number:
+            target = i
+            break
+
+    if target is None:
+        print(f"Issue #{issue_number} not found among open issues.")
+        return False
+
+    older = [i for i in issues if i["number"] < issue_number]
+    dup = find_duplicate(target, older, threshold)
+    if dup:
+        ratio = difflib.SequenceMatcher(
+            None,
+            normalize_title(target["title"]),
+            normalize_title(dup["title"]),
+        ).ratio()
+        print(
+            f"#{target['number']}: \"{target['title']}\"\n"
+            f"  -> duplicate of #{dup['number']}: \"{dup['title']}\" "
+            f"({ratio:.0%} similar)"
+        )
+        close_as_duplicate(issue_number, dup["number"], repo, dry_run)
+        return True
+
+    print(f"#{issue_number}: no duplicate found above threshold {threshold}")
+    return False
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Detect and close duplicate GitHub issues")
+    mode = parser.add_mutually_exclusive_group(required=True)
+    mode.add_argument("--scan", action="store_true", help="Scan all open issues")
+    mode.add_argument("--issue-number", type=int, help="Check a single issue number")
+    parser.add_argument("--threshold", type=float, default=0.85, help="Similarity threshold (0-1)")
+    parser.add_argument("--close", action="store_true", help="Actually close duplicates (default is dry-run)")
+    parser.add_argument("--repo", type=str, help="Repository (owner/repo). Auto-detected if omitted.")
+    args = parser.parse_args()
+
+    dry_run = not args.close
+
+    if dry_run:
+        print("=== DRY RUN MODE (pass --close to actually close issues) ===\n")
+
+    print("Fetching open issues...")
+    issues = fetch_open_issues(args.repo)
+    print(f"Found {len(issues)} open issues.\n")
+
+    if args.scan:
+        count = scan_all(issues, args.threshold, args.repo, dry_run)
+        print(f"\nTotal duplicates {'found' if dry_run else 'closed'}: {count}")
+    else:
+        found = check_single(args.issue_number, issues, args.threshold, args.repo, dry_run)
+        sys.exit(0 if found else 0)  # Always exit 0; finding no dup is not an error
+
+
+if __name__ == "__main__":
+    main()

.github/workflows/auto_update_price_and_context_window.yml

@@ -7,6 +7,7 @@ on:
 
 jobs:
   auto_update_price_and_context_window:
+    if: github.repository == 'BerriAI/litellm'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3

.github/workflows/check_duplicate_issues.yml

@@ -20,10 +20,33 @@ jobs:
           reaction: eyes
           comment: |
             **⚠️ Potential duplicate detected**
-            
+
             This issue appears similar to existing issue(s):
             {{#issues}}
             - [#{{number}}]({{html_url}}) - {{title}} ({{accuracy}}% similar)
             {{/issues}}
-            
+
             Please review the linked issue(s) to see if they address your concern. If this is not a duplicate, please provide additional context to help us understand the difference.
+
+      - name: Checkout close script
+        if: github.event.action == 'opened'
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: .github/scripts
+
+      - name: Set up Python
+        if: github.event.action == 'opened'
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Auto-close if high-confidence duplicate
+        if: github.event.action == 'opened'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          python3 .github/scripts/close_duplicate_issues.py \
+            --issue-number ${{ github.event.issue.number }} \
+            --repo ${{ github.repository }} \
+            --threshold 0.85 \
+            --close

.github/workflows/codeql.yml

@@ -0,0 +1,53 @@
+name: "CodeQL"
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  schedule:
+    # Run weekly on Sundays at 04:00 UTC
+    - cron: "0 4 * * 0"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  analyze:
+    if: github.event_name != 'schedule' || github.repository == 'BerriAI/litellm'
+    name: Analyze (${{ matrix.language }})
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    permissions:
+      security-events: write
+      packages: read
+      actions: read
+      contents: read
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - language: actions
+            build-mode: none
+          - language: javascript-typescript
+            build-mode: none
+          - language: python
+            build-mode: none
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v3
+        with:
+          languages: ${{ matrix.language }}
+          build-mode: ${{ matrix.build-mode }}
+          config-file: ./.github/codeql/codeql-config.yml
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v3
+        with:
+          category: "/language:${{ matrix.language }}"

.github/workflows/codspeed.yml

@@ -0,0 +1,44 @@
+name: CodSpeed Benchmarks
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+  # Allow CodSpeed to trigger backtest performance analysis
+  # in order to generate initial data
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  id-token: write
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  benchmarks:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install dependencies
+        run: |
+          pip install -e "."
+          pip install pytest pytest-codspeed==4.3.0
+
+      - name: Run benchmarks
+        uses: CodSpeedHQ/action@v4
+        with:
+          mode: simulation
+          run: pytest tests/benchmarks/ --codspeed

.github/workflows/create_daily_staging_branch.yml

@@ -7,6 +7,7 @@ on:
 
 jobs:
   create-staging-branch:
+    if: github.repository == 'BerriAI/litellm'
     runs-on: ubuntu-latest
 
     steps:
@@ -41,3 +42,40 @@ jobs:
             git push origin $BRANCH_NAME
             echo "Successfully created and pushed branch: $BRANCH_NAME"
           fi
+
+  create-internal-dev-branch:
+    if: github.repository == 'BerriAI/litellm'
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Create internal dev branch
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          # Configure Git user
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+
+          # Generate branch name with MM_DD_YYYY format
+          BRANCH_NAME="litellm_internal_dev_$(date +'%m_%d_%Y')"
+          echo "Creating branch: $BRANCH_NAME"
+
+          # Fetch all branches
+          git fetch --all
+
+          # Check if the branch already exists
+          if git show-ref --verify --quiet refs/remotes/origin/$BRANCH_NAME; then
+            echo "Branch $BRANCH_NAME already exists. Skipping creation."
+          else
+            echo "Creating new branch: $BRANCH_NAME"
+            # Create the new branch from main
+            git checkout -b $BRANCH_NAME origin/main
+            # Push the new branch
+            git push origin $BRANCH_NAME
+            echo "Successfully created and pushed branch: $BRANCH_NAME"
+          fi

.github/workflows/ghcr_deploy.yml

@@ -299,6 +299,15 @@ jobs:
             ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-spend_logs:main-stable', env.REGISTRY) || '' }}
           platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
 
+  run-observatory-tests:
+    if: github.event.inputs.release_type == 'rc' || github.event.inputs.release_type == 'stable'
+    needs: [docker-hub-deploy]
+    uses: ./.github/workflows/run_observatory_tests.yml
+    with:
+      tag: ${{ github.event.inputs.tag }}
+      commit_hash: ${{ github.event.inputs.commit_hash }}
+    secrets: inherit
+
   build-and-push-helm-chart:
     if: github.event.inputs.release_type  != 'dev'
     needs: [docker-hub-deploy, build-and-push-image, build-and-push-image-database]
@@ -360,7 +369,8 @@ jobs:
   release:
     name: "New LiteLLM Release"
     needs: [docker-hub-deploy, build-and-push-image, build-and-push-image-database]
-
+    permissions:
+      contents: write
     runs-on: "ubuntu-latest"
    
     steps:

.github/workflows/interpret_load_test.py

@@ -123,7 +123,7 @@ if __name__ == "__main__":
         + docker_run_command
         + "\n\n"
         + "### Don't want to maintain your internal proxy? get in touch 🎉"
-        + "\nHosted Proxy Alpha: https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat"
+        + "\nHosted Proxy Alpha: https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions"
         + "\n\n"
         + "## Load Test LiteLLM Proxy Results"
         + "\n\n"

.github/workflows/publish_enterprise.yml

@@ -0,0 +1,94 @@
+name: Publish litellm-enterprise to PyPI
+
+on:
+  workflow_dispatch:
+    inputs:
+      bump:
+        description: "Version bump type"
+        required: true
+        default: "patch"
+        type: choice
+        options:
+          - patch
+          - minor
+          - major
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    if: github.repository == 'BerriAI/litellm'
+    permissions:
+      contents: write
+      pull-requests: write
+    defaults:
+      run:
+        working-directory: enterprise
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install Poetry
+        run: pip install poetry
+
+      - name: Bump version
+        id: bump
+        run: |
+          OLD=$(poetry version -s)
+          poetry version ${{ github.event.inputs.bump }}
+          NEW=$(poetry version -s)
+          echo "old=$OLD" >> $GITHUB_OUTPUT
+          echo "new=$NEW" >> $GITHUB_OUTPUT
+
+      - name: Update version refs in root pyproject.toml and requirements.txt
+        run: |
+          OLD=${{ steps.bump.outputs.old }}
+          NEW=${{ steps.bump.outputs.new }}
+          sed -i "s/litellm-enterprise = {version = \"${OLD}\"/litellm-enterprise = {version = \"${NEW}\"/" ../pyproject.toml
+          sed -i "s/litellm-enterprise==${OLD}/litellm-enterprise==${NEW}/" ../requirements.txt
+
+      - name: Update poetry.lock
+        working-directory: .
+        run: poetry lock
+
+      - name: Build
+        run: poetry build
+
+      - name: Commit version bump and create PR
+        id: create-pr
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          cd ..
+          BRANCH="bump/enterprise-${{ steps.bump.outputs.new }}"
+          git checkout -b "$BRANCH"
+          git add enterprise/pyproject.toml pyproject.toml requirements.txt poetry.lock
+          git commit -m "bump: litellm-enterprise ${{ steps.bump.outputs.old }} → ${{ steps.bump.outputs.new }}"
+          git push origin "$BRANCH" --force
+          gh pr create \
+            --title "bump: litellm-enterprise ${{ steps.bump.outputs.old }} → ${{ steps.bump.outputs.new }}" \
+            --body "Version bump for litellm-enterprise. Merge to update main." \
+            --head "$BRANCH" \
+            --base main \
+          || true
+          PR_URL=$(gh pr list --head "$BRANCH" --json url -q '.[0].url')
+          echo "pr_url=$PR_URL" >> $GITHUB_OUTPUT
+        env:
+          GH_TOKEN: ${{ github.token }}
+
+      - name: Enable auto-merge
+        run: |
+          gh pr merge "${{ steps.create-pr.outputs.pr_url }}" --auto --squash
+        env:
+          GH_TOKEN: ${{ github.token }}
+
+      - name: Publish to PyPI
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_ENTERPRISE }}
+        run: |
+          pip install twine
+          twine upload dist/litellm_enterprise-${{ steps.bump.outputs.new }}*

.github/workflows/publish_proxy_extras.yml

@@ -0,0 +1,74 @@
+name: Publish litellm-proxy-extras to PyPI
+
+on:
+  workflow_dispatch:
+    inputs:
+      bump:
+        description: "Version bump type"
+        required: true
+        default: "patch"
+        type: choice
+        options:
+          - patch
+          - minor
+          - major
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    if: github.repository == 'BerriAI/litellm'
+    permissions:
+      contents: write
+    defaults:
+      run:
+        working-directory: litellm-proxy-extras
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install Poetry
+        run: pip install poetry
+
+      - name: Bump version
+        id: bump
+        run: |
+          OLD=$(poetry version -s)
+          poetry version ${{ github.event.inputs.bump }}
+          NEW=$(poetry version -s)
+          echo "old=$OLD" >> $GITHUB_OUTPUT
+          echo "new=$NEW" >> $GITHUB_OUTPUT
+
+      - name: Update version refs in root pyproject.toml and requirements.txt
+        run: |
+          OLD=${{ steps.bump.outputs.old }}
+          NEW=${{ steps.bump.outputs.new }}
+          sed -i "s/litellm-proxy-extras = {version = \"${OLD}\"/litellm-proxy-extras = {version = \"${NEW}\"/" ../pyproject.toml
+          sed -i "s/litellm-proxy-extras==${OLD}/litellm-proxy-extras==${NEW}/" ../requirements.txt
+
+      - name: Update poetry.lock
+        working-directory: .
+        run: poetry lock
+
+      - name: Build
+        run: poetry build
+
+      - name: Commit version bump
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          cd ..
+          git add litellm-proxy-extras/pyproject.toml pyproject.toml requirements.txt poetry.lock
+          git commit -m "bump: litellm-proxy-extras ${{ steps.bump.outputs.old }} → ${{ steps.bump.outputs.new }}"
+          git push
+
+      - name: Publish to PyPI
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_PUBLISH_PASSWORD }}
+        run: |
+          pip install twine
+          twine upload dist/litellm_proxy_extras-${{ steps.bump.outputs.new }}*

.github/workflows/regenerate-poetry-lock.yml

@@ -0,0 +1,80 @@
+name: Regenerate poetry.lock
+
+# Runs whenever pyproject.toml is merged into main (the most common cause of
+# the "pyproject.toml changed significantly since poetry.lock was last generated"
+# CI failure).  Can also be triggered manually.
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - pyproject.toml
+  workflow_dispatch:
+
+permissions:
+  contents: write       # needed to push the auto/regenerate-poetry-lock-* branch
+  pull-requests: write  # needed to open the PR and enable auto-merge
+
+jobs:
+  regenerate-lock:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install Poetry
+        run: pip install poetry
+
+      - name: Regenerate poetry.lock
+        run: poetry lock
+
+      - name: Check whether poetry.lock actually changed
+        id: diff
+        run: |
+          if git diff --quiet poetry.lock; then
+            echo "changed=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Open PR with the refreshed lock file
+        if: steps.diff.outputs.changed == 'true'
+        id: open-pr
+        run: |
+          BRANCH="auto/regenerate-poetry-lock-$(date +'%Y%m%d%H%M%S')"
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git checkout -b "$BRANCH"
+          git add poetry.lock
+          git commit -m "chore: regenerate poetry.lock to match pyproject.toml"
+          git push -f origin "$BRANCH"
+
+          cat > /tmp/pr-body.md << 'BODY'
+          Automated regeneration of `poetry.lock` after `pyproject.toml` was updated on `main`.
+
+          Fixes the recurring CI failure:
+          ```
+          pyproject.toml changed significantly since poetry.lock was last generated.
+          Run `poetry lock` to fix the lock file.
+          ```
+          BODY
+
+          PR_URL=$(gh pr create \
+            --title "chore: regenerate poetry.lock to match pyproject.toml" \
+            --body-file /tmp/pr-body.md \
+            --head "$BRANCH" \
+            --base main)
+          echo "pr_url=$PR_URL" >> "$GITHUB_OUTPUT"
+        env:
+          GH_TOKEN: ${{ github.token }}
+
+      - name: Enable auto-merge
+        if: steps.diff.outputs.changed == 'true'
+        run: |
+          gh pr merge "${{ steps.open-pr.outputs.pr_url }}" --auto --squash
+        env:
+          GH_TOKEN: ${{ github.token }}

.github/workflows/run_observatory_tests.yml

@@ -0,0 +1,225 @@
+name: Run Observatory Tests
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: "Docker image tag to test (e.g. v1.61.0.rc1)"
+        required: true
+        type: string
+      commit_hash:
+        description: "Commit hash (defaults to HEAD of current branch)"
+        required: false
+        type: string
+  workflow_call:
+    inputs:
+      tag:
+        description: "Docker image tag to test"
+        required: true
+        type: string
+      commit_hash:
+        description: "Commit hash of the release"
+        required: true
+        type: string
+
+permissions:
+  contents: read
+
+env:
+  LITELLM_MASTER_KEY: ${{ secrets.LITELLM_MASTER_KEY_STAGING }}
+
+jobs:
+  observatory-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Validate tag input
+        env:
+          TAG: ${{ inputs.tag }}
+        run: |
+          if [[ ! "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+ ]]; then
+            echo "Invalid tag format: $TAG (expected vX.Y.Z...)"
+            exit 1
+          fi
+
+      - name: Start LiteLLM container
+        env:
+          TAG: ${{ inputs.tag }}
+          AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
+          AZURE_API_BASE: ${{ secrets.AZURE_API_BASE }}
+        run: |
+          docker run -d \
+            --name litellm-rc \
+            -p 4000:4000 \
+            -v "${{ github.workspace }}/.github/observatory/litellm_config.yaml:/app/config.yaml" \
+            -e LITELLM_MASTER_KEY="${LITELLM_MASTER_KEY}" \
+            -e AZURE_API_KEY="${AZURE_API_KEY}" \
+            -e AZURE_API_BASE="${AZURE_API_BASE}" \
+            "litellm/litellm:${TAG}" \
+            --config /app/config.yaml --port 4000
+
+      - name: Wait for LiteLLM health check
+        run: |
+          echo "Waiting for LiteLLM to be ready..."
+          for i in $(seq 1 30); do
+            if curl -s -f http://localhost:4000/health/liveliness > /dev/null 2>&1; then
+              echo "LiteLLM is healthy"
+              exit 0
+            fi
+            echo "Attempt $i/30 - not ready yet, waiting 10s..."
+            sleep 10
+          done
+          echo "LiteLLM failed to start within 5 minutes"
+          docker logs litellm-rc
+          exit 1
+
+      - name: Start cloudflared tunnel
+        run: |
+          # Install cloudflared
+          curl -sL https://github.com/cloudflare/cloudflared/releases/download/2025.2.1/cloudflared-linux-amd64 -o /usr/local/bin/cloudflared
+          chmod +x /usr/local/bin/cloudflared
+
+          # Start a quick tunnel (no account needed) and capture the URL
+          cloudflared tunnel --url http://localhost:4000 --no-autoupdate > /tmp/cloudflared.log 2>&1 &
+          CLOUDFLARED_PID=$!
+          echo "CLOUDFLARED_PID=$CLOUDFLARED_PID" >> $GITHUB_ENV
+
+          # Wait for tunnel URL to appear in logs
+          echo "Waiting for tunnel URL..."
+          for i in $(seq 1 30); do
+            TUNNEL_URL=$(grep -oP 'https://[a-z0-9-]+\.trycloudflare\.com' /tmp/cloudflared.log | head -1 || true)
+            if [ -n "$TUNNEL_URL" ]; then
+              echo "Tunnel URL: $TUNNEL_URL"
+              echo "TUNNEL_URL=$TUNNEL_URL" >> $GITHUB_ENV
+              exit 0
+            fi
+            sleep 2
+          done
+          echo "Failed to get tunnel URL"
+          cat /tmp/cloudflared.log
+          exit 1
+
+      - name: Verify tunnel connectivity
+        run: |
+          echo "Testing tunnel at ${{ env.TUNNEL_URL }}..."
+          # Quick tunnels need time for DNS propagation; retry to avoid
+          # transient NXDOMAIN (curl exit code 6) on first attempt.
+          for i in $(seq 1 10); do
+            if curl -sf "${{ env.TUNNEL_URL }}/health/liveliness" > /dev/null 2>&1; then
+              echo "Tunnel is working (attempt $i)"
+              exit 0
+            fi
+            echo "Attempt $i/10 - tunnel not routable yet, waiting 5s..."
+            sleep 5
+          done
+          echo "Tunnel failed to become reachable after 50s"
+          cat /tmp/cloudflared.log
+          exit 1
+
+      - name: Trigger observatory test run
+        id: trigger
+        env:
+          OBSERVATORY_URL: ${{ secrets.OBSERVATORY_URL }}
+          OBSERVATORY_API_KEY: ${{ secrets.OBSERVATORY_API_KEY }}
+        run: |
+          PAYLOAD=$(jq -n \
+            --arg url "${TUNNEL_URL}" \
+            --arg key "${LITELLM_MASTER_KEY}" \
+            '{
+              deployment_url: $url,
+              api_key: $key,
+              test_suite: "TestOAIAzureRelease",
+              models: ["gpt-4o-mini", "gpt-4o"]
+            }')
+          RESPONSE=$(curl -s -w "\n%{http_code}" -X POST "${OBSERVATORY_URL}/run-test" \
+            -H "Content-Type: application/json" \
+            -H "X-LiteLLM-Observatory-API-Key: ${OBSERVATORY_API_KEY}" \
+            -d "$PAYLOAD")
+          HTTP_CODE=$(echo "$RESPONSE" | tail -1)
+          BODY=$(echo "$RESPONSE" | head -n -1)
+          echo "Response ($HTTP_CODE): $BODY"
+          if [ "$HTTP_CODE" -ge 400 ]; then
+            echo "Failed to trigger test run"
+            exit 1
+          fi
+
+          # Extract request_id for polling this specific run
+          REQUEST_ID=$(echo "$BODY" | jq -r '.results.request_id')
+          if [ -z "$REQUEST_ID" ] || [ "$REQUEST_ID" = "null" ]; then
+            echo "Failed to extract request_id from response"
+            exit 1
+          fi
+          echo "Request ID: $REQUEST_ID"
+          echo "request_id=$REQUEST_ID" >> $GITHUB_OUTPUT
+
+      - name: Poll for test completion
+        id: poll
+        env:
+          OBSERVATORY_URL: ${{ secrets.OBSERVATORY_URL }}
+          OBSERVATORY_API_KEY: ${{ secrets.OBSERVATORY_API_KEY }}
+          REQUEST_ID: ${{ steps.trigger.outputs.request_id }}
+        run: |
+          TIMEOUT=900  # 15 minutes
+          INTERVAL=30
+          ELAPSED=0
+          while [ $ELAPSED -lt $TIMEOUT ]; do
+            STATUS=$(curl -s "${OBSERVATORY_URL}/run-status/${REQUEST_ID}" \
+              -H "X-LiteLLM-Observatory-API-Key: ${OBSERVATORY_API_KEY}")
+            RUN_STATUS=$(echo "$STATUS" | jq -r '.status')
+            echo "Run status (${ELAPSED}s elapsed): $RUN_STATUS"
+
+            if [ "$RUN_STATUS" = "completed" ] || [ "$RUN_STATUS" = "failed" ]; then
+              echo "Test finished with status: $RUN_STATUS"
+              echo "$STATUS" > /tmp/observatory_result.json
+              exit 0
+            fi
+
+            sleep $INTERVAL
+            ELAPSED=$((ELAPSED + INTERVAL))
+          done
+          echo "Timed out waiting for test to complete after ${TIMEOUT}s"
+          exit 1
+
+      - name: Verify test results
+        run: |
+          RESULT=$(cat /tmp/observatory_result.json)
+          echo "Full result: $RESULT"
+
+          STATUS=$(echo "$RESULT" | jq -r '.status')
+          TEST_PASSED=$(echo "$RESULT" | jq -r '.result.test_passed // false')
+          FAILURE_RATE=$(echo "$RESULT" | jq -r '.result.failure_rate // "N/A"')
+          ERROR=$(echo "$RESULT" | jq -r '.error // empty')
+
+          echo "Status: $STATUS"
+          echo "Test passed: $TEST_PASSED"
+          echo "Failure rate: $FAILURE_RATE"
+
+          if [ -n "$ERROR" ]; then
+            echo "Error: $ERROR"
+          fi
+
+          if [ "$STATUS" = "failed" ]; then
+            echo "Test run failed"
+            exit 1
+          fi
+
+          if [ "$TEST_PASSED" != "true" ]; then
+            echo "Tests did not pass (failure rate: $FAILURE_RATE)"
+            exit 1
+          fi
+
+          echo "All tests passed!"
+
+      - name: Print LiteLLM logs on failure
+        if: failure()
+        run: |
+          docker logs litellm-rc 2>/dev/null || true
+          cat /tmp/cloudflared.log 2>/dev/null || true
+
+      - name: Cleanup
+        if: always()
+        run: |
+          kill "${{ env.CLOUDFLARED_PID }}" 2>/dev/null || true
+          docker rm -f litellm-rc 2>/dev/null || true

.github/workflows/scan_duplicate_issues.yml

@@ -0,0 +1,47 @@
+name: Scan Duplicate Issues (One-Time)
+
+on:
+  workflow_dispatch:
+    inputs:
+      threshold:
+        description: "Similarity threshold (0-1)"
+        required: false
+        default: "0.85"
+      close:
+        description: "Actually close duplicates (false = dry run)"
+        required: false
+        type: boolean
+        default: false
+
+jobs:
+  scan:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      contents: read
+    steps:
+      - name: Checkout scripts
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: .github/scripts
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Scan for duplicate issues
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          INPUT_THRESHOLD: ${{ inputs.threshold }}
+          INPUT_CLOSE: ${{ inputs.close }}
+        run: |
+          CLOSE_FLAG=""
+          if [ "$INPUT_CLOSE" = "true" ]; then
+            CLOSE_FLAG="--close"
+          fi
+          python3 .github/scripts/close_duplicate_issues.py \
+            --scan \
+            --repo ${{ github.repository }} \
+            --threshold "$INPUT_THRESHOLD" \
+            $CLOSE_FLAG

.github/workflows/stale.yml

@@ -7,6 +7,7 @@ on:
 
 jobs:
   stale:
+    if: github.repository == 'BerriAI/litellm'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/stale@v8

.github/workflows/test-linting.yml

@@ -28,16 +28,18 @@ jobs:
         find . -type d -name "__pycache__" -exec rm -rf {} + || true
         find . -name "*.pyc" -delete || true
 
+    - name: Check poetry.lock is up to date
+      run: |
+        poetry check --lock || (echo "❌ poetry.lock is out of sync with pyproject.toml. Run 'poetry lock' locally and commit the result." && exit 1)
+
     - name: Install dependencies
       run: |
-        poetry lock
         poetry install --with dev
-        poetry run pip install openai==1.100.1
 
-    - name: Run Black formatting
+    - name: Check Black formatting
       run: |
         cd litellm
-        poetry run black .
+        poetry run black --check --exclude '/enterprise/' .
         cd ..
 
     - name: Debug - Check file state
@@ -74,3 +76,35 @@ jobs:
     - name: Check import safety
       run: |
         poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
+
+  secret-scan:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      contents: read
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.12'
+
+    - name: Run secret scan test
+      run: |
+        pip install pytest
+        pytest tests/litellm/test_no_hardcoded_secrets.py -v
+
+    - name: Run ggshield secret scan
+      env:
+        GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }}
+      run: |
+        if [ -n "$GITGUARDIAN_API_KEY" ]; then
+          pip install ggshield
+          ggshield secret scan repo .
+        else
+          echo "GITGUARDIAN_API_KEY not set, skipping ggshield scan"
+        fi

.github/workflows/test-litellm-matrix.yml

@@ -12,44 +12,107 @@ concurrency:
 jobs:
   test:
     runs-on: ubuntu-latest
-    timeout-minutes: 15
+    timeout-minutes: 20  # Increased from 15 to 20
     strategy:
       fail-fast: false
       matrix:
         test-group:
           # tests/test_litellm split by subdirectory (~560 files total)
-          - name: "llms"
-            path: "tests/test_litellm/llms"
-            workers: 4
+          # Vertex AI tests separated for better isolation (prevent auth/env pollution)
+          - name: "llms-vertex"
+            path: "tests/test_litellm/llms/vertex_ai"
+            workers: 1
+            reruns: 2
+          - name: "llms-other"
+            path: "tests/test_litellm/llms --ignore=tests/test_litellm/llms/vertex_ai"
+            workers: 2
+            reruns: 2
           # tests/test_litellm/proxy split by subdirectory (~180 files total)
           - name: "proxy-guardrails"
             path: "tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/management_helpers"
-            workers: 4
+            workers: 2
+            reruns: 2
           - name: "proxy-core"
             path: "tests/test_litellm/proxy/auth tests/test_litellm/proxy/client tests/test_litellm/proxy/db tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine"
-            workers: 4
+            workers: 2
+            reruns: 2
           - name: "proxy-misc"
             path: "tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py"
-            workers: 4
+            workers: 2
+            reruns: 2
           - name: "integrations"
             path: "tests/test_litellm/integrations"
-            workers: 4
+            workers: 2
+            reruns: 3  # Integration tests tend to be flakier
           - name: "core-utils"
             path: "tests/test_litellm/litellm_core_utils"
             workers: 2
-          - name: "other"
-            path: "tests/test_litellm/caching tests/test_litellm/responses tests/test_litellm/secret_managers tests/test_litellm/vector_stores tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/enterprise tests/test_litellm/experimental_mcp_client tests/test_litellm/google_genai tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/router_strategy tests/test_litellm/router_utils tests/test_litellm/types"
-            workers: 4
+            reruns: 1
+          - name: "other-1"
+            # responses (5942) + caching (1723) + types (819) ≈ 8.5k lines
+            path: "tests/test_litellm/responses tests/test_litellm/caching tests/test_litellm/types"
+            workers: 2
+            reruns: 2
+          - name: "other-2"
+            # enterprise (3062) + google_genai (2511) + router_utils (1982) ≈ 7.6k lines
+            path: "tests/test_litellm/enterprise tests/test_litellm/google_genai tests/test_litellm/router_utils"
+            workers: 2
+            reruns: 2
+          - name: "other-3"
+            # remaining dirs ≈ 8.0k lines
+            path: "tests/test_litellm/router_strategy tests/test_litellm/secret_managers tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/experimental_mcp_client tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/vector_stores"
+            workers: 2
+            reruns: 2
           - name: "root"
             path: "tests/test_litellm/test_*.py"
-            workers: 4
+            workers: 2
+            reruns: 2
           # tests/proxy_unit_tests split alphabetically (~48 files total)
-          - name: "proxy-unit-a"
-            path: "tests/proxy_unit_tests/test_[a-o]*.py"
+          - name: "proxy-unit-a1"
+            # test_[a-j]*.py: jwt (1564) + auth_checks (978) + google_gemini (478) + e2e_pod_lock (437) + rest
+            path: "tests/proxy_unit_tests/test_[a-j]*.py"
             workers: 2
-          - name: "proxy-unit-b"
-            path: "tests/proxy_unit_tests/test_[p-z]*.py"
+            reruns: 1
+          - name: "proxy-unit-a2"
+            # test_[k-o]*.py: key_generate_prisma (4346) + key_generate_dynamodb + models_fallback
+            path: "tests/proxy_unit_tests/test_[k-o]*.py"
             workers: 2
+            reruns: 1
+          - name: "proxy-unit-b1"
+            # lighter config/utility proxy tests (prisma, project, prompt, proxy_[c-r]*)
+            path: "tests/proxy_unit_tests/test_prisma*.py tests/proxy_unit_tests/test_project*.py tests/proxy_unit_tests/test_prompt*.py tests/proxy_unit_tests/test_proxy_[c-r]*.py"
+            workers: 2
+            reruns: 1
+          - name: "proxy-unit-b2"
+            # proxy_server.py alone (2750 lines) - isolated to avoid blocking smaller tests
+            path: "tests/proxy_unit_tests/test_proxy_server.py"
+            workers: 2
+            reruns: 1
+          - name: "proxy-unit-b3"
+            # proxy_server_* (618) + proxy_setting_guardrails (71) - smaller server-related tests
+            path: "tests/proxy_unit_tests/test_proxy_server_*.py tests/proxy_unit_tests/test_proxy_setting_guardrails.py"
+            workers: 2
+            reruns: 1
+          - name: "proxy-unit-b4"
+            # proxy_utils.py alone (2339 lines) - isolated to avoid blocking token counter
+            path: "tests/proxy_unit_tests/test_proxy_utils.py"
+            workers: 2
+            reruns: 1
+          - name: "proxy-unit-b5"
+            # proxy_token_counter (1279) - runs independently from utils
+            path: "tests/proxy_unit_tests/test_proxy_token_counter.py"
+            workers: 2
+            reruns: 1
+          - name: "proxy-unit-b6"
+            # test_[r-t]*.py: response_polling (1399) + search_api_logging (202) + server_root (64) + skills_db (261) + realtime_cache (62)
+            path: "tests/proxy_unit_tests/test_[r-t]*.py"
+            workers: 2
+            reruns: 1
+          - name: "proxy-unit-b7"
+            # test_[u-z]*.py: user_api_key_auth (1136) + zero_cost (590) + update_spend (305) + unit_test_* (206) + ui_path (157)
+            path: "tests/proxy_unit_tests/test_[u-z]*.py"
+            workers: 2
+            reruns: 1
 
     name: test (${{ matrix.test-group.name }})
 
@@ -79,12 +142,17 @@ jobs:
         run: |
           poetry config virtualenvs.in-project true
           poetry install --with dev,proxy-dev --extras "proxy semantic-router"
-          poetry run pip install pytest-retry==1.6.3 pytest-xdist google-genai==1.22.0 \
+          # pytest-rerunfailures and pytest-xdist are in pyproject.toml dev dependencies
+          poetry run pip install google-genai==1.22.0 \
             google-cloud-aiplatform>=1.38 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core
 
       - name: Setup litellm-enterprise
         run: |
-          cd enterprise && poetry run pip install -e . && cd ..
+          poetry run pip install --force-reinstall --no-deps -e enterprise/
+
+      - name: Generate Prisma client
+        run: |
+          poetry run prisma generate --schema litellm/proxy/schema.prisma
 
       - name: Run tests - ${{ matrix.test-group.name }}
         run: |
@@ -92,4 +160,7 @@ jobs:
             --tb=short -vv \
             --maxfail=10 \
             -n ${{ matrix.test-group.workers }} \
+            --reruns ${{ matrix.test-group.reruns }} \
+            --reruns-delay 1 \
+            --dist=loadscope \
             --durations=20

.github/workflows/test-litellm.yml

@@ -38,13 +38,11 @@ jobs:
         poetry run pip install "google-genai==1.22.0"
         poetry run pip install "google-cloud-aiplatform>=1.38"
         poetry run pip install "fastapi-offline==1.7.3"
-        poetry run pip install "python-multipart==0.0.22"
+        poetry run pip install "python-multipart>=0.0.20"
         poetry run pip install "openapi-core"
     - name: Setup litellm-enterprise as local package
       run: |
-        cd enterprise
-        poetry run pip install -e .
-        cd ..
+        poetry run pip install --force-reinstall --no-deps -e enterprise/
     - name: Run tests
       run: |
         poetry run pytest tests/test_litellm --tb=short -vv --maxfail=10 -n 4 --durations=50

.github/workflows/test-mcp.yml

@@ -40,9 +40,7 @@ jobs:
 
     - name: Setup litellm-enterprise as local package
       run: |
-        cd enterprise
-        python -m pip install -e .
-        cd ..
+        poetry run pip install --force-reinstall --no-deps -e enterprise/
 
     - name: Run MCP tests
       run: |

.github/workflows/test-proxy-e2e-azure-batches.yml

@@ -0,0 +1,90 @@
+name: Proxy E2E Azure Batches Tests
+
+on:
+  pull_request:
+    branches: [main]
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  proxy_e2e_azure_batches_tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+
+    services:
+      postgres:
+        image: postgres:15
+        env:
+          POSTGRES_USER: llmproxy
+          POSTGRES_PASSWORD: dbpassword9090
+          POSTGRES_DB: litellm
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+
+      - name: Cache Poetry dependencies
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/pypoetry
+            ~/.cache/pip
+            .venv
+          key: ${{ runner.os }}-poetry-e2e-batches-${{ hashFiles('poetry.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-poetry-e2e-batches-
+            ${{ runner.os }}-poetry-
+
+      - name: Install dependencies
+        run: |
+          poetry config virtualenvs.in-project true
+          poetry install --with dev,proxy-dev --extras "proxy"
+          poetry run pip install psycopg2-binary uvicorn fastapi httpx tenacity
+
+      - name: Setup litellm-enterprise
+        run: |
+          poetry run pip install --force-reinstall --no-deps -e enterprise/
+
+      - name: Generate Prisma client
+        run: |
+          poetry run prisma generate --schema litellm/proxy/schema.prisma
+
+      - name: Run Prisma migrations
+        env:
+          DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
+        run: |
+          cd litellm/proxy
+          poetry run prisma migrate deploy --schema schema.prisma
+          cd ../..
+
+      - name: Run Azure Batch E2E Tests
+        env:
+          DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
+          USE_LOCAL_LITELLM: "true"
+          USE_MOCK_MODELS: "true"
+          USE_STATE_TRACKER: "true"
+          LITELLM_LOG: DEBUG
+        run: |
+          poetry run pytest tests/proxy_e2e_azure_batches_tests/test_proxy_e2e_azure_batches.py \
+            -vv -s -k "test_e2e_managed_batch" \
+            --tb=short \
+            --maxfail=3 \
+            --durations=10
+

.github/workflows/test_server_root_path.yml

@@ -26,7 +26,7 @@ jobs:
         uses: docker/build-push-action@v5
         with:
           context: .
-          file: ./docker/Dockerfile.database
+          file: ./docker/Dockerfile.non_root
           tags: litellm-test:${{ github.sha }}
           load: true
           cache-from: type=gha

.gitignore

@@ -89,6 +89,7 @@ tests/test_custom_dir/*
 test.py
 
 litellm_config.yaml
+!.github/observatory/litellm_config.yaml
 .cursor
 .vscode/launch.json
 litellm/proxy/to_delete_loadtest_work/*

.pre-commit-config.yaml

@@ -14,12 +14,12 @@ repos:
         types: [python]
         files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py
         exclude: ^litellm/__init__.py$
-    # -   id: black
-    #     name: black
-    #     entry: poetry run black
-    #     language: system
-    #     types: [python]
-    #     files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py
+    -   id: black
+        name: black
+        entry: poetry run black
+        language: system
+        types: [python]
+        files: (litellm/|litellm_proxy_extras/).*\.py
 -   repo: https://github.com/pycqa/flake8
     rev: 7.0.0  # The version of flake8 to use
     hooks:

AGENTS.md

@@ -109,6 +109,8 @@ Key files:
 - `litellm/proxy/auth/` - Authentication logic
 - `litellm/proxy/management_endpoints/` - Admin API endpoints
 
+**Database (proxy)**: Use Prisma model methods (`prisma_client.db.<model>.upsert`, `.find_many`, `.find_unique`, etc.), not raw SQL (`execute_raw`/`query_raw`). See COMMON PITFALLS for details.
+
 ## MCP (MODEL CONTEXT PROTOCOL) SUPPORT
 
 LiteLLM supports MCP for agent workflows:
@@ -174,6 +176,43 @@ When opening issues or pull requests, follow these templates:
 3. **Rate Limits**: Respect provider rate limits in tests
 4. **Memory Usage**: Be mindful of memory usage in streaming scenarios
 5. **Dependencies**: Keep dependencies minimal and well-justified
+6. **UI/Backend Contract Mismatch**: When adding a new entity type to the UI, always check whether the backend endpoint accepts a single value or an array. Match the UI control accordingly (single-select vs. multi-select) to avoid silently dropping user selections
+7. **Missing Tests for New Entity Types**: When adding a new entity type (e.g., in `EntityUsage`, `UsageViewSelect`), always add corresponding tests in the existing test files and update any icon/component mocks
+8. **Raw SQL in proxy DB code**: Do not use `execute_raw` or `query_raw` for proxy database access. Use Prisma model methods (e.g. `prisma_client.db.litellm_tooltable.upsert()`, `.find_many()`, `.find_unique()`) so behavior stays consistent with the schema, the client stays mockable in tests, and you avoid the pitfalls of hand-written SQL (parameter ordering, type casting, schema drift)
+
+8. **Do not hardcode model-specific flags**: Put model-specific capability flags in `model_prices_and_context_window.json` and read them via `get_model_info` (or existing helpers like `supports_reasoning`). This prevents users from needing to upgrade LiteLLM each time a new model supports a feature.
+
+   **Example of BAD** (hardcoded model checks):
+
+   ```python
+   @staticmethod
+   def _is_effort_supported_model(model: str) -> bool:
+       """Check if the model supports the output_config.effort parameter..."""
+       model_lower = model.lower()
+       if AnthropicConfig._is_claude_4_6_model(model):
+           return True
+       return any(
+           v in model_lower for v in ("opus-4-5", "opus_4_5", "opus-4.5", "opus_4.5")
+       )
+   ```
+
+   **Example of GOOD** (config-driven or helper that reads from config):
+
+   ```python
+   if (
+       "claude-3-7-sonnet" in model
+       or AnthropicConfig._is_claude_4_6_model(model)
+       or supports_reasoning(
+           model=model,
+           custom_llm_provider=self.custom_llm_provider,
+       )
+   ):
+       ...
+   ```
+
+   Using helpers like `supports_reasoning` (which read from `model_prices_and_context_window.json` / `get_model_info`) allows future model updates to "just work" without code changes.
+
+9. **Never close HTTP/SDK clients on cache eviction**: Do not add `close()`, `aclose()`, or `create_task(close_fn())` inside `LLMClientCache._remove_key()` or any cache eviction path. Evicted clients may still be held by in-flight requests; closing them causes `RuntimeError: Cannot send a request, as the client has been closed.` in production after the cache TTL (1 hour) expires. Connection cleanup is handled at shutdown by `close_litellm_async_clients()`. See PR #22247 for the full incident history.
 
 ## HELPFUL RESOURCES
 
@@ -187,4 +226,49 @@ When opening issues or pull requests, follow these templates:
 - Check similar provider implementations
 - Ensure comprehensive test coverage
 - Update documentation appropriately
-- Consider backward compatibility impact 
+- Consider backward compatibility impact
+
+## Cursor Cloud specific instructions
+
+### Environment
+
+- Poetry is installed in `~/.local/bin`; the update script ensures it is on `PATH`.
+- Python 3.12, Node 22 are pre-installed.
+- The virtual environment lives under `~/.cache/pypoetry/virtualenvs/`.
+
+### Running the proxy server
+
+Start the proxy with a config file:
+
+```bash
+poetry run litellm --config dev_config.yaml --port 4000
+```
+
+The proxy takes ~15-20 seconds to fully start (it runs Prisma migrations on boot). Wait for `/health` to return before sending requests. Without a PostgreSQL `DATABASE_URL`, the proxy connects to a default Neon dev database embedded in the `litellm-proxy-extras` package.
+
+### Running tests
+
+See `CLAUDE.md` and the `Makefile` for standard commands. Key notes:
+
+- `psycopg-binary` must be installed (`poetry run pip install psycopg-binary`) because the pytest-postgresql plugin requires it and the lock file only includes `psycopg` (no binary).
+- `openapi-core` must be installed (`poetry run pip install openapi-core`) for the OpenAPI compliance tests in `tests/test_litellm/interactions/`.
+- The `--timeout` pytest flag is NOT available; don't pass it.
+- Unit tests: `poetry run pytest tests/test_litellm/ -x -vv -n 4`
+- Black `--check` may report pre-existing formatting issues; this does not block test runs.
+- If `poetry install` fails with "pyproject.toml changed significantly since poetry.lock was last generated", run `poetry lock` first to regenerate the lock file.
+
+### Lint
+
+```bash
+cd litellm && poetry run ruff check .
+```
+
+Ruff is the primary fast linter. For the full lint suite (including mypy, black, circular imports), run `make lint` per `CLAUDE.md`.
+
+### UI Dashboard development
+
+- The UI is at `ui/litellm-dashboard/`. Run `npm run dev` from that directory for the Next.js dev server on port 3000.
+- The proxy at port 4000 serves a **pre-built** static UI from `litellm/proxy/_experimental/out/`. After making UI code changes, you must run `npm run build` in the dashboard directory and copy the output: `cp -r ui/litellm-dashboard/out/* litellm/proxy/_experimental/out/` for the proxy to serve the updated UI.
+- SVGs used as provider logos (loaded via `<img>` tags) must NOT use `fill="currentColor"` — replace with an explicit color like `#000000` or use the `-color` variant from lobehub icons, since CSS color inheritance does not work inside `<img>` elements.
+- Provider logos live in `ui/litellm-dashboard/public/assets/logos/` (source) and `litellm/proxy/_experimental/out/assets/logos/` (pre-built). Both locations must have the file for it to work in dev and proxy-served modes.
+- UI Vitest tests: `cd ui/litellm-dashboard && npx vitest run`

CLAUDE.md

@@ -91,19 +91,74 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components:
 - Async/await patterns throughout
 - Type hints required for all public APIs
 - **Avoid imports within methods** — place all imports at the top of the file (module-level). Inline imports inside functions/methods make dependencies harder to trace and hurt readability. The only exception is avoiding circular imports where absolutely necessary.
+- **Use dict spread for immutable copies** — prefer `{**original, "key": new_value}` over `dict(obj)` + mutation. The spread produces the final dict in one step and makes intent clear.
+- **Guard at resolution time** — when resolving an optional value through a fallback chain (`a or b or ""`), raise immediately if the resolved result being empty is an error. Don't pass empty strings or sentinel values downstream for the callee to deal with.
+- **Extract complex comprehensions to named helpers** — a set/dict comprehension that calls into the DB or manager (e.g. "which of these server IDs are OAuth2?") belongs in a named helper function, not inline in the caller.
+- **FastAPI parameter declarations** — mark required query/form params with `= Query(...)` / `= Form(...)` explicitly when other params in the same handler are optional. Mixing `str` (required) with `Optional[str] = None` in the same signature causes silent 422s when the required param is missing.
 
 ### Testing Strategy
 - Unit tests in `tests/test_litellm/`
 - Integration tests for each provider in `tests/llm_translation/`
 - Proxy tests in `tests/proxy_unit_tests/`
 - Load tests in `tests/load_tests/`
+- **Always add tests when adding new entity types or features** — if the existing test file covers other entity types, add corresponding tests for the new one
+- **Keep monkeypatch stubs in sync with real signatures** — when a function gains a new optional parameter, update every `fake_*` / `stub_*` in tests that patch it to also accept that kwarg (even as `**kwargs`). Stale stubs fail with `unexpected keyword argument` and mask real bugs.
+- **Test all branches of name→ID resolution** — when adding server/resource lookup that resolves names to UUIDs, test: (1) name resolves and UUID is allowed, (2) name resolves but UUID is not allowed, (3) name does not resolve at all. The silent-fallback path is where access-control bugs hide.
+
+### UI / Backend Consistency
+- When wiring a new UI entity type to an existing backend endpoint, verify the backend API contract (single value vs. array, required vs. optional params) and ensure the UI controls match — e.g., use a single-select dropdown when the backend accepts a single value, not a multi-select
+
+### MCP OAuth / OpenAPI Transport Mapping
+- `TRANSPORT.OPENAPI` is a UI-only concept. The backend only accepts `"http"`, `"sse"`, or `"stdio"`. Always map it to `"http"` before any API call (including pre-OAuth temp-session calls).
+- FastAPI validation errors return `detail` as an array of `{loc, msg, type}` objects. Error extractors must handle: array (map `.msg`), string, nested `{error: string}`, and fallback.
+- When an MCP server already has `authorization_url` stored, skip OAuth discovery (`_discovery_metadata`) — the server URL for OpenAPI MCPs is the spec file, not the API base, and fetching it causes timeouts.
+- `client_id` should be optional in the `/authorize` endpoint — if the server has a stored `client_id` in credentials, use that. Never require callers to re-supply it.
+
+### MCP Credential Storage
+- OAuth credentials and BYOK credentials share the `litellm_mcpusercredentials` table, distinguished by a `"type"` field in the JSON payload (`"oauth2"` vs plain string).
+- When deleting OAuth credentials, check type before deleting to avoid accidentally deleting a BYOK credential for the same `(user_id, server_id)` pair.
+- Always pass the raw `expires_at` timestamp to the client — never set it to `None` for expired credentials. Let the frontend compute the "Expired" display state from the timestamp.
+- Use `RecordNotFoundError` (not bare `except Exception`) when catching "already deleted" in credential delete endpoints.
+
+### Browser Storage Safety (UI)
+- Never write LiteLLM access tokens or API keys to `localStorage` — use `sessionStorage` only. `localStorage` survives browser close and is readable by any injected script (XSS).
+- Shared utility functions (e.g. `extractErrorMessage`) belong in `src/utils/` — never define them inline in hooks or duplicate them across files.
 
 ### Database Migrations
 - Prisma handles schema migrations
 - Migration files auto-generated with `prisma migrate dev`
 - Always test migrations against both PostgreSQL and SQLite
 
+### Proxy database access
+- **Do not write raw SQL** for proxy DB operations. Use Prisma model methods instead of `execute_raw` / `query_raw`.
+- Use the generated client: `prisma_client.db.<model>` (e.g. `litellm_tooltable`, `litellm_usertable`) with `.upsert()`, `.find_many()`, `.find_unique()`, `.update()`, `.update_many()` as appropriate. This avoids schema/client drift, keeps code testable with simple mocks, and matches patterns used in spend logs and other proxy code.
+- **No N+1 queries.** Never query the DB inside a loop. Batch-fetch with `{"in": ids}` and distribute in-memory.
+- **Batch writes.** Use `create_many`/`update_many`/`delete_many` instead of individual calls (these return counts only; `update_many`/`delete_many` no-op silently on missing rows). When multiple separate writes target the same table (e.g. in `batch_()`), order by primary key to avoid deadlocks.
+- **Push work to the DB.** Filter, sort, group, and aggregate in SQL, not Python. Verify Prisma generates the expected SQL — e.g. prefer `group_by` over `find_many(distinct=...)` which does client-side processing.
+- **Bound large result sets.** Prisma materializes full results in memory. For results over ~10 MB, paginate with `take`/`skip` or `cursor`/`take`, always with an explicit `order`. Prefer cursor-based pagination (`skip` is O(n)). Don't paginate naturally small result sets.
+- **Limit fetched columns on wide tables.** Use `select` to fetch only needed fields — returns a partial object, so downstream code must not access unselected fields.
+- **Check index coverage.** For new or modified queries, check `schema.prisma` for a supporting index. Prefer extending an existing index (e.g. `@@index([a])` → `@@index([a, b])`) over adding a new one, unless it's a `@@unique`. Only add indexes for large/frequent queries.
+- **Keep schema files in sync.** Apply schema changes to all `schema.prisma` copies (`schema.prisma`, `litellm/proxy/`, `litellm-proxy-extras/`, `litellm-js/spend-logs/` for SpendLogs) with a migration under `litellm-proxy-extras/litellm_proxy_extras/migrations/`.
+
+### Setup Wizard (`litellm/setup_wizard.py`)
+- The wizard is implemented as a single `SetupWizard` class with `@staticmethod` methods — keep it that way. No module-level functions except `run_setup_wizard()` (the public entrypoint) and pure helpers (color, ANSI).
+- Use `litellm.utils.check_valid_key(model, api_key)` for credential validation — never roll a custom completion call.
+- Do not hardcode provider env-key names or model lists that already exist in the codebase. Add a `test_model` field to each provider entry to drive `check_valid_key`; set it to `None` for providers that can't be validated with a single API key (Azure, Bedrock, Ollama).
+
 ### Enterprise Features
 - Enterprise-specific code in `enterprise/` directory
 - Optional features enabled via environment variables
-- Separate licensing and authentication for enterprise features
+- Separate licensing and authentication for enterprise features
+
+### HTTP Client Cache Safety
+- **Never close HTTP/SDK clients on cache eviction.** `LLMClientCache._remove_key()` must not call `close()`/`aclose()` on evicted clients — they may still be used by in-flight requests. Doing so causes `RuntimeError: Cannot send a request, as the client has been closed.` after the 1-hour TTL expires. Cleanup happens at shutdown via `close_litellm_async_clients()`.
+
+### Troubleshooting: DB schema out of sync after proxy restart
+`litellm-proxy-extras` runs `prisma migrate deploy` on startup using **its own** bundled migration files, which may lag behind schema changes in the current worktree. Symptoms: `Unknown column`, `Invalid prisma invocation`, or missing data on new fields.
+
+**Diagnose:** Run `\d "TableName"` in psql and compare against `schema.prisma` — missing columns confirm the issue.
+
+**Fix options:**
+1. **Create a Prisma migration** (permanent) — run `prisma migrate dev --name <description>` in the worktree. The generated file will be picked up by `prisma migrate deploy` on next startup.
+2. **Apply manually for local dev** — `psql -d litellm -c "ALTER TABLE ... ADD COLUMN IF NOT EXISTS ..."` after each proxy start. Fine for dev, not for production.
+3. **Update litellm-proxy-extras** — if the package is installed from PyPI, its migration directory must include the new file. Either update the package or run the migration manually until the next release ships it.

Dockerfile

@@ -39,7 +39,7 @@ RUN pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt
 # ensure pyjwt is used, not jwt
 RUN pip uninstall jwt -y
 RUN pip uninstall PyJWT -y
-RUN pip install PyJWT==2.9.0 --no-cache-dir
+RUN pip install PyJWT==2.12.0 --no-cache-dir
 
 # Runtime stage
 FROM $LITELLM_RUNTIME_IMAGE AS runtime
@@ -49,7 +49,7 @@ USER root
 
 # Install runtime dependencies (libsndfile needed for audio processing on ARM64)
 RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \
-    npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 && \
+    npm install -g npm@latest tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
     # SECURITY FIX: npm bundles tar, glob, and brace-expansion at multiple nested
     # levels inside its dependency tree. `npm install -g <pkg>` only creates a
     # SEPARATE global package, it does NOT replace npm's internal copies.
@@ -64,7 +64,21 @@ RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile
     find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
     done && \
-    npm cache clean --force
+    find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
+    done && \
+    find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
+    done && \
+    # SECURITY FIX: patch npm's own package.json metadata so scanners see the
+    # actual installed versions instead of the stale declared dependencies.
+    find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
+        sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null && \
+    npm cache clean --force && \
+    # Remove the apk-tracked npm so its stale SBOM metadata (tar 7.5.9) is
+    # no longer visible to image scanners.  The globally installed npm@latest
+    # at /usr/local/lib/node_modules/npm/ remains fully functional.
+    { apk del --no-cache npm 2>/dev/null || true; }
 
 WORKDIR /app
 # Copy the current directory contents into the container at /app
@@ -90,14 +104,21 @@ RUN find /usr/lib -type f -path "*/tornado/test/*" -delete && \
 # npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
 # Patch every copy of tar, glob, and brace-expansion inside that tree.
 RUN GLOBAL="$(npm root -g)" && \
-    find /usr/lib -path "*/nodejs_wheel/*/node_modules/tar" -type d | while read d; do \
+    [ -n "$GLOBAL" ] || { echo "ERROR: npm root -g returned empty; aborting"; exit 1; } && \
+    find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
     done && \
-    find /usr/lib -path "*/nodejs_wheel/*/node_modules/glob" -type d | while read d; do \
+    find /usr/lib -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
     done && \
-    find /usr/lib -path "*/nodejs_wheel/*/node_modules/@isaacs/brace-expansion" -type d | while read d; do \
+    find /usr/lib -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
+    done && \
+    find /usr/lib -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
+    done && \
+    find /usr/lib -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
     done
 
 # Install semantic_router and aurelio-sdk using script

README.md

@@ -28,6 +28,9 @@
     <a href="https://www.litellm.ai/support">
         <img src="https://img.shields.io/static/v1?label=Chat%20on&message=Slack&color=black&logo=Slack&style=flat-square" alt="Slack">
     </a>
+    <a href="https://codspeed.io/BerriAI/litellm?utm_source=badge">
+        <img src="https://img.shields.io/endpoint?url=https://codspeed.io/badge.json" alt="CodSpeed"/>
+    </a>
 </h4>
 
 <img width="2688" height="1600" alt="Group 7154 (1)" src="https://github.com/user-attachments/assets/c5ee0412-6fb5-4fb6-ab5b-bafae4209ca6" />
@@ -203,7 +206,7 @@ curl -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
 {
   "mcpServers": {
     "LiteLLM": {
-      "url": "http://localhost:4000/mcp",
+      "url": "http://localhost:4000/mcp/",
       "headers": {
         "x-litellm-api-key": "Bearer sk-1234"
       }
@@ -399,7 +402,7 @@ Support for more providers. Missing a provider or LLM Platform, raise a [feature
 # Enterprise
 For companies that need better security, user management and professional support
 
-[Talk to founders](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
+[Talk to founders](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
 
 This covers:
 - ✅ **Features under the [LiteLLM Commercial License](https://docs.litellm.ai/docs/proxy/enterprise):**

ci_cd/security_scans.sh

@@ -10,13 +10,13 @@ echo "Starting security scans for LiteLLM..."
 # Function to install Trivy and required tools
 install_trivy() {
     echo "Installing Trivy and required tools..."
+    TRIVY_VERSION="0.69.3"
     sudo apt-get update
-    sudo apt-get install -y wget apt-transport-https gnupg lsb-release jq curl
-    wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | sudo apt-key add -
-    echo "deb https://aquasecurity.github.io/trivy-repo/deb $(lsb_release -sc) main" | sudo tee -a /etc/apt/sources.list.d/trivy.list
-    sudo apt-get update
-    sudo apt-get install trivy
-    echo "Trivy and required tools installed successfully"
+    sudo apt-get install -y wget jq curl bsdmainutils
+    wget -qO trivy.deb "https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.deb"
+    sudo dpkg -i trivy.deb
+    rm trivy.deb
+    echo "Trivy ${TRIVY_VERSION} installed successfully"
 }
 
 # Function to install Grype
@@ -158,6 +158,14 @@ run_grype_scans() {
         "CVE-2025-11468" # No fix available yet
         "CVE-2026-1299" # Python 3.13 email module header injection - not applicable, LiteLLM doesn't use BytesGenerator for email serialization
         "CVE-2026-0775" # npm cli incorrect permission assignment - no fix available yet, npm is only used at build/prisma-generate time
+        "GHSA-3ppc-4f35-3m26" # minimatch ReDoS via repeated wildcards - from nodejs_wheel bundled npm, not used in application runtime code
+        "GHSA-83g3-92jg-28cx" # tar arbitrary file read/write via hardlink - from nodejs_wheel bundled npm, not used in application runtime code
+        "CVE-2026-25639" # axios - full fix requires 1.x major version bump; pinned to >=0.30.2 to clear other axios CVEs, upgrade to 1.x in follow-up
+        "CVE-2026-2297" # Python 3.13 SourcelessFileLoader audit hook bypass - no fix available in base image
+        "GHSA-qffp-2rhf-9h96" # tar hardlink path traversal - from nodejs_wheel bundled npm, not used in application runtime code
+        "CVE-2026-2673" # OpenSSL 3.6.1 TLS 1.3 key exchange group negotiation issue - no fix available yet
+        "CVE-2026-3644" # Python 3.13 vulnerability - no fix available in base image
+        "CVE-2026-4224" # Python 3.13 Expat parser stack overflow in ElementDeclHandler - no fix available in base image
     )
 
     # Build JSON array of allowlisted CVE IDs for jq

cookbook/benchmark/readme.md

@@ -178,4 +178,4 @@ Benchmark Results for 'When will BerriAI IPO?':
 ```
 
 ## Support 
-**🤝 Schedule a 1-on-1 Session:** Book a [1-on-1 session](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) with Krrish and Ishaan, the founders, to discuss any issues, provide feedback, or explore how we can improve LiteLLM for you.
+**🤝 Schedule a 1-on-1 Session:** Book a [1-on-1 session](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) with Krrish and Ishaan, the founders, to discuss any issues, provide feedback, or explore how we can improve LiteLLM for you.

cookbook/gollem_go_agent_framework/README.md

@@ -0,0 +1,119 @@
+# Gollem Go Agent Framework with LiteLLM
+
+A working example showing how to use [gollem](https://github.com/fugue-labs/gollem), a production-grade Go agent framework, with LiteLLM as a proxy gateway. This lets Go developers access 100+ LLM providers through a single proxy while keeping compile-time type safety for tools and structured output.
+
+## Quick Start
+
+### 1. Start LiteLLM Proxy
+
+```bash
+# Simple start with a single model
+litellm --model gpt-4o
+
+# Or with the example config for multi-provider access
+litellm --config proxy_config.yaml
+```
+
+### 2. Run the examples
+
+```bash
+# Install Go dependencies
+go mod tidy
+
+# Basic agent
+go run ./basic
+
+# Agent with type-safe tools
+go run ./tools
+
+# Streaming responses
+go run ./streaming
+```
+
+## Configuration
+
+The included `proxy_config.yaml` sets up three providers through LiteLLM:
+
+```yaml
+model_list:
+  - model_name: gpt-4o          # OpenAI
+  - model_name: claude-sonnet    # Anthropic
+  - model_name: gemini-pro       # Google Vertex AI
+```
+
+Switch providers in Go by changing a single string — no code changes needed:
+
+```go
+model := openai.NewLiteLLM("http://localhost:4000",
+    openai.WithModel("gpt-4o"),        // OpenAI
+    // openai.WithModel("claude-sonnet"),  // Anthropic
+    // openai.WithModel("gemini-pro"),     // Google
+)
+```
+
+## Examples
+
+### `basic/` — Basic Agent
+
+Connects gollem to LiteLLM and runs a simple prompt. Demonstrates the `NewLiteLLM` constructor and basic agent creation.
+
+### `tools/` — Type-Safe Tools
+
+Shows gollem's compile-time type-safe tool framework working through LiteLLM's tool-use passthrough. The tool parameters are Go structs with JSON tags — the schema is generated automatically at compile time.
+
+### `streaming/` — Streaming Responses
+
+Real-time token streaming using Go 1.23+ range-over-function iterators, proxied through LiteLLM's SSE passthrough.
+
+## How It Works
+
+Gollem's `openai.NewLiteLLM()` constructor creates an OpenAI-compatible provider pointed at your LiteLLM proxy. Since LiteLLM speaks the OpenAI API protocol, everything works out of the box:
+
+- **Chat completions** — standard request/response
+- **Tool use** — LiteLLM passes tool definitions and calls through transparently
+- **Streaming** — Server-Sent Events proxied through LiteLLM
+- **Structured output** — JSON schema response format works with supporting models
+
+```
+Go App (gollem) → LiteLLM Proxy → OpenAI / Anthropic / Google / ...
+```
+
+## Why Use This?
+
+- **Type-safe Go**: Compile-time type checking for tools, structured output, and agent configuration — no runtime surprises
+- **Single proxy, many models**: Switch between OpenAI, Anthropic, Google, and 100+ other providers by changing a model name string
+- **Zero-dependency core**: gollem's core has no external dependencies — just stdlib
+- **Single binary deployment**: `go build` produces one binary, no pip/venv/Docker needed
+- **Cost tracking & rate limiting**: LiteLLM handles cost tracking, rate limits, and fallbacks at the proxy layer
+
+## Environment Variables
+
+```bash
+# Required for providers you want to use (set in LiteLLM config or env)
+export OPENAI_API_KEY="sk-..."
+export ANTHROPIC_API_KEY="sk-ant-..."
+
+# Optional: point to a non-default LiteLLM proxy
+export LITELLM_PROXY_URL="http://localhost:4000"
+```
+
+## Troubleshooting
+
+**Connection errors?**
+- Make sure LiteLLM is running: `litellm --model gpt-4o`
+- Check the URL is correct (default: `http://localhost:4000`)
+
+**Model not found?**
+- Verify the model name matches what's configured in LiteLLM
+- Run `curl http://localhost:4000/models` to see available models
+
+**Tool calls not working?**
+- Ensure the underlying model supports tool use (GPT-4o, Claude, Gemini)
+- Check LiteLLM logs for any provider-specific errors
+
+## Learn More
+
+- [gollem GitHub](https://github.com/fugue-labs/gollem)
+- [gollem API Reference](https://pkg.go.dev/github.com/fugue-labs/gollem/core)
+- [LiteLLM Proxy Docs](https://docs.litellm.ai/docs/simple_proxy)
+- [LiteLLM Supported Models](https://docs.litellm.ai/docs/providers)

cookbook/gollem_go_agent_framework/basic/main.go

@@ -0,0 +1,41 @@
+// Basic gollem agent connected to a LiteLLM proxy.
+//
+// Usage:
+//
+//	litellm --model gpt-4o   # start proxy in another terminal
+//	go run ./basic
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/fugue-labs/gollem/core"
+	"github.com/fugue-labs/gollem/provider/openai"
+)
+
+func main() {
+	proxyURL := "http://localhost:4000"
+	if u := os.Getenv("LITELLM_PROXY_URL"); u != "" {
+		proxyURL = u
+	}
+
+	// Connect to LiteLLM proxy. NewLiteLLM creates an OpenAI-compatible
+	// provider pointed at the given URL.
+	model := openai.NewLiteLLM(proxyURL,
+		openai.WithModel("gpt-4o"), // any model name configured in LiteLLM
+	)
+
+	// Create and run a simple agent.
+	agent := core.NewAgent[string](model,
+		core.WithSystemPrompt[string]("You are a helpful assistant. Be concise."),
+	)
+
+	result, err := agent.Run(context.Background(), "Explain quantum computing in two sentences.")
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Println(result.Output)
+}

cookbook/gollem_go_agent_framework/go.mod

@@ -0,0 +1,5 @@
+module github.com/BerriAI/litellm/cookbook/gollem_go_agent_framework
+
+go 1.25.1
+
+require github.com/fugue-labs/gollem v0.1.0

cookbook/gollem_go_agent_framework/go.sum

@@ -0,0 +1,2 @@
+github.com/fugue-labs/gollem v0.1.0 h1:QexYnvkb44QZFEljgAePqMIGZjgsbk0Y5GJ2jYYgfa8=
+github.com/fugue-labs/gollem v0.1.0/go.mod h1:htW1YO81uysSKVOkYJtxhGCFrzm+36HBFxEWuECoHKQ=

cookbook/gollem_go_agent_framework/proxy_config.yaml

@@ -0,0 +1,16 @@
+model_list:
+  - model_name: gpt-4o
+    litellm_params:
+      model: openai/gpt-4o
+      api_key: os.environ/OPENAI_API_KEY
+
+  - model_name: claude-sonnet
+    litellm_params:
+      model: anthropic/claude-sonnet-4-20250514
+      api_key: os.environ/ANTHROPIC_API_KEY
+
+  - model_name: gemini-pro
+    litellm_params:
+      model: vertex_ai/gemini-2.0-flash
+      vertex_project: my-project
+      vertex_location: us-central1

cookbook/gollem_go_agent_framework/streaming/main.go

@@ -0,0 +1,56 @@
+// Streaming responses from gollem through LiteLLM.
+//
+// Uses Go 1.23+ range-over-function iterators for real-time token
+// streaming via LiteLLM's SSE passthrough.
+//
+// Usage:
+//
+//	litellm --model gpt-4o
+//	go run ./streaming
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/fugue-labs/gollem/core"
+	"github.com/fugue-labs/gollem/provider/openai"
+)
+
+func main() {
+	proxyURL := "http://localhost:4000"
+	if u := os.Getenv("LITELLM_PROXY_URL"); u != "" {
+		proxyURL = u
+	}
+
+	model := openai.NewLiteLLM(proxyURL,
+		openai.WithModel("gpt-4o"),
+	)
+
+	agent := core.NewAgent[string](model)
+
+	// RunStream returns a streaming result that yields tokens as they arrive.
+	stream, err := agent.RunStream(context.Background(), "Write a haiku about distributed systems")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// StreamText yields text chunks in real-time.
+	// The boolean argument controls whether deltas (true) or accumulated
+	// text (false) is returned.
+	fmt.Print("Response: ")
+	for text, err := range stream.StreamText(true) {
+		if err != nil {
+			log.Fatal(err)
+		}
+		fmt.Print(text)
+	}
+	fmt.Println()
+
+	// After streaming completes, the final response is available.
+	resp := stream.Response()
+	fmt.Printf("\nTokens used: input=%d, output=%d\n",
+		resp.Usage.InputTokens, resp.Usage.OutputTokens)
+}

cookbook/gollem_go_agent_framework/tools/main.go

@@ -0,0 +1,64 @@
+// Gollem agent with type-safe tools through LiteLLM.
+//
+// The tool parameters are Go structs — gollem generates the JSON schema
+// automatically at compile time. LiteLLM passes tool definitions through
+// transparently to the underlying provider.
+//
+// Usage:
+//
+//	litellm --model gpt-4o
+//	go run ./tools
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/fugue-labs/gollem/core"
+	"github.com/fugue-labs/gollem/provider/openai"
+)
+
+// WeatherParams defines the tool's input schema via struct tags.
+// The JSON schema is generated at compile time — no runtime reflection needed.
+type WeatherParams struct {
+	City string `json:"city" description:"City name to get weather for"`
+	Unit string `json:"unit,omitempty" description:"Temperature unit: celsius or fahrenheit"`
+}
+
+func main() {
+	proxyURL := "http://localhost:4000"
+	if u := os.Getenv("LITELLM_PROXY_URL"); u != "" {
+		proxyURL = u
+	}
+
+	model := openai.NewLiteLLM(proxyURL,
+		openai.WithModel("gpt-4o"),
+	)
+
+	// Define a type-safe tool. The function signature enforces correct types.
+	weatherTool := core.FuncTool[WeatherParams](
+		"get_weather",
+		"Get current weather for a city",
+		func(ctx context.Context, p WeatherParams) (string, error) {
+			unit := p.Unit
+			if unit == "" {
+				unit = "fahrenheit"
+			}
+			// In production, call a real weather API here.
+			return fmt.Sprintf("Weather in %s: 72°F (22°C), sunny", p.City), nil
+		},
+	)
+
+	agent := core.NewAgent[string](model,
+		core.WithTools[string](weatherTool),
+		core.WithSystemPrompt[string]("You are a helpful weather assistant. Use the get_weather tool to answer weather questions."),
+	)
+
+	result, err := agent.Run(context.Background(), "What's the weather like in San Francisco and Tokyo?")
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Println(result.Output)
+}

cookbook/mock_prompt_management_server/README.md

@@ -0,0 +1,293 @@
+# Mock Prompt Management Server
+
+A reference implementation of the [LiteLLM Generic Prompt Management API](https://docs.litellm.ai/docs/adding_provider/generic_prompt_management_api).
+
+This FastAPI server demonstrates how to build a prompt management API that integrates with LiteLLM without requiring a PR to the LiteLLM repository.
+
+## Quick Start
+
+### 1. Install Dependencies
+
+```bash
+pip install fastapi uvicorn pydantic
+```
+
+### 2. Start the Server
+
+```bash
+python mock_prompt_management_server.py
+```
+
+The server will start on `http://localhost:8080`
+
+### 3. Test the Endpoint
+
+```bash
+# Get a prompt
+curl "http://localhost:8080/beta/litellm_prompt_management?prompt_id=hello-world-prompt"
+
+# Get a prompt with authentication
+curl "http://localhost:8080/beta/litellm_prompt_management?prompt_id=hello-world-prompt" \
+  -H "Authorization: Bearer test-token-12345"
+
+# List all prompts
+curl "http://localhost:8080/prompts"
+
+# Get prompt variables
+curl "http://localhost:8080/prompts/hello-world-prompt/variables"
+```
+
+## Using with LiteLLM
+
+### Configuration
+
+Create a `config.yaml` file:
+
+```yaml
+model_list:
+  - model_name: gpt-3.5-turbo
+    litellm_params:
+      model: openai/gpt-3.5-turbo
+      api_key: os.environ/OPENAI_API_KEY
+
+prompts:
+  - prompt_id: "hello-world-prompt"
+    litellm_params:
+      prompt_integration: "generic_prompt_management"
+      api_base: http://localhost:8080
+      api_key: test-token-12345
+```
+
+### Start LiteLLM Proxy
+
+```bash
+litellm --config config.yaml
+```
+
+### Make a Request
+
+```bash
+curl http://0.0.0.0:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer sk-1234" \
+  -d '{
+    "model": "gpt-3.5-turbo",
+    "prompt_id": "hello-world-prompt",
+    "prompt_variables": {
+      "domain": "data science",
+      "task": "analyzing customer behavior"
+    },
+    "messages": [
+      {"role": "user", "content": "Please help me get started"}
+    ]
+  }'
+```
+
+## Available Prompts
+
+The server includes several example prompts:
+
+| Prompt ID | Description | Variables |
+|-----------|-------------|-----------|
+| `hello-world-prompt` | Basic helpful assistant | `domain`, `task` |
+| `code-review-prompt` | Code review assistant | `years_experience`, `language`, `code` |
+| `customer-support-prompt` | Customer support agent | `company_name`, `customer_message` |
+| `data-analysis-prompt` | Data analysis expert | `analysis_type`, `dataset_name`, `data` |
+| `creative-writing-prompt` | Creative writing assistant | `genre`, `length`, `topic` |
+
+## Authentication
+
+The server supports optional Bearer token authentication. Valid tokens for testing:
+
+- `test-token-12345`
+- `dev-token-67890`
+- `prod-token-abcdef`
+
+If no `Authorization` header is provided, requests are allowed (for testing purposes).
+
+## API Endpoints
+
+### LiteLLM Spec Endpoints
+
+#### `GET /beta/litellm_prompt_management`
+
+Get a prompt by ID (required by LiteLLM).
+
+**Query Parameters:**
+- `prompt_id` (required): The prompt ID
+- `project_name` (optional): Project filter
+- `slug` (optional): Slug filter
+- `version` (optional): Version filter
+
+**Response:**
+```json
+{
+  "prompt_id": "hello-world-prompt",
+  "prompt_template": [
+    {
+      "role": "system",
+      "content": "You are a helpful assistant specialized in {domain}."
+    },
+    {
+      "role": "user",
+      "content": "Help me with: {task}"
+    }
+  ],
+  "prompt_template_model": "gpt-4",
+  "prompt_template_optional_params": {
+    "temperature": 0.7,
+    "max_tokens": 500
+  }
+}
+```
+
+### Convenience Endpoints (Not in LiteLLM Spec)
+
+#### `GET /health`
+
+Health check endpoint.
+
+#### `GET /prompts`
+
+List all available prompts.
+
+#### `GET /prompts/{prompt_id}/variables`
+
+Get all variables used in a prompt template.
+
+#### `POST /prompts`
+
+Create a new prompt (in-memory only, for testing).
+
+## Example: Full Integration Test
+
+### 1. Start the Mock Server
+
+```bash
+python mock_prompt_management_server.py
+```
+
+### 2. Test with Python
+
+```python
+from litellm import completion
+
+# The completion will:
+# 1. Fetch the prompt from your API
+# 2. Replace {domain} with "machine learning"
+# 3. Replace {task} with "building a recommendation system"
+# 4. Merge with your messages
+# 5. Use the model and params from the prompt
+
+response = completion(
+    model="gpt-4",
+    prompt_id="hello-world-prompt",
+    prompt_variables={
+        "domain": "machine learning",
+        "task": "building a recommendation system"
+    },
+    messages=[
+        {"role": "user", "content": "I have user behavior data from the past year."}
+    ],
+    # Configure the generic prompt manager
+    generic_prompt_config={
+        "api_base": "http://localhost:8080",
+        "api_key": "test-token-12345",
+    }
+)
+
+print(response.choices[0].message.content)
+```
+
+## Customization
+
+### Adding New Prompts
+
+Edit the `PROMPTS_DB` dictionary in `mock_prompt_management_server.py`:
+
+```python
+PROMPTS_DB = {
+    "my-custom-prompt": {
+        "prompt_id": "my-custom-prompt",
+        "prompt_template": [
+            {
+                "role": "system",
+                "content": "You are a {role}."
+            },
+            {
+                "role": "user",
+                "content": "{user_input}"
+            }
+        ],
+        "prompt_template_model": "gpt-4",
+        "prompt_template_optional_params": {
+            "temperature": 0.8,
+            "max_tokens": 1000
+        }
+    }
+}
+```
+
+### Using a Database
+
+Replace the `PROMPTS_DB` dictionary with database queries:
+
+```python
+@app.get("/beta/litellm_prompt_management")
+async def get_prompt(prompt_id: str):
+    # Fetch from database
+    prompt = await db.prompts.find_one({"prompt_id": prompt_id})
+    
+    if not prompt:
+        raise HTTPException(status_code=404, detail="Prompt not found")
+    
+    return PromptResponse(**prompt)
+```
+
+### Adding Access Control
+
+Use the custom query parameters for access control:
+
+```python
+@app.get("/beta/litellm_prompt_management")
+async def get_prompt(
+    prompt_id: str,
+    project_name: Optional[str] = None,
+    user_id: Optional[str] = None,
+    authorization: Optional[str] = Header(None)
+):
+    token = verify_api_key(authorization)
+    
+    # Check if user has access to this project
+    if not has_project_access(token, project_name):
+        raise HTTPException(status_code=403, detail="Access denied")
+    
+    # Fetch and return prompt
+    ...
+```
+
+## Production Considerations
+
+Before deploying to production:
+
+1. **Use a real database** instead of in-memory storage
+2. **Implement proper authentication** with JWT tokens or API keys
+3. **Add rate limiting** to prevent abuse
+4. **Use HTTPS** for encrypted communication
+5. **Add logging and monitoring** for observability
+6. **Implement caching** for frequently accessed prompts
+7. **Add versioning** for prompt management
+8. **Implement access control** based on teams/users
+9. **Add input validation** for all parameters
+10. **Use environment variables** for configuration
+
+## Related Documentation
+
+- [Generic Prompt Management API Documentation](https://docs.litellm.ai/docs/adding_provider/generic_prompt_management_api)
+- [LiteLLM Prompt Management](https://docs.litellm.ai/docs/proxy/prompt_management)
+- [Generic Guardrail API](https://docs.litellm.ai/docs/adding_provider/generic_guardrail_api)
+
+## Questions?
+
+This is a reference implementation for the LiteLLM Generic Prompt Management API. For questions or issues, please open an issue on the [LiteLLM GitHub repository](https://github.com/BerriAI/litellm).
+

cookbook/mock_prompt_management_server/mock_prompt_management_server.py

@@ -0,0 +1,390 @@
+#!/usr/bin/env python3
+"""
+Mock Prompt Management API Server
+
+This is a FastAPI server that implements the LiteLLM Generic Prompt Management API
+for testing and demonstration purposes.
+
+Usage:
+    python mock_prompt_management_server.py
+
+The server will start on http://localhost:8080
+
+Test the endpoint:
+    curl "http://localhost:8080/beta/litellm_prompt_management?prompt_id=hello-world-prompt"
+"""
+
+import os
+import json
+from typing import Any, Dict, List, Optional
+
+from fastapi import FastAPI, HTTPException, Header, Query, status
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel, Field
+
+# ============================================================================
+# Response Models
+# ============================================================================
+
+
+class MessageContent(BaseModel):
+    """A single message in the prompt template"""
+
+    role: str = Field(..., description="Message role (system, user, assistant)")
+    content: str = Field(
+        ..., description="Message content with optional {variable} placeholders"
+    )
+
+
+class PromptResponse(BaseModel):
+    """Response format for the prompt management API"""
+
+    prompt_id: str = Field(..., description="The ID of the prompt")
+    prompt_template: List[MessageContent] = Field(
+        ..., description="Array of messages in OpenAI format"
+    )
+    prompt_template_model: Optional[str] = Field(
+        None, description="Optional model to use for this prompt"
+    )
+    prompt_template_optional_params: Optional[Dict[str, Any]] = Field(
+        None, description="Optional parameters like temperature, max_tokens, etc."
+    )
+
+
+# ============================================================================
+# Mock Prompt Database
+# ============================================================================
+
+PROMPTS_DB = {
+    "hello-world-prompt": {
+        "prompt_id": "hello-world-prompt",
+        "prompt_template": [
+            {
+                "role": "system",
+                "content": "You are a helpful assistant specialized in {domain}.",
+            },
+            {"role": "user", "content": "Help me with: {task}"},
+        ],
+        "prompt_template_model": "gpt-4",
+        "prompt_template_optional_params": {"temperature": 0.7, "max_tokens": 500},
+    },
+    "code-review-prompt": {
+        "prompt_id": "code-review-prompt",
+        "prompt_template": [
+            {
+                "role": "system",
+                "content": "You are an expert code reviewer with {years_experience} years of experience in {language}.",
+            },
+            {
+                "role": "user",
+                "content": "Please review the following code for bugs, security issues, and best practices:\n\n{code}",
+            },
+        ],
+        "prompt_template_model": "gpt-4-turbo",
+        "prompt_template_optional_params": {
+            "temperature": 0.3,
+            "max_tokens": 1500,
+        },
+    },
+    "customer-support-prompt": {
+        "prompt_id": "customer-support-prompt",
+        "prompt_template": [
+            {
+                "role": "system",
+                "content": "You are a friendly customer support agent for {company_name}. Always be professional, empathetic, and solution-oriented.",
+            },
+            {
+                "role": "user",
+                "content": "Customer inquiry: {customer_message}",
+            },
+        ],
+        "prompt_template_model": "gpt-3.5-turbo",
+        "prompt_template_optional_params": {
+            "temperature": 0.8,
+            "max_tokens": 800,
+            "top_p": 0.9,
+        },
+    },
+    "data-analysis-prompt": {
+        "prompt_id": "data-analysis-prompt",
+        "prompt_template": [
+            {
+                "role": "system",
+                "content": "You are a data scientist expert in {analysis_type} analysis.",
+            },
+            {
+                "role": "user",
+                "content": "Analyze the following data and provide insights:\n\nDataset: {dataset_name}\nData: {data}",
+            },
+        ],
+        "prompt_template_model": "gpt-4",
+        "prompt_template_optional_params": {
+            "temperature": 0.5,
+            "max_tokens": 2000,
+        },
+    },
+    "creative-writing-prompt": {
+        "prompt_id": "creative-writing-prompt",
+        "prompt_template": [
+            {
+                "role": "system",
+                "content": "You are a creative writer specializing in {genre} fiction.",
+            },
+            {
+                "role": "user",
+                "content": "Write a {length} story about: {topic}",
+            },
+        ],
+        "prompt_template_model": "gpt-4",
+        "prompt_template_optional_params": {
+            "temperature": 0.9,
+            "max_tokens": 3000,
+            "top_p": 0.95,
+        },
+    },
+}
+
+# Valid API tokens for authentication (in production, use a secure token store)
+VALID_API_TOKENS = {
+    "test-token-12345",
+    "dev-token-67890",
+    "prod-token-abcdef",
+}
+
+# ============================================================================
+# FastAPI App
+# ============================================================================
+
+app = FastAPI(
+    title="Mock Prompt Management API",
+    description="A mock server implementing the LiteLLM Generic Prompt Management API",
+    version="1.0.0",
+)
+
+
+def verify_api_key(authorization: Optional[str] = Header(None)) -> bool:
+    """
+    Verify the API key from the Authorization header.
+
+    Args:
+        authorization: Authorization header (Bearer token)
+
+    Returns:
+        True if valid, raises HTTPException if invalid
+    """
+    if authorization is None:
+        # Allow requests without authentication for testing
+        return True
+
+    # Extract token from "Bearer <token>"
+    if not authorization.startswith("Bearer "):
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Invalid authorization header format. Expected 'Bearer <token>'",
+        )
+
+    token = authorization.replace("Bearer ", "").strip()
+
+    if token not in VALID_API_TOKENS:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Invalid API key",
+        )
+
+    return True
+
+
+@app.get("/beta/litellm_prompt_management", response_model=PromptResponse)
+async def get_prompt(
+    prompt_id: str = Query(..., description="The ID of the prompt to fetch"),
+    project_name: Optional[str] = Query(
+        None, description="Optional project name filter"
+    ),
+    slug: Optional[str] = Query(None, description="Optional slug filter"),
+    version: Optional[str] = Query(None, description="Optional version filter"),
+    authorization: Optional[str] = Header(None),
+) -> PromptResponse:
+    """
+    Get a prompt by ID with optional filtering.
+
+    This endpoint implements the LiteLLM Generic Prompt Management API specification.
+
+    Args:
+        prompt_id: The ID of the prompt to fetch
+        project_name: Optional project name for filtering
+        slug: Optional slug for filtering
+        version: Optional version for filtering
+        authorization: Optional Bearer token for authentication
+
+    Returns:
+        PromptResponse with the prompt template and configuration
+
+    Raises:
+        HTTPException: 401 if authentication fails, 404 if prompt not found
+    """
+    # Verify authentication
+    verify_api_key(authorization)
+
+    # Log the request parameters (useful for debugging)
+    print(f"Fetching prompt: {prompt_id}")
+    if project_name:
+        print(f"  Project: {project_name}")
+    if slug:
+        print(f"  Slug: {slug}")
+    if version:
+        print(f"  Version: {version}")
+
+    # Check if prompt exists
+    if prompt_id not in PROMPTS_DB:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Prompt '{prompt_id}' not found. Available prompts: {list(PROMPTS_DB.keys())}",
+        )
+
+    # Get the prompt from the database
+    prompt_data = PROMPTS_DB[prompt_id]
+
+    # Optional: Apply filtering based on project_name, slug, or version
+    # In a real implementation, you might use these to filter prompts by access control
+    # or to fetch specific versions from your database
+
+    return PromptResponse(**prompt_data)
+
+
+@app.get("/health")
+async def health_check():
+    """Health check endpoint"""
+    return {
+        "status": "healthy",
+        "service": "mock-prompt-management-api",
+        "version": "1.0.0",
+    }
+
+
+@app.get("/prompts")
+async def list_prompts(authorization: Optional[str] = Header(None)):
+    """
+    List all available prompts.
+
+    This is a convenience endpoint (not part of the LiteLLM spec) for
+    discovering available prompts.
+    """
+    # Verify authentication
+    verify_api_key(authorization)
+
+    prompts_list = [
+        {
+            "prompt_id": pid,
+            "model": p.get("prompt_template_model"),
+            "has_variables": any(
+                "{" in msg.get("content", "") for msg in p.get("prompt_template", [])
+            ),
+        }
+        for pid, p in PROMPTS_DB.items()
+    ]
+
+    return {"prompts": prompts_list, "total": len(prompts_list)}
+
+
+@app.get("/prompts/{prompt_id}/variables")
+async def get_prompt_variables(
+    prompt_id: str, authorization: Optional[str] = Header(None)
+):
+    """
+    Get all variables in a prompt template.
+
+    This is a convenience endpoint (not part of the LiteLLM spec) for
+    discovering what variables a prompt expects.
+    """
+    # Verify authentication
+    verify_api_key(authorization)
+
+    if prompt_id not in PROMPTS_DB:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Prompt '{prompt_id}' not found",
+        )
+
+    prompt_data = PROMPTS_DB[prompt_id]
+    variables = set()
+
+    # Extract variables from the prompt template
+    import re
+
+    for message in prompt_data["prompt_template"]:
+        content = message.get("content", "")
+        # Find all {variable} patterns
+        found_vars = re.findall(r"\{(\w+)\}", content)
+        variables.update(found_vars)
+
+    return {
+        "prompt_id": prompt_id,
+        "variables": sorted(list(variables)),
+        "example_usage": {
+            "prompt_id": prompt_id,
+            "prompt_variables": {var: f"<{var}_value>" for var in variables},
+        },
+    }
+
+
+@app.post("/prompts")
+async def create_prompt(
+    prompt: PromptResponse, authorization: Optional[str] = Header(None)
+):
+    """
+    Create a new prompt (convenience endpoint for testing).
+
+    This is NOT part of the LiteLLM spec - it's just for testing purposes.
+    """
+    # Verify authentication
+    verify_api_key(authorization)
+
+    if prompt.prompt_id in PROMPTS_DB:
+        raise HTTPException(
+            status_code=status.HTTP_409_CONFLICT,
+            detail=f"Prompt '{prompt.prompt_id}' already exists",
+        )
+
+    PROMPTS_DB[prompt.prompt_id] = prompt.dict()
+
+    return {
+        "status": "created",
+        "prompt_id": prompt.prompt_id,
+        "message": "Prompt created successfully (in-memory only)",
+    }
+
+
+# ============================================================================
+# Main
+# ============================================================================
+
+if __name__ == "__main__":
+    import uvicorn
+
+    print("=" * 70)
+    print("Mock Prompt Management API Server")
+    print("=" * 70)
+    print(f"\nStarting server on http://localhost:8080")
+    print(f"\nAvailable prompts: {len(PROMPTS_DB)}")
+    for prompt_id in PROMPTS_DB.keys():
+        print(f"  - {prompt_id}")
+    print(f"\nValid API tokens: {len(VALID_API_TOKENS)}")
+    print("  - test-token-12345")
+    print("  - dev-token-67890")
+    print("  - prod-token-abcdef")
+    print("\nEndpoints:")
+    print("  GET  /beta/litellm_prompt_management?prompt_id=<id>  (LiteLLM spec)")
+    print("  GET  /health                                          (health check)")
+    print("  GET  /prompts                                         (list all prompts)")
+    print(
+        "  GET  /prompts/{id}/variables                          (get prompt variables)"
+    )
+    print("  POST /prompts                                         (create prompt)")
+    print("\nExample usage:")
+    print(
+        '  curl "http://localhost:8080/beta/litellm_prompt_management?prompt_id=hello-world-prompt"'
+    )
+    print("\nPress CTRL+C to stop the server")
+    print("=" * 70)
+
+    uvicorn.run(app, host="0.0.0.0", port=8080, log_level="info")

deploy/charts/litellm-helm/README.md

@@ -36,6 +36,10 @@ If `db.useStackgresOperator` is used (not yet implemented):
 | `serviceAccount.create`     | Whether or not to create a Kubernetes Service Account for this deployment. The default is `false` because LiteLLM has no need to access the Kubernetes API.                                                                                   | `false`                   |
 | `service.type`              | Kubernetes Service type (e.g. `LoadBalancer`, `ClusterIP`, etc.)                                                                                                                                                                              | `ClusterIP`               |
 | `service.port`              | TCP port that the Kubernetes Service will listen on. Also the TCP port within the Pod that the proxy will listen on.                                                                                                                          | `4000`                    |
+| `livenessProbe.*`           | Liveness probe settings for the LiteLLM container (`path`, `periodSeconds`, `timeoutSeconds`, thresholds, and initial delay).                                                                                                                  | See `values.yaml`         |
+| `readinessProbe.*`          | Readiness probe settings for the LiteLLM container (`path`, `periodSeconds`, `timeoutSeconds`, thresholds, and initial delay).                                                                                                                 | See `values.yaml`         |
+| `startupProbe.*`            | Startup probe settings for the LiteLLM container (`path`, `periodSeconds`, `timeoutSeconds`, thresholds, and initial delay).                                                                                                                   | See `values.yaml`         |
+| `resources.*`               | CPU/memory requests and limits for the LiteLLM container.                                                                                                                                                                                       | `{}`                      |
 | `service.loadBalancerClass` | Optional LoadBalancer implementation class (only used when `service.type` is `LoadBalancer`)                                                                                                                                                  | `""`                      |
 | `ingress.labels`            | Additional labels for the Ingress resource                                                                                                                                                                                                    | `{}`                      |
 | `ingress.*`                 | See [values.yaml](./values.yaml) for example settings                                                                                                                                                                                         | N/A                       |

deploy/charts/litellm-helm/templates/_helpers.tpl

@@ -61,6 +61,20 @@ Create the name of the service account to use
 {{- end }}
 {{- end }}
 
+{{/*
+Create the service account name used by migration jobs.
+When Helm hooks are enabled, pre-install/pre-upgrade hooks run before normal resources.
+If this chart is creating the ServiceAccount, it is not yet available for the hook job,
+so fall back to "default" (or an explicit override) to avoid a cyclic dependency.
+*/}}
+{{- define "litellm.migrationServiceAccountName" -}}
+{{- if and .Values.migrationJob.hooks.helm.enabled .Values.serviceAccount.create }}
+{{- default "default" .Values.migrationJob.serviceAccountName }}
+{{- else }}
+{{- include "litellm.serviceAccountName" . }}
+{{- end }}
+{{- end }}
+
 {{/*
 Get redis service name
 */}}

deploy/charts/litellm-helm/templates/configmap-litellm.yaml

@@ -6,4 +6,4 @@ metadata:
 data:
   config.yaml: |
 {{ .Values.proxy_config | toYaml | indent 6 }}
-{{- end }}
+{{- end }}

deploy/charts/litellm-helm/templates/deployment.yaml

@@ -13,9 +13,16 @@ spec:
   {{- if and (not .Values.keda.enabled) (not .Values.autoscaling.enabled) }}
   replicas: {{ .Values.replicaCount }}
   {{- end }}
+  {{- with .Values.strategy }}
+  strategy:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
   selector:
     matchLabels:
       {{- include "litellm.selectorLabels" . | nindent 6 }}
+  {{- if .Values.deploymentMinReadySeconds }}
+  minReadySeconds: {{ .Values.deploymentMinReadySeconds }}
+  {{- end }}
   template:
     metadata:
       annotations:
@@ -158,18 +165,31 @@ spec:
             {{- end }}
           livenessProbe:
             httpGet:
-              path: /health/liveliness
+              path: {{ .Values.livenessProbe.path | quote }}
               port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
+            initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
+            periodSeconds: {{ .Values.livenessProbe.periodSeconds }}
+            timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }}
+            successThreshold: {{ .Values.livenessProbe.successThreshold }}
+            failureThreshold: {{ .Values.livenessProbe.failureThreshold }}
           readinessProbe:
             httpGet:
-              path: /health/readiness
+              path: {{ .Values.readinessProbe.path | quote }}
               port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
+            initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
+            periodSeconds: {{ .Values.readinessProbe.periodSeconds }}
+            timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }}
+            successThreshold: {{ .Values.readinessProbe.successThreshold }}
+            failureThreshold: {{ .Values.readinessProbe.failureThreshold }}
           startupProbe:
             httpGet:
-              path: /health/readiness
+              path: {{ .Values.startupProbe.path | quote }}
               port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
-            failureThreshold: 30
-            periodSeconds: 10
+            initialDelaySeconds: {{ .Values.startupProbe.initialDelaySeconds }}
+            periodSeconds: {{ .Values.startupProbe.periodSeconds }}
+            timeoutSeconds: {{ .Values.startupProbe.timeoutSeconds }}
+            successThreshold: {{ .Values.startupProbe.successThreshold }}
+            failureThreshold: {{ .Values.startupProbe.failureThreshold }}
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
           volumeMounts:
@@ -235,4 +255,4 @@ spec:
       {{- if .Values.topologySpreadConstraints }}
       topologySpreadConstraints:
         {{- toYaml .Values.topologySpreadConstraints | nindent 8 }}
-      {{- end }}
+      {{- end }}

deploy/charts/litellm-helm/templates/migrations-job.yaml

@@ -34,7 +34,7 @@ spec:
       imagePullSecrets:
         {{- toYaml . | nindent 8 }}
       {{- end }}
-      serviceAccountName: {{ include "litellm.serviceAccountName" . }}
+      serviceAccountName: {{ include "litellm.migrationServiceAccountName" . }}
       {{- with .Values.migrationJob.extraInitContainers }}
       initContainers:
         {{- toYaml . | nindent 8 }}

deploy/charts/litellm-helm/tests/deployment_tests.yaml

@@ -159,4 +159,163 @@ tests:
           value: -c
       - equal:
           path: spec.template.spec.containers[0].lifecycle.preStop.exec.command[2]
-          value: echo "Container stopping"
+          value: echo "Container stopping"
+  - it: should render background health check settings from proxy_config.general_settings
+    template: configmap-litellm.yaml
+    set:
+      proxy_config.general_settings.background_health_checks: true
+      proxy_config.general_settings.health_check_interval: 240
+      proxy_config.general_settings.health_check_concurrency: 16
+      proxy_config.general_settings.health_check_details: false
+    asserts:
+      - matchRegex:
+          path: data["config.yaml"]
+          pattern: '(?m)^\s*background_health_checks:\s*true$'
+      - matchRegex:
+          path: data["config.yaml"]
+          pattern: '(?m)^\s*health_check_interval:\s*240$'
+      - matchRegex:
+          path: data["config.yaml"]
+          pattern: '(?m)^\s*health_check_concurrency:\s*16$'
+      - matchRegex:
+          path: data["config.yaml"]
+          pattern: '(?m)^\s*health_check_details:\s*false$'
+  - it: should allow overriding liveness, readiness, and startup probes
+    template: deployment.yaml
+    set:
+      livenessProbe:
+        path: /custom/livez
+        initialDelaySeconds: 5
+        periodSeconds: 15
+        timeoutSeconds: 5
+        successThreshold: 1
+        failureThreshold: 5
+      readinessProbe:
+        path: /custom/readyz
+        initialDelaySeconds: 10
+        periodSeconds: 20
+        timeoutSeconds: 6
+        successThreshold: 1
+        failureThreshold: 6
+      startupProbe:
+        path: /custom/startupz
+        initialDelaySeconds: 15
+        periodSeconds: 25
+        timeoutSeconds: 7
+        successThreshold: 1
+        failureThreshold: 40
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.httpGet.path
+          value: /custom/livez
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.timeoutSeconds
+          value: 5
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.httpGet.path
+          value: /custom/readyz
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.timeoutSeconds
+          value: 6
+      - equal:
+          path: spec.template.spec.containers[0].startupProbe.httpGet.path
+          value: /custom/startupz
+      - equal:
+          path: spec.template.spec.containers[0].startupProbe.failureThreshold
+          value: 40
+  - it: should render container resources from values
+    template: deployment.yaml
+    set:
+      resources:
+        limits:
+          cpu: 500m
+          memory: 2Gi
+        requests:
+          cpu: 250m
+          memory: 1Gi
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].resources.limits.cpu
+          value: 500m
+      - equal:
+          path: spec.template.spec.containers[0].resources.limits.memory
+          value: 2Gi
+      - equal:
+          path: spec.template.spec.containers[0].resources.requests.cpu
+          value: 250m
+      - equal:
+          path: spec.template.spec.containers[0].resources.requests.memory
+          value: 1Gi
+  - it: should keep default probes and empty resources unchanged
+    template: deployment.yaml
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.httpGet.path
+          value: /health/liveliness
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.initialDelaySeconds
+          value: 0
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.periodSeconds
+          value: 10
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.timeoutSeconds
+          value: 1
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.successThreshold
+          value: 1
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.failureThreshold
+          value: 3
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.httpGet.path
+          value: /health/readiness
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.initialDelaySeconds
+          value: 0
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.periodSeconds
+          value: 10
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.timeoutSeconds
+          value: 1
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.successThreshold
+          value: 1
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.failureThreshold
+          value: 3
+      - equal:
+          path: spec.template.spec.containers[0].startupProbe.httpGet.path
+          value: /health/readiness
+      - equal:
+          path: spec.template.spec.containers[0].startupProbe.initialDelaySeconds
+          value: 0
+      - equal:
+          path: spec.template.spec.containers[0].startupProbe.periodSeconds
+          value: 10
+      - equal:
+          path: spec.template.spec.containers[0].startupProbe.timeoutSeconds
+          value: 1
+      - equal:
+          path: spec.template.spec.containers[0].startupProbe.successThreshold
+          value: 1
+      - equal:
+          path: spec.template.spec.containers[0].startupProbe.failureThreshold
+          value: 30
+      - equal:
+          path: spec.template.spec.containers[0].resources
+          value: {}
+  - it: should be able to set minReadySeconds
+    template: deployment.yaml
+    set:
+      deploymentMinReadySeconds: 5
+    asserts:
+      - equal:
+          path: spec.minReadySeconds
+          value: 5
+  - it: should have minReadySeconds absent when deploymentMinReadySeconds is not set
+    template: deployment.yaml
+    asserts:
+      - notExists:
+          path: spec.minReadySeconds

deploy/charts/litellm-helm/tests/migrations-job_tests.yaml

@@ -124,4 +124,67 @@ tests:
       - notContains:
           path: spec.template.spec.containers[0].env
           content:
-            name: DATABASE_URL
+            name: DATABASE_URL
+
+  - it: should use default service account for helm hooks when serviceAccount.create is true
+    template: migrations-job.yaml
+    set:
+      migrationJob:
+        enabled: true
+        hooks:
+          helm:
+            enabled: true
+      serviceAccount:
+        create: true
+    asserts:
+      - equal:
+          path: spec.template.spec.serviceAccountName
+          value: default
+
+  - it: should use migrationJob.serviceAccountName override for helm hooks when serviceAccount.create is true
+    template: migrations-job.yaml
+    set:
+      migrationJob:
+        enabled: true
+        serviceAccountName: migration-sa
+        hooks:
+          helm:
+            enabled: true
+      serviceAccount:
+        create: true
+    asserts:
+      - equal:
+          path: spec.template.spec.serviceAccountName
+          value: migration-sa
+
+  - it: should use chart service account when helm hooks are disabled
+    template: migrations-job.yaml
+    set:
+      migrationJob:
+        enabled: true
+        hooks:
+          helm:
+            enabled: false
+      serviceAccount:
+        create: true
+        name: my-custom-sa
+    asserts:
+      - equal:
+          path: spec.template.spec.serviceAccountName
+          value: my-custom-sa
+
+  - it: should use pre-existing service account when helm hooks are enabled but serviceAccount.create is false
+    template: migrations-job.yaml
+    set:
+      migrationJob:
+        enabled: true
+        hooks:
+          helm:
+            enabled: true
+      serviceAccount:
+        create: false
+        name: pre-existing-sa
+    asserts:
+      - equal:
+          path: spec.template.spec.serviceAccountName
+          value: pre-existing-sa

deploy/charts/litellm-helm/values.yaml

@@ -31,10 +31,20 @@ serviceAccount:
 # annotations for litellm deployment
 deploymentAnnotations: {}
 deploymentLabels: {}
+deploymentMinReadySeconds: 0
+
 # annotations for litellm pods
 podAnnotations: {}
 podLabels: {}
 
+# -- Deployment strategy configuration
+# Example:
+#   type: RollingUpdate
+#   rollingUpdate:
+#     maxUnavailable: 0
+#     maxSurge: 1
+strategy: {}
+
 terminationGracePeriodSeconds: 90
 topologySpreadConstraints:
   []
@@ -84,6 +94,31 @@ service:
 separateHealthApp: false
 separateHealthPort: 8081
 
+# Probe tuning for proxy container
+livenessProbe:
+  path: /health/liveliness
+  initialDelaySeconds: 0
+  periodSeconds: 10
+  timeoutSeconds: 1
+  successThreshold: 1
+  failureThreshold: 3
+
+readinessProbe:
+  path: /health/readiness
+  initialDelaySeconds: 0
+  periodSeconds: 10
+  timeoutSeconds: 1
+  successThreshold: 1
+  failureThreshold: 3
+
+startupProbe:
+  path: /health/readiness
+  initialDelaySeconds: 0
+  periodSeconds: 10
+  timeoutSeconds: 1
+  successThreshold: 1
+  failureThreshold: 30
+
 ingress:
   enabled: false
   className: "nginx"
@@ -274,6 +309,10 @@ migrationJob:
   retries: 3 # Number of retries for the Job in case of failure
   backoffLimit: 4 # Backoff limit for Job restarts
   disableSchemaUpdate: false # Skip schema migrations for specific environments. When True, the job will exit with code 0.
+  # Optional service account for the migration job.
+  # Only used when migrationJob.hooks.helm.enabled=true and serviceAccount.create=true.
+  # In that case, pre-install/pre-upgrade hooks run before normal resources, so this defaults to "default".
+  serviceAccountName: ""
   annotations: {}
   ttlSecondsAfterFinished: 120
   resources: {}

dev_config.yaml

@@ -0,0 +1,13 @@
+model_list:
+  - model_name: fake-openai-endpoint
+    litellm_params:
+      model: openai/fake-model
+      api_key: fake-key
+      api_base: https://exampleopenaiendpoint-production.up.railway.app/
+
+general_settings:
+  master_key: sk-1234
+
+litellm_settings:
+  drop_params: True
+  telemetry: False

docker/Dockerfile.custom_ui

@@ -5,8 +5,21 @@ FROM ghcr.io/berriai/litellm:litellm_fwd_server_root_path-dev
 WORKDIR /app
 
 # Install Node.js and npm (adjust version as needed)
-RUN apt-get update && apt-get install -y nodejs npm && \
-    npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 && \
+RUN apt-get update && apt-get upgrade -y \
+        libxml2 \
+        libexpat1 \
+        openssl \
+        libssl3 \
+        git \
+        libkrb5-3 \
+        libglib2.0-0 \
+        wget \
+        libaom3 \
+        libxslt1.1 \
+        libgnutls30 \
+        libc6 && \
+    apt-get install -y nodejs npm && \
+    npm install -g npm@latest tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
     GLOBAL="$(npm root -g)" && \
     find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
@@ -17,7 +30,16 @@ RUN apt-get update && apt-get install -y nodejs npm && \
     find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
     done && \
-    npm cache clean --force
+    find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
+    done && \
+    find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
+    done && \
+    find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
+        sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null && \
+    npm cache clean --force && \
+    apt-get purge -y npm
 
 # Copy the UI source into the container
 COPY ./ui/litellm-dashboard /app/ui/litellm-dashboard

docker/Dockerfile.database

@@ -50,7 +50,7 @@ USER root
 
 # Install runtime dependencies
 RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \
-    npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 && \
+    npm install -g npm@latest tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
     GLOBAL="$(npm root -g)" && \
     find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
@@ -61,7 +61,16 @@ RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile
     find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
     done && \
-    npm cache clean --force
+    find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
+    done && \
+    find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
+    done && \
+    find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
+        sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null && \
+    npm cache clean --force && \
+    { apk del --no-cache npm 2>/dev/null || true; }
 
 WORKDIR /app
 # Copy the current directory contents into the container at /app
@@ -79,14 +88,21 @@ RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl
 # npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
 # Patch every copy of tar, glob, and brace-expansion inside that tree.
 RUN GLOBAL="$(npm root -g)" && \
-    find /usr/lib -path "*/nodejs_wheel/*/node_modules/tar" -type d | while read d; do \
+    [ -n "$GLOBAL" ] || { echo "ERROR: npm root -g returned empty; aborting"; exit 1; } && \
+    find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
     done && \
-    find /usr/lib -path "*/nodejs_wheel/*/node_modules/glob" -type d | while read d; do \
+    find /usr/lib -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
     done && \
-    find /usr/lib -path "*/nodejs_wheel/*/node_modules/@isaacs/brace-expansion" -type d | while read d; do \
+    find /usr/lib -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
+    done && \
+    find /usr/lib -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
+    done && \
+    find /usr/lib -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
     done
 
 # Install semantic_router and aurelio-sdk using script
@@ -96,7 +112,7 @@ RUN sed -i 's/\r$//' docker/install_auto_router.sh && chmod +x docker/install_au
 # ensure pyjwt is used, not jwt
 RUN pip uninstall jwt -y
 RUN pip uninstall PyJWT -y
-RUN pip install PyJWT==2.9.0 --no-cache-dir
+RUN pip install PyJWT==2.12.0 --no-cache-dir
 
 # Build Admin UI (runtime stage)
 # Convert Windows line endings to Unix and make executable

docker/Dockerfile.dev

@@ -31,7 +31,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
 # Fix JWT dependency conflicts early
 RUN pip uninstall jwt -y || true && \
     pip uninstall PyJWT -y || true && \
-    pip install PyJWT==2.9.0 --no-cache-dir
+    pip install PyJWT==2.12.0 --no-cache-dir
 
 # Copy only necessary files for build
 COPY pyproject.toml README.md schema.prisma poetry.lock ./
@@ -56,13 +56,26 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
 USER root
 
 # Install only runtime dependencies
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    libssl3 \
+RUN apt-get update && apt-get upgrade -y \
+        libxml2 \
+        libexpat1 \
+        openssl \
+        libssl3 \
+        git \
+        libkrb5-3 \
+        libglib2.0-0 \
+        wget \
+        libaom3 \
+        libxslt1.1 \
+        libgnutls30 \
+        libc6 \
+    && apt-get install -y --no-install-recommends \
+        libssl3 \
     libatomic1 \
     nodejs \
     npm \
     && rm -rf /var/lib/apt/lists/* \
-    && npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 \
+    && npm install -g npm@latest tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \
     && GLOBAL="$(npm root -g)" \
     && find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
           rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
@@ -73,7 +86,16 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     && find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
           rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
        done \
-    && npm cache clean --force
+    && find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
+          rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
+       done \
+    && find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
+          rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
+       done \
+    && find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
+        sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null \
+    && npm cache clean --force \
+    && apt-get purge -y npm
 
 WORKDIR /app
 
@@ -95,14 +117,21 @@ RUN pip install --no-cache-dir *.whl /wheels/* --no-index --find-links=/wheels/
 # npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
 # Patch every copy of tar, glob, and brace-expansion inside that tree.
 RUN GLOBAL="$(npm root -g)" && \
-    find /usr/lib -path "*/nodejs_wheel/*/node_modules/tar" -type d | while read d; do \
+    [ -n "$GLOBAL" ] || { echo "ERROR: npm root -g returned empty; aborting"; exit 1; } && \
+    find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
     done && \
-    find /usr/lib -path "*/nodejs_wheel/*/node_modules/glob" -type d | while read d; do \
+    find /usr/lib -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
     done && \
-    find /usr/lib -path "*/nodejs_wheel/*/node_modules/@isaacs/brace-expansion" -type d | while read d; do \
+    find /usr/lib -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
+    done && \
+    find /usr/lib -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
+    done && \
+    find /usr/lib -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
     done
 
 # Generate prisma client and set permissions

docker/Dockerfile.non_root

@@ -32,7 +32,7 @@ RUN for i in 1 2 3; do \
 # Cache Python dependencies
 COPY requirements.txt .
 RUN pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt \
-  && pip wheel --no-cache-dir --wheel-dir=/wheels/ "semantic_router==0.1.11" "aurelio-sdk==0.0.19" "PyJWT==2.9.0"
+  && pip wheel --no-cache-dir --wheel-dir=/wheels/ "semantic_router==0.1.11" "aurelio-sdk==0.0.19" "PyJWT==2.12.0"
 
 # Copy source after dependency layers
 COPY . .
@@ -80,7 +80,7 @@ ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
     XDG_CACHE_HOME=/app/.cache \
     PATH="/usr/lib/python3.13/site-packages/nodejs/bin:${PATH}"
 
-RUN pip install --no-cache-dir prisma==0.11.0 nodejs-wheel-binaries==24.12.0 \
+RUN pip install --no-cache-dir prisma==0.11.0 nodejs-wheel-binaries==24.13.1 \
     && mkdir -p /app/.cache/npm
 
 RUN NPM_CONFIG_CACHE=/app/.cache/npm \
@@ -105,7 +105,8 @@ RUN for i in 1 2 3; do \
   && for i in 1 2 3; do \
     apk add --no-cache python3 py3-pip bash openssl tzdata nodejs npm supervisor && break || sleep 5; \
     done \
-  && npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 \
+  && apk upgrade --no-cache nodejs \
+  && npm install -g npm@latest tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \
   && GLOBAL="$(npm root -g)" \
   && find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
@@ -116,7 +117,16 @@ RUN for i in 1 2 3; do \
   && find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
      done \
-  && npm cache clean --force
+  && find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
+     done \
+  && find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
+     done \
+  && find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
+        sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null \
+  && npm cache clean --force \
+  && { apk del --no-cache npm 2>/dev/null || true; }
 
 # Copy artifacts from builder
 COPY --from=builder /app/requirements.txt /app/requirements.txt
@@ -162,14 +172,21 @@ RUN pip install --no-index --find-links=/wheels/ -r requirements.txt && \
 # npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
 # Patch every copy of tar, glob, and brace-expansion inside that tree.
 RUN GLOBAL="$(npm root -g)" && \
-    find /usr/lib -path "*/nodejs_wheel/*/node_modules/tar" -type d | while read d; do \
+    [ -n "$GLOBAL" ] || { echo "ERROR: npm root -g returned empty; aborting"; exit 1; } && \
+    find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
     done && \
-    find /usr/lib -path "*/nodejs_wheel/*/node_modules/glob" -type d | while read d; do \
+    find /usr/lib -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
     done && \
-    find /usr/lib -path "*/nodejs_wheel/*/node_modules/@isaacs/brace-expansion" -type d | while read d; do \
+    find /usr/lib -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
+    done && \
+    find /usr/lib -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
+    done && \
+    find /usr/lib -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
     done
 
 # Permissions, cleanup, and Prisma prep
@@ -181,7 +198,7 @@ RUN sed -i 's/\r$//' docker/entrypoint.sh && \
     chown -R nobody:nogroup /app /var/lib/litellm/ui /var/lib/litellm/assets /nonexistent /.npm && \
     pip uninstall jwt -y || true && \
     pip uninstall PyJWT -y || true && \
-    pip install --no-index --find-links=/wheels/ PyJWT==2.10.1 --no-cache-dir && \
+    pip install --no-index --find-links=/wheels/ PyJWT==2.12.0 --no-cache-dir && \
     rm -rf /wheels && \
     PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \
     chown -R nobody:nogroup $PRISMA_PATH && \

docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md

@@ -3,18 +3,9 @@ slug: anthropic_advanced_features
 title: "Day 0 Support: Claude 4.5 Opus (+Advanced Features)"
 date: 2025-11-25T10:00:00
 authors:
-  - name: Sameer Kankute
-    title: SWE @ LiteLLM (LLM Translation)
-    url: https://www.linkedin.com/in/sameer-kankute/
-    image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
-  - name: Krrish Dholakia
-    title: "CEO, LiteLLM"
-    url: https://www.linkedin.com/in/krish-d/
-    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
-  - name: Ishaan Jaff
-    title: "CTO, LiteLLM"
-    url: https://www.linkedin.com/in/reffajnaahsi/
-    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+  - sameer
+  - krrish
+  - ishaan-alt
 description: "Guide to Claude Opus 4.5 and advanced features in LiteLLM: Tool Search, Programmatic Tool Calling, and Effort Parameter."
 tags: [anthropic, claude, tool search, programmatic tool calling, effort, advanced features]
 hide_table_of_contents: false
@@ -25,6 +16,8 @@ import TabItem from '@theme/TabItem';
 
 This guide covers Anthropic's latest model (Claude Opus 4.5) and its advanced features now available in LiteLLM: Tool Search, Programmatic Tool Calling, Tool Input Examples, and the Effort Parameter.
 
+{/* truncate */}
+
 ---
 
 | Feature | Supported Models |

docs/my-website/blog/anthropic_wildcard_model_access_incident/index.md

@@ -0,0 +1,138 @@
+---
+slug: anthropic-wildcard-model-access-incident
+title: "Incident Report: Wildcard Blocking New Models After Cost Map Reload"
+date: 2026-02-23T10:00:00
+authors:
+  - sameer
+  - krrish
+  - ishaan-alt
+tags: [incident-report, proxy, auth, model-access]
+hide_table_of_contents: false
+---
+
+**Date:** Feb 23, 2026  
+**Duration:** ~3 hours  
+**Severity:** High (for users with provider wildcard access rules)  
+**Status:** Resolved
+
+## Summary
+
+When a new Anthropic model (e.g. `claude-sonnet-4-6`) was added to the LiteLLM model cost map and a cost map reload was triggered, requests to the new model were rejected with:
+
+```
+key not allowed to access model. This key can only access models=['anthropic/*']. Tried to access claude-sonnet-4-6.
+```
+
+The reload updated `litellm.model_cost` correctly but never re-ran `add_known_models()`, so `litellm.anthropic_models` (the in-memory set used by the wildcard resolver) remained stale. The new model was invisible to the `anthropic/*` wildcard even though the cost map knew about it.
+
+- **LLM calls:** All requests to newly-added Anthropic models were blocked with a 401.
+- **Existing models:** Unaffected — only models missing from the stale provider set were impacted.
+- **Other providers:** Same bug class existed for any provider wildcard (e.g. `openai/*`, `gemini/*`).
+
+{/* truncate */}
+
+---
+
+## Background
+
+LiteLLM supports provider-level wildcard access rules. When an admin configures a key or team with `models=['anthropic/*']`, any model whose provider resolves to `anthropic` should be allowed. The resolution happens in `_model_custom_llm_provider_matches_wildcard_pattern`:
+
+```mermaid
+flowchart TD
+    A["1. Request arrives for claude-sonnet-4-6"] --> B["2. Auth check: can this key call this model?
+    proxy/auth/auth_checks.py"]
+    B --> C["3. Key has models=['anthropic/*']
+    → wildcard match attempted"]
+    C --> D["4. get_llm_provider('claude-sonnet-4-6')
+    checks litellm.anthropic_models set"]
+    D -->|"model IN set"| E["5a. ✅ Provider = 'anthropic'
+    → 'anthropic/claude-sonnet-4-6' matches 'anthropic/*'"]
+    D -->|"model NOT IN set"| F["5b. ❌ Provider unknown
+    → exception raised → wildcard returns False"]
+    E --> G["6. Request allowed"]
+    F --> H["6. 401: key not allowed to access model"]
+
+    style E fill:#d4edda,stroke:#28a745
+    style F fill:#f8d7da,stroke:#dc3545
+    style H fill:#f8d7da,stroke:#dc3545
+    style D fill:#fff3cd,stroke:#ffc107
+```
+
+`litellm.anthropic_models` is a Python `set` populated at import time by `add_known_models()`. It is the source `get_llm_provider()` consults to map a bare model name like `claude-sonnet-4-6` to the provider string `"anthropic"`.
+
+---
+
+## Root Cause
+
+`add_known_models()` is called **once** at module import time. Both reload paths in `proxy_server.py` updated `litellm.model_cost` with the fresh map but never called `add_known_models()` again:
+
+```python
+# Before the fix — both reload paths looked like this:
+new_model_cost_map = get_model_cost_map(url=model_cost_map_url)
+litellm.model_cost = new_model_cost_map          # ✅ cost map updated
+_invalidate_model_cost_lowercase_map()           # ✅ cache cleared
+# ❌ add_known_models() never called
+#    → litellm.anthropic_models still has the old set
+#    → new model not in the set
+#    → get_llm_provider() raises for the new model
+#    → wildcard match returns False
+#    → 401 for every request to the new model
+```
+
+The gap existed in two places:
+1. `_check_and_reload_model_cost_map` — the periodic automatic reload (every 10 s)
+2. The `/reload/model_cost_map` admin endpoint — the manual reload
+
+**Timeline:**
+
+1. New model (`claude-sonnet-4-6`) added to `model_prices_and_context_window.json`
+2. Admin triggers cost map reload via UI → `litellm.model_cost` updated
+3. Users with `anthropic/*` wildcard keys attempt requests to `claude-sonnet-4-6`
+4. `get_llm_provider('claude-sonnet-4-6')` raises → wildcard returns False → 401
+5. Admin reloads cost map again — same result (root cause not addressed)
+6. ~3 hours of investigation → root cause identified → fix deployed
+
+---
+
+## The Fix
+
+After each reload, `add_known_models()` is called with the freshly fetched map passed explicitly. Passing the map directly (rather than relying on the module-level reference) removes any ambiguity about which dict is iterated:
+
+```python
+# After the fix — both reload paths now do:
+new_model_cost_map = get_model_cost_map(url=model_cost_map_url)
+litellm.model_cost = new_model_cost_map
+_invalidate_model_cost_lowercase_map()
+litellm.add_known_models(model_cost_map=new_model_cost_map)  # ✅ sets repopulated
+```
+
+`add_known_models()` was also updated to accept an optional explicit map so callers cannot accidentally iterate a stale module-level reference:
+
+```python
+# Before
+def add_known_models():
+    for key, value in model_cost.items():   # reads module global — ambiguous after reload
+        ...
+
+# After
+def add_known_models(model_cost_map: Optional[Dict] = None):
+    _map = model_cost_map if model_cost_map is not None else model_cost
+    for key, value in _map.items():         # always iterates the map you just fetched
+        ...
+```
+
+After the fix, the provider sets (`anthropic_models`, `open_ai_chat_completion_models`, etc.) are always consistent with `litellm.model_cost` immediately after every reload. New models become accessible via wildcard rules without any proxy restart.
+
+---
+
+## Remediation
+
+| # | Action | Status | Code |
+|---|---|---|---|
+| 1 | Call `add_known_models(model_cost_map=...)` in the periodic reload path | ✅ Done | [`proxy_server.py#L4393`](https://github.com/BerriAI/litellm/blob/main/litellm/proxy/proxy_server.py#L4393) |
+| 2 | Call `add_known_models(model_cost_map=...)` in the `/reload/model_cost_map` endpoint | ✅ Done | [`proxy_server.py#L11904`](https://github.com/BerriAI/litellm/blob/main/litellm/proxy/proxy_server.py#L11904) |
+| 3 | Update `add_known_models()` to accept an explicit map parameter | ✅ Done | [`__init__.py#L617`](https://github.com/BerriAI/litellm/blob/main/litellm/__init__.py#L617) |
+| 4 | Regression test: `add_known_models(model_cost_map=...)` populates provider sets | ✅ Done | [`test_auth_checks.py`](https://github.com/BerriAI/litellm/blob/main/tests/proxy_unit_tests/test_auth_checks.py) |
+| 5 | Regression test: `anthropic/*` wildcard grants/denies access correctly after reload | ✅ Done | [`test_auth_checks.py`](https://github.com/BerriAI/litellm/blob/main/tests/proxy_unit_tests/test_auth_checks.py) |
+
+---

docs/my-website/blog/authors.yml

@@ -4,6 +4,12 @@ litellm:
   url: https://github.com/BerriAI/litellm
   image_url: https://github.com/BerriAI.png
 
+sameer:
+  name: Sameer Kankute
+  title: SWE @ LiteLLM (LLM Translation)
+  url: https://www.linkedin.com/in/sameer-kankute/
+  image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
+
 krrish:
   name: Krrish Dholakia
   title: CEO, LiteLLM
@@ -22,3 +28,21 @@ ishaan-alt:
   title: CTO, LiteLLM
   url: https://www.linkedin.com/in/reffajnaahsi/
   image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+
+ryan:
+  name: Ryan Crabbe
+  title: Performance Engineer, LiteLLM
+  url: https://www.linkedin.com/in/ryan-crabbe-0b9687214
+  image_url: https://media.licdn.com/dms/image/v2/D5603AQHt1t9Z4BJ6Gw/profile-displayphoto-shrink_400_400/profile-displayphoto-shrink_400_400/0/1724453682340?e=1772064000&v=beta&t=VXdmr13rsNB05wyA2F1TENOB5UuDHUZ0FCHTolNyR5M
+
+alexsander:
+  name: Alexsander Hamir
+  title: Performance Engineer, LiteLLM
+  url: https://www.linkedin.com/in/alexsander-baptista/
+  image_url: https://github.com/AlexsanderHamir.png
+
+yuneng:
+  name: Yuneng Jiang
+  title: SWE @ LiteLLM (Full Stack)
+  url: https://www.linkedin.com/in/yuneng-david-jiang-455676139/
+  image_url: https://avatars.githubusercontent.com/u/171294688?v=4

docs/my-website/blog/claude_code_beta_headers/index.md

@@ -3,18 +3,9 @@ slug: claude-code-beta-headers-incident
 title: "Incident Report: Invalid beta headers with Claude Code"
 date: 2026-02-16T10:00:00
 authors:
-  - name: Sameer Kankute
-    title: SWE @ LiteLLM (LLM Translation)
-    url: https://www.linkedin.com/in/sameer-kankute/
-    image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
-  - name: Ishaan Jaff
-    title: "CTO, LiteLLM"
-    url: https://www.linkedin.com/in/reffajnaahsi/
-    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
-  - name: Krrish Dholakia
-    title: "CEO, LiteLLM"
-    url: https://www.linkedin.com/in/krish-d/
-    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
+  - sameer
+  - ishaan-alt
+  - krrish
 tags: [incident-report, anthropic, stability]
 hide_table_of_contents: false
 ---
@@ -24,6 +15,8 @@ hide_table_of_contents: false
 **Severity:** High
 **Status:** Resolved
 
+> **Note:** This fix will be available starting from `v1.81.13-nightly` or higher of LiteLLM.
+
 ## Summary
 
 Claude Code began sending unsupported Anthropic beta headers to non-Anthropic providers (Bedrock, Azure AI, Vertex AI), causing `invalid beta flag` errors. LiteLLM was forwarding all beta headers without provider-specific validation. Users experienced request failures when routing Claude Code requests through LiteLLM to these providers.
@@ -171,5 +164,5 @@ curl -X POST "https://your-proxy-url/reload/anthropic_beta_headers" \
 
 ## Related documentation
 
-- [Managing Anthropic Beta Headers](../proxy/sync_anthropic_beta_headers.md) - Complete configuration guide
+- [Managing Anthropic Beta Headers](../../docs/proxy/sync_anthropic_beta_headers) - Complete configuration guide
 - [`anthropic_beta_headers_config.json`](https://github.com/BerriAI/litellm/blob/main/litellm/anthropic_beta_headers_config.json) - Current configuration file

docs/my-website/blog/claude_opus_4_6/index.md

@@ -3,18 +3,9 @@ slug: claude_opus_4_6
 title: "Day 0 Support: Claude Opus 4.6"
 date: 2026-02-05T10:00:00
 authors:
-  - name: Sameer Kankute
-    title: SWE @ LiteLLM (LLM Translation)
-    url: https://www.linkedin.com/in/sameer-kankute/
-    image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
-  - name: Ishaan Jaff
-    title: "CTO, LiteLLM"
-    url: https://www.linkedin.com/in/reffajnaahsi/
-    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
-  - name: Krrish Dholakia
-    title: "CEO, LiteLLM"
-    url: https://www.linkedin.com/in/krish-d/
-    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
+  - sameer
+  - ishaan-alt
+  - krrish
 description: "Day 0 support for Claude Opus 4.6 on LiteLLM AI Gateway - use across Anthropic, Azure, Vertex AI, and Bedrock."
 tags: [anthropic, claude, opus 4.6]
 hide_table_of_contents: false
@@ -25,6 +16,8 @@ import TabItem from '@theme/TabItem';
 
 LiteLLM now supports Claude Opus 4.6 on Day 0. Use it across Anthropic, Azure, Vertex AI, and Bedrock through the LiteLLM AI Gateway.
 
+{/* truncate */}
+
 ## Docker Image
 
 ```bash

docs/my-website/blog/claude_sonnet_4_6/index.md

@@ -0,0 +1,279 @@
+---
+slug: claude_sonnet_4_6
+title: "Day 0 Support: Claude Sonnet 4.6"
+date: 2026-02-17T10:00:00
+authors:
+  - ishaan-alt
+  - krrish
+description: "Day 0 support for Claude Sonnet 4.6 on LiteLLM AI Gateway - use across Anthropic, Azure, Vertex AI, and Bedrock."
+tags: [anthropic, claude, sonnet 4.6]
+hide_table_of_contents: false
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+LiteLLM now supports Claude Sonnet 4.6 on Day 0. Use it across Anthropic, Azure, Vertex AI, and Bedrock through the LiteLLM AI Gateway.
+
+{/* truncate */}
+
+## Docker Image
+
+```bash
+docker pull ghcr.io/berriai/litellm:v1.81.3-stable.sonnet-4-6
+```
+
+## Usage - Anthropic
+
+<Tabs>
+<TabItem value="proxy" label="LiteLLM Proxy">
+
+**1. Setup config.yaml**
+
+```yaml
+model_list:
+  - model_name: claude-sonnet-4-6
+    litellm_params:
+      model: anthropic/claude-sonnet-4-6
+      api_key: os.environ/ANTHROPIC_API_KEY
+```
+
+**2. Start the proxy**
+
+```bash
+docker run -d \
+  -p 4000:4000 \
+  -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
+  -v $(pwd)/config.yaml:/app/config.yaml \
+  ghcr.io/berriai/litellm:v1.81.3-stable.sonnet-4-6 \
+  --config /app/config.yaml
+```
+
+**3. Test it!**
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+--header 'Content-Type: application/json' \
+--header 'Authorization: Bearer $LITELLM_KEY' \
+--data '{
+  "model": "claude-sonnet-4-6",
+  "messages": [
+    {
+      "role": "user",
+      "content": "what llm are you"
+    }
+  ]
+}'
+```
+
+</TabItem>
+
+<TabItem value="sdk" label="LiteLLM SDK">
+
+```python
+from litellm import completion
+
+response = completion(
+    model="anthropic/claude-sonnet-4-6",
+    messages=[{"role": "user", "content": "what llm are you"}]
+)
+print(response.choices[0].message.content)
+```
+
+</TabItem>
+</Tabs>
+
+## Usage - Azure
+
+<Tabs>
+<TabItem value="proxy" label="LiteLLM Proxy">
+
+**1. Setup config.yaml**
+
+```yaml
+model_list:
+  - model_name: claude-sonnet-4-6
+    litellm_params:
+      model: azure_ai/claude-sonnet-4-6
+      api_key: os.environ/AZURE_AI_API_KEY
+      api_base: os.environ/AZURE_AI_API_BASE  # https://<resource>.services.ai.azure.com
+```
+
+**2. Start the proxy**
+
+```bash
+docker run -d \
+  -p 4000:4000 \
+  -e AZURE_AI_API_KEY=$AZURE_AI_API_KEY \
+  -e AZURE_AI_API_BASE=$AZURE_AI_API_BASE \
+  -v $(pwd)/config.yaml:/app/config.yaml \
+  ghcr.io/berriai/litellm:v1.81.3-stable.sonnet-4-6 \
+  --config /app/config.yaml
+```
+
+**3. Test it!**
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+--header 'Content-Type: application/json' \
+--header 'Authorization: Bearer $LITELLM_KEY' \
+--data '{
+  "model": "claude-sonnet-4-6",
+  "messages": [
+    {
+      "role": "user",
+      "content": "what llm are you"
+    }
+  ]
+}'
+```
+
+</TabItem>
+
+<TabItem value="sdk" label="LiteLLM SDK">
+
+```python
+from litellm import completion
+
+response = completion(
+    model="azure_ai/claude-sonnet-4-6",
+    api_key="your-azure-api-key",
+    api_base="https://<resource>.services.ai.azure.com",
+    messages=[{"role": "user", "content": "what llm are you"}]
+)
+print(response.choices[0].message.content)
+```
+
+</TabItem>
+</Tabs>
+
+## Usage - Vertex AI
+
+<Tabs>
+<TabItem value="proxy" label="LiteLLM Proxy">
+
+**1. Setup config.yaml**
+
+```yaml
+model_list:
+  - model_name: claude-sonnet-4-6
+    litellm_params:
+      model: vertex_ai/claude-sonnet-4-6
+      vertex_project: os.environ/VERTEX_PROJECT
+      vertex_location: us-east5
+```
+
+**2. Start the proxy**
+
+```bash
+docker run -d \
+  -p 4000:4000 \
+  -e VERTEX_PROJECT=$VERTEX_PROJECT \
+  -e GOOGLE_APPLICATION_CREDENTIALS=/app/credentials.json \
+  -v $(pwd)/config.yaml:/app/config.yaml \
+  -v $(pwd)/credentials.json:/app/credentials.json \
+  ghcr.io/berriai/litellm:v1.81.3-stable.sonnet-4-6 \
+  --config /app/config.yaml
+```
+
+**3. Test it!**
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+--header 'Content-Type: application/json' \
+--header 'Authorization: Bearer $LITELLM_KEY' \
+--data '{
+  "model": "claude-sonnet-4-6",
+  "messages": [
+    {
+      "role": "user",
+      "content": "what llm are you"
+    }
+  ]
+}'
+```
+
+</TabItem>
+
+<TabItem value="sdk" label="LiteLLM SDK">
+
+```python
+from litellm import completion
+
+response = completion(
+    model="vertex_ai/claude-sonnet-4-6",
+    vertex_project="your-project-id",
+    vertex_location="us-east5",
+    messages=[{"role": "user", "content": "what llm are you"}]
+)
+print(response.choices[0].message.content)
+```
+
+</TabItem>
+</Tabs>
+
+## Usage - Bedrock
+
+<Tabs>
+<TabItem value="proxy" label="LiteLLM Proxy">
+
+**1. Setup config.yaml**
+
+```yaml
+model_list:
+  - model_name: claude-sonnet-4-6
+    litellm_params:
+      model: bedrock/anthropic.claude-sonnet-4-6-v1
+      aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
+      aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
+      aws_region_name: us-east-1
+```
+
+**2. Start the proxy**
+
+```bash
+docker run -d \
+  -p 4000:4000 \
+  -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
+  -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
+  -v $(pwd)/config.yaml:/app/config.yaml \
+  ghcr.io/berriai/litellm:v1.81.3-stable.sonnet-4-6 \
+  --config /app/config.yaml
+```
+
+**3. Test it!**
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+--header 'Content-Type: application/json' \
+--header 'Authorization: Bearer $LITELLM_KEY' \
+--data '{
+  "model": "claude-sonnet-4-6",
+  "messages": [
+    {
+      "role": "user",
+      "content": "what llm are you"
+    }
+  ]
+}'
+```
+
+</TabItem>
+
+<TabItem value="sdk" label="LiteLLM SDK">
+
+```python
+from litellm import completion
+
+response = completion(
+    model="bedrock/anthropic.claude-sonnet-4-6-v1",
+    aws_access_key_id="your-access-key",
+    aws_secret_access_key="your-secret-key",
+    aws_region_name="us-east-1",
+    messages=[{"role": "user", "content": "what llm are you"}]
+)
+print(response.choices[0].message.content)
+```
+
+</TabItem>
+</Tabs>

docs/my-website/blog/fastapi_middleware_performance/index.mdx

@@ -3,18 +3,9 @@ slug: fastapi-middleware-performance
 title: "Your Middleware Could Be a Bottleneck"
 date: 2026-02-07T10:00:00
 authors:
-  - name: Krrish Dholakia
-    title: "CEO, LiteLLM"
-    url: https://www.linkedin.com/in/krish-d/
-    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
-  - name: Ishaan Jaff
-    title: "CTO, LiteLLM"
-    url: https://www.linkedin.com/in/reffajnaahsi/
-    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
-  - name: Ryan Crabbe
-    title: "Performance Engineer, LiteLLM"
-    url: https://www.linkedin.com/in/ryan-crabbe-0b9687214
-    image_url: https://media.licdn.com/dms/image/v2/D5603AQHt1t9Z4BJ6Gw/profile-displayphoto-shrink_400_400/profile-displayphoto-shrink_400_400/0/1724453682340?e=1772064000&v=beta&t=VXdmr13rsNB05wyA2F1TENOB5UuDHUZ0FCHTolNyR5M
+  - krrish
+  - ishaan-alt
+  - ryan
 description: "How we improved LiteLLM proxy latency and throughput by replacing a single middleware base class"
 tags: [performance, fastapi, middleware]
 hide_table_of_contents: false

docs/my-website/blog/gemin_3.1/index.md

@@ -0,0 +1,142 @@
+---
+slug: gemini_3_1_pro
+title: "DAY 0 Support: Gemini 3.1 Pro on LiteLLM"
+date: 2026-02-19T10:00:00
+authors:
+  - sameer
+  - krrish
+  - ishaan-alt
+description: "Guide to using Gemini 3.1 Pro on LiteLLM Proxy and SDK with day 0 support."
+tags: [gemini, day 0 support, llms]
+hide_table_of_contents: false
+---
+
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Gemini 3.1 Pro Day 0 Support 
+
+LiteLLM now supports `gemini-3.1-pro-preview` and all the new API changes along with it.
+
+{/* truncate */}
+
+## Deploy this version
+
+<Tabs>
+<TabItem value="docker" label="Docker">
+
+``` showLineNumbers title="docker run litellm"
+docker run \
+-e STORE_MODEL_IN_DB=True \
+-p 4000:4000 \
+ghcr.io/berriai/litellm:main-v1.81.9-stable.gemini.3.1-pro
+```
+
+</TabItem>
+
+<TabItem value="pip" label="Pip">
+
+``` showLineNumbers title="pip install litellm"
+pip install litellm==v1.81.9-stable.gemini.3.1-pro
+```
+
+</TabItem>
+</Tabs>
+
+## What's New
+
+### 1. New Thinking Levels: `thinkingLevel` with MINIMAL & MEDIUM
+
+Gemini 3.1 Pro introduces support for **medium** thinking level
+
+LiteLLM automatically maps the OpenAI `reasoning_effort` parameter to Gemini's `thinkingLevel`, so you can use familiar `reasoning_effort` values (`minimal`, `low`, `medium`, `high`) without changing your code!
+
+---
+## Supported Endpoints
+
+LiteLLM provides **full end-to-end support** for Gemini 3.1 Pro on:
+
+- ✅ `/v1/chat/completions` - OpenAI-compatible chat completions endpoint
+- ✅ `/v1/responses` - OpenAI Responses API endpoint (streaming and non-streaming)
+- ✅ [`/v1/messages`](../../docs/anthropic_unified) - Anthropic-compatible messages endpoint
+- ✅ `/v1/generateContent` – [Google Gemini API](../../docs/generateContent) compatible endpoint 
+
+All endpoints support:
+- Streaming and non-streaming responses
+- Function calling with thought signatures
+- Multi-turn conversations
+- All Gemini 3-specific features
+- Conversion of provider specific thinking related param to thinkingLevel
+
+## Quick Start
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+**Basic Usage with MEDIUM thinking (NEW)**
+
+```python
+from litellm import completion
+
+# No need to make any changes to your code as we map openai reasoning param to thinkingLevel
+response = completion(
+    model="gemini/gemini-3.1-pro-preview",
+    messages=[{"role": "user", "content": "Solve this complex math problem: 25 * 4 + 10"}],
+    reasoning_effort="medium",  # NEW: MEDIUM thinking level
+)
+
+print(response.choices[0].message.content)
+```
+
+</TabItem>
+
+<TabItem value="proxy" label="PROXY">
+
+**1. Setup config.yaml**
+
+```yaml
+model_list:
+  - model_name: gemini-3.1-pro-preview
+    litellm_params:
+      model: gemini/gemini-3.1-pro-preview
+      api_key: os.environ/GEMINI_API_KEY
+  - model_name: vertex-gemini-3.1-pro-preview
+    litellm_params:
+      model: vertex_ai/gemini-3.1-pro-preview
+```
+
+**2. Start proxy**
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+**3. Call with MEDIUM thinking**
+
+```bash
+curl -X POST http://localhost:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer <YOUR-LITELLM-KEY>" \
+  -d '{
+    "model": "gemini-3.1-pro-preview",
+    "messages": [{"role": "user", "content": "Complex reasoning task"}],
+    "reasoning_effort": "medium"
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+---
+
+## `reasoning_effort` Mapping for Gemini 3+
+
+| reasoning_effort | thinking_level | 
+|------------------|----------------|
+| `minimal` | `minimal` |
+| `low` | `low` |
+| `medium` | `medium` |
+| `high` | `high` |
+| `disable` | `minimal` |
+| `none` | `minimal` |

docs/my-website/blog/gemini_3/index.md

@@ -3,18 +3,9 @@ slug: gemini_3
 title: "DAY 0 Support: Gemini 3 on LiteLLM"
 date: 2025-11-19T10:00:00
 authors:
-  - name: Sameer Kankute
-    title: SWE @ LiteLLM (LLM Translation)
-    url: https://www.linkedin.com/in/sameer-kankute/
-    image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
-  - name: Krrish Dholakia
-    title: "CEO, LiteLLM"
-    url: https://www.linkedin.com/in/krish-d/
-    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
-  - name: Ishaan Jaff
-    title: "CTO, LiteLLM"
-    url: https://www.linkedin.com/in/reffajnaahsi/
-    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+  - sameer
+  - krrish
+  - ishaan-alt
 description: "Common questions and best practices for using gemini-3-pro-preview with LiteLLM Proxy and SDK."
 tags: [gemini, day 0 support, llms]
 hide_table_of_contents: false
@@ -29,6 +20,8 @@ This guide covers common questions and best practices for using `gemini-3-pro-pr
 
 :::
 
+{/* truncate */}
+
 ## Quick Start
 
 <Tabs>
@@ -976,8 +969,7 @@ messages.append(response.choices[0].message)  # ✅ Includes thought signatures
 
 ## Additional Resources
 
-- [Gemini Provider Documentation](../gemini.md)
-- [Thought Signatures Guide](../gemini.md#thought-signatures)
-- [Reasoning Content Documentation](../../reasoning_content.md)
-- [Function Calling Guide](../../function_calling.md)
-
+- [Gemini Provider Documentation](../../docs/providers/gemini)
+- [Thought Signatures Guide](../../docs/providers/gemini#thought-signatures)
+- [Reasoning Content Documentation](../../docs/reasoning_content)
+- [Function Calling Guide](../../docs/completion/function_call)

docs/my-website/blog/gemini_3_1_flash_lite/index.md

@@ -0,0 +1,168 @@
+---
+slug: gemini_3_1_flash_lite_preview
+title: "DAY 0 Support: Gemini 3.1 Flash Lite Preview on LiteLLM"
+date: 2026-03-03T08:00:00
+authors:
+  - sameer
+  - krrish
+  - ishaan-alt
+description: "Guide to using Gemini 3.1 Flash Lite Preview on LiteLLM Proxy and SDK with day 0 support."
+tags: [gemini, day 0 support, llms, supernova]
+hide_table_of_contents: false
+---
+
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Gemini 3.1 Flash Lite Preview Day 0 Support 
+
+LiteLLM now supports `gemini-3.1-flash-lite-preview` with full day 0 support!
+
+:::note
+If you only want cost tracking, you need no change in your current Litellm version. But if you want the support for new features introduced along with it like thinking levels, you will need to use v1.80.8-stable.1 or above.
+:::
+
+{/* truncate */}
+
+## Deploy this version
+
+<Tabs>
+<TabItem value="docker" label="Docker">
+
+``` showLineNumbers title="docker run litellm"
+docker run \
+-e STORE_MODEL_IN_DB=True \
+-p 4000:4000 \
+ghcr.io/berriai/litellm:main-v1.80.8-stable.1
+```
+
+</TabItem>
+
+<TabItem value="pip" label="Pip">
+
+``` showLineNumbers title="pip install litellm"
+pip install litellm==v1.80.8-stable.1
+```
+
+</TabItem>
+</Tabs>
+
+## What's New
+
+Supports all four thinking levels:
+- **MINIMAL**: Ultra-fast responses with minimal reasoning
+- **LOW**: Simple instruction following
+- **MEDIUM**: Balanced reasoning for complex tasks
+- **HIGH**: Maximum reasoning depth (dynamic)
+
+---
+
+## Quick Start
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+**Basic Usage**
+
+```python
+from litellm import completion
+
+response = completion(
+    model="gemini/gemini-3.1-flash-lite-preview",
+    messages=[{"role": "user", "content": "Extract key entities from this text: ..."}],
+)
+
+print(response.choices[0].message.content)
+```
+
+**With Thinking Levels**
+
+```python
+from litellm import completion
+
+# Use MEDIUM thinking for complex reasoning tasks
+response = completion(
+    model="gemini/gemini-3.1-flash-lite-preview",
+    messages=[{"role": "user", "content": "Analyze this dataset and identify patterns"}],
+    reasoning_effort="medium",  # low, medium , high
+)
+
+print(response.choices[0].message.content)
+```
+
+</TabItem>
+
+<TabItem value="proxy" label="PROXY">
+
+**1. Setup config.yaml**
+
+```yaml
+model_list:
+  - model_name: gemini-3.1-flash-lite
+    litellm_params:
+      model: gemini/gemini-3.1-flash-lite-preview
+      api_key: os.environ/GEMINI_API_KEY
+  
+  # Or use Vertex AI
+  - model_name: vertex-gemini-3.1-flash-lite
+    litellm_params:
+      model: vertex_ai/gemini-3.1-flash-lite-preview
+      vertex_project: your-project-id
+      vertex_location: us-central1
+```
+
+**2. Start proxy**
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+**3. Make requests**
+
+```bash
+curl -X POST http://localhost:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer <YOUR-LITELLM-KEY>" \
+  -d '{
+    "model": "gemini-3.1-flash-lite",
+    "messages": [{"role": "user", "content": "Extract structured data from this text"}],
+    "reasoning_effort": "low"
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+---
+
+## Supported Endpoints
+
+LiteLLM provides **full end-to-end support** for Gemini 3.1 Flash Lite Preview on:
+
+- ✅ `/v1/chat/completions` - OpenAI-compatible chat completions endpoint
+- ✅ `/v1/responses` - OpenAI Responses API endpoint (streaming and non-streaming)
+- ✅ [`/v1/messages`](../../docs/anthropic_unified) - Anthropic-compatible messages endpoint
+- ✅ `/v1/generateContent` – [Google Gemini API](../../docs/generateContent) compatible endpoint 
+
+All endpoints support:
+- Streaming and non-streaming responses
+- Function calling with thought signatures
+- Multi-turn conversations
+- All Gemini 3-specific features (thinking levels, thought signatures)
+- Full multimodal support (text, image, audio, video)
+
+---
+
+## `reasoning_effort` Mapping for Gemini 3.1
+
+LiteLLM automatically maps OpenAI's `reasoning_effort` parameter to Gemini's `thinkingLevel`:
+
+| reasoning_effort | thinking_level | Use Case |
+|------------------|----------------|----------|
+| `minimal` | `minimal` | Ultra-fast responses, simple queries |
+| `low` | `low` | Basic instruction following |
+| `medium` | `medium` | Balanced reasoning for moderate complexity |
+| `high` | `high` | Maximum reasoning depth, complex problems |
+| `disable` | `minimal` | Disable extended reasoning |
+| `none` | `minimal` | No extended reasoning |

docs/my-website/blog/gemini_3_flash/index.md

@@ -3,18 +3,9 @@ slug: gemini_3_flash
 title: "DAY 0 Support: Gemini 3 Flash on LiteLLM"
 date: 2025-12-17T10:00:00
 authors:
-  - name: Sameer Kankute
-    title: SWE @ LiteLLM (LLM Translation)
-    url: https://www.linkedin.com/in/sameer-kankute/
-    image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
-  - name: Krrish Dholakia
-    title: "CEO, LiteLLM"
-    url: https://www.linkedin.com/in/krish-d/
-    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
-  - name: Ishaan Jaff
-    title: "CTO, LiteLLM"
-    url: https://www.linkedin.com/in/reffajnaahsi/
-    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+  - sameer
+  - krrish
+  - ishaan-alt
 description: "Guide to using Gemini 3 Flash on LiteLLM Proxy and SDK with day 0 support."
 tags: [gemini, day 0 support, llms]
 hide_table_of_contents: false
@@ -32,6 +23,8 @@ LiteLLM now supports `gemini-3-flash-preview` and all the new API changes along
 If you only want cost tracking, you need no change in your current Litellm version. But if you want the support for new features introduced along with it like thinking levels, you will need to use v1.80.8-stable.1 or above.
 :::
 
+{/* truncate */}
+
 ## Deploy this version
 
 <Tabs>
@@ -80,7 +73,7 @@ LiteLLM provides **full end-to-end support** for Gemini 3 Flash on:
 - ✅ `/v1/chat/completions` - OpenAI-compatible chat completions endpoint
 - ✅ `/v1/responses` - OpenAI Responses API endpoint (streaming and non-streaming)
 - ✅ [`/v1/messages`](../../docs/anthropic_unified) - Anthropic-compatible messages endpoint
-- ✅ `/v1/generateContent` – [Google Gemini API](../../docs/generateContent.md) compatible endpoint 
+- ✅ `/v1/generateContent` – [Google Gemini API](../../docs/generateContent) compatible endpoint 
 All endpoints support:
 - Streaming and non-streaming responses
 - Function calling with thought signatures
@@ -252,4 +245,3 @@ If using this model via vertex_ai, keep the location as global as this is the on
 | `high` | `high` |
 | `disable` | `minimal` |
 | `none` | `minimal` |
-

docs/my-website/blog/gemini_embedding_2_multimodal/index.md

@@ -0,0 +1,168 @@
+---
+slug: gemini_embedding_2_multimodal
+title: "Gemini Embedding 2 Preview: Multimodal Embeddings on LiteLLM"
+date: 2025-03-11T10:00:00
+authors:
+  - sameer
+description: "Generate embeddings from text, images, audio, video, and PDFs with gemini-embedding-2-preview on LiteLLM via Gemini API and Vertex AI."
+tags: [gemini, embeddings, multimodal, vertex ai]
+hide_table_of_contents: false
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Gemini Embedding 2 Preview: Multimodal Embeddings
+
+LiteLLM now supports **multimodal embeddings** with `gemini-embedding-2-preview`—generating a single embedding from a mix of text, images, audio, video, and PDF content. Available via both the **Gemini API** (API key) and **Vertex AI** (GCP credentials).
+
+{/* truncate */}
+
+## Supported Input Types
+
+| Modality | Supported Formats | 
+|----------|-------------------|
+| **Text** | Plain text |
+| **Image** | PNG, JPEG | 
+| **Audio** | MP3, WAV | 
+| **Video** | MP4, MOV | 
+| **Documents** | PDF | 
+
+## Input Formats
+
+LiteLLM accepts three input formats for multimodal content:
+
+1. **Data URIs** – Base64-encoded inline: `data:image/png;base64,<encoded_data>`
+2. **GCS URLs** – Cloud Storage paths (Vertex AI): `gs://bucket/path/to/file.png`
+3. **Gemini File References** – Pre-uploaded files (Gemini API): `files/abc123`
+
+## Quick Start
+
+<Tabs>
+<TabItem value="gemini" label="Gemini API">
+
+```python
+from litellm import embedding
+import os
+
+os.environ["GEMINI_API_KEY"] = "your-api-key"
+
+# Text + Image (base64)
+response = embedding(
+    model="gemini/gemini-embedding-2-preview",
+    input=[
+        "The food was delicious and the waiter...",
+        "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAIAQMAAAD+wSzIAAAABlBMVEX///+/v7+jQ3Y5AAAADklEQVQI12P4AIX8EAgALgAD/aNpbtEAAAAASUVORK5CYII"
+    ],
+)
+print(response)
+```
+
+</TabItem>
+
+<TabItem value="vertex" label="Vertex AI">
+
+```python
+import litellm
+from litellm import embedding
+
+litellm.vertex_project = "your-project-id"
+litellm.vertex_location = "us-central1"
+
+# Text + Image (GCS URL)
+response = embedding(
+    model="vertex_ai/gemini-embedding-2-preview",
+    input=[
+        "Describe this image",
+        "gs://my-bucket/images/photo.png"
+    ],
+)
+print(response)
+```
+
+</TabItem>
+
+<TabItem value="proxy" label="LiteLLM Proxy">
+
+**1. Config (config.yaml)**
+
+```yaml
+model_list:
+  - model_name: gemini-embedding-2-preview
+    litellm_params:
+      model: gemini/gemini-embedding-2-preview
+      api_key: os.environ/GEMINI_API_KEY
+  - model_name: vertex-gemini-embedding-2-preview
+    litellm_params:
+      model: vertex_ai/gemini-embedding-2-preview
+      vertex_project: os.environ/VERTEXAI_PROJECT
+      vertex_location: os.environ/VERTEXAI_LOCATION
+
+general_settings:
+  master_key: sk-1234
+```
+
+**2. Start proxy**
+
+```bash
+litellm --config config.yaml
+```
+
+**3. Call embeddings**
+
+```bash
+curl -X POST http://localhost:4000/embeddings \
+  -H "Authorization: Bearer sk-1234" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "gemini-embedding-2-preview",
+    "input": [
+      "The food was delicious and the waiter...",
+      "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAIAQMAAAD+wSzIAAAABlBMVEX///+/v7+jQ3Y5AAAADklEQVQI12P4AIX8EAgALgAD/aNpbtEAAAAASUVORK5CYII"
+    ]
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+## Input Format Examples
+
+| Format | Example | Provider |
+|--------|---------|----------|
+| **Data URI** | `data:image/png;base64,...` | Gemini, Vertex AI |
+| **GCS URL** | `gs://bucket/path/image.png` | Vertex AI |
+| **File reference** | `files/abc123` | Gemini API only |
+
+### Supported MIME Types for Data URIs
+
+- **Images:** `image/png`, `image/jpeg`
+- **Audio:** `audio/mpeg`, `audio/wav`
+- **Video:** `video/mp4`, `video/quicktime`
+- **Documents:** `application/pdf`
+
+### GCS URL MIME Inference
+
+For Vertex AI, MIME types are inferred from file extensions:
+
+- `.png` → `image/png`
+- `.jpg` / `.jpeg` → `image/jpeg`
+- `.mp3` → `audio/mpeg`
+- `.wav` → `audio/wav`
+- `.mp4` → `video/mp4`
+- `.mov` → `video/quicktime`
+- `.pdf` → `application/pdf`
+
+## Optional Parameters
+
+| Parameter | Description | Maps to |
+|-----------|-------------|---------|
+| `dimensions` | Output embedding size | `outputDimensionality` |
+
+```python
+response = embedding(
+    model="gemini/gemini-embedding-2-preview",
+    input=["text to embed"],
+    dimensions=768,  # Optional: control output vector size
+)
+```

docs/my-website/blog/gpt_5_3_codex/index.md

@@ -0,0 +1,138 @@
+---
+slug: gpt_5_3_codex
+title: "Day 0 Support: GPT-5.3-Codex"
+date: 2026-02-24T10:00:00
+authors:
+  - sameer
+  - krrish
+  - ishaan-alt
+description: "Day 0 support for GPT-5.3-Codex on LiteLLM, including phase parameter handling for Responses API."
+tags: [openai, gpt-5.3-codex, codex, day 0 support]
+hide_table_of_contents: false
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+LiteLLM now supports GPT-5.3-Codex on Day 0, including support for the new assistant `phase` metadata on Responses API output items.
+
+{/* truncate */}
+
+## Why `phase` matters for GPT-5.3-Codex
+
+`phase` appears on assistant output items and helps distinguish preamble/commentary turns from final closeout responses.
+
+Reference: [Phase parameter docs](https://developers.openai.com/api/reference/overview)
+
+Supported values:
+- `null`
+- `"commentary"`
+- `"final_answer"`
+
+Important:
+- Persist assistant output items with `phase` exactly as returned.
+- Send those assistant items back on the next turn.
+- Do **not** add `phase` to user messages.
+
+## Docker Image
+
+```bash
+docker pull ghcr.io/berriai/litellm:v1.81.12-stable.gpt-5.3
+```
+
+## Usage 
+
+<Tabs>
+<TabItem value="proxy" label="LiteLLM Proxy">
+
+**1. Setup config.yaml**
+
+```yaml
+model_list:
+  - model_name: gpt-5.3-codex
+    litellm_params:
+      model: openai/gpt-5.3-codex
+```
+
+**2. Start the proxy**
+
+```bash
+docker run -d \
+  -p 4000:4000 \
+  -e ANTHROPIC_API_KEY=$OPENAI_API_KEY \
+  -v $(pwd)/config.yaml:/app/config.yaml \
+  ghcr.io/berriai/litellm:v1.81.12-stable.gpt-5.3 \
+  --config /app/config.yaml
+```
+
+
+**3. Test it**
+
+```bash
+curl -X POST "http://0.0.0.0:4000/v1/responses" \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer $LITELLM_KEY" \
+  -d '{
+    "model": "gpt-5.3-codex",
+    "input": "Write a Python script that checks if a number is prime."
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+## Python Example: Persist `phase` with OpenAI Client + LiteLLM Base URL
+
+```python
+from openai import OpenAI
+
+client = OpenAI(
+    base_url="http://0.0.0.0:4000/v1",  # LiteLLM Proxy
+    api_key="your-litellm-api-key",
+)
+
+items = []  # Persist this per conversation/thread
+
+
+def _item_get(item, key, default=None):
+    if isinstance(item, dict):
+        return item.get(key, default)
+    return getattr(item, key, default)
+
+
+def run_turn(user_text: str):
+    global items
+
+    # User message: no phase field
+    items.append(
+        {
+            "type": "message",
+            "role": "user",
+            "content": [{"type": "input_text", "text": user_text}],
+        }
+    )
+
+    resp = client.responses.create(
+        model="gpt-5.3-codex",
+        input=items,
+    )
+
+    # Persist assistant output items verbatim, including phase
+    for out_item in (resp.output or []):
+        items.append(out_item)
+
+    # Optional: inspect latest phase for UI/telemetry routing
+    latest_phase = None
+    for out_item in reversed(resp.output or []):
+        if _item_get(out_item, "type") == "output_item.done" and _item_get(out_item, "phase") is not None:
+            latest_phase = _item_get(out_item, "phase")
+            break
+
+    return resp, latest_phase
+```
+
+## Notes
+
+- Use `/v1/responses` for GPT Codex models.
+- Preserve full assistant output history for best multi-turn behavior.
+- If `phase` metadata is dropped during history reconstruction, output quality can degrade on long-running tasks.

docs/my-website/blog/gpt_5_4/index.md

@@ -0,0 +1,90 @@
+---
+slug: gpt_5_4
+title: "Day 0 Support: GPT-5.4"
+date: 2026-03-05T10:00:00
+authors:
+  - sameer
+  - krrish
+  - ishaan-alt
+description: "GPT-5.4 model support in LiteLLM"
+tags: [openai, gpt-5.4, completion]
+hide_table_of_contents: false
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+LiteLLM now supports fully GPT-5.4!
+
+{/* truncate */}
+
+## Docker Image
+
+```bash
+docker pull ghcr.io/berriai/litellm:v1.81.14-stable.gpt-5.4_patch
+```
+
+## Usage
+
+<Tabs>
+<TabItem value="proxy" label="LiteLLM Proxy">
+
+**1. Setup config.yaml**
+
+```yaml
+model_list:
+  - model_name: gpt-5.4
+    litellm_params:
+      model: openai/gpt-5.4
+      api_key: os.environ/OPENAI_API_KEY
+```
+
+**2. Start the proxy**
+
+```bash
+docker run -d \
+  -p 4000:4000 \
+  -e OPENAI_API_KEY=$OPENAI_API_KEY \
+  -v $(pwd)/config.yaml:/app/config.yaml \
+  ghcr.io/berriai/litellm:v1.81.14-stable.gpt-5.4_patch \
+  --config /app/config.yaml
+```
+
+**3. Test it**
+
+```bash
+curl -X POST "http://0.0.0.0:4000/chat/completions" \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer $LITELLM_KEY" \
+  -d '{
+    "model": "gpt-5.4",
+    "messages": [
+      {"role": "user", "content": "Write a Python function to check if a number is prime."}
+    ]
+  }'
+```
+
+</TabItem>
+<TabItem value="sdk" label="LiteLLM SDK">
+
+```python
+from litellm import completion
+
+response = completion(
+    model="openai/gpt-5.4",
+    messages=[
+        {"role": "user", "content": "Write a Python function to check if a number is prime."}
+    ],
+)
+
+print(response.choices[0].message.content)
+```
+
+</TabItem>
+</Tabs>
+
+## Notes
+
+- Restart your container to get the cost tracking for this model.
+- Use `/responses` for better model performance.
+- GPT-5.4 supports reasoning, function calling, vision, and tool-use — see the [OpenAI provider docs](../../docs/providers/openai) for advanced usage.

docs/my-website/blog/gpt_5_4_mini_nano/index.md

@@ -0,0 +1,106 @@
+---
+slug: gpt_5_4_mini_nano
+title: "Day 0 Support: GPT-5.4-mini and GPT-5.4-nano"
+date: 2026-03-17T10:00:00
+authors:
+  - name: Sameer Kankute
+    title: SWE @ LiteLLM (LLM Translation)
+    url: https://www.linkedin.com/in/sameer-kankute/
+    image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
+  - name: Krrish Dholakia
+    title: "CEO, LiteLLM"
+    url: https://www.linkedin.com/in/krish-d/
+    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
+  - name: Ishaan Jaff
+    title: "CTO, LiteLLM"
+    url: https://www.linkedin.com/in/reffajnaahsi/
+    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+description: "GPT-5.4-mini and GPT-5.4-nano model support in LiteLLM"
+tags: [openai, gpt-5.4-mini, gpt-5.4-nano, completion]
+hide_table_of_contents: false
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+LiteLLM now supports GPT-5.4-mini and GPT-5.4-nano — cost-effective models for simple completions and high-throughput workloads.
+
+:::note
+If you're on **v1.82.3-stable** or above, you don't need any update to use these models.
+:::
+
+## Usage
+
+<Tabs>
+<TabItem value="proxy" label="LiteLLM Proxy">
+
+**1. Setup config.yaml**
+
+```yaml
+model_list:
+  - model_name: gpt-5.4-mini
+    litellm_params:
+      model: openai/gpt-5.4-mini
+      api_key: os.environ/OPENAI_API_KEY
+  - model_name: gpt-5.4-nano
+    litellm_params:
+      model: openai/gpt-5.4-nano
+      api_key: os.environ/OPENAI_API_KEY
+```
+
+**2. Start the proxy**
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+**3. Test it**
+
+```bash
+# GPT-5.4-mini
+curl -X POST "http://localhost:4000/v1/chat/completions" \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer $LITELLM_KEY" \
+  -d '{
+    "model": "gpt-5.4-mini",
+    "messages": [{"role": "user", "content": "What is the capital of France?"}]
+  }'
+
+# GPT-5.4-nano
+curl -X POST "http://localhost:4000/v1/chat/completions" \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer $LITELLM_KEY" \
+  -d '{
+    "model": "gpt-5.4-nano",
+    "messages": [{"role": "user", "content": "What is 2 + 2?"}]
+  }'
+```
+
+</TabItem>
+<TabItem value="sdk" label="LiteLLM SDK">
+
+```python
+from litellm import completion
+
+# GPT-5.4-mini
+response = completion(
+    model="openai/gpt-5.4-mini",
+    messages=[{"role": "user", "content": "What is the capital of France?"}],
+)
+print(response.choices[0].message.content)
+
+# GPT-5.4-nano
+response = completion(
+    model="openai/gpt-5.4-nano",
+    messages=[{"role": "user", "content": "What is 2 + 2?"}],
+)
+print(response.choices[0].message.content)
+```
+
+</TabItem>
+</Tabs>
+
+## Notes
+
+- Both models support function calling, vision, and tool-use — see the [OpenAI provider docs](../../docs/providers/openai) for advanced usage.
+- GPT-5.4-nano is the most cost-effective option for simple tasks; GPT-5.4-mini offers a balance of speed and capability.

docs/my-website/blog/guardrail_logging_secret_exposure_incident/index.md

@@ -0,0 +1,78 @@
+---
+slug: guardrail-logging-secret-exposure-incident
+title: "Incident Report: Guardrail logging exposed secret headers in spend logs and traces"
+date: 2026-03-18T10:00:00
+authors:
+  - litellm
+tags: [incident-report, security, guardrails]
+hide_table_of_contents: false
+---
+
+**Date:** March 18, 2026
+**Duration:** Unknown
+**Severity:** High
+**Status:** Resolved
+
+## Summary
+
+When a custom guardrail returned the full LiteLLM request/data dictionary, the guardrail response logged by LiteLLM could include `secret_fields.raw_headers`, including plaintext `Authorization` headers containing API keys or other credentials.
+
+This information could then propagate to logging and observability surfaces that consume guardrail metadata, including:
+
+- **Spend logs in the LiteLLM UI:** visible to admins with access to spend-log data
+- **OpenTelemetry traces:** visible to anyone with access to the relevant telemetry backend
+
+LLM calls, proxy routing, and provider execution were not blocked by this bug. The impact was exposure of sensitive request headers in observability and logging paths.
+
+{/* truncate */}
+
+---
+
+## Background
+
+LiteLLM keeps internal request data (including request headers) for use during the call. That data is not meant to be written to logs or telemetry.
+
+When custom guardrails run, their outcomes are logged so they can appear in spend logs, OpenTelemetry traces, and other observability backends. If a guardrail returned the full request payload instead of a minimal result, that internal request data could be included in what was logged. Before the fix, the guardrail logging path did not strip that data before sending it to those systems.
+
+```mermaid
+flowchart TD
+    inboundRequest["1. Incoming proxy request"] --> storeSecrets["2. Store internal request data"]
+    storeSecrets --> guardrailRuns["3. Custom guardrail runs"]
+    guardrailRuns --> fullDataReturn["4. Guardrail returns full request payload"]
+    fullDataReturn --> loggingBuild["5. Build guardrail log payload"]
+    loggingBuild --> spendLogs["6a. Persist to spend logs / UI"]
+    loggingBuild --> otelTraces["6b. Attach to OTEL guardrail spans"]
+```
+
+---
+
+## Root Cause
+
+The root cause was incomplete sanitization in the guardrail logging path. When building the payload that gets sent to spend logs and traces, LiteLLM prepared guardrail responses for logging but did not strip internal request data (such as headers) from them. If a guardrail returned a response that included that data, it was passed through to the logging and observability systems unchanged.
+
+---
+
+## Impact
+
+This issue required all of the following:
+
+1. A custom guardrail returned the full LiteLLM request/data dictionary, or another response object containing `secret_fields`.
+2. LiteLLM logged that guardrail response through the standard guardrail logging path.
+3. An operator, admin, or telemetry consumer had access to the resulting logs or traces.
+
+When those conditions were met, sensitive values could become visible through:
+
+- **Spend logs / UI responses:** guardrail metadata could be included in spend-log payloads rendered in the admin UI.
+- **OpenTelemetry traces:** `guardrail_response` could be written as a span attribute on guardrail spans.
+- **Other downstream observability backends:** any integration consuming the same guardrail metadata could receive the leaked values.
+
+This was a logging and telemetry exposure bug. It did not let callers bypass auth, access other tenants directly, or change model behavior, but it could expose plaintext credentials to people with access to those observability systems.
+
+---
+
+## Guidance For Users
+
+- Upgrade to LiteLLM 1.82.3+.
+- If you operated custom guardrails that return the full request/data dict, review whether spend logs or telemetry traces were retained during the affected period.
+- Rotate any credentials that may have appeared in `Authorization` or other forwarded request headers in those systems.
+- Apply least-privilege access controls to spend-log views and telemetry backends that may contain request-derived metadata.

docs/my-website/blog/httpx_cache_eviction_incident/index.md

@@ -0,0 +1,126 @@
+---
+slug: httpx-cache-eviction-incident
+title: "Incident Report: Cache Eviction Closes In-Use httpx Clients"
+date: 2026-02-27T10:00:00
+authors:
+  - ryan
+  - ishaan-alt
+  - krrish
+tags: [incident-report, caching, stability]
+hide_table_of_contents: false
+---
+
+**Date:** February 27, 2026
+**Duration:** ~6 days (Feb 21 merge -> Feb 27 fix)
+**Severity:** High
+**Status:** Resolved
+
+> **Note:** This fix is available starting from LiteLLM `v1.81.14.rc.2` or higher.
+
+## Summary
+
+A change to improve Redis connection pool cleanup introduced a regression that closed **httpx clients** that were still actively being used by the proxy. The `LLMClientCache` (an in-memory TTL cache) stores both Redis clients *and* httpx clients under the same eviction policy. When a cache entry expired or was evicted, the new cleanup code called `aclose()`/`close()` on the evicted value which worked correctly for Redis clients, but destroyed httpx clients that other parts of the system still held references to and were actively using for LLM API calls.
+
+**Impact:** Any proxy instance that hit the cache TTL (default 10 minutes) or capacity limit (200 entries) would have its httpx clients closed out from under it, causing requests to LLM providers to fail with connection errors.
+
+{/* truncate */}
+
+---
+
+## Background
+
+`LLMClientCache` extends `InMemoryCache` and is used to cache SDK clients (OpenAI, Anthropic, etc.) to avoid re-creating them on every request. These clients are keyed by configuration + event loop ID. The cache has:
+
+- **Max size:** 200 entries
+- **Default TTL:** 10 minutes
+
+When the cache is full or entries expire, `InMemoryCache.evict_cache()` calls `_remove_key()` to drop entries.
+
+The cached values are a mix of:
+- **Redis/async Redis clients** — owned exclusively by the cache, safe to close on eviction
+- **httpx-backed SDK clients** (OpenAI, Anthropic, etc.) — shared references, still in use by router/model instances
+
+---
+
+## Root Cause
+
+[PR #21717](https://github.com/BerriAI/litellm/pull/21717) overrode `_remove_key()` in `LLMClientCache` to close async clients on eviction:
+
+<details>
+<summary>Problematic code added in PR #21717</summary>
+
+```python
+class LLMClientCache(InMemoryCache):
+    def _remove_key(self, key: str) -> None:
+        value = self.cache_dict.get(key)
+        super()._remove_key(key)
+        if value is not None:
+            close_fn = getattr(value, "aclose", None) or getattr(value, "close", None)
+            if close_fn and asyncio.iscoroutinefunction(close_fn):
+                try:
+                    asyncio.get_running_loop().create_task(close_fn())
+                except RuntimeError:
+                    pass
+            elif close_fn and callable(close_fn):
+                try:
+                    close_fn()
+                except Exception:
+                    pass
+```
+
+</details>
+
+The intent was correct for Redis clients — prevent connection pool leaks when cached Redis clients expire. But `LLMClientCache` also stores httpx-backed SDK clients (e.g., `AsyncOpenAI`, `AsyncAnthropic`). These clients:
+
+1. Have an `aclose()` method (inherited from httpx)
+2. Are still held by references elsewhere in the codebase (router, model instances)
+3. Were being closed without any check on whether they were still in use
+
+So when the cache evicted an entry, it would call `aclose()` on an httpx client that was still being used for active LLM requests → closed transport → connection errors.
+
+---
+
+## The Fix
+
+[PR #22247](https://github.com/BerriAI/litellm/pull/22247) removed the `_remove_key` override entirely:
+
+<details>
+<summary>The fix (PR #22247)</summary>
+
+```diff
+ class LLMClientCache(InMemoryCache):
+-    def _remove_key(self, key: str) -> None:
+-        """Close async clients before evicting them to prevent connection pool leaks."""
+-        value = self.cache_dict.get(key)
+-        super()._remove_key(key)
+-        if value is not None:
+-            close_fn = getattr(value, "aclose", None) or getattr(
+-                value, "close", None
+-            )
+-            ...
+-
+     def update_cache_key_with_event_loop(self, key):
+```
+
+</details>
+
+The eviction now simply drops the reference and lets Python's GC handle cleanup, which is safe because:
+- httpx clients that are still referenced elsewhere stay alive
+- Unreferenced clients get cleaned up by GC naturally
+
+The other improvements from PR #21717 were kept:
+- **`max_connections` respected for URL-based Redis configs**, previously silently dropped
+- **`disconnect()` now closes both sync and async Redis clients**, sync client was previously leaked
+- **Connection pool passthrough**, when a pool is provided with a URL config, it's used directly instead of creating a duplicate
+
+---
+
+## Remediation
+
+| Action | Status | Code |
+|--------|--------|------|
+| Remove `_remove_key` override that closes shared clients on eviction | ✅ Done | [PR #22247](https://github.com/BerriAI/litellm/pull/22247) |
+| Add e2e test: evicted client still usable (capacity) | ✅ Done | [PR #22313](https://github.com/BerriAI/litellm/pull/22313) |
+| Add e2e test: expired client still usable (TTL) | ✅ Done | [PR #22313](https://github.com/BerriAI/litellm/pull/22313) |
+
+The e2e tests go through `get_async_httpx_client()` the same code path the proxy uses in production and assert the client is still functional after eviction. These run in CI on every PR against `main`. If anyone modifies `LLMClientCache` eviction behavior, overrides `_remove_key`, or adds any form of client cleanup on eviction, these tests will fail regardless of the implementation approach.

docs/my-website/blog/litellm_observatory/index.md

@@ -3,18 +3,9 @@ slug: litellm-observatory
 title: "Improve release stability with 24 hour load tests"
 date: 2026-02-06T10:00:00
 authors:
-  - name: Alexsander Hamir
-    title: "Performance Engineer, LiteLLM"
-    url: https://www.linkedin.com/in/alexsander-baptista/
-    image_url: https://github.com/AlexsanderHamir.png
-  - name: Krrish Dholakia
-    title: "CEO, LiteLLM"
-    url: https://www.linkedin.com/in/krish-d/
-    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
-  - name: Ishaan Jaff
-    title: "CTO, LiteLLM"
-    url: https://www.linkedin.com/in/reffajnaahsi/
-    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+  - alexsander
+  - krrish
+  - ishaan-alt
 description: "How we built a long-running, release-validation system to catch regressions before they reach users."
 tags: [testing, observability, reliability, releases]
 hide_table_of_contents: false
@@ -28,6 +19,8 @@ As LiteLLM adoption has grown, so have expectations around reliability, performa
 
 This post introduces **LiteLLM Observatory**, a long-running release-validation system we built to catch regressions before they reach users.
 
+{/* truncate */}
+
 ---
 
 ## Why We Built the Observatory
@@ -133,4 +126,3 @@ Reliability is an ongoing investment.
 LiteLLM Observatory is one of several systems we’re building to continuously raise the bar on release quality and operational safety. As LiteLLM evolves, so will our validation tooling, informed by real-world usage and lessons learned.
 
 We’ll continue to share those improvements openly as we go.
-

docs/my-website/blog/minimax_m2_5/index.md

@@ -3,18 +3,9 @@ slug: minimax_m2_5
 title: "Day 0 Support: MiniMax-M2.5"
 date: 2026-02-12T10:00:00
 authors:
-  - name: Sameer Kankute
-    title: SWE @ LiteLLM (LLM Translation)
-    url: https://www.linkedin.com/in/sameer-kankute/
-    image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
-  - name: Krrish Dholakia
-    title: "CEO, LiteLLM"
-    url: https://www.linkedin.com/in/krish-d/
-    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
-  - name: Ishaan Jaff
-    title: "CTO, LiteLLM"
-    url: https://www.linkedin.com/in/reffajnaahsi/
-    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+  - sameer
+  - krrish
+  - ishaan-alt
 description: "Day 0 support for MiniMax-M2.5 on LiteLLM"
 tags: [minimax, M2.5, llm]
 hide_table_of_contents: false
@@ -25,6 +16,8 @@ import TabItem from '@theme/TabItem';
 
 LiteLLM now supports MiniMax-M2.5 on Day 0. Use it across OpenAI-compatible and Anthropic-compatible APIs through the LiteLLM AI Gateway.
 
+{/* truncate */}
+
 ## Supported Models
 
 LiteLLM supports the following MiniMax models:

docs/my-website/blog/model_cost_map_incident/index.md

@@ -3,10 +3,7 @@ slug: model-cost-map-incident
 title: "Incident Report: Invalid model cost map on main"
 date: 2026-02-10T10:00:00
 authors:
-  - name: Ishaan Jaffer
-    title: "CTO, LiteLLM"
-    url: https://www.linkedin.com/in/ishaanjaffer/
-    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+  - ishaan
 tags: [incident-report, stability]
 hide_table_of_contents: false
 ---

docs/my-website/blog/realtime_webrtc_http_endpoints/index.md

@@ -0,0 +1,111 @@
+---
+slug: realtime_webrtc_http_endpoints
+title: "Realtime WebRTC HTTP Endpoints"
+date: 2026-03-12T10:00:00
+authors:
+  - sameer
+  - krrish
+  - ishaan-alt
+description: "Use the LiteLLM proxy to route OpenAI-style WebRTC realtime via HTTP: client_secrets and SDP exchange."
+tags: [realtime, webrtc, proxy, openai]
+hide_table_of_contents: false
+---
+
+import WebRTCTester from '@site/src/components/WebRTCTester';
+
+Connect to the Realtime API via WebRTC from browser/mobile clients. LiteLLM handles auth and key management.
+
+{/* truncate */}
+
+## How it works
+
+![WebRTC flow: Browser, LiteLLM Proxy, and OpenAI/Azure](../../img/webrtc_flow.png)
+
+**Flow of generating ephemeral token**
+
+![Ephemeral token flow: Browser requests token, LiteLLM gets real token from OpenAI, returns encrypted token](../../img/ephemeral_token.png)
+
+
+## Proxy Setup
+
+```yaml
+model_list:
+  - model_name: gpt-4o-realtime
+    litellm_params:
+      model: openai/gpt-4o-realtime-preview-2024-12-17
+      api_key: os.environ/OPENAI_API_KEY
+    model_info:
+      mode: realtime
+```
+
+**Azure:** use `model: azure/gpt-4o-realtime-preview`, `api_key`, `api_base`.
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+## Try it live
+
+<WebRTCTester />
+
+## Client Usage
+
+**1. Get token** - `POST /v1/realtime/client_secrets` with LiteLLM API key and `{ model }`.
+
+**2. WebRTC handshake** - Create `RTCPeerConnection`, add mic track, create data channel `oai-events`, send SDP offer to `POST /v1/realtime/calls` with `Authorization: Bearer <encrypted_token>` and `Content-Type: application/sdp`.
+
+**3. Events** - Use the data channel for `session.update` and other events.
+
+<details>
+<summary>Full code example</summary>
+
+```javascript
+// 1. Token
+const r = await fetch("http://proxy:4000/v1/realtime/client_secrets", {
+  method: "POST",
+  headers: { "Authorization": "Bearer sk-litellm-key", "Content-Type": "application/json" },
+  body: JSON.stringify({ model: "gpt-4o-realtime" }),
+});
+const { client_secret } = await r.json();
+const token = client_secret.value;
+
+// 2. WebRTC
+const pc = new RTCPeerConnection();
+const audio = document.createElement("audio");
+audio.autoplay = true;
+pc.ontrack = (e) => (audio.srcObject = e.streams[0]);
+const ms = await navigator.mediaDevices.getUserMedia({ audio: true });
+pc.addTrack(ms.getTracks()[0]);
+const dc = pc.createDataChannel("oai-events");
+const offer = await pc.createOffer();
+await pc.setLocalDescription(offer);
+
+const sdpRes = await fetch("http://proxy:4000/v1/realtime/calls", {
+  method: "POST",
+  headers: { "Authorization": `Bearer ${token}`, "Content-Type": "application/sdp" },
+  body: offer.sdp,
+});
+await pc.setRemoteDescription({ type: "answer", sdp: await sdpRes.text() });
+
+// 3. Events
+dc.send(JSON.stringify({ type: "session.update", session: { instructions: "..." } }));
+```
+
+</details>
+
+## FAQ
+
+**Q: What do I do if I get a 401 Token expired error?**  
+A: Tokens are short-lived. Get a fresh token right before creating the WebRTC offer.
+
+**Q: Which key should I use for `/v1/realtime/calls`?**  
+A: Use the **encrypted token** from `client_secrets`, not your raw API key.
+
+**Q: Should I pass the `model` parameter when making the call?**  
+A: No, the encrypted token already encodes all routing information including model.
+
+**Q: How do I resolve Azure `api-version` errors?**  
+A: Set the correct `api_version` in `litellm_params` (or via the `AZURE_API_VERSION` environment variable), along with the right `api_base` and deployment values.
+
+**Q: What if I get no audio?**  
+A: Make sure you grant microphone permission, ensure `pc.ontrack` assigns the audio element with `autoplay` enabled, check your network/firewall for WebRTC traffic, and inspect the browser console for ICE or SDP errors.

docs/my-website/blog/responses_api_encrypted_content_incident/index.md

@@ -0,0 +1,312 @@
+---
+slug: responses-api-encrypted-content-incident
+title: "Incident Report: Encrypted Content Failures in Multi-Region Responses API Load Balancing"
+date: 2026-02-24T10:00:00
+authors:
+  - sameer
+  - krrish
+  - ishaan-alt
+tags: [incident-report, proxy, responses-api, load-balancing]
+hide_table_of_contents: false
+---
+
+**Date:** Feb 24, 2026  
+**Duration:** Ongoing (until fix deployed)  
+**Severity:** High (for users load balancing Responses API across different API keys)  
+**Status:** Resolved
+
+## Summary
+
+When load balancing OpenAI's Responses API across deployments with **different API keys** (e.g., different Azure regions or OpenAI organizations), follow-up requests containing encrypted content items (like `rs_...` reasoning items) would fail with:
+
+```json
+{
+  "error": {
+    "message": "The encrypted content for item rs_0d09d6e56879e76500699d6feee41c8197bd268aae76141f87 could not be verified. Reason: Encrypted content organization_id did not match the target organization.",
+    "type": "invalid_request_error",
+    "code": "invalid_encrypted_content"
+  }
+}
+```
+
+Encrypted content items are cryptographically tied to the API key's organization that created them. When the router load balanced a follow-up request to a deployment with a different API key, decryption failed.
+
+- **Responses API calls with encrypted content:** Complete failure when routed to wrong deployment
+- **Initial requests:** Unaffected — only follow-up requests containing encrypted items failed
+- **Other API endpoints:** No impact — chat completions, embeddings, etc. functioned normally
+
+{/* truncate */}
+
+---
+
+## Background
+
+OpenAI's Responses API can return encrypted "reasoning items" (with IDs like `rs_...`) that contain intermediate reasoning steps. These items are encrypted with the organization's key and can only be decrypted by the same organization's API key.
+
+When load balancing across deployments with different API keys, the existing affinity mechanisms were insufficient:
+
+- **`responses_api_deployment_check`**: Requires `previous_response_id` which some clients (like Codex) don't provide
+- **`deployment_affinity`**: Too broad — pins *all* requests from a user to one deployment, reducing effective quota by the number of users
+- **`session_affinity`**: Requires explicit session IDs and still reduces quota
+
+```mermaid
+flowchart TD
+    A["1. Initial request to Responses API
+    router.aresponses()"] --> B["2. Router load balances to Deployment A
+    (API Key 1, Azure East US)"]
+    B --> C["3. Response contains encrypted item
+    rs_abc123 (encrypted with Org 1 key)"]
+    C --> D["4. Follow-up request includes rs_abc123 in input"]
+    D --> E["5. Router load balances to Deployment B
+    (API Key 2, Azure West Europe)"]
+    E -->|"Different API key"| F["6. ❌ Deployment B cannot decrypt rs_abc123
+    Error: invalid_encrypted_content"]
+    
+    D -.->|"With encrypted_content_affinity"| G["5b. Router detects rs_abc123 was created by Deployment A"]
+    G --> H["6b. ✅ Routes to Deployment A (bypasses rate limits)
+    Request succeeds"]
+
+    style F fill:#f8d7da,stroke:#dc3545
+    style H fill:#d4edda,stroke:#28a745
+    style E fill:#fff3cd,stroke:#ffc107
+    style G fill:#d4edda,stroke:#28a745
+```
+
+---
+
+## Root Cause
+
+LiteLLM's router had no mechanism to track which deployment created specific encrypted content items and route follow-up requests accordingly. The router treated all deployments as interchangeable, leading to decryption failures when encrypted content crossed organizational boundaries.
+
+**The Problem Flow:**
+
+1. User calls `router.aresponses()` with model `gpt-5.1-codex`
+2. Router load balances to Deployment A (Azure East US, API Key 1)
+3. Response contains encrypted reasoning item `rs_abc123` (encrypted with Org 1's key)
+4. User makes follow-up request with `rs_abc123` in the input
+5. Router load balances to Deployment B (Azure West Europe, API Key 2)
+6. Deployment B tries to decrypt `rs_abc123` with Org 2's key → **fails**
+
+**Why Existing Solutions Didn't Work:**
+
+- **`previous_response_id`**: Not provided by all clients (e.g., Codex)
+- **`deployment_affinity`**: Pins *all* user requests to one deployment → reduces quota to 1/N where N = number of deployments
+- **`session_affinity`**: Requires explicit session management and still reduces quota
+
+**Timeline:**
+
+1. Users configured multi-region Responses API load balancing with different API keys
+2. Initial requests succeeded, but follow-up requests with encrypted content failed intermittently
+3. Error rate correlated with number of deployments (more deployments = higher chance of routing to wrong one)
+4. Investigation revealed encrypted content was organization-bound
+5. Existing affinity mechanisms deemed unsuitable (quota reduction, missing `previous_response_id`)
+6. New solution designed and implemented: `encrypted_content_affinity`
+
+---
+
+## The Fix
+
+Implemented a new `encrypted_content_affinity` pre-call check that intelligently tracks encrypted content and routes follow-up requests **only when necessary**.
+
+### Implementation
+
+**1. Encoding `model_id` into output items** ([`responses/utils.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/responses/utils.py))
+
+The same approach used for `previous_response_id` affinity — no cache needed. When a response contains output items with `encrypted_content`, LiteLLM encodes the originating deployment's `model_id` in **two places** for redundancy:
+
+1. **Into the item ID** (if present): `rs_abc123` → `encitem_{base64("litellm:model_id:{model_id};item_id:rs_abc123")}`
+2. **Into the encrypted_content itself**: Wraps the content with `litellm_enc:{base64("model_id:{model_id}")};{original_encrypted_content}`
+
+```python
+# Encoding item IDs (when present)
+def _build_encrypted_item_id(model_id: str, item_id: str) -> str:
+    assembled = f"litellm:model_id:{model_id};item_id:{item_id}"
+    encoded = base64.b64encode(assembled.encode("utf-8")).decode("utf-8")
+    return f"encitem_{encoded}"
+
+# Wrapping encrypted_content (always, for redundancy)
+def _wrap_encrypted_content_with_model_id(encrypted_content: str, model_id: str) -> str:
+    metadata = f"model_id:{model_id}"
+    encoded_metadata = base64.b64encode(metadata.encode("utf-8")).decode("utf-8")
+    return f"litellm_enc:{encoded_metadata};{encrypted_content}"
+```
+
+**Why wrap encrypted_content directly?** Some clients (like Codex) don't consistently send item IDs in follow-up requests, but they always send the `encrypted_content` itself. By embedding `model_id` into the content, affinity works even when IDs are missing.
+
+**Streaming responses:** The wrapping logic is applied to both:
+- Final response objects (non-streaming)
+- Individual streaming events (`response.output_item.added`, `response.output_item.done`)
+
+This ensures clients receiving streaming responses get wrapped content they can send back.
+
+Before forwarding to the upstream provider, LiteLLM restores the original item IDs and unwraps encrypted_content so the provider never sees the encoded form:
+
+```python
+# In responses/main.py — before calling the handler
+input = ResponsesAPIRequestUtils._restore_encrypted_content_item_ids_in_input(input)
+```
+
+**2. `EncryptedContentAffinityCheck` — routing only** ([`encrypted_content_affinity_check.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py))
+
+No `async_log_success_event` or cache lookups — the `model_id` is decoded directly from the item ID or encrypted_content:
+
+```python
+class EncryptedContentAffinityCheck(CustomLogger):
+    async def async_filter_deployments(self, model, healthy_deployments, ...):
+        """Extract model_id from input items (ID or encrypted_content) and pin to that deployment."""
+        for item in request_kwargs.get("input", []):
+            # Try to extract model_id from two sources:
+            model_id = self._extract_model_id_from_input(item)
+            
+            if model_id:
+                deployment = self._find_deployment_by_model_id(
+                    healthy_deployments, model_id
+                )
+                if deployment:
+                    request_kwargs["_encrypted_content_affinity_pinned"] = True
+                    return [deployment]
+        return healthy_deployments
+    
+    def _extract_model_id_from_input(self, item: dict) -> Optional[str]:
+        """Extract model_id from either encoded ID or wrapped encrypted_content."""
+        # 1. Try decoding from item ID (if present)
+        item_id = item.get("id", "")
+        if item_id:
+            decoded = ResponsesAPIRequestUtils._decode_encrypted_item_id(item_id)
+            if decoded:
+                return decoded["model_id"]
+        
+        # 2. Try unwrapping from encrypted_content (fallback for clients that omit IDs)
+        encrypted_content = item.get("encrypted_content", "")
+        if encrypted_content and encrypted_content.startswith("litellm_enc:"):
+            model_id, _ = ResponsesAPIRequestUtils._unwrap_encrypted_content_with_model_id(
+                encrypted_content
+            )
+            return model_id
+        
+        return None
+```
+
+**3. Rate Limit Bypass** ([`router.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/router.py))
+
+When encrypted content requires a specific deployment, RPM/TPM limits are bypassed (the request would fail on any other deployment anyway):
+
+```python
+# In async_get_available_deployment, after filtering healthy deployments:
+if (
+    request_kwargs.get("_encrypted_content_affinity_pinned")
+    and len(healthy_deployments) == 1
+):
+    return healthy_deployments[0]  # Bypass routing strategy (RPM/TPM checks)
+```
+
+**3. Configuration**
+
+```yaml
+router_settings:
+  routing_strategy: usage-based-routing-v2
+  enable_pre_call_checks: true
+  optional_pre_call_checks:
+    - encrypted_content_affinity
+  deployment_affinity_ttl_seconds: 86400  # 24 hours
+```
+
+### Key Benefits
+
+✅ **No quota reduction**: Only pins requests containing encrypted items  
+✅ **Bypasses rate limits**: When encrypted content requires a specific deployment, RPM/TPM limits don't block it  
+✅ **No `previous_response_id` required**: Works by encoding `model_id` directly into the item ID  
+✅ **No cache required**: `model_id` is decoded on-the-fly from the item ID — no Redis, no TTL  
+✅ **Globally safe**: Can be enabled for all models; non-Responses-API calls are unaffected  
+✅ **Surgical precision**: Normal requests continue to load balance freely
+
+---
+
+## Remediation
+
+| # | Action | Status | Code |
+|---|---|---|---|
+| 1 | Encode `model_id` into encrypted-content item IDs on response | ✅ Done | [`responses/utils.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/responses/utils.py) |
+| 2 | Restore original item IDs before forwarding to upstream provider | ✅ Done | [`responses/main.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/responses/main.py) |
+| 3 | `EncryptedContentAffinityCheck`: decode item IDs to route (no cache) | ✅ Done | [`encrypted_content_affinity_check.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py) |
+| 4 | Add `encrypted_content_affinity` to `OptionalPreCallChecks` type | ✅ Done | [`types/router.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/types/router.py) |
+| 5 | Implement rate limit bypass for affinity-pinned requests | ✅ Done | [`router.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/router.py) |
+| 6 | Unit tests: encoding/decoding utilities, routing, RPM bypass | ✅ Done | [`test_encrypted_content_affinity_check.py`](https://github.com/BerriAI/litellm/blob/main/litellm/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py) |
+| 7 | Documentation: Responses API guide, load balancing guide, config reference | ✅ Done | [Docs](https://docs.litellm.ai/docs/response_api#encrypted-content-affinity-multi-region-load-balancing) |
+| 8 | **[Mar 3]** Fix streaming events to wrap encrypted_content | ✅ Done | [`responses/streaming_iterator.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/responses/streaming_iterator.py) |
+
+---
+
+## Follow-up Fix: Streaming Responses (Mar 3, 2026)
+
+### The Issue
+
+After the initial fix was deployed, users reported that the `invalid_encrypted_content` error **still occurred** when using streaming responses with clients like Codex. Investigation revealed:
+
+- ✅ Non-streaming responses: `encrypted_content` was correctly wrapped with `litellm_enc:` prefix
+- ❌ Streaming responses: Individual `response.output_item.added` and `response.output_item.done` events contained **raw, unwrapped** `encrypted_content`
+
+Since Codex and other clients consume responses as streams, they received unwrapped content in these events and sent it back in follow-up requests, causing the affinity check to fail.
+
+### The Root Cause
+
+The `_update_encrypted_content_item_ids_in_response` function only modified the **final** response object, which is used for non-streaming responses. For streaming responses, individual chunks are processed by `ResponsesAPIStreamingIterator._process_chunk`, which was **not** applying the wrapping logic to streaming events.
+
+### The Fix
+
+Modified `litellm/litellm/responses/streaming_iterator.py` to wrap `encrypted_content` in streaming events:
+
+```python
+# In ResponsesAPIStreamingIterator._process_chunk
+if (
+    self.litellm_metadata
+    and self.litellm_metadata.get("encrypted_content_affinity_enabled")
+):
+    event_type = getattr(openai_responses_api_chunk, "type", None)
+    if event_type in (
+        ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED,
+        ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
+    ):
+        item = getattr(openai_responses_api_chunk, "item", None)
+        if item:
+            encrypted_content = getattr(item, "encrypted_content", None)
+            if encrypted_content and isinstance(encrypted_content, str):
+                model_id = (
+                    self.litellm_metadata.get("model_info", {}).get("id")
+                    if self.litellm_metadata
+                    else None
+                )
+                if model_id:
+                    wrapped_content = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id(
+                        encrypted_content, model_id
+                    )
+                    setattr(item, "encrypted_content", wrapped_content)
+```
+
+This ensures that **all** `encrypted_content` sent to clients (streaming or non-streaming) is wrapped with `model_id` metadata, enabling consistent affinity routing.
+
+---
+
+## Migration Guide
+
+### Before (Using `deployment_affinity`)
+
+```yaml
+router_settings:
+  optional_pre_call_checks:
+    - deployment_affinity  # ❌ Reduces quota by number of users
+```
+
+**Problem:** All requests from a user pin to one deployment, reducing effective quota to 1/N.
+
+### After (Using `encrypted_content_affinity`)
+
+```yaml
+router_settings:
+  optional_pre_call_checks:
+    - encrypted_content_affinity  # ✅ Only pins requests with encrypted content
+```
+
+**Benefit:** Normal requests load balance freely, only encrypted content requests pin when necessary.
+
+---

docs/my-website/blog/server_root_path/index.md

@@ -0,0 +1,146 @@
+---
+slug: server-root-path-incident
+title: "Incident Report: SERVER_ROOT_PATH regression broke UI routing"
+date: 2026-02-21T10:00:00
+authors:
+  - yuneng
+  - ishaan-alt
+  - krrish
+tags: [incident-report, ui, stability]
+hide_table_of_contents: false
+---
+
+**Date:** January 22, 2026
+**Duration:** ~4 days (until fix merged January 26, 2026)
+**Severity:** High
+**Status:** Resolved
+
+> **Note:** This fix is available starting from LiteLLM `v1.81.3.rc.6` or higher.
+
+## Summary
+
+A PR ([`#19467`](https://github.com/BerriAI/litellm/pull/19467)) accidentally removed the `root_path=server_root_path` parameter from the FastAPI app initialization in `proxy_server.py`. This caused the proxy to ignore the `SERVER_ROOT_PATH` environment variable when serving the UI. Users who deploy LiteLLM behind a reverse proxy with a path prefix (e.g., `/api/v1` or `/llmproxy`) found that all UI pages returned 404 Not Found.
+
+- **LLM API calls:** No impact. API routing was unaffected.
+- **UI pages:** All UI pages returned 404 for deployments using `SERVER_ROOT_PATH`.
+- **Swagger/OpenAPI docs:** Broken when accessed through the configured root path.
+
+{/* truncate */}
+
+---
+
+## Background
+
+Many LiteLLM deployments run behind a reverse proxy (e.g., Nginx, Traefik, AWS ALB) that routes traffic to LiteLLM under a path prefix. FastAPI's `root_path` parameter tells the application about this prefix so it can correctly serve static files, generate URLs, and handle routing.
+
+```mermaid
+sequenceDiagram
+    participant User as User Browser
+    participant RP as Reverse Proxy
+    participant LP as LiteLLM Proxy
+
+    User->>RP: GET /llmproxy/ui/
+    RP->>LP: GET /ui/ (X-Forwarded-Prefix: /llmproxy)
+
+    Note over LP: Before regression:<br/>FastAPI root_path="/llmproxy"<br/>→ Serves UI correctly
+
+    Note over LP: After regression:<br/>FastAPI root_path=""<br/>→ UI assets resolve to wrong paths<br/>→ 404 Not Found
+```
+
+The `root_path` parameter was present in `proxy_server.py` since early versions of LiteLLM. It was removed as a side effect of PR [#19467](https://github.com/BerriAI/litellm/pull/19467), which was intended to fix a different UI 404 issue.
+
+---
+
+## Root cause
+
+PR [#19467](https://github.com/BerriAI/litellm/pull/19467) (`73d49f8`) removed the `root_path=server_root_path` line from the `FastAPI()` constructor in `proxy_server.py`:
+
+```diff
+ app = FastAPI(
+     docs_url=_get_docs_url(),
+     redoc_url=_get_redoc_url(),
+     title=_title,
+     description=_description,
+     version=version,
+-    root_path=server_root_path,
+     lifespan=proxy_startup_event,
+ )
+```
+
+Without `root_path`, FastAPI treated all requests as if the application was mounted at `/`, causing path mismatches for any deployment using `SERVER_ROOT_PATH`.
+
+The regression went undetected because:
+
+1. **No automated test** verified that `root_path` was set on the FastAPI app.
+2. **No manual test procedure** existed for `SERVER_ROOT_PATH` functionality.
+3. **Default deployments** (without `SERVER_ROOT_PATH`) were unaffected, so most CI tests passed.
+
+---
+
+## Remediation
+
+| #   | Action                                                                                            | Status  | Code                                                                                                                       |
+| --- | ------------------------------------------------------------------------------------------------- | ------- | -------------------------------------------------------------------------------------------------------------------------- |
+| 1   | Restore `root_path=server_root_path` in FastAPI app initialization                                | ✅ Done | [`#19790`](https://github.com/BerriAI/litellm/pull/19790) (`5426b3c`)                                                      |
+| 2   | Add unit tests for `get_server_root_path()` and FastAPI app initialization                        | ✅ Done | [`test_server_root_path.py`](https://github.com/BerriAI/litellm/blob/main/tests/proxy_unit_tests/test_server_root_path.py) |
+| 3   | Add CI workflow that builds Docker image and tests UI routing with `SERVER_ROOT_PATH` on every PR | ✅ Done | [`test_server_root_path.yml`](https://github.com/BerriAI/litellm/blob/main/.github/workflows/test_server_root_path.yml)    |
+| 4   | Document manual test procedure for `SERVER_ROOT_PATH`                                             | ✅ Done | [Discussion #8495](https://github.com/BerriAI/litellm/discussions/8495)                                                    |
+
+---
+
+## CI workflow details
+
+The new [`test_server_root_path.yml`](https://github.com/BerriAI/litellm/blob/main/.github/workflows/test_server_root_path.yml) workflow runs on every PR against `main`. It:
+
+1. Builds the LiteLLM Docker image
+2. Starts a container with `SERVER_ROOT_PATH` set (tests both `/api/v1` and `/llmproxy`)
+3. Verifies the UI returns valid HTML at `{ROOT_PATH}/ui/`
+4. Fails the workflow if the UI is unreachable
+
+```mermaid
+flowchart TD
+    A["PR opened/updated"] --> B["Build Docker image"]
+    B --> C["Start container with SERVER_ROOT_PATH=/api/v1"]
+    B --> D["Start container with SERVER_ROOT_PATH=/llmproxy"]
+    C --> E["curl {ROOT_PATH}/ui/ → expect HTML"]
+    D --> F["curl {ROOT_PATH}/ui/ → expect HTML"]
+    E -->|"HTML found"| G["✅ Pass"]
+    E -->|"404 or no HTML"| H["❌ Fail Workflow"]
+    F -->|"HTML found"| G
+    F -->|"404 or no HTML"| H
+
+    style G fill:#d4edda,stroke:#28a745
+    style H fill:#f8d7da,stroke:#dc3545
+```
+
+This prevents future regressions where changes to `proxy_server.py` accidentally break `SERVER_ROOT_PATH` support.
+
+---
+
+## Timeline
+
+| Time (UTC)         | Event                                                                                                                                                        |
+| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| Jan 22, 2026 04:20 | PR [#19467](https://github.com/BerriAI/litellm/pull/19467) merged, removing `root_path=server_root_path`                                                     |
+| Jan 22–26          | Users on nightly builds report UI 404 errors when using `SERVER_ROOT_PATH`                                                                                   |
+| Jan 26, 2026 17:48 | Fix PR [#19790](https://github.com/BerriAI/litellm/pull/19790) merged, restoring `root_path=server_root_path`                                                |
+| Feb 18, 2026       | CI workflow [`test_server_root_path.yml`](https://github.com/BerriAI/litellm/blob/main/.github/workflows/test_server_root_path.yml) added to run on every PR |
+
+---
+
+## Resolution steps for users
+
+For users still experiencing issues, update to the latest LiteLLM version:
+
+```bash
+pip install --upgrade litellm
+```
+
+Verify your `SERVER_ROOT_PATH` is correctly set:
+
+```bash
+# In your environment or docker-compose.yml
+SERVER_ROOT_PATH="/your-prefix"
+```
+
+Then confirm the UI is accessible at `http://your-host:4000/your-prefix/ui/`.

docs/my-website/blog/sub_millisecond_proxy_overhead/index.md

@@ -3,18 +3,9 @@ slug: sub-millisecond-proxy-overhead
 title: "Achieving Sub-Millisecond Proxy Overhead"
 date: 2026-02-02T10:00:00
 authors:
-  - name: Alexsander Hamir
-    title: "Performance Engineer, LiteLLM"
-    url: https://www.linkedin.com/in/alexsander-baptista/
-    image_url: https://github.com/AlexsanderHamir.png
-  - name: Krrish Dholakia
-    title: "CEO, LiteLLM"
-    url: https://www.linkedin.com/in/krish-d/
-    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
-  - name: Ishaan Jaff
-    title: "CTO, LiteLLM"
-    url: https://www.linkedin.com/in/reffajnaahsi/
-    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+  - alexsander
+  - krrish
+  - ishaan-alt
 description: "Our Q1 performance target and architectural direction for achieving sub-millisecond proxy overhead on modest hardware."
 tags: [performance, architecture]
 hide_table_of_contents: false
@@ -32,6 +23,8 @@ Proxy overhead refers to the latency introduced by LiteLLM itself, independent o
 
 To measure it, we run the same workload directly against the provider and through LiteLLM at identical QPS (for example, 1,000 QPS) and compare the latency delta. To reduce noise, the load generator, LiteLLM, and a mock LLM endpoint all run on the same machine, ensuring the difference reflects proxy overhead rather than network latency.
 
+{/* truncate */}
+
 ---
 
 ## Where We're Coming From

docs/my-website/blog/video_characters_litellm/index.md

@@ -0,0 +1,121 @@
+---
+slug: video_characters_api
+title: "New Video Characters, Edit and Extension API support"
+date: 2026-03-16T10:00:00
+authors:
+  - sameer
+  - krrish
+  - ishaan-alt
+description: "LiteLLM now supports creating, retrieving, and managing reusable video characters across multiple video generations."
+tags: [videos, characters, proxy, routing]
+hide_table_of_contents: false
+---
+
+LiteLLM now supoports videos character, edit and extension apis.
+
+{/* truncate */}
+
+## What's New
+
+Four new endpoints for video character operations:
+- **Create character** - Upload a video to create a reusable asset
+- **Get character** - Retrieve character metadata
+- **Edit video** - Modify generated videos
+- **Extend video** - Continue clips with character consistency
+
+**Available from:** LiteLLM v1.83.0+
+
+## Quick Example
+
+```python
+import litellm
+
+# Create character from video
+character = litellm.avideo_create_character(
+    name="Luna",
+    video=open("luna.mp4", "rb"),
+    custom_llm_provider="openai",
+    model="sora-2"
+)
+print(f"Character: {character.id}")
+
+# Use in generation
+video = litellm.avideo(
+    model="sora-2",
+    prompt="Luna dances through a magical forest.",
+    characters=[{"id": character.id}],
+    seconds="8"
+)
+
+# Get character info
+fetched = litellm.avideo_get_character(
+    character_id=character.id,
+    custom_llm_provider="openai"
+)
+
+# Edit with character preserved
+edited = litellm.avideo_edit(
+    video_id=video.id,
+    prompt="Add warm golden lighting"
+)
+
+# Extend sequence
+extended = litellm.avideo_extension(
+    video_id=video.id,
+    prompt="Luna waves goodbye",
+    seconds="5"
+)
+```
+
+## Via Proxy
+
+```bash
+# Create character
+curl -X POST "http://localhost:4000/v1/videos/characters" \
+  -H "Authorization: Bearer sk-litellm-key" \
+  -F "video=@luna.mp4" \
+  -F "name=Luna"
+
+# Get character
+curl -X GET "http://localhost:4000/v1/videos/characters/char_abc123def456" \
+  -H "Authorization: Bearer sk-litellm-key"
+
+# Edit video
+curl -X POST "http://localhost:4000/v1/videos/edits" \
+  -H "Authorization: Bearer sk-litellm-key" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "video": {"id": "video_xyz789"},
+    "prompt": "Add warm golden lighting and enhance colors"
+  }'
+
+# Extend video
+curl -X POST "http://localhost:4000/v1/videos/extensions" \
+  -H "Authorization: Bearer sk-litellm-key" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "video": {"id": "video_xyz789"},
+    "prompt": "Luna waves goodbye and walks into the sunset",
+    "seconds": "5"
+  }'
+```
+
+## Managed Character IDs
+
+LiteLLM automatically encodes provider and model metadata into character IDs:
+
+**What happens:**
+```
+Upload character "Luna" with model "sora-2" on OpenAI
+  ↓
+LiteLLM creates: char_abc123def456 (contains provider + model_id)
+  ↓
+When you reference it later, LiteLLM decodes automatically
+  ↓
+Router knows exactly which deployment to use
+```
+
+**Behind the scenes:**
+- Character ID format: `character_<base64_encoded_metadata>`
+- Metadata includes: provider, model_id, original_character_id
+- Transparent to you - just use the ID, LiteLLM handles routing

docs/my-website/blog/vllm_embeddings_incident/index.md

@@ -0,0 +1,108 @@
+---
+slug: vllm-embeddings-incident
+title: "Incident Report: vLLM Embeddings Broken by encoding_format Parameter"
+date: 2026-02-18T10:00:00
+authors:
+  - sameer
+  - krrish
+  - ishaan-alt
+tags: [incident-report, embeddings, vllm]
+hide_table_of_contents: false
+---
+
+**Date:** Feb 16, 2026
+**Duration:** ~3 hours
+**Severity:** High (for vLLM embedding users)
+**Status:** Resolved
+
+## Summary
+
+A commit ([`dbcae4a`](https://github.com/BerriAI/litellm/commit/dbcae4aca5836770d0e9cd43abab0333c3d61ab2)) intended to fix OpenAI SDK behavior broke vLLM embeddings by explicitly passing `encoding_format=None` in API requests. vLLM rejects this with error: `"unknown variant \`\`, expected float or base64"`.
+
+- **vLLM embedding calls:** Complete failure - all requests rejected
+- **Other providers:** No impact - OpenAI and other providers functioned normally
+- **Other vLLM functionality:** No impact - only embeddings were affected
+
+{/* truncate */}
+
+---
+
+## Background
+
+The `encoding_format` parameter for embeddings specifies whether vectors should be returned as `float` arrays or `base64` encoded strings. Different providers have different expectations:
+
+- **OpenAI SDK:** If `encoding_format` is omitted, the SDK adds a default value of `"float"`
+- **vLLM:** Strictly validates `encoding_format` - only accepts `"float"`, `"base64"`, or complete omission. Rejects `None` or empty string values.
+
+```mermaid
+flowchart TD
+    A["1. User calls litellm.embedding()
+    litellm/main.py"] --> B["2. Transform request for provider
+    litellm/llms/openai_like/embedding/handler.py"]
+    B --> C["3. Send request to vLLM endpoint"]
+    C -->|"encoding_format omitted"| D["4a. ✅ vLLM processes request"]
+    C -->|"encoding_format='float' or 'base64'"| D
+    C -->|"encoding_format=None or ''"| E["4b. ❌ vLLM rejects with error:
+    'unknown variant, expected float or base64'"]
+
+    style D fill:#d4edda,stroke:#28a745
+    style E fill:#f8d7da,stroke:#dc3545
+    style B fill:#fff3cd,stroke:#ffc107
+```
+
+---
+
+## Root cause
+
+A well-intentioned fix for OpenAI SDK behavior inadvertently broke vLLM embeddings:
+
+**The Breaking Change ([`dbcae4a`](https://github.com/BerriAI/litellm/commit/dbcae4aca5836770d0e9cd43abab0333c3d61ab2)):**
+
+In `litellm/main.py`, the code was changed to explicitly set `encoding_format=None` instead of omitting it:
+
+```python
+# Added in dbcae4a
+if encoding_format is not None:
+    optional_params["encoding_format"] = encoding_format
+else:
+    # Omitting causes openai sdk to add default value of "float"
+    optional_params["encoding_format"] = None
+```
+
+This fix worked correctly for OpenAI - explicitly passing `None` prevented the SDK from adding its default value. However, vLLM's strict parameter validation rejected `None` values, causing all embedding requests to fail.
+
+---
+
+## The Fix
+
+Fix deployed ([`55348dd`](https://github.com/BerriAI/litellm/commit/55348dd9c51b5b028f676d25ad023b8f052fc071)). The solution filters out `None` and empty string values from `optional_params` before sending requests to OpenAI-like providers (including vLLM).
+
+**In `litellm/llms/openai_like/embedding/handler.py`:**
+
+```python
+# Before (broken)
+data = {"model": model, "input": input, **optional_params}
+
+# After (fixed)
+filtered_optional_params = {k: v for k, v in optional_params.items() if v not in (None, '')}
+data = {"model": model, "input": input, **filtered_optional_params}
+```
+
+This ensures:
+- Valid values (`"float"`, `"base64"`) are preserved and sent
+- `None` and empty string values are filtered out (parameter omitted entirely)
+- OpenAI SDK no longer adds defaults because liteLLM handles the parameter upstream
+
+---
+
+## Remediation
+
+| # | Action | Status | Code |
+|---|---|---|---|
+| 1 | Filter `None` and empty string values in OpenAI-like embedding handler | ✅ Done | [`handler.py#L108`](https://github.com/BerriAI/litellm/blob/main/litellm/llms/openai_like/embedding/handler.py#L108) |
+| 2 | Unit tests for parameter filtering (None, empty string, valid values) | ✅ Done | [`test_openai_like_embedding.py`](https://github.com/BerriAI/litellm/blob/main/tests/test_litellm/llms/openai_like/embedding/test_openai_like_embedding.py) |
+| 3 | Transformation tests for hosted_vllm embedding config | ✅ Done | [`test_hosted_vllm_embedding_transformation.py`](https://github.com/BerriAI/litellm/blob/main/tests/test_litellm/llms/hosted_vllm/embedding/test_hosted_vllm_embedding_transformation.py) |
+| 4 | E2E tests with actual vLLM endpoint | ✅ Done | [`test_hosted_vllm_embedding_e2e.py`](https://github.com/BerriAI/litellm/blob/main/tests/test_litellm/llms/hosted_vllm/embedding/test_hosted_vllm_embedding_e2e.py) |
+| 5 | Validate JSON payload structure matches vLLM expectations | ✅ Done | Tests verify exact JSON sent to endpoint |
+
+---

docs/my-website/docs/a2a.md

@@ -20,6 +20,7 @@ Add A2A Agents on LiteLLM AI Gateway, Invoke agents in A2A Protocol, track reque
 | Logging | ✅ |
 | Load Balancing | ✅ |
 | Streaming | ✅ |
+| [Iteration Budgets](a2a_iteration_budgets) | ✅ |
 
 
 :::tip

docs/my-website/docs/a2a_agent_headers.md

@@ -0,0 +1,252 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# A2A Agent Authentication Headers
+
+Forward authentication credentials (Bearer tokens, API keys, etc.) from clients to backend A2A agents.
+
+## Overview
+
+When LiteLLM proxies a request to a backend A2A agent, the agent may require its own authentication headers. There are three ways to supply them:
+
+| Method | Who configures | How it works |
+|---|---|---|
+| **Static headers** | Admin (UI / API) | Always sent, regardless of client request |
+| **Forward client headers** | Admin (UI / API) | Header names to extract from client request and forward |
+| **Convention-based** | Client (no admin config) | Client sends `x-a2a-{agent_name}-{header}` — automatically routed |
+
+All three methods can be combined. **Static headers always win** on key conflicts.
+
+---
+
+## Method 1 — Static Headers
+
+Admin-configured headers that are always sent to the backend agent. Use this for server-to-server tokens or internal credentials that clients should never see or override.
+
+<Tabs>
+<TabItem value="ui" label="UI">
+
+1. Go to **Agents** in the LiteLLM dashboard.
+2. Create or edit an agent.
+3. Open the **Authentication Headers** panel.
+4. Under **Static Headers**, click **Add Static Header** and fill in the header name and value.
+
+</TabItem>
+<TabItem value="api" label="REST API">
+
+```bash
+curl -X POST http://localhost:4000/v1/agents \
+  -H "Authorization: Bearer sk-admin" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "agent_name": "my-agent",
+    "agent_card_params": { ... },
+    "static_headers": {
+      "Authorization": "Bearer internal-server-token",
+      "X-Internal-Service": "litellm-proxy"
+    }
+  }'
+```
+
+To update an existing agent:
+
+```bash
+curl -X PATCH http://localhost:4000/v1/agents/{agent_id} \
+  -H "Authorization: Bearer sk-admin" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "static_headers": {
+      "Authorization": "Bearer new-token"
+    }
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+**Client call — no special headers needed:**
+
+```bash
+curl -X POST http://localhost:4000/a2a/my-agent \
+  -H "Authorization: Bearer sk-client-key" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "jsonrpc": "2.0", "id": "1", "method": "message/send",
+    "params": { "message": { "role": "user", "parts": [{"kind": "text", "text": "Hello"}], "messageId": "msg-1" } }
+  }'
+```
+
+The backend agent receives `Authorization: Bearer internal-server-token` without the client ever knowing the value.
+
+---
+
+## Method 2 — Forward Client Headers
+
+Admin specifies a list of header **names**. When the client sends a request that includes those headers, LiteLLM extracts their values and forwards them to the backend agent. The client controls the values; the admin controls which headers are eligible to be forwarded.
+
+<Tabs>
+<TabItem value="ui" label="UI">
+
+1. Go to **Agents** in the LiteLLM dashboard.
+2. Create or edit an agent.
+3. Open the **Authentication Headers** panel.
+4. Under **Forward Client Headers**, type header names and press **Enter** (e.g. `x-api-key`, `Authorization`).
+
+</TabItem>
+<TabItem value="api" label="REST API">
+
+```bash
+curl -X POST http://localhost:4000/v1/agents \
+  -H "Authorization: Bearer sk-admin" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "agent_name": "my-agent",
+    "agent_card_params": { ... },
+    "extra_headers": ["x-api-key", "x-user-token"]
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+**Client call — include the forwarded headers:**
+
+```bash
+curl -X POST http://localhost:4000/a2a/my-agent \
+  -H "Authorization: Bearer sk-client-key" \
+  -H "x-api-key: user-secret-value" \
+  -H "Content-Type: application/json" \
+  -d '{ ... }'
+```
+
+The backend agent receives `x-api-key: user-secret-value`.
+
+:::note
+Header name matching is **case-insensitive**. If the client sends `X-API-Key` and `extra_headers` lists `x-api-key`, they match.
+:::
+
+---
+
+## Method 3 — Convention-Based Forwarding
+
+Clients can forward headers to a specific agent without any admin pre-configuration by using the naming convention:
+
+```
+x-a2a-{agent_name_or_id}-{header_name}: value
+```
+
+LiteLLM parses these headers automatically and routes them to the matching agent only.
+
+**Examples:**
+
+| Client header sent | Agent name/ID | Forwarded as |
+|---|---|---|
+| `x-a2a-my-agent-authorization: Bearer tok` | `my-agent` | `authorization: Bearer tok` |
+| `x-a2a-my-agent-x-api-key: secret` | `my-agent` | `x-api-key: secret` |
+| `x-a2a-abc123-authorization: Bearer tok` | agent ID `abc123` | `authorization: Bearer tok` |
+
+```bash
+curl -X POST http://localhost:4000/a2a/my-agent \
+  -H "Authorization: Bearer sk-client-key" \
+  -H "x-a2a-my-agent-authorization: Bearer agent-specific-token" \
+  -H "Content-Type: application/json" \
+  -d '{ ... }'
+```
+
+The `x-a2a-other-agent-authorization` header sent in the same request is **not** forwarded to `my-agent` — it is silently ignored.
+
+:::tip Matches both agent name and agent ID
+Both the human-readable name (e.g. `my-agent`) and the UUID (e.g. `abc123-...`) are valid. Use whichever is convenient for the client.
+:::
+
+---
+
+## Merge Precedence
+
+When multiple methods supply the same header name, **static headers win**:
+
+```
+dynamic (forwarded/convention)  →  merged  ←  static (overlays, wins)
+```
+
+Example:
+
+| Source | `Authorization` value |
+|---|---|
+| Client sends (via `extra_headers` or convention) | `Bearer client-token` |
+| Admin-configured `static_headers` | `Bearer server-token` |
+| **What the backend agent receives** | **`Bearer server-token`** |
+
+This ensures admin-controlled credentials cannot be overridden by client requests.
+
+---
+
+## Combining All Three Methods
+
+```bash
+# Register agent with static + forwarded headers
+curl -X POST http://localhost:4000/v1/agents \
+  -H "Authorization: Bearer sk-admin" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "agent_name": "my-agent",
+    "agent_card_params": { ... },
+    "static_headers": {
+      "X-Internal-Token": "secret123"
+    },
+    "extra_headers": ["x-user-id"]
+  }'
+
+# Client call using all three mechanisms
+curl -X POST http://localhost:4000/a2a/my-agent \
+  -H "Authorization: Bearer sk-client-key" \
+  -H "x-user-id: user-42" \
+  -H "x-a2a-my-agent-x-request-id: req-abc" \
+  -H "Content-Type: application/json" \
+  -d '{ ... }'
+```
+
+The backend agent receives:
+
+```
+X-Internal-Token: secret123          ← static header (always)
+x-user-id: user-42                   ← forwarded (in extra_headers)
+x-request-id: req-abc                ← convention-based (x-a2a-my-agent-*)
+X-LiteLLM-Trace-Id: <uuid>           ← LiteLLM internal
+X-LiteLLM-Agent-Id: <agent-id>       ← LiteLLM internal
+```
+
+---
+
+## Header Isolation
+
+Each agent invocation uses an isolated HTTP connection. Headers configured for agent A are **never** sent to agent B, even if both agents are running and receiving requests simultaneously.
+
+---
+
+## API Reference
+
+### `POST /v1/agents` / `PATCH /v1/agents/{agent_id}`
+
+| Field | Type | Description |
+|---|---|---|
+| `static_headers` | `object` | `{"Header-Name": "value"}` — always forwarded |
+| `extra_headers` | `string[]` | Header names to extract from client request and forward |
+
+### Agent Response
+
+Both fields are returned in `GET /v1/agents` and `GET /v1/agents/{agent_id}`:
+
+```json
+{
+  "agent_id": "...",
+  "agent_name": "my-agent",
+  "static_headers": { "X-Internal-Token": "secret123" },
+  "extra_headers": ["x-user-id"],
+  ...
+}
+```
+
+:::caution
+`static_headers` values are stored in the database and returned by the API. Treat them as you would any credential — do not store sensitive long-lived tokens here if your API is publicly accessible. Consider using short-lived tokens or environment-injected secrets instead.
+:::

docs/my-website/docs/a2a_iteration_budgets.md

@@ -0,0 +1,188 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Agent Iteration Budgets
+
+Control runaway costs from agentic loops with per-session iteration and budget caps.
+
+## Overview
+
+When agents run agentic loops, they can make unbounded LLM calls, causing unexpected costs. LiteLLM provides two controls:
+
+| Control | Description |
+|---------|-------------|
+| **Max Iterations** | Hard cap on the number of LLM calls per session |
+| **Max Budget Per Session** | Dollar cap per session (identified by `x-litellm-trace-id`) |
+
+Both controls require a `session_id` (sent via `x-litellm-trace-id` header or `metadata.session_id`) to track calls within a session.
+
+## Trace-ID Enforcement
+
+LiteLLM supports two independent trace-id flags, configured in `litellm_params` on the agent:
+
+| Flag | Description |
+|------|-------------|
+| `require_trace_id_on_calls_to_agent` | Requires callers invoking this agent to include `x-litellm-trace-id`. Use when the agent should only be called as a sub-agent with a trace context. Returns **400** if missing. |
+| `require_trace_id_on_calls_by_agent` | Requires all LLM/MCP calls made **by** this agent (via its virtual key) to include `x-litellm-trace-id`. This is what enables `max_iterations` and `max_budget_per_session` tracking. Returns **400** if missing. |
+
+## Configuring via UI
+
+When creating an agent in the LiteLLM Admin UI:
+
+1. Navigate to the **Agents** tab and click **Add Agent**
+2. In the **Agent Settings** step, expand the **Tracing** section
+3. Toggle **Require x-litellm-trace-id on calls BY this agent** to enable session tracking
+4. Set **Max Iterations** to cap the number of LLM calls per session
+5. Set **Max Budget Per Session ($)** to cap spend per session
+
+The trace-id flags are stored on the agent's `litellm_params`. Budget controls (`max_iterations`, `max_budget_per_session`) are stored in the virtual key's metadata.
+
+## Configuring via API
+
+Set trace-id enforcement on the agent itself:
+
+```bash
+curl -X POST 'http://localhost:4000/v1/agents' \
+  -H 'Authorization: Bearer sk-1234' \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "agent_name": "my-research-agent",
+    "agent_card_params": {
+      "name": "my-research-agent",
+      "description": "A research agent with budget controls",
+      "url": "http://my-agent:8080",
+      "version": "1.0.0"
+    },
+    "litellm_params": {
+      "require_trace_id_on_calls_to_agent": true,
+      "require_trace_id_on_calls_by_agent": true
+    }
+  }'
+```
+
+Budget controls are set on the agent's `litellm_params` (not on individual keys), so they apply across all keys for the agent:
+
+```bash
+curl -X POST 'http://localhost:4000/v1/agents' \
+  -H 'Authorization: Bearer sk-1234' \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "agent_name": "my-research-agent",
+    "agent_card_params": {
+      "name": "my-research-agent",
+      "description": "A research agent with budget controls",
+      "url": "http://my-agent:8080",
+      "version": "1.0.0"
+    },
+    "litellm_params": {
+      "require_trace_id_on_calls_by_agent": true,
+      "max_iterations": 25,
+      "max_budget_per_session": 5.00
+    }
+  }'
+```
+
+## How It Works
+
+### Session Tracking
+
+Callers identify their session by including a `session_id` in one of these ways:
+- **Header**: `x-litellm-trace-id: my-session-123`
+- **Metadata**: `{"metadata": {"session_id": "my-session-123"}}`
+
+### Max Iterations
+
+When `max_iterations` is set in agent `litellm_params`:
+- Each LLM call for a session increments a counter
+- When the counter exceeds `max_iterations`, the request receives a **429 Too Many Requests**
+- Counters expire after 1 hour by default (configurable via `LITELLM_MAX_ITERATIONS_TTL` env var)
+
+### Max Budget Per Session
+
+When `max_budget_per_session` is set in agent `litellm_params`:
+- After each successful LLM call, the response cost is accumulated for the session
+- Before each call, the accumulated spend is checked against the budget
+- When spend exceeds the budget, the request receives a **429 Too Many Requests**
+- Session spend counters expire after 1 hour by default (configurable via `LITELLM_MAX_BUDGET_PER_SESSION_TTL` env var)
+
+## Example
+
+Create an agent with max 25 iterations and a $5 budget cap:
+
+<Tabs>
+<TabItem value="ui" label="Via UI">
+
+1. Go to **Agents** → **Add Agent**
+2. Configure your agent (name, model, etc.)
+3. In **Agent Settings**, expand the **Tracing** section
+4. Toggle on **Require x-litellm-trace-id on calls BY this agent**
+5. Set **Max Iterations** to `25`
+6. Set **Max Budget Per Session** to `5.00`
+7. Proceed to create a new key for the agent
+8. Click **Create Agent**
+
+</TabItem>
+<TabItem value="api" label="Via API">
+
+```bash
+# 1. Create the agent with trace-id enforcement
+curl -X POST 'http://localhost:4000/v1/agents' \
+  -H 'Authorization: Bearer sk-1234' \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "agent_name": "my-research-agent",
+    "agent_card_params": {
+      "name": "my-research-agent",
+      "description": "A research agent with budget controls",
+      "url": "http://my-agent:8080",
+      "version": "1.0.0"
+    },
+    "litellm_params": {
+      "require_trace_id_on_calls_by_agent": true
+    }
+  }'
+
+# 2. Create a key for the agent
+curl -X POST 'http://localhost:4000/key/generate' \
+  -H 'Authorization: Bearer sk-1234' \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "agent_id": "<agent_id_from_step_1>",
+    "key_alias": "my-research-agent-key"
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+### Making Calls with Session Tracking
+
+```bash
+curl -X POST 'http://localhost:4000/chat/completions' \
+  -H 'Authorization: Bearer sk-agent-key-xxx' \
+  -H 'x-litellm-trace-id: session-abc-123' \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "model": "gpt-4o",
+    "messages": [{"role": "user", "content": "Hello"}]
+  }'
+```
+
+After 25 calls or $5 spent within this session, subsequent requests will receive:
+
+```json
+{
+  "error": {
+    "message": "Session budget exceeded for session session-abc-123. Current spend: $5.0032, max_budget_per_session: $5.00.",
+    "type": "budget_exceeded",
+    "code": 429
+  }
+}
+```
+
+## Environment Variables
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `LITELLM_MAX_ITERATIONS_TTL` | `3600` (1 hour) | TTL in seconds for session iteration counters |
+| `LITELLM_MAX_BUDGET_PER_SESSION_TTL` | `3600` (1 hour) | TTL in seconds for session budget counters |

docs/my-website/docs/adding_provider/generic_guardrail_api.md

@@ -237,12 +237,42 @@ litellm_settings:
         mode: pre_call  # or post_call, during_call
         api_base: https://your-guardrail-api.com
         api_key: os.environ/YOUR_GUARDRAIL_API_KEY  # optional
+        unreachable_fallback: fail_closed  # default: fail_closed. Set to fail_open to proceed if the guardrail endpoint is unreachable (network errors, or HTTP 502/503/504 from an upstream proxy/LB).
         additional_provider_specific_params:
           # your custom parameters
           threshold: 0.8
           language: "en"
 ```
 
+### Static and dynamic headers
+
+You can send two kinds of headers to your guardrail endpoint:
+
+- **Static headers** (`headers`): A key/value map sent with **every** request to your guardrail. Use this for fixed values (e.g. API keys, `X-Service-Name`). Configure in `litellm_params`:
+
+  ```yaml
+  litellm_params:
+    guardrail: generic_guardrail_api
+    api_base: https://your-guardrail-api.com
+    headers:
+      X-Service-Name: "my-app"
+      X-API-Key: "secret"
+  ```
+
+- **Dynamic headers** (`extra_headers`): A list of **header names** that are forwarded from the **client request** to your guardrail. Only headers in this list (plus a small default allowlist such as `x-litellm-*`) have their values sent; others are sent as `[present]`. Use this to pass through client-provided headers (e.g. `x-request-id`, `x-correlation-id`). Configure in `litellm_params`:
+
+  ```yaml
+  litellm_params:
+    guardrail: generic_guardrail_api
+    api_base: https://your-guardrail-api.com
+    extra_headers:
+      - x-request-id
+      - x-correlation-id
+      - x-custom-auth
+  ```
+
+This mirrors the [MCP static and extra headers](/docs/mcp#forwarding-custom-headers-to-mcp-servers) behavior.
+
 ### Example: Pillar Security
 
 [Pillar Security](https://pillar.security) uses the Generic Guardrail API to provide comprehensive AI security scanning including prompt injection protection, PII/PCI detection, secret detection, and content moderation.

docs/my-website/docs/adding_provider/generic_prompt_management_api.md

@@ -0,0 +1,576 @@
+# [BETA] Generic Prompt Management API - Integrate Without a PR
+
+## The Problem
+
+As a prompt management provider, integrating with LiteLLM traditionally requires:
+- Making a PR to the LiteLLM repository
+- Waiting for review and merge
+- Maintaining provider-specific code in LiteLLM's codebase
+- Updating the integration for changes to your API
+
+## The Solution
+
+The **Generic Prompt Management API** lets you integrate with LiteLLM **instantly** by implementing a simple API endpoint. No PR required.
+
+### Key Benefits
+
+1. **No PR Needed** - Deploy and integrate immediately
+3. **Simple Contract** - One GET endpoint, standard JSON response
+4. **Variable Substitution** - Support for prompt variables with `{variable}` syntax
+5. **Custom Parameters** - Pass provider-specific query params via config
+6. **Full Control** - You own and maintain your prompt management API
+7. **Model & Parameters Override** - Optionally override model and parameters from your prompts
+
+## Get Started in 3 Steps
+
+### Step 1: Configure LiteLLM
+
+Add to your `config.yaml`:
+
+```yaml
+prompts:
+  - prompt_id: "simple_prompt"
+    litellm_params:
+      prompt_integration: "generic_prompt_management"
+      api_base: http://localhost:8080
+      api_key: os.environ/YOUR_API_KEY
+```
+
+### Step 2: Implement Your API Endpoint
+
+```python
+from fastapi import FastAPI
+from pydantic import BaseModel
+
+app = FastAPI()
+
+@app.get("/beta/litellm_prompt_management")
+async def get_prompt(prompt_id: str):
+    return {
+        "prompt_id": prompt_id,
+        "prompt_template": [
+            {"role": "system", "content": "You are a helpful assistant."},
+            {"role": "user", "content": "Help me with {task}"}
+        ],
+        "prompt_template_model": "gpt-4",
+        "prompt_template_optional_params": {"temperature": 0.7}
+    }
+```
+
+### Step 3: Use in Your App
+
+```python
+from litellm import completion
+
+response = completion(
+    model="gpt-4",
+    prompt_id="simple_prompt",
+    prompt_variables={"task": "data analysis"},
+    messages=[{"role": "user", "content": "I have sales data"}]
+)
+```
+
+That's it! LiteLLM fetches your prompt, applies variables, and makes the request
+
+## API Contract
+
+### Endpoint
+
+Implement `GET /beta/litellm_prompt_management`
+
+### Request Format
+
+Your endpoint will receive a GET request with query parameters:
+
+```
+GET /beta/litellm_prompt_management?prompt_id={prompt_id}&{custom_params}
+```
+
+**Query Parameters:**
+- `prompt_id` (required): The ID of the prompt to fetch
+- Custom parameters: Any additional parameters you configured in `provider_specific_query_params`
+
+**Example:**
+```
+GET /beta/litellm_prompt_management?prompt_id=hello-world-prompt-2bac&project_name=litellm&slug=hello-world-prompt-2bac
+```
+
+### Response Format
+
+```json
+{
+  "prompt_id": "hello-world-prompt-2bac",
+  "prompt_template": [
+    {
+      "role": "system",
+      "content": "You are a helpful assistant specialized in {domain}."
+    },
+    {
+      "role": "user",
+      "content": "Help me with {task}"
+    }
+  ],
+  "prompt_template_model": "gpt-4",
+  "prompt_template_optional_params": {
+    "temperature": 0.7,
+    "max_tokens": 500,
+    "top_p": 0.9
+  }
+}
+```
+
+**Response Fields:**
+- `prompt_id` (string, required): The ID of the prompt
+- `prompt_template` (array, required): Array of OpenAI-format messages with optional `{variable}` placeholders
+- `prompt_template_model` (string, optional): Model to use for this prompt (overrides client model unless `ignore_prompt_manager_model: true`)
+- `prompt_template_optional_params` (object, optional): Additional parameters like temperature, max_tokens, etc. (merged with client params unless `ignore_prompt_manager_optional_params: true`)
+
+## LiteLLM Configuration
+
+Add to `config.yaml`:
+
+```yaml
+model_list:
+  - model_name: gpt-3.5-turbo
+    litellm_params:
+      model: openai/gpt-3.5-turbo
+      api_key: os.environ/OPENAI_API_KEY
+
+prompts:
+  - prompt_id: "simple_prompt"
+    litellm_params:
+      prompt_integration: "generic_prompt_management"
+      provider_specific_query_params:
+        project_name: litellm
+        slug: hello-world-prompt-2bac
+      api_base: http://localhost:8080
+      api_key: os.environ/YOUR_PROMPT_API_KEY  # optional
+      ignore_prompt_manager_model: true  # optional, keep client's model
+      ignore_prompt_manager_optional_params: true  # optional, don't merge prompt manager's params (e.g. temperature, max_tokens, etc.)
+```
+
+### Configuration Parameters
+
+- `prompt_integration`: Must be `"generic_prompt_management"`
+- `provider_specific_query_params`: Custom query parameters sent to your API (optional)
+- `api_base`: Base URL of your prompt management API
+- `api_key`: Optional API key for authentication (sent as `Bearer` token)
+- `ignore_prompt_manager_model`: If `true`, use the model specified by client instead of prompt's model (default: `false`)
+- `ignore_prompt_manager_optional_params`: If `true`, don't merge prompt's optional params with client params (default: `false`)
+
+## Usage
+
+### Using with LiteLLM SDK
+
+**Basic usage with prompt ID:**
+
+```python
+from litellm import completion
+
+response = completion(
+    model="gpt-4",
+    prompt_id="simple_prompt",
+    messages=[{"role": "user", "content": "Additional message"}]
+)
+```
+
+**With prompt variables:**
+
+```python
+response = completion(
+    model="gpt-4",
+    prompt_id="simple_prompt",
+    prompt_variables={
+        "domain": "data science",
+        "task": "analyzing customer churn"
+    },
+    messages=[{"role": "user", "content": "Please provide a detailed analysis"}]
+)
+```
+
+The prompt template will have `{domain}` replaced with "data science" and `{task}` replaced with "analyzing customer churn".
+
+### Using with LiteLLM Proxy
+
+**1. Start the proxy with your config:**
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+**2. Make requests with prompt_id:**
+
+```bash
+curl http://0.0.0.0:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer sk-1234" \
+  -d '{
+    "model": "gpt-4",
+    "prompt_id": "simple_prompt",
+    "prompt_variables": {
+      "domain": "healthcare",
+      "task": "patient risk assessment"
+    },
+    "messages": [
+      {"role": "user", "content": "Analyze the following data..."}
+    ]
+  }'
+```
+
+**3. Using with OpenAI SDK:**
+
+```python
+from openai import OpenAI
+
+client = OpenAI(
+    base_url="http://0.0.0.0:4000",
+    api_key="sk-1234"
+)
+
+response = client.chat.completions.create(
+    model="gpt-4",
+    messages=[
+        {"role": "user", "content": "Analyze the data"}
+    ],
+    extra_body={
+        "prompt_id": "simple_prompt",
+        "prompt_variables": {
+            "domain": "finance",
+            "task": "fraud detection"
+        }
+    }
+)
+```
+
+## Implementation Example
+
+See [mock_prompt_management_server.py](https://github.com/BerriAI/litellm/blob/main/cookbook/mock_prompt_management_server/mock_prompt_management_server.py) for a complete reference implementation with multiple example prompts, authentication, and convenience endpoints.
+
+**Minimal FastAPI example:**
+
+```python
+from fastapi import FastAPI, HTTPException, Header
+from typing import Optional, Dict, Any, List
+from pydantic import BaseModel
+
+app = FastAPI()
+
+# In-memory prompt storage (replace with your database)
+PROMPTS = {
+    "hello-world-prompt": {
+        "prompt_id": "hello-world-prompt",
+        "prompt_template": [
+            {
+                "role": "system",
+                "content": "You are a helpful assistant specialized in {domain}."
+            },
+            {
+                "role": "user", 
+                "content": "Help me with: {task}"
+            }
+        ],
+        "prompt_template_model": "gpt-4",
+        "prompt_template_optional_params": {
+            "temperature": 0.7,
+            "max_tokens": 500
+        }
+    },
+    "code-review-prompt": {
+        "prompt_id": "code-review-prompt",
+        "prompt_template": [
+            {
+                "role": "system",
+                "content": "You are an expert code reviewer. Review code for {language}."
+            },
+            {
+                "role": "user",
+                "content": "Review the following code:\n\n{code}"
+            }
+        ],
+        "prompt_template_model": "gpt-4-turbo",
+        "prompt_template_optional_params": {
+            "temperature": 0.3,
+            "max_tokens": 1000
+        }
+    }
+}
+
+class PromptResponse(BaseModel):
+    prompt_id: str
+    prompt_template: List[Dict[str, str]]
+    prompt_template_model: Optional[str] = None
+    prompt_template_optional_params: Optional[Dict[str, Any]] = None
+
+@app.get("/beta/litellm_prompt_management", response_model=PromptResponse)
+async def get_prompt(
+    prompt_id: str,
+    authorization: Optional[str] = Header(None),
+    project_name: Optional[str] = None,
+    slug: Optional[str] = None,
+):
+    """
+    Get a prompt by ID with optional filtering by project_name and slug.
+    
+    Args:
+        prompt_id: The ID of the prompt to fetch
+        authorization: Optional Bearer token for authentication
+        project_name: Optional project name filter
+        slug: Optional slug filter
+    """
+    
+    # Optional: Validate authorization
+    if authorization:
+        token = authorization.replace("Bearer ", "")
+        # Validate your token here
+        if not is_valid_token(token):
+            raise HTTPException(status_code=401, detail="Invalid API key")
+    
+    # Optional: Apply additional filtering based on custom params
+    if project_name or slug:
+        # You can use these parameters to filter or validate access
+        # For example, check if the user has access to this project
+        pass
+    
+    # Fetch the prompt from your storage
+    if prompt_id not in PROMPTS:
+        raise HTTPException(
+            status_code=404,
+            detail=f"Prompt '{prompt_id}' not found"
+        )
+    
+    prompt_data = PROMPTS[prompt_id]
+    
+    return PromptResponse(**prompt_data)
+
+def is_valid_token(token: str) -> bool:
+    """Validate API token - implement your logic here"""
+    # Example: Check against your database or secret store
+    valid_tokens = ["your-secret-token", "another-valid-token"]
+    return token in valid_tokens
+
+# Optional: Health check endpoint
+@app.get("/health")
+async def health_check():
+    return {"status": "healthy"}
+
+# Optional: List all prompts endpoint
+@app.get("/prompts")
+async def list_prompts(authorization: Optional[str] = Header(None)):
+    """List all available prompts"""
+    if authorization:
+        token = authorization.replace("Bearer ", "")
+        if not is_valid_token(token):
+            raise HTTPException(status_code=401, detail="Invalid API key")
+    
+    return {
+        "prompts": [
+            {"prompt_id": pid, "model": p.get("prompt_template_model")}
+            for pid, p in PROMPTS.items()
+        ]
+    }
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=8080)
+```
+
+### Running the Example Server
+
+1. Install dependencies:
+```bash
+pip install fastapi uvicorn
+```
+
+2. Save the code above to `prompt_server.py`
+
+3. Run the server:
+```bash
+python prompt_server.py
+```
+
+4. Test the endpoint:
+```bash
+curl "http://localhost:8080/beta/litellm_prompt_management?prompt_id=hello-world-prompt&project_name=litellm&slug=hello-world-prompt-2bac"
+```
+
+Expected response:
+```json
+{
+  "prompt_id": "hello-world-prompt",
+  "prompt_template": [
+    {
+      "role": "system",
+      "content": "You are a helpful assistant specialized in {domain}."
+    },
+    {
+      "role": "user",
+      "content": "Help me with: {task}"
+    }
+  ],
+  "prompt_template_model": "gpt-4",
+  "prompt_template_optional_params": {
+    "temperature": 0.7,
+    "max_tokens": 500
+  }
+}
+```
+
+## Advanced Features
+
+### Variable Substitution
+
+LiteLLM automatically substitutes variables in your prompt templates using the `{variable}` syntax. Both `{variable}` and `{{variable}}` formats are supported.
+
+**Example prompt template:**
+```json
+{
+  "prompt_template": [
+    {
+      "role": "system",
+      "content": "You are an expert in {domain} with {years} years of experience."
+    }
+  ]
+}
+```
+
+**Client request:**
+```python
+completion(
+    model="gpt-4",
+    prompt_id="expert_prompt",
+    prompt_variables={
+        "domain": "machine learning",
+        "years": "10"
+    }
+)
+```
+
+**Result:**
+```
+"You are an expert in machine learning with 10 years of experience."
+```
+
+### Caching
+
+LiteLLM automatically caches fetched prompts in memory. The cache key includes:
+- `prompt_id`
+- `prompt_label` (if provided)
+- `prompt_version` (if provided)
+
+This means your API endpoint is only called once per unique prompt configuration.
+
+### Model Override Behavior
+
+**Default behavior (without `ignore_prompt_manager_model`):**
+```yaml
+prompts:
+  - prompt_id: "my_prompt"
+    litellm_params:
+      prompt_integration: "generic_prompt_management"
+      api_base: http://localhost:8080
+```
+
+If your API returns `"prompt_template_model": "gpt-4"`, LiteLLM will use `gpt-4` regardless of what the client specified.
+
+**With `ignore_prompt_manager_model: true`:**
+```yaml
+prompts:
+  - prompt_id: "my_prompt"
+    litellm_params:
+      prompt_integration: "generic_prompt_management"
+      api_base: http://localhost:8080
+      ignore_prompt_manager_model: true
+```
+
+LiteLLM will use the model specified by the client, ignoring the prompt's model.
+
+### Parameter Merging Behavior
+
+**Default behavior (without `ignore_prompt_manager_optional_params`):**
+
+Client params are merged with prompt params, with prompt params taking precedence:
+```python
+# Prompt returns: {"temperature": 0.7, "max_tokens": 500}
+# Client sends: {"temperature": 0.9, "top_p": 0.95}
+# Final params: {"temperature": 0.7, "max_tokens": 500, "top_p": 0.95}
+```
+
+**With `ignore_prompt_manager_optional_params: true`:**
+
+Only client params are used:
+```python
+# Prompt returns: {"temperature": 0.7, "max_tokens": 500}
+# Client sends: {"temperature": 0.9, "top_p": 0.95}
+# Final params: {"temperature": 0.9, "top_p": 0.95}
+```
+
+## Security Considerations
+
+1. **Authentication**: Use the `api_key` parameter to secure your prompt management API
+2. **Authorization**: Implement team/user-based access control using the custom query parameters
+3. **Rate Limiting**: Add rate limiting to prevent abuse of your API
+4. **Input Validation**: Validate all query parameters before processing
+5. **HTTPS**: Always use HTTPS in production for encrypted communication
+6. **Secrets**: Store API keys in environment variables, not in config files
+
+## Use Cases
+
+✅ **Use Generic Prompt Management API when:**
+- You want instant integration without waiting for PRs
+- You maintain your own prompt management service
+- You need full control over prompt versioning and updates
+- You want to build custom prompt management features
+- You need to integrate with your internal systems
+
+✅ **Common scenarios:**
+- Internal prompt management system for your organization
+- Multi-tenant prompt management with team-based access control
+- A/B testing different prompt versions
+- Prompt experimentation and analytics
+- Integration with existing prompt engineering workflows
+
+## When to Use This
+
+✅ **Use Generic Prompt Management API when:**
+- You want instant integration without waiting for PRs
+- You maintain your own prompt management service
+- You need full control over updates and features
+- You want custom prompt storage and versioning logic
+
+❌ **Make a PR when:**
+- You want deeper integration with LiteLLM internals
+- Your integration requires complex LiteLLM-specific logic
+- You want to be featured as a built-in provider
+- You're building a reusable integration for the community
+
+## Troubleshooting
+
+### Prompt not found
+- Verify the `prompt_id` matches exactly (case-sensitive)
+- Check that your API endpoint is accessible from LiteLLM
+- Verify authentication if using `api_key`
+
+### Variables not substituted
+- Ensure variables use `{variable}` or `{{variable}}` syntax
+- Check that variable names in `prompt_variables` match template exactly
+- Variables are case-sensitive
+
+### Model not being overridden
+- Check if `ignore_prompt_manager_model: true` is set in config
+- Verify your API is returning `prompt_template_model` in the response
+
+### Parameters not being applied
+- Check if `ignore_prompt_manager_optional_params: true` is set
+- Verify your API is returning `prompt_template_optional_params`
+- Ensure parameter names match OpenAI's parameter names
+
+## Questions?
+
+This is a **beta API**. We're actively improving it based on feedback. Open an issue or PR if you need additional capabilities.
+
+## Related Documentation
+
+- [Prompt Management Overview](../proxy/prompt_management.md)
+- [Generic Guardrail API](./generic_guardrail_api.md)
+- [LiteLLM Proxy Setup](../proxy/quick_start.md)
+

docs/my-website/docs/anthropic_count_tokens.md

@@ -138,6 +138,7 @@ The `/v1/messages/count_tokens` endpoint automatically routes to the appropriate
 | Provider | Token Counting Method |
 |----------|----------------------|
 | Anthropic | [Anthropic Token Counting API](https://docs.anthropic.com/en/docs/build-with-claude/token-counting) |
+| OpenAI | [OpenAI Responses API `/input_tokens`](https://platform.openai.com/docs/api-reference/responses/input-tokens) — see [Token Counting](./count_tokens.md) |
 | Vertex AI (Claude) | Vertex AI Partner Models Token Counter |
 | Bedrock (Claude) | AWS Bedrock CountTokens API |
 | Gemini | Google AI Studio countTokens API |

docs/my-website/docs/anthropic_unified/index.md

@@ -506,12 +506,15 @@ Request body will be in the Anthropic messages API format. **litellm follows the
   A system prompt providing context or specific instructions to the model.
 - **temperature** (number):  
   Controls randomness in the model's responses. Valid range: `0 < temperature < 1`.
-- **thinking** (object):  
+- **thinking** (object):
   Configuration for enabling extended thinking. If enabled, it includes:
-  - **budget_tokens** (integer):  
+  - **budget_tokens** (integer):
     Minimum of 1024 tokens (and less than `max_tokens`).
-  - **type** (enum):  
+  - **type** (enum):
     E.g., `"enabled"`.
+  - **summary** (string, optional):
+    Enables the summary style for thinking blocks. Possible values: `"auto"`, `"concise"`, `"detailed"`, `"disabled"`.
+    When routing to non-Anthropic providers (e.g., `openai/gpt-5.1`), the `summary` value is preserved and forwarded to the downstream API.
 - **tool_choice** (object):  
   Instructs how the model should utilize any provided tools.
 - **tools** (array of objects):  

docs/my-website/docs/anthropic_unified/messages_to_responses_mapping.md

@@ -0,0 +1,120 @@
+# v1/messages → /responses Parameter Mapping
+
+When you send a request to `/v1/messages` targeting an OpenAI or Azure model, LiteLLM internally routes it through the OpenAI Responses API. This page documents exactly how every parameter gets translated in both directions.
+
+The transformation lives in `litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py`.
+
+
+## Request: Anthropic → Responses API
+
+### Top-level parameters
+
+| Anthropic (`/v1/messages`) | Responses API | Notes |
+|---|---|---|
+| `model` | `model` | Passed through as-is |
+| `messages` | `input` | Structurally transformed — see the messages section below |
+| `system` (string) | `instructions` | Passed as a plain string |
+| `system` (list of content blocks) | `instructions` | Text blocks are joined with `\n`; non-text blocks are ignored |
+| `max_tokens` | `max_output_tokens` | Renamed |
+| `temperature` | `temperature` | Passed through as-is |
+| `top_p` | `top_p` | Passed through as-is |
+| `tools` | `tools` | Format-translated — see the tools section below |
+| `tool_choice` | `tool_choice` | Type-remapped — see the tool_choice section below |
+| `thinking` | `reasoning` | Budget tokens mapped to effort level — see the thinking section below |
+| `output_format` or `output_config.format` | `text` | Wrapped as `{"format": {"type": "json_schema", "name": "structured_output", "schema": ..., "strict": true}}` |
+| `context_management` | `context_management` | Converted from Anthropic dict to OpenAI array format — see the context_management section below |
+| `metadata.user_id` | `user` | Extracted from the metadata object and truncated to 64 characters |
+| `stop_sequences` | ❌ Not mapped | Dropped silently |
+| `top_k` | ❌ Not mapped | Dropped silently |
+| `speed` | ❌ Not mapped | Only used to set Anthropic beta headers on the native path |
+
+
+### How messages get converted
+
+Each Anthropic message is expanded into one or more Responses API input items. The key difference is that `tool_result` and `tool_use` blocks become **top-level items** in the input array rather than being nested inside a message.
+
+| Anthropic message | Responses API input item |
+|---|---|
+| `user` role, string content | `{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "..."}]}` |
+| `user` role, `{"type": "text"}` block | `{"type": "input_text", "text": "..."}` inside a user message |
+| `user` role, `{"type": "image", "source": {"type": "base64"}}` | `{"type": "input_image", "image_url": "data:<media_type>;base64,<data>"}` inside a user message |
+| `user` role, `{"type": "image", "source": {"type": "url"}}` | `{"type": "input_image", "image_url": "<url>"}` inside a user message |
+| `user` role, `{"type": "tool_result"}` block | Top-level `{"type": "function_call_output", "call_id": "...", "output": "..."}` — pulled out of the message entirely |
+| `assistant` role, string content | `{"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "..."}]}` |
+| `assistant` role, `{"type": "text"}` block | `{"type": "output_text", "text": "..."}` inside an assistant message |
+| `assistant` role, `{"type": "tool_use"}` block | Top-level `{"type": "function_call", "call_id": "<id>", "name": "...", "arguments": "<JSON string>"}` — pulled out of the message entirely |
+| `assistant` role, `{"type": "thinking"}` block | `{"type": "output_text", "text": "<thinking text>"}` inside an assistant message |
+
+
+### tools
+
+| Anthropic tool | Responses API tool |
+|---|---|
+| Any tool where `type` starts with `"web_search"` or `name == "web_search"` | `{"type": "web_search_preview"}` |
+| All other tools | `{"type": "function", "name": "...", "description": "...", "parameters": <input_schema>}` |
+
+
+### tool_choice
+
+| Anthropic `tool_choice.type` | Responses API `tool_choice` |
+|---|---|
+| `"auto"` | `{"type": "auto"}` |
+| `"any"` | `{"type": "required"}` |
+| `"tool"` | `{"type": "function", "name": "<tool name>"}` |
+
+
+### thinking → reasoning
+
+The `budget_tokens` value is mapped to a string effort level. `summary` is always set to `"detailed"`.
+
+| `thinking.budget_tokens` | `reasoning.effort` |
+|---|---|
+| >= 10000 | `"high"` |
+| >= 5000 | `"medium"` |
+| >= 2000 | `"low"` |
+| < 2000 | `"minimal"` |
+
+If `thinking.type` is anything other than `"enabled"`, the `reasoning` field is not sent at all.
+
+
+### context_management
+
+Anthropic uses a nested dict with an `edits` array. OpenAI uses a flat array of compaction objects.
+
+```
+Anthropic input:
+{
+  "edits": [
+    {
+      "type": "compact_20260112",
+      "trigger": {"type": "input_tokens", "value": 150000}
+    }
+  ]
+}
+
+Responses API output:
+[
+  {"type": "compaction", "compact_threshold": 150000}
+]
+```
+
+
+## Response: Responses API → Anthropic
+
+When the Responses API reply comes back, LiteLLM converts it into an Anthropic `AnthropicMessagesResponse`.
+
+| Responses API field | Anthropic response field | Notes |
+|---|---|---|
+| `response.id` | `id` | |
+| `response.model` | `model` | Falls back to `"unknown-model"` if missing |
+| `ResponseReasoningItem` — `summary[*].text` | `content` block `{"type": "thinking", "thinking": "..."}` | Each non-empty summary text becomes a thinking block |
+| `ResponseOutputMessage` — `content[*]` where `type == "output_text"` | `content` block `{"type": "text", "text": "..."}` | |
+| `ResponseFunctionToolCall` — `{call_id, name, arguments}` | `content` block `{"type": "tool_use", "id": "...", "name": "...", "input": {...}}` | `arguments` is JSON-parsed back into a dict |
+| Any `function_call` present in output | `stop_reason: "tool_use"` | |
+| `response.status == "incomplete"` | `stop_reason: "max_tokens"` | Takes precedence over the default |
+| Everything else | `stop_reason: "end_turn"` | Default |
+| `response.usage.input_tokens` | `usage.input_tokens` | |
+| `response.usage.output_tokens` | `usage.output_tokens` | |
+| *(hardcoded)* | `type: "message"` | Always set |
+| *(hardcoded)* | `role: "assistant"` | Always set |
+| *(hardcoded)* | `stop_sequence: null` | Always null on this path |

docs/my-website/docs/apply_guardrail.md

@@ -11,6 +11,7 @@ This endpoint supports various guardrail types including:
 - **Presidio** - PII detection and masking
 - **Bedrock** - AWS Bedrock guardrails for content moderation
 - **Lakera** - AI safety guardrails
+- **PANW Prisma AIRS** - Threat detection, DLP, and policy enforcement
 - **Custom guardrails** - User-defined guardrails
 
 ## Configuration

docs/my-website/docs/audio_transcription.md

@@ -13,7 +13,7 @@ import TabItem from '@theme/TabItem';
 | Fallbacks | ✅ | Works between supported models |
 | Loadbalancing | ✅ | Works between supported models |
 | Guardrails | ✅ | Applies to output transcribed text (non-streaming only) |
-| Supported Providers | `openai`, `azure`, `vertex_ai`, `gemini`, `deepgram`, `groq`, `fireworks_ai`, `ovhcloud` | |
+| Supported Providers | `openai`, `azure`, `vertex_ai`, `gemini`, `deepgram`, `groq`, `fireworks_ai`, `ovhcloud`, `mistral` | |
 
 ## Quick Start
 
@@ -126,6 +126,7 @@ transcript = client.audio.transcriptions.create(
 - [Fireworks AI](./providers/fireworks_ai.md#audio-transcription)
 - [Groq](./providers/groq.md#speech-to-text---whisper)
 - [Deepgram](./providers/deepgram.md)
+- [Mistral (Voxtral)](./providers/mistral.md#audio-transcription)
 - [OVHcloud AI Endpoints](./providers/ovhcloud.md)
 
 ---

docs/my-website/docs/benchmarks.md

@@ -5,6 +5,44 @@ import Image from '@theme/IdealImage';
 
 Benchmarks for LiteLLM Gateway (Proxy Server) tested against a fake OpenAI endpoint.
 
+## Setting Up Benchmarking with Network Mock
+
+The fastest way to benchmark proxy overhead is using `network_mock` mode. This intercepts outbound requests at the httpx transport layer and returns canned responses, no need for setting up a mock provider. 
+
+**1. Create a proxy config:**
+
+```yaml
+model_list:
+  - model_name: db-openai-endpoint
+    litellm_params:
+      model: openai/gpt-4o
+      api_key: "sk-fake-key"
+      api_base: "https://api.openai.com"
+
+litellm_settings:
+  network_mock: true
+  callbacks: []
+  num_retries: 0
+  request_timeout: 30
+
+general_settings:
+  master_key: "sk-1234"
+```
+
+**2. Start the proxy:**
+
+```bash
+litellm --config benchmark_config.yaml --port 4000 --num_workers 8
+```
+
+**3. Run the benchmark script:**
+
+```bash
+python scripts/benchmark_mock.py --requests 2000 --max-concurrent 200 --runs 3
+```
+
+This measures pure proxy overhead on the hot path without any network latency to a real or fake provider.
+
 ## Setting Up a Fake OpenAI Endpoint
 
 For load testing and benchmarking, you can use a fake OpenAI proxy server. LiteLLM provides:

docs/my-website/docs/caching/all_caches.md

@@ -297,6 +297,7 @@ litellm.cache = Cache(
     similarity_threshold=0.7, # similarity threshold for cache hits, 0 == no similarity, 1 = exact matches, 0.5 == 50% similarity
     qdrant_quantization_config ="binary", # can be one of 'binary', 'product' or 'scalar' quantizations that is supported by qdrant
     qdrant_semantic_cache_embedding_model="text-embedding-ada-002", # this model is passed to litellm.embedding(), any litellm.embedding() model is supported here
+    qdrant_semantic_cache_vector_size=1536, # vector size for the embedding model, must match the dimensionality of the embedding model used
 )
 
 response1 = completion(
@@ -635,6 +636,7 @@ def __init__(
     qdrant_quantization_config: Optional[str] = None,
     qdrant_semantic_cache_embedding_model="text-embedding-ada-002",
 
+    qdrant_semantic_cache_vector_size: Optional[int] = None,
     **kwargs
 ):
 ```

docs/my-website/docs/completion/message_sanitization.md

@@ -0,0 +1,465 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Message Sanitization for Tool Calling for anthropic models
+
+**Automatically fix common message formatting issues when using tool calling with `modify_params=True`**
+
+LiteLLM can automatically sanitize messages to handle common issues that occur during tool calling workflows, especially when using OpenAI-compatible clients with providers that have strict message format requirements (like Anthropic Claude).
+
+## Overview
+
+When `litellm.modify_params = True` is enabled, LiteLLM automatically sanitizes messages to fix three common issues:
+
+1. **Orphaned Tool Calls** - Assistant messages with tool_calls but missing tool results
+2. **Orphaned Tool Results** - Tool messages that reference non-existent tool_call_ids
+3. **Empty Message Content** - Messages with empty or whitespace-only text content
+
+This ensures your tool calling workflows work seamlessly across different LLM providers without manual message validation.
+
+## Why Message Sanitization?
+
+Different LLM providers have varying requirements for message formats, especially during tool calling:
+
+- **Anthropic Claude** requires every tool_call to have a corresponding tool result
+- Some providers reject messages with empty content
+- OpenAI-compatible clients may not always maintain perfect message consistency
+
+Without sanitization, these issues cause API errors that interrupt your workflows. With `modify_params=True`, LiteLLM handles these edge cases automatically.
+
+## Quick Start
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+import litellm
+
+# Enable automatic message sanitization
+litellm.modify_params = True
+
+# This will work even if messages have formatting issues
+response = litellm.completion(
+    model="anthropic/claude-3-5-sonnet-20241022",
+    messages=[
+        {"role": "user", "content": "What's the weather in Boston?"},
+        {
+            "role": "assistant",
+            "tool_calls": [
+                {
+                    "id": "call_123",
+                    "type": "function",
+                    "function": {"name": "get_weather", "arguments": '{"city": "Boston"}'}
+                }
+            ]
+            # Missing tool result - LiteLLM will add a dummy result automatically
+        },
+        {"role": "user", "content": "Thanks!"}
+    ],
+    tools=[{
+        "type": "function",
+        "function": {
+            "name": "get_weather",
+            "description": "Get weather for a city",
+            "parameters": {
+                "type": "object",
+                "properties": {"city": {"type": "string"}},
+                "required": ["city"]
+            }
+        }
+    }]
+)
+```
+
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+```yaml
+litellm_settings:
+  modify_params: true  # Enable automatic message sanitization
+
+model_list:
+  - model_name: claude-3-5-sonnet
+    litellm_params:
+      model: anthropic/claude-3-5-sonnet-20241022
+```
+
+</TabItem>
+</Tabs>
+
+## Sanitization Cases
+
+### Case A: Orphaned Tool Calls (Missing Tool Results)
+
+**Problem:** An assistant message contains `tool_calls`, but no corresponding tool result messages follow.
+
+**Solution:** LiteLLM automatically adds dummy tool result messages for any missing tool results.
+
+**Example:**
+
+```python
+import litellm
+litellm.modify_params = True
+
+# Messages with orphaned tool calls
+messages = [
+    {"role": "user", "content": "Search for Python tutorials"},
+    {
+        "role": "assistant",
+        "tool_calls": [
+            {
+                "id": "call_abc123",
+                "type": "function",
+                "function": {"name": "web_search", "arguments": '{"query": "Python tutorials"}'}
+            }
+        ]
+    },
+    # Missing tool result here!
+    {"role": "user", "content": "What about JavaScript?"}
+]
+
+# LiteLLM automatically adds:
+# {
+#     "role": "tool",
+#     "tool_call_id": "call_abc123",
+#     "content": "[System: Tool execution skipped/interrupted by user. No result provided for tool 'web_search'.]"
+# }
+
+response = litellm.completion(
+    model="anthropic/claude-3-5-sonnet-20241022",
+    messages=messages,
+    tools=[...]
+)
+```
+
+**When this happens:**
+- User interrupts tool execution
+- Client loses tool results due to network issues
+- Conversation flow changes before tool completes
+- Multi-turn conversations where tools are optional
+
+### Case B: Orphaned Tool Results (Invalid tool_call_id)
+
+**Problem:** A tool message references a `tool_call_id` that doesn't exist in any previous assistant message.
+
+**Solution:** LiteLLM automatically removes these orphaned tool result messages.
+
+**Example:**
+
+```python
+import litellm
+litellm.modify_params = True
+
+# Messages with orphaned tool result
+messages = [
+    {"role": "user", "content": "Hello"},
+    {"role": "assistant", "content": "Hi! How can I help?"},
+    {
+        "role": "tool",
+        "tool_call_id": "call_nonexistent",  # This tool_call_id doesn't exist!
+        "content": "Some result"
+    }
+]
+
+# LiteLLM automatically removes the orphaned tool message
+
+response = litellm.completion(
+    model="anthropic/claude-3-5-sonnet-20241022",
+    messages=messages
+)
+```
+
+**When this happens:**
+- Message history is manually edited
+- Tool results are duplicated or mismatched
+- Conversation state is restored incorrectly
+- Messages are merged from different conversations
+
+### Case C: Empty Message Content
+
+**Problem:** User or assistant messages have empty or whitespace-only content.
+
+**Solution:** LiteLLM replaces empty content with a system placeholder message.
+
+**Example:**
+
+```python
+import litellm
+litellm.modify_params = True
+
+# Messages with empty content
+messages = [
+    {"role": "user", "content": ""},  # Empty content
+    {"role": "assistant", "content": "   "},  # Whitespace only
+]
+
+# LiteLLM automatically replaces with:
+# {"role": "user", "content": "[System: Empty message content sanitised to satisfy protocol]"}
+# {"role": "assistant", "content": "[System: Empty message content sanitised to satisfy protocol]"}
+
+response = litellm.completion(
+    model="anthropic/claude-3-5-sonnet-20241022",
+    messages=messages
+)
+```
+
+**When this happens:**
+- UI sends empty messages
+- Content is stripped during preprocessing
+- Placeholder messages in conversation history
+- Edge cases in message construction
+
+## Configuration
+
+### Enable Globally
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+import litellm
+
+# Enable for all completion calls
+litellm.modify_params = True
+```
+
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+```yaml
+litellm_settings:
+  modify_params: true
+```
+
+</TabItem>
+<TabItem value="env" label="Environment Variable">
+
+```bash
+export LITELLM_MODIFY_PARAMS=True
+```
+
+</TabItem>
+</Tabs>
+
+### Enable Per-Request
+
+```python
+import litellm
+
+# Enable only for specific requests
+response = litellm.completion(
+    model="anthropic/claude-3-5-sonnet-20241022",
+    messages=messages,
+    modify_params=True  # Override global setting
+)
+```
+
+## Supported Providers
+
+Message sanitization currently works with:
+
+- ✅ Anthropic (Claude)
+
+**Note:** While the sanitization logic is provider-agnostic, it is currently only applied in the Anthropic message transformation pipeline. Support for additional providers may be added in future releases.
+
+## Implementation Details
+
+### How It Works
+
+The message sanitization process runs **before** messages are converted to provider-specific formats:
+
+1. **Input:** OpenAI-format messages with potential issues
+2. **Sanitization:** Three helper functions process the messages:
+   - `_sanitize_empty_text_content()` - Fixes empty content
+   - `_add_missing_tool_results()` - Adds dummy tool results
+   - `_is_orphaned_tool_result()` - Identifies orphaned results
+3. **Output:** Clean, provider-compatible messages
+
+### Code Reference
+
+The sanitization logic is implemented in:
+- `litellm/litellm_core_utils/prompt_templates/factory.py`
+- Function: `sanitize_messages_for_tool_calling()`
+
+### Logging
+
+When sanitization occurs, LiteLLM logs debug messages:
+
+```python
+import litellm
+litellm.set_verbose = True  # Enable debug logging
+
+# You'll see logs like:
+# "_add_missing_tool_results: Found 1 orphaned tool calls. Adding dummy tool results."
+# "_is_orphaned_tool_result: Found orphaned tool result with tool_call_id=call_123"
+# "_sanitize_empty_text_content: Replaced empty text content in user message"
+```
+
+## Best Practices
+
+### 1. Enable for Production Workflows
+
+```python
+# Recommended for production
+litellm.modify_params = True
+
+# Ensures robust handling of edge cases
+response = litellm.completion(
+    model="anthropic/claude-3-5-sonnet-20241022",
+    messages=messages,
+    tools=tools
+)
+```
+
+### 2. Preserve Tool Results When Possible
+
+While sanitization handles missing tool results, it's better to provide actual results:
+
+```python
+# Good: Provide actual tool results
+messages = [
+    {"role": "user", "content": "Search for Python"},
+    {"role": "assistant", "tool_calls": [...]},
+    {"role": "tool", "tool_call_id": "call_123", "content": "Actual search results"}
+]
+
+# Fallback: Sanitization adds dummy result if missing
+messages = [
+    {"role": "user", "content": "Search for Python"},
+    {"role": "assistant", "tool_calls": [...]},
+    # Missing tool result - sanitization adds dummy
+]
+```
+
+### 3. Monitor Sanitization Events
+
+Use logging to track when sanitization occurs:
+
+```python
+import litellm
+import logging
+
+# Enable debug logging
+litellm.set_verbose = True
+logging.basicConfig(level=logging.DEBUG)
+
+# Track sanitization events in your application
+response = litellm.completion(
+    model="anthropic/claude-3-5-sonnet-20241022",
+    messages=messages
+)
+```
+
+### 4. Test Edge Cases
+
+Ensure your application handles sanitized messages correctly:
+
+```python
+import litellm
+litellm.modify_params = True
+
+# Test orphaned tool calls
+test_messages = [
+    {"role": "user", "content": "Test"},
+    {"role": "assistant", "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "test", "arguments": "{}"}}]},
+    {"role": "user", "content": "Continue"}  # No tool result
+]
+
+response = litellm.completion(
+    model="anthropic/claude-3-5-sonnet-20241022",
+    messages=test_messages,
+    tools=[...]
+)
+
+# Verify the response handles the dummy tool result appropriately
+```
+
+## Related Features
+
+- **[Drop Params](./drop_params.md)** - Drop unsupported parameters for specific providers
+- **[Message Trimming](./message_trimming.md)** - Trim messages to fit token limits
+- **[Function Calling](./function_call.md)** - Complete guide to tool/function calling
+- **[Reasoning Content](../reasoning_content.md)** - Extended thinking with tool calling
+
+## Troubleshooting
+
+### Sanitization Not Working
+
+**Issue:** Messages still cause errors despite `modify_params=True`
+
+**Solution:**
+1. Verify `modify_params` is enabled:
+   ```python
+   import litellm
+   print(litellm.modify_params)  # Should be True
+   ```
+
+2. Check if the issue is provider-specific:
+   ```python
+   litellm.set_verbose = True  # Enable debug logging
+   ```
+
+3. Ensure you're using a recent version of LiteLLM:
+   ```bash
+   pip install --upgrade litellm
+   ```
+
+### Unexpected Dummy Tool Results
+
+**Issue:** Dummy tool results appear when you expect actual results
+
+**Cause:** Tool result messages are missing or have incorrect `tool_call_id`
+
+**Solution:**
+1. Verify tool result messages have correct `tool_call_id`:
+   ```python
+   # Correct
+   {"role": "tool", "tool_call_id": "call_123", "content": "result"}
+   
+   # Incorrect - will be treated as orphaned
+   {"role": "tool", "tool_call_id": "wrong_id", "content": "result"}
+   ```
+
+2. Ensure tool results immediately follow assistant messages with tool_calls
+
+### Performance Impact
+
+**Issue:** Concerned about performance overhead
+
+**Details:** Message sanitization has minimal performance impact:
+- Runs in O(n) time where n = number of messages
+- Only processes messages when `modify_params=True`
+- Typically adds < 1ms to request processing time
+
+## FAQ
+
+**Q: Does sanitization modify my original messages?**
+
+A: No, sanitization creates a new list of messages. Your original messages remain unchanged.
+
+**Q: Can I disable specific sanitization cases?**
+
+A: Currently, all three cases are handled together when `modify_params=True`. To disable sanitization entirely, set `modify_params=False`.
+
+**Q: What happens to the dummy tool results?**
+
+A: Dummy tool results are sent to the LLM provider along with other messages. The model sees them as regular tool results with informative error messages.
+
+**Q: Does this work with streaming?**
+
+A: Yes, message sanitization works with both streaming and non-streaming requests.
+
+**Q: Is this related to `drop_params`?**
+
+A: No, they're separate features:
+- `modify_params` - Modifies/fixes message content and structure
+- `drop_params` - Removes unsupported API parameters
+
+Both can be enabled simultaneously.
+
+## See Also
+
+- [Reasoning Content with Tool Calling](../reasoning_content.md)
+- [Function Calling Guide](./function_call.md)
+- [Bedrock Provider Documentation](../providers/bedrock.md)
+- [Anthropic Provider Documentation](../providers/anthropic.md)

docs/my-website/docs/completion/output.md

@@ -51,6 +51,28 @@ Here's what an example response looks like
 }
 ```
 
+## Native Finish Reason
+
+LiteLLM maps all provider-specific `finish_reason` values to OpenAI-compatible values (`stop`, `length`, `tool_calls`, `function_call`, `content_filter`). When the original provider value differs from the mapped value, it is preserved in `provider_specific_fields["native_finish_reason"]`.
+
+This is useful for agent loops that need to distinguish between different stop conditions (e.g., Gemini's `MALFORMED_FUNCTION_CALL` vs a normal `stop`).
+
+```python
+response = completion(model="gemini/gemini-2.0-flash", messages=messages)
+
+choice = response.choices[0]
+print(choice.finish_reason)  # "stop" (OpenAI-compatible)
+
+# Access the original provider value when it differs:
+if hasattr(choice, "provider_specific_fields") and choice.provider_specific_fields:
+    native = choice.provider_specific_fields.get("native_finish_reason")
+    if native == "MALFORMED_FUNCTION_CALL":
+        # Handle malformed function call differently from a normal stop
+        pass
+```
+
+When the provider already returns an OpenAI-compatible value (e.g., `stop`), `native_finish_reason` is not set.
+
 ## Additional Attributes
 
 You can also access information like latency. 

docs/my-website/docs/completion/prompt_caching.md

@@ -6,6 +6,8 @@ import TabItem from '@theme/TabItem';
 Supported Providers:
 - OpenAI (`openai/`)
 - Anthropic API (`anthropic/`)
+- Google AI Studio (`gemini/`)
+- Vertex AI (`vertex_ai/`, `vertex_ai_beta/`)
 - Bedrock (`bedrock/`, `bedrock/invoke/`, `bedrock/converse`) ([All models bedrock supports prompt caching on](https://docs.aws.amazon.com/bedrock/latest/userguide/prompt-caching.html))
 - Deepseek API (`deepseek/`)
 
@@ -63,7 +65,6 @@ for _ in range(2):
                     }
                 ],
             },
-            # marked for caching with the cache_control parameter, so that this checkpoint can read from the previous cache.
             {
                 "role": "user",
                 "content": [
@@ -77,7 +78,6 @@ for _ in range(2):
                 "role": "assistant",
                 "content": "Certainly! the key terms and conditions are the following: the contract is 1 year long for $10/mo",
             },
-            # The final turn is marked with cache-control, for continuing in followups.
             {
                 "role": "user",
                 "content": [
@@ -112,16 +112,16 @@ model_list:
         api_key: os.environ/OPENAI_API_KEY
 ```
 
-2. Start proxy 
+2. Start proxy
 
 ```bash
 litellm --config /path/to/config.yaml
 ```
 
-3. Test it! 
+3. Test it!
 
 ```python
-from openai import OpenAI 
+from openai import OpenAI
 import os
 
 client = OpenAI(
@@ -144,7 +144,6 @@ for _ in range(2):
                     }
                 ],
             },
-            # marked for caching with the cache_control parameter, so that this checkpoint can read from the previous cache.
             {
                 "role": "user",
                 "content": [
@@ -158,7 +157,6 @@ for _ in range(2):
                 "role": "assistant",
                 "content": "Certainly! the key terms and conditions are the following: the contract is 1 year long for $10/mo",
             },
-            # The final turn is marked with cache-control, for continuing in followups.
             {
                 "role": "user",
                 "content": [
@@ -183,13 +181,85 @@ assert response.usage.prompt_tokens_details.cached_tokens > 0
 </TabItem>
 </Tabs>
 
+### OpenAI `prompt_cache_key` and `prompt_cache_retention`
+
+OpenAI prompt caching is [**automatic**](https://platform.openai.com/docs/guides/prompt-caching) — no `cache_control` message annotations are needed. Any request with 1024+ prompt tokens is eligible for caching.
+
+OpenAI also supports two optional parameters for more control over caching behavior:
+
+- **`prompt_cache_key`** (string) — A routing hint that improves cache hit rates for requests sharing long common prefixes. Requests with the same cache key are routed to the same backend, increasing the likelihood of a cache hit.
+- **`prompt_cache_retention`** (`"in_memory"` or `"24h"`) — Controls cache TTL. Default is `"in_memory"` (5–10 min). Set to `"24h"` for extended caching that offloads KV tensors to GPU-local storage.
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+from litellm import completion
+import os
+
+os.environ["OPENAI_API_KEY"] = ""
+
+response = completion(
+    model="gpt-4o",
+    messages=[
+        {
+            "role": "system",
+            "content": "You are an AI assistant tasked with analyzing legal documents. "
+            + "Here is the full text of a complex legal agreement " * 400,
+        },
+        {
+            "role": "user",
+            "content": "What are the key terms and conditions?",
+        },
+    ],
+    prompt_cache_key="legal-doc-analysis",
+    prompt_cache_retention="24h",
+)
+print(response.usage)
+```
+
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+```python
+from openai import OpenAI
+
+client = OpenAI(
+    api_key="LITELLM_PROXY_KEY",
+    base_url="LITELLM_PROXY_BASE",
+)
+
+response = client.chat.completions.create(
+    model="gpt-4o",
+    messages=[
+        {
+            "role": "system",
+            "content": "You are an AI assistant tasked with analyzing legal documents. "
+            + "Here is the full text of a complex legal agreement " * 400,
+        },
+        {
+            "role": "user",
+            "content": "What are the key terms and conditions?",
+        },
+    ],
+    extra_body={
+        "prompt_cache_key": "legal-doc-analysis",
+        "prompt_cache_retention": "24h",
+    },
+)
+print(response.usage)
+```
+
+</TabItem>
+</Tabs>
+
 ### Anthropic Example 
 
 Anthropic charges for cache writes. 
 
 Specify the content to cache with `"cache_control": {"type": "ephemeral"}`.
 
-If you pass that in for any other llm provider, it will be ignored. 
+This same format also works for [Gemini / Vertex AI](#google-ai-studio--vertex-ai-gemini-example). For other providers, it will be ignored.
 
 <Tabs>
 <TabItem value="sdk" label="SDK">
@@ -288,6 +358,208 @@ print(response.usage)
 </TabItem>
 </Tabs>
 
+### Google AI Studio / Vertex AI (Gemini) Example
+
+Use the same Anthropic-style `cache_control` format — LiteLLM automatically translates it to Google's [context caching API](https://ai.google.dev/api/caching).
+
+**How it works under the hood:**
+1. Messages with `cache_control` are separated and sent to Google's `cachedContents` API
+2. The cached content ID is then passed as `cachedContent` in the Gemini request body
+3. Works across all three providers: `gemini/` (Google AI Studio), `vertex_ai/`, and `vertex_ai_beta/`
+4. Requires a minimum of **1024 tokens** in the cached content — below that, caching is silently skipped
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+from litellm import completion
+import os
+
+os.environ["GEMINI_API_KEY"] = ""
+
+response = completion(
+    model="gemini/gemini-2.5-flash",
+    messages=[
+        {
+            "role": "system",
+            "content": [
+                {
+                    "type": "text",
+                    "text": "You are an AI assistant tasked with analyzing legal documents.",
+                },
+                {
+                    "type": "text",
+                    "text": "Here is the full text of a complex legal agreement" * 400,
+                    "cache_control": {"type": "ephemeral"},
+                },
+            ],
+        },
+        {
+            "role": "user",
+            "content": "what are the key terms and conditions in this agreement?",
+        },
+    ],
+)
+
+print(response.usage)
+```
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+1. Setup config.yaml
+
+```yaml
+model_list:
+    - model_name: gemini-2.5-flash
+      litellm_params:
+        model: gemini/gemini-2.5-flash
+        api_key: os.environ/GEMINI_API_KEY
+```
+
+2. Start proxy
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+3. Test it!
+
+```python
+from openai import OpenAI
+
+client = OpenAI(
+    api_key="LITELLM_PROXY_KEY",  # sk-1234
+    base_url="LITELLM_PROXY_BASE",  # http://0.0.0.0:4000
+)
+
+response = client.chat.completions.create(
+    model="gemini-2.5-flash",
+    messages=[
+        {
+            "role": "system",
+            "content": [
+                {
+                    "type": "text",
+                    "text": "You are an AI assistant tasked with analyzing legal documents.",
+                },
+                {
+                    "type": "text",
+                    "text": "Here is the full text of a complex legal agreement" * 400,
+                    "cache_control": {"type": "ephemeral"},
+                },
+            ],
+        },
+        {
+            "role": "user",
+            "content": "what are the key terms and conditions in this agreement?",
+        },
+    ],
+)
+
+print(response.usage)
+```
+
+</TabItem>
+</Tabs>
+
+#### Vertex AI
+
+For Vertex AI, use `vertex_ai/` prefix:
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+from litellm import completion
+
+response = completion(
+    model="vertex_ai/gemini-2.5-flash",
+    vertex_project="my-gcp-project",
+    vertex_location="us-central1",
+    messages=[
+        {
+            "role": "system",
+            "content": [
+                {
+                    "type": "text",
+                    "text": "You are an AI assistant tasked with analyzing legal documents.",
+                },
+                {
+                    "type": "text",
+                    "text": "Here is the full text of a complex legal agreement" * 400,
+                    "cache_control": {"type": "ephemeral"},
+                },
+            ],
+        },
+        {
+            "role": "user",
+            "content": "what are the key terms and conditions in this agreement?",
+        },
+    ],
+)
+
+print(response.usage)
+```
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+1. Setup config.yaml
+
+```yaml
+model_list:
+    - model_name: gemini-2.5-flash
+      litellm_params:
+        model: vertex_ai/gemini-2.5-flash
+        vertex_project: my-gcp-project
+        vertex_location: us-central1
+```
+
+2. Start proxy
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+3. Test it!
+
+```python
+from openai import OpenAI
+
+client = OpenAI(
+    api_key="LITELLM_PROXY_KEY",  # sk-1234
+    base_url="LITELLM_PROXY_BASE",  # http://0.0.0.0:4000
+)
+
+response = client.chat.completions.create(
+    model="gemini-2.5-flash",
+    messages=[
+        {
+            "role": "system",
+            "content": [
+                {
+                    "type": "text",
+                    "text": "You are an AI assistant tasked with analyzing legal documents.",
+                },
+                {
+                    "type": "text",
+                    "text": "Here is the full text of a complex legal agreement" * 400,
+                    "cache_control": {"type": "ephemeral"},
+                },
+            ],
+        },
+        {
+            "role": "user",
+            "content": "what are the key terms and conditions in this agreement?",
+        },
+    ],
+)
+
+print(response.usage)
+```
+
+</TabItem>
+</Tabs>
+
 ### Deepeek Example 
 
 Works the same as OpenAI. 

docs/my-website/docs/completion/usage.md

@@ -50,3 +50,51 @@ for chunk in completion:
   print(chunk.choices[0].delta)
 
 ```
+
+### Proxy: Always Include Streaming Usage
+
+When using the LiteLLM Proxy, you can configure it to automatically include usage information in all streaming responses, even if the client doesn't send `stream_options={"include_usage": True}`.
+
+#### Configuration
+
+Add the following to your config.yaml:
+
+```yaml
+general_settings:
+  always_include_stream_usage: true
+```
+
+Alternatively, configure it through the UI:
+
+1. Navigate to the LiteLLM Proxy UI
+2. Go to `Settings` >  `Router Settings` > `General`
+3. Find the `always_include_stream_usage` setting
+4. Toggle it to `true`
+5. Click `Update` to save
+
+#### How it works
+
+When `always_include_stream_usage` is enabled:
+- All streaming requests will automatically have `stream_options={"include_usage": True}` added
+- Clients will receive usage information in the final chunk, even if they didn't explicitly request it
+- If a client already provides `stream_options`, `include_usage: True` will be added without overwriting other options
+- Non-streaming requests are not affected
+
+#### Example
+
+With this setting enabled, a simple streaming request like:
+
+```bash
+curl -X POST http://localhost:4000/v1/chat/completions \
+  -H "Authorization: Bearer sk-1234" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "gpt-4o",
+    "messages": [{"role": "user", "content": "Hello!"}],
+    "stream": true
+  }'
+```
+
+Will automatically receive usage information in the response, without needing to explicitly include `stream_options`.
+
+```

docs/my-website/docs/completion/web_fetch.md

@@ -115,6 +115,11 @@ print(response)
 
 Web fetch is available on the following Anthropic API models:
 
+- `claude-opus-4-6` (Claude Opus 4.6)
+- `claude-sonnet-4-6` (Claude Sonnet 4.6)
+- `claude-opus-4-5` (Claude Opus 4.5)
+- `claude-sonnet-4-5` (Claude Sonnet 4.5)
+- `claude-haiku-4-5` (Claude Haiku 4.5)
 - `claude-opus-4-1-20250805` (Claude Opus 4.1)
 - `claude-opus-4-20250514` (Claude Opus 4)
 - `claude-sonnet-4-20250514` (Claude Sonnet 4)