Merge branch 'main' into main

.circleci/requirements.txt

@@ -17,4 +17,5 @@ mcp==1.25.0    # for MCP server
 semantic_router==0.1.10 # for auto-routing with litellm
 fastuuid==0.12.0
 responses==0.25.7  # for proxy client tests
-pytest-retry==1.6.3  # for automatic test retries
+pytest-retry==1.6.3  # for automatic test retries
+litellm-proxy-extras  # for prisma migrations

.github/codeql/codeql-config.yml

@@ -0,0 +1,22 @@
+name: "LiteLLM CodeQL config"
+
+# Use security-extended suite instead of security-and-quality to avoid
+# result sets > 2 GiB on this codebase that cause fatal OOM failures.
+queries:
+  - uses: security-extended
+
+# These two queries are security queries included in security-extended that
+# individually produce result sets > 2 GiB on this codebase, causing fatal
+# OOM failures. Exclude them as a safety net until CI confirms they no longer
+# OOM; drop these exclusions in a follow-up once verified.
+query-filters:
+  - exclude:
+      id: py/clear-text-logging-sensitive-data  # CWE-312 — > 2 GiB result set
+  - exclude:
+      id: py/polynomial-redos                   # CWE-730 — > 2 GiB result set
+
+paths-ignore:
+  - tests
+  - docs
+  - "**/*.md"
+  - litellm/proxy/_experimental/out

.github/observatory/litellm_config.yaml

@@ -0,0 +1,19 @@
+# LiteLLM Observatory Test Configuration
+# This config is used by CI to spin up a temporary LiteLLM instance
+# for running observatory tests against RC/stable releases.
+#
+# Add model definitions for the providers you want to test.
+# Provider API keys are injected via environment variables in CI.
+
+model_list:
+  - model_name: gpt-4o
+    litellm_params:
+      model: azure/gpt-4o
+      api_key: os.environ/AZURE_API_KEY
+      api_base: os.environ/AZURE_API_BASE
+
+  - model_name: gpt-4o-mini
+    litellm_params:
+      model: azure/gpt-4o-mini
+      api_key: os.environ/AZURE_API_KEY
+      api_base: os.environ/AZURE_API_BASE

.github/pull_request_template.md

@@ -6,11 +6,15 @@
 
 **Please complete all items before asking a LiteLLM maintainer to review your PR**
 
-- [ ] I have Added testing in the [`tests/litellm/`](https://github.com/BerriAI/litellm/tree/main/tests/litellm) directory, **Adding at least 1 test is a hard requirement** - [see details](https://docs.litellm.ai/docs/extras/contributing_code)
+- [ ] I have Added testing in the [`tests/test_litellm/`](https://github.com/BerriAI/litellm/tree/main/tests/test_litellm) directory, **Adding at least 1 test is a hard requirement** - [see details](https://docs.litellm.ai/docs/extras/contributing_code)
 - [ ] My PR passes all unit tests on [`make test-unit`](https://docs.litellm.ai/docs/extras/contributing_code)
 - [ ] My PR's scope is as isolated as possible, it only solves 1 specific problem
 - [ ] I have requested a Greptile review by commenting `@greptileai` and received a **Confidence Score of at least 4/5** before requesting a maintainer review
 
+## Delays in PR merge?
+
+If you're seeing a delay in your PR being merged, ping the LiteLLM Team on [Slack (#pr-review)](https://join.slack.com/t/litellmossslack/shared_invite/zt-3o7nkuyfr-p_kbNJj8taRfXGgQI1~YyA).
+
 ## CI (LiteLLM team)
 
 > **CI status guideline:**

.github/scripts/close_duplicate_issues.py

@@ -0,0 +1,208 @@
+#!/usr/bin/env python3
+"""
+Detect and close duplicate GitHub issues using title similarity.
+
+Modes:
+  --scan           Compare all open issues against each other (batch)
+  --issue-number N Check a single issue against older open issues
+
+Requires the `gh` CLI to be authenticated.
+"""
+
+import argparse
+import difflib
+import json
+import re
+import subprocess
+import sys
+
+
+def normalize_title(title: str) -> str:
+    """Strip common prefixes, lowercase, and collapse whitespace."""
+    title = re.sub(
+        r"^\[?(bug|feature request|enhancement|question|docs)[:\]]?\s*",
+        "",
+        title,
+        flags=re.IGNORECASE,
+    )
+    return " ".join(title.lower().split())
+
+
+def gh(*args: str) -> str:
+    """Run a gh CLI command and return stdout."""
+    result = subprocess.run(
+        ["gh", *args],
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    return result.stdout
+
+
+def fetch_open_issues(repo: str | None) -> list[dict]:
+    """Fetch all open issues (excluding PRs) via gh api --paginate."""
+    if repo:
+        endpoint = f"repos/{repo}/issues?state=open&per_page=100&sort=created&direction=asc"
+    else:
+        endpoint = "repos/{owner}/{repo}/issues?state=open&per_page=100&sort=created&direction=asc"
+    cmd = ["api", "--paginate", endpoint]
+
+    raw = gh(*cmd)
+    # gh --paginate concatenates JSON arrays, so we may get multiple arrays
+    issues = []
+    for line in raw.strip().splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        parsed = json.loads(line)
+        if isinstance(parsed, list):
+            issues.extend(parsed)
+        else:
+            issues.append(parsed)
+
+    # Filter out pull requests (they also appear in the issues endpoint)
+    return [i for i in issues if "pull_request" not in i]
+
+
+def close_as_duplicate(
+    issue_number: int, duplicate_of: int, repo: str | None, dry_run: bool
+) -> None:
+    """Close an issue as duplicate of another, adding a comment and label."""
+    repo_args = ["--repo", repo] if repo else []
+
+    if dry_run:
+        print(f"  [DRY RUN] Would close #{issue_number} as duplicate of #{duplicate_of}")
+        return
+
+    # Add comment
+    comment_body = (
+        f"Closing as duplicate of #{duplicate_of}.\n\n"
+        "If you believe this is not a duplicate, please reopen and add context "
+        "explaining how this differs."
+    )
+    gh("issue", "comment", str(issue_number), "--body", comment_body, *repo_args)
+
+    # Add label
+    gh("issue", "edit", str(issue_number), "--add-label", "duplicate", *repo_args)
+
+    # Close with not_planned reason
+    gh(
+        "api",
+        f"repos/{repo or '{owner}/{repo}'}/issues/{issue_number}",
+        "-X",
+        "PATCH",
+        "-f",
+        "state=closed",
+        "-f",
+        "state_reason=not_planned",
+    )
+
+    print(f"  Closed #{issue_number} as duplicate of #{duplicate_of}")
+
+
+def find_duplicate(
+    issue: dict, candidates: list[dict], threshold: float
+) -> dict | None:
+    """Return the first candidate whose normalized title is above threshold."""
+    norm = normalize_title(issue["title"])
+    for candidate in candidates:
+        if candidate["number"] == issue["number"]:
+            continue
+        cand_norm = normalize_title(candidate["title"])
+        ratio = difflib.SequenceMatcher(None, norm, cand_norm).ratio()
+        if ratio >= threshold:
+            return candidate
+    return None
+
+
+def scan_all(issues: list[dict], threshold: float, repo: str | None, dry_run: bool) -> int:
+    """Compare every issue against all older issues. Returns count of duplicates found."""
+    # Sort oldest first
+    issues.sort(key=lambda i: i["number"])
+    closed_count = 0
+
+    for idx, issue in enumerate(issues):
+        older = issues[:idx]
+        if not older:
+            continue
+        dup = find_duplicate(issue, older, threshold)
+        if dup:
+            ratio = difflib.SequenceMatcher(
+                None,
+                normalize_title(issue["title"]),
+                normalize_title(dup["title"]),
+            ).ratio()
+            print(
+                f"#{issue['number']}: \"{issue['title']}\"\n"
+                f"  -> duplicate of #{dup['number']}: \"{dup['title']}\" "
+                f"({ratio:.0%} similar)"
+            )
+            close_as_duplicate(issue["number"], dup["number"], repo, dry_run)
+            closed_count += 1
+
+    return closed_count
+
+
+def check_single(
+    issue_number: int, issues: list[dict], threshold: float, repo: str | None, dry_run: bool
+) -> bool:
+    """Check a single issue against all older open issues. Returns True if duplicate found."""
+    target = None
+    for i in issues:
+        if i["number"] == issue_number:
+            target = i
+            break
+
+    if target is None:
+        print(f"Issue #{issue_number} not found among open issues.")
+        return False
+
+    older = [i for i in issues if i["number"] < issue_number]
+    dup = find_duplicate(target, older, threshold)
+    if dup:
+        ratio = difflib.SequenceMatcher(
+            None,
+            normalize_title(target["title"]),
+            normalize_title(dup["title"]),
+        ).ratio()
+        print(
+            f"#{target['number']}: \"{target['title']}\"\n"
+            f"  -> duplicate of #{dup['number']}: \"{dup['title']}\" "
+            f"({ratio:.0%} similar)"
+        )
+        close_as_duplicate(issue_number, dup["number"], repo, dry_run)
+        return True
+
+    print(f"#{issue_number}: no duplicate found above threshold {threshold}")
+    return False
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Detect and close duplicate GitHub issues")
+    mode = parser.add_mutually_exclusive_group(required=True)
+    mode.add_argument("--scan", action="store_true", help="Scan all open issues")
+    mode.add_argument("--issue-number", type=int, help="Check a single issue number")
+    parser.add_argument("--threshold", type=float, default=0.85, help="Similarity threshold (0-1)")
+    parser.add_argument("--close", action="store_true", help="Actually close duplicates (default is dry-run)")
+    parser.add_argument("--repo", type=str, help="Repository (owner/repo). Auto-detected if omitted.")
+    args = parser.parse_args()
+
+    dry_run = not args.close
+
+    if dry_run:
+        print("=== DRY RUN MODE (pass --close to actually close issues) ===\n")
+
+    print("Fetching open issues...")
+    issues = fetch_open_issues(args.repo)
+    print(f"Found {len(issues)} open issues.\n")
+
+    if args.scan:
+        count = scan_all(issues, args.threshold, args.repo, dry_run)
+        print(f"\nTotal duplicates {'found' if dry_run else 'closed'}: {count}")
+    else:
+        found = check_single(args.issue_number, issues, args.threshold, args.repo, dry_run)
+        sys.exit(0 if found else 0)  # Always exit 0; finding no dup is not an error
+
+
+if __name__ == "__main__":
+    main()

.github/workflows/auto_update_price_and_context_window.yml

@@ -7,6 +7,7 @@ on:
 
 jobs:
   auto_update_price_and_context_window:
+    if: github.repository == 'BerriAI/litellm'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3

.github/workflows/check_duplicate_issues.yml

@@ -27,3 +27,26 @@ jobs:
             {{/issues}}
 
             Please review the linked issue(s) to see if they address your concern. If this is not a duplicate, please provide additional context to help us understand the difference.
+
+      - name: Checkout close script
+        if: github.event.action == 'opened'
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: .github/scripts
+
+      - name: Set up Python
+        if: github.event.action == 'opened'
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Auto-close if high-confidence duplicate
+        if: github.event.action == 'opened'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          python3 .github/scripts/close_duplicate_issues.py \
+            --issue-number ${{ github.event.issue.number }} \
+            --repo ${{ github.repository }} \
+            --threshold 0.85 \
+            --close

.github/workflows/codeql.yml

@@ -0,0 +1,52 @@
+name: "CodeQL"
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  schedule:
+    # Run weekly on Sundays at 04:00 UTC
+    - cron: "0 4 * * 0"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  analyze:
+    name: Analyze (${{ matrix.language }})
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    permissions:
+      security-events: write
+      packages: read
+      actions: read
+      contents: read
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - language: actions
+            build-mode: none
+          - language: javascript-typescript
+            build-mode: none
+          - language: python
+            build-mode: none
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v3
+        with:
+          languages: ${{ matrix.language }}
+          build-mode: ${{ matrix.build-mode }}
+          config-file: ./.github/codeql/codeql-config.yml
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v3
+        with:
+          category: "/language:${{ matrix.language }}"

.github/workflows/codspeed.yml

@@ -0,0 +1,44 @@
+name: CodSpeed Benchmarks
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+  # Allow CodSpeed to trigger backtest performance analysis
+  # in order to generate initial data
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  id-token: write
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  benchmarks:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install dependencies
+        run: |
+          pip install -e "."
+          pip install pytest pytest-codspeed==4.3.0
+
+      - name: Run benchmarks
+        uses: CodSpeedHQ/action@v4
+        with:
+          mode: simulation
+          run: pytest tests/benchmarks/ --codspeed

.github/workflows/create_daily_staging_branch.yml

@@ -41,3 +41,39 @@ jobs:
             git push origin $BRANCH_NAME
             echo "Successfully created and pushed branch: $BRANCH_NAME"
           fi
+
+  create-internal-dev-branch:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Create internal dev branch
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          # Configure Git user
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+
+          # Generate branch name with MM_DD_YYYY format
+          BRANCH_NAME="litellm_internal_dev_$(date +'%m_%d_%Y')"
+          echo "Creating branch: $BRANCH_NAME"
+
+          # Fetch all branches
+          git fetch --all
+
+          # Check if the branch already exists
+          if git show-ref --verify --quiet refs/remotes/origin/$BRANCH_NAME; then
+            echo "Branch $BRANCH_NAME already exists. Skipping creation."
+          else
+            echo "Creating new branch: $BRANCH_NAME"
+            # Create the new branch from main
+            git checkout -b $BRANCH_NAME origin/main
+            # Push the new branch
+            git push origin $BRANCH_NAME
+            echo "Successfully created and pushed branch: $BRANCH_NAME"
+          fi

.github/workflows/ghcr_deploy.yml

@@ -299,6 +299,15 @@ jobs:
             ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-spend_logs:main-stable', env.REGISTRY) || '' }}
           platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
 
+  run-observatory-tests:
+    if: github.event.inputs.release_type == 'rc' || github.event.inputs.release_type == 'stable'
+    needs: [docker-hub-deploy]
+    uses: ./.github/workflows/run_observatory_tests.yml
+    with:
+      tag: ${{ github.event.inputs.tag }}
+      commit_hash: ${{ github.event.inputs.commit_hash }}
+    secrets: inherit
+
   build-and-push-helm-chart:
     if: github.event.inputs.release_type  != 'dev'
     needs: [docker-hub-deploy, build-and-push-image, build-and-push-image-database]
@@ -360,7 +369,8 @@ jobs:
   release:
     name: "New LiteLLM Release"
     needs: [docker-hub-deploy, build-and-push-image, build-and-push-image-database]
-
+    permissions:
+      contents: write
     runs-on: "ubuntu-latest"
    
     steps:

.github/workflows/publish_enterprise.yml

@@ -0,0 +1,94 @@
+name: Publish litellm-enterprise to PyPI
+
+on:
+  workflow_dispatch:
+    inputs:
+      bump:
+        description: "Version bump type"
+        required: true
+        default: "patch"
+        type: choice
+        options:
+          - patch
+          - minor
+          - major
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    if: github.repository == 'BerriAI/litellm'
+    permissions:
+      contents: write
+      pull-requests: write
+    defaults:
+      run:
+        working-directory: enterprise
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install Poetry
+        run: pip install poetry
+
+      - name: Bump version
+        id: bump
+        run: |
+          OLD=$(poetry version -s)
+          poetry version ${{ github.event.inputs.bump }}
+          NEW=$(poetry version -s)
+          echo "old=$OLD" >> $GITHUB_OUTPUT
+          echo "new=$NEW" >> $GITHUB_OUTPUT
+
+      - name: Update version refs in root pyproject.toml and requirements.txt
+        run: |
+          OLD=${{ steps.bump.outputs.old }}
+          NEW=${{ steps.bump.outputs.new }}
+          sed -i "s/litellm-enterprise = {version = \"${OLD}\"/litellm-enterprise = {version = \"${NEW}\"/" ../pyproject.toml
+          sed -i "s/litellm-enterprise==${OLD}/litellm-enterprise==${NEW}/" ../requirements.txt
+
+      - name: Update poetry.lock
+        working-directory: .
+        run: poetry lock
+
+      - name: Build
+        run: poetry build
+
+      - name: Commit version bump and create PR
+        id: create-pr
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          cd ..
+          BRANCH="bump/enterprise-${{ steps.bump.outputs.new }}"
+          git checkout -b "$BRANCH"
+          git add enterprise/pyproject.toml pyproject.toml requirements.txt poetry.lock
+          git commit -m "bump: litellm-enterprise ${{ steps.bump.outputs.old }} → ${{ steps.bump.outputs.new }}"
+          git push origin "$BRANCH" --force
+          gh pr create \
+            --title "bump: litellm-enterprise ${{ steps.bump.outputs.old }} → ${{ steps.bump.outputs.new }}" \
+            --body "Version bump for litellm-enterprise. Merge to update main." \
+            --head "$BRANCH" \
+            --base main \
+          || true
+          PR_URL=$(gh pr list --head "$BRANCH" --json url -q '.[0].url')
+          echo "pr_url=$PR_URL" >> $GITHUB_OUTPUT
+        env:
+          GH_TOKEN: ${{ github.token }}
+
+      - name: Enable auto-merge
+        run: |
+          gh pr merge "${{ steps.create-pr.outputs.pr_url }}" --auto --squash
+        env:
+          GH_TOKEN: ${{ github.token }}
+
+      - name: Publish to PyPI
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_ENTERPRISE }}
+        run: |
+          pip install twine
+          twine upload dist/litellm_enterprise-${{ steps.bump.outputs.new }}*

.github/workflows/publish_proxy_extras.yml

@@ -0,0 +1,74 @@
+name: Publish litellm-proxy-extras to PyPI
+
+on:
+  workflow_dispatch:
+    inputs:
+      bump:
+        description: "Version bump type"
+        required: true
+        default: "patch"
+        type: choice
+        options:
+          - patch
+          - minor
+          - major
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    if: github.repository == 'BerriAI/litellm'
+    permissions:
+      contents: write
+    defaults:
+      run:
+        working-directory: litellm-proxy-extras
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install Poetry
+        run: pip install poetry
+
+      - name: Bump version
+        id: bump
+        run: |
+          OLD=$(poetry version -s)
+          poetry version ${{ github.event.inputs.bump }}
+          NEW=$(poetry version -s)
+          echo "old=$OLD" >> $GITHUB_OUTPUT
+          echo "new=$NEW" >> $GITHUB_OUTPUT
+
+      - name: Update version refs in root pyproject.toml and requirements.txt
+        run: |
+          OLD=${{ steps.bump.outputs.old }}
+          NEW=${{ steps.bump.outputs.new }}
+          sed -i "s/litellm-proxy-extras = {version = \"${OLD}\"/litellm-proxy-extras = {version = \"${NEW}\"/" ../pyproject.toml
+          sed -i "s/litellm-proxy-extras==${OLD}/litellm-proxy-extras==${NEW}/" ../requirements.txt
+
+      - name: Update poetry.lock
+        working-directory: .
+        run: poetry lock
+
+      - name: Build
+        run: poetry build
+
+      - name: Commit version bump
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          cd ..
+          git add litellm-proxy-extras/pyproject.toml pyproject.toml requirements.txt poetry.lock
+          git commit -m "bump: litellm-proxy-extras ${{ steps.bump.outputs.old }} → ${{ steps.bump.outputs.new }}"
+          git push
+
+      - name: Publish to PyPI
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_PUBLISH_PASSWORD }}
+        run: |
+          pip install twine
+          twine upload dist/litellm_proxy_extras-${{ steps.bump.outputs.new }}*

.github/workflows/run_observatory_tests.yml

@@ -0,0 +1,225 @@
+name: Run Observatory Tests
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: "Docker image tag to test (e.g. v1.61.0.rc1)"
+        required: true
+        type: string
+      commit_hash:
+        description: "Commit hash (defaults to HEAD of current branch)"
+        required: false
+        type: string
+  workflow_call:
+    inputs:
+      tag:
+        description: "Docker image tag to test"
+        required: true
+        type: string
+      commit_hash:
+        description: "Commit hash of the release"
+        required: true
+        type: string
+
+permissions:
+  contents: read
+
+env:
+  LITELLM_MASTER_KEY: ${{ secrets.LITELLM_MASTER_KEY_STAGING }}
+
+jobs:
+  observatory-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Validate tag input
+        env:
+          TAG: ${{ inputs.tag }}
+        run: |
+          if [[ ! "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+ ]]; then
+            echo "Invalid tag format: $TAG (expected vX.Y.Z...)"
+            exit 1
+          fi
+
+      - name: Start LiteLLM container
+        env:
+          TAG: ${{ inputs.tag }}
+          AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
+          AZURE_API_BASE: ${{ secrets.AZURE_API_BASE }}
+        run: |
+          docker run -d \
+            --name litellm-rc \
+            -p 4000:4000 \
+            -v "${{ github.workspace }}/.github/observatory/litellm_config.yaml:/app/config.yaml" \
+            -e LITELLM_MASTER_KEY="${LITELLM_MASTER_KEY}" \
+            -e AZURE_API_KEY="${AZURE_API_KEY}" \
+            -e AZURE_API_BASE="${AZURE_API_BASE}" \
+            "litellm/litellm:${TAG}" \
+            --config /app/config.yaml --port 4000
+
+      - name: Wait for LiteLLM health check
+        run: |
+          echo "Waiting for LiteLLM to be ready..."
+          for i in $(seq 1 30); do
+            if curl -s -f http://localhost:4000/health/liveliness > /dev/null 2>&1; then
+              echo "LiteLLM is healthy"
+              exit 0
+            fi
+            echo "Attempt $i/30 - not ready yet, waiting 10s..."
+            sleep 10
+          done
+          echo "LiteLLM failed to start within 5 minutes"
+          docker logs litellm-rc
+          exit 1
+
+      - name: Start cloudflared tunnel
+        run: |
+          # Install cloudflared
+          curl -sL https://github.com/cloudflare/cloudflared/releases/download/2025.2.1/cloudflared-linux-amd64 -o /usr/local/bin/cloudflared
+          chmod +x /usr/local/bin/cloudflared
+
+          # Start a quick tunnel (no account needed) and capture the URL
+          cloudflared tunnel --url http://localhost:4000 --no-autoupdate > /tmp/cloudflared.log 2>&1 &
+          CLOUDFLARED_PID=$!
+          echo "CLOUDFLARED_PID=$CLOUDFLARED_PID" >> $GITHUB_ENV
+
+          # Wait for tunnel URL to appear in logs
+          echo "Waiting for tunnel URL..."
+          for i in $(seq 1 30); do
+            TUNNEL_URL=$(grep -oP 'https://[a-z0-9-]+\.trycloudflare\.com' /tmp/cloudflared.log | head -1 || true)
+            if [ -n "$TUNNEL_URL" ]; then
+              echo "Tunnel URL: $TUNNEL_URL"
+              echo "TUNNEL_URL=$TUNNEL_URL" >> $GITHUB_ENV
+              exit 0
+            fi
+            sleep 2
+          done
+          echo "Failed to get tunnel URL"
+          cat /tmp/cloudflared.log
+          exit 1
+
+      - name: Verify tunnel connectivity
+        run: |
+          echo "Testing tunnel at ${{ env.TUNNEL_URL }}..."
+          # Quick tunnels need time for DNS propagation; retry to avoid
+          # transient NXDOMAIN (curl exit code 6) on first attempt.
+          for i in $(seq 1 10); do
+            if curl -sf "${{ env.TUNNEL_URL }}/health/liveliness" > /dev/null 2>&1; then
+              echo "Tunnel is working (attempt $i)"
+              exit 0
+            fi
+            echo "Attempt $i/10 - tunnel not routable yet, waiting 5s..."
+            sleep 5
+          done
+          echo "Tunnel failed to become reachable after 50s"
+          cat /tmp/cloudflared.log
+          exit 1
+
+      - name: Trigger observatory test run
+        id: trigger
+        env:
+          OBSERVATORY_URL: ${{ secrets.OBSERVATORY_URL }}
+          OBSERVATORY_API_KEY: ${{ secrets.OBSERVATORY_API_KEY }}
+        run: |
+          PAYLOAD=$(jq -n \
+            --arg url "${TUNNEL_URL}" \
+            --arg key "${LITELLM_MASTER_KEY}" \
+            '{
+              deployment_url: $url,
+              api_key: $key,
+              test_suite: "TestOAIAzureRelease",
+              models: ["gpt-4o-mini", "gpt-4o"]
+            }')
+          RESPONSE=$(curl -s -w "\n%{http_code}" -X POST "${OBSERVATORY_URL}/run-test" \
+            -H "Content-Type: application/json" \
+            -H "X-LiteLLM-Observatory-API-Key: ${OBSERVATORY_API_KEY}" \
+            -d "$PAYLOAD")
+          HTTP_CODE=$(echo "$RESPONSE" | tail -1)
+          BODY=$(echo "$RESPONSE" | head -n -1)
+          echo "Response ($HTTP_CODE): $BODY"
+          if [ "$HTTP_CODE" -ge 400 ]; then
+            echo "Failed to trigger test run"
+            exit 1
+          fi
+
+          # Extract request_id for polling this specific run
+          REQUEST_ID=$(echo "$BODY" | jq -r '.results.request_id')
+          if [ -z "$REQUEST_ID" ] || [ "$REQUEST_ID" = "null" ]; then
+            echo "Failed to extract request_id from response"
+            exit 1
+          fi
+          echo "Request ID: $REQUEST_ID"
+          echo "request_id=$REQUEST_ID" >> $GITHUB_OUTPUT
+
+      - name: Poll for test completion
+        id: poll
+        env:
+          OBSERVATORY_URL: ${{ secrets.OBSERVATORY_URL }}
+          OBSERVATORY_API_KEY: ${{ secrets.OBSERVATORY_API_KEY }}
+          REQUEST_ID: ${{ steps.trigger.outputs.request_id }}
+        run: |
+          TIMEOUT=900  # 15 minutes
+          INTERVAL=30
+          ELAPSED=0
+          while [ $ELAPSED -lt $TIMEOUT ]; do
+            STATUS=$(curl -s "${OBSERVATORY_URL}/run-status/${REQUEST_ID}" \
+              -H "X-LiteLLM-Observatory-API-Key: ${OBSERVATORY_API_KEY}")
+            RUN_STATUS=$(echo "$STATUS" | jq -r '.status')
+            echo "Run status (${ELAPSED}s elapsed): $RUN_STATUS"
+
+            if [ "$RUN_STATUS" = "completed" ] || [ "$RUN_STATUS" = "failed" ]; then
+              echo "Test finished with status: $RUN_STATUS"
+              echo "$STATUS" > /tmp/observatory_result.json
+              exit 0
+            fi
+
+            sleep $INTERVAL
+            ELAPSED=$((ELAPSED + INTERVAL))
+          done
+          echo "Timed out waiting for test to complete after ${TIMEOUT}s"
+          exit 1
+
+      - name: Verify test results
+        run: |
+          RESULT=$(cat /tmp/observatory_result.json)
+          echo "Full result: $RESULT"
+
+          STATUS=$(echo "$RESULT" | jq -r '.status')
+          TEST_PASSED=$(echo "$RESULT" | jq -r '.result.test_passed // false')
+          FAILURE_RATE=$(echo "$RESULT" | jq -r '.result.failure_rate // "N/A"')
+          ERROR=$(echo "$RESULT" | jq -r '.error // empty')
+
+          echo "Status: $STATUS"
+          echo "Test passed: $TEST_PASSED"
+          echo "Failure rate: $FAILURE_RATE"
+
+          if [ -n "$ERROR" ]; then
+            echo "Error: $ERROR"
+          fi
+
+          if [ "$STATUS" = "failed" ]; then
+            echo "Test run failed"
+            exit 1
+          fi
+
+          if [ "$TEST_PASSED" != "true" ]; then
+            echo "Tests did not pass (failure rate: $FAILURE_RATE)"
+            exit 1
+          fi
+
+          echo "All tests passed!"
+
+      - name: Print LiteLLM logs on failure
+        if: failure()
+        run: |
+          docker logs litellm-rc 2>/dev/null || true
+          cat /tmp/cloudflared.log 2>/dev/null || true
+
+      - name: Cleanup
+        if: always()
+        run: |
+          kill "${{ env.CLOUDFLARED_PID }}" 2>/dev/null || true
+          docker rm -f litellm-rc 2>/dev/null || true

.github/workflows/scan_duplicate_issues.yml

@@ -0,0 +1,47 @@
+name: Scan Duplicate Issues (One-Time)
+
+on:
+  workflow_dispatch:
+    inputs:
+      threshold:
+        description: "Similarity threshold (0-1)"
+        required: false
+        default: "0.85"
+      close:
+        description: "Actually close duplicates (false = dry run)"
+        required: false
+        type: boolean
+        default: false
+
+jobs:
+  scan:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      contents: read
+    steps:
+      - name: Checkout scripts
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: .github/scripts
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Scan for duplicate issues
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          INPUT_THRESHOLD: ${{ inputs.threshold }}
+          INPUT_CLOSE: ${{ inputs.close }}
+        run: |
+          CLOSE_FLAG=""
+          if [ "$INPUT_CLOSE" = "true" ]; then
+            CLOSE_FLAG="--close"
+          fi
+          python3 .github/scripts/close_duplicate_issues.py \
+            --scan \
+            --repo ${{ github.repository }} \
+            --threshold "$INPUT_THRESHOLD" \
+            $CLOSE_FLAG

.github/workflows/test-linting.yml

@@ -28,16 +28,18 @@ jobs:
         find . -type d -name "__pycache__" -exec rm -rf {} + || true
         find . -name "*.pyc" -delete || true
 
+    - name: Check poetry.lock is up to date
+      run: |
+        poetry check --lock || (echo "❌ poetry.lock is out of sync with pyproject.toml. Run 'poetry lock' locally and commit the result." && exit 1)
+
     - name: Install dependencies
       run: |
-        poetry lock
         poetry install --with dev
-        poetry run pip install openai==1.100.1
 
-    - name: Run Black formatting
+    - name: Check Black formatting
       run: |
         cd litellm
-        poetry run black .
+        poetry run black --check --exclude '/enterprise/' .
         cd ..
 
     - name: Debug - Check file state
@@ -74,3 +76,35 @@ jobs:
     - name: Check import safety
       run: |
         poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
+
+  secret-scan:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      contents: read
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.12'
+
+    - name: Run secret scan test
+      run: |
+        pip install pytest
+        pytest tests/litellm/test_no_hardcoded_secrets.py -v
+
+    - name: Run ggshield secret scan
+      env:
+        GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }}
+      run: |
+        if [ -n "$GITGUARDIAN_API_KEY" ]; then
+          pip install ggshield
+          ggshield secret scan repo .
+        else
+          echo "GITGUARDIAN_API_KEY not set, skipping ggshield scan"
+        fi

.github/workflows/test-litellm.yml

@@ -38,7 +38,7 @@ jobs:
         poetry run pip install "google-genai==1.22.0"
         poetry run pip install "google-cloud-aiplatform>=1.38"
         poetry run pip install "fastapi-offline==1.7.3"
-        poetry run pip install "python-multipart==0.0.22"
+        poetry run pip install "python-multipart>=0.0.20"
         poetry run pip install "openapi-core"
     - name: Setup litellm-enterprise as local package
       run: |

.github/workflows/test-proxy-e2e-azure-batches.yml

@@ -0,0 +1,90 @@
+name: Proxy E2E Azure Batches Tests
+
+on:
+  pull_request:
+    branches: [main]
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  proxy_e2e_azure_batches_tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+
+    services:
+      postgres:
+        image: postgres:15
+        env:
+          POSTGRES_USER: llmproxy
+          POSTGRES_PASSWORD: dbpassword9090
+          POSTGRES_DB: litellm
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+
+      - name: Cache Poetry dependencies
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/pypoetry
+            ~/.cache/pip
+            .venv
+          key: ${{ runner.os }}-poetry-e2e-batches-${{ hashFiles('poetry.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-poetry-e2e-batches-
+            ${{ runner.os }}-poetry-
+
+      - name: Install dependencies
+        run: |
+          poetry config virtualenvs.in-project true
+          poetry install --with dev,proxy-dev --extras "proxy"
+          poetry run pip install psycopg2-binary uvicorn fastapi httpx tenacity
+
+      - name: Setup litellm-enterprise
+        run: |
+          poetry run pip install --force-reinstall --no-deps -e enterprise/
+
+      - name: Generate Prisma client
+        run: |
+          poetry run prisma generate --schema litellm/proxy/schema.prisma
+
+      - name: Run Prisma migrations
+        env:
+          DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
+        run: |
+          cd litellm/proxy
+          poetry run prisma migrate deploy --schema schema.prisma
+          cd ../..
+
+      - name: Run Azure Batch E2E Tests
+        env:
+          DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
+          USE_LOCAL_LITELLM: "true"
+          USE_MOCK_MODELS: "true"
+          USE_STATE_TRACKER: "true"
+          LITELLM_LOG: DEBUG
+        run: |
+          poetry run pytest tests/proxy_e2e_azure_batches_tests/test_proxy_e2e_azure_batches.py \
+            -vv -s -k "test_e2e_managed_batch" \
+            --tb=short \
+            --maxfail=3 \
+            --durations=10
+

.gitignore

@@ -89,6 +89,7 @@ tests/test_custom_dir/*
 test.py
 
 litellm_config.yaml
+!.github/observatory/litellm_config.yaml
 .cursor
 .vscode/launch.json
 litellm/proxy/to_delete_loadtest_work/*

.pre-commit-config.yaml

@@ -14,12 +14,12 @@ repos:
         types: [python]
         files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py
         exclude: ^litellm/__init__.py$
-    # -   id: black
-    #     name: black
-    #     entry: poetry run black
-    #     language: system
-    #     types: [python]
-    #     files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py
+    -   id: black
+        name: black
+        entry: poetry run black
+        language: system
+        types: [python]
+        files: (litellm/|litellm_proxy_extras/).*\.py
 -   repo: https://github.com/pycqa/flake8
     rev: 7.0.0  # The version of flake8 to use
     hooks:

AGENTS.md

@@ -109,6 +109,8 @@ Key files:
 - `litellm/proxy/auth/` - Authentication logic
 - `litellm/proxy/management_endpoints/` - Admin API endpoints
 
+**Database (proxy)**: Use Prisma model methods (`prisma_client.db.<model>.upsert`, `.find_many`, `.find_unique`, etc.), not raw SQL (`execute_raw`/`query_raw`). See COMMON PITFALLS for details.
+
 ## MCP (MODEL CONTEXT PROTOCOL) SUPPORT
 
 LiteLLM supports MCP for agent workflows:
@@ -174,6 +176,43 @@ When opening issues or pull requests, follow these templates:
 3. **Rate Limits**: Respect provider rate limits in tests
 4. **Memory Usage**: Be mindful of memory usage in streaming scenarios
 5. **Dependencies**: Keep dependencies minimal and well-justified
+6. **UI/Backend Contract Mismatch**: When adding a new entity type to the UI, always check whether the backend endpoint accepts a single value or an array. Match the UI control accordingly (single-select vs. multi-select) to avoid silently dropping user selections
+7. **Missing Tests for New Entity Types**: When adding a new entity type (e.g., in `EntityUsage`, `UsageViewSelect`), always add corresponding tests in the existing test files and update any icon/component mocks
+8. **Raw SQL in proxy DB code**: Do not use `execute_raw` or `query_raw` for proxy database access. Use Prisma model methods (e.g. `prisma_client.db.litellm_tooltable.upsert()`, `.find_many()`, `.find_unique()`) so behavior stays consistent with the schema, the client stays mockable in tests, and you avoid the pitfalls of hand-written SQL (parameter ordering, type casting, schema drift)
+
+8. **Do not hardcode model-specific flags**: Put model-specific capability flags in `model_prices_and_context_window.json` and read them via `get_model_info` (or existing helpers like `supports_reasoning`). This prevents users from needing to upgrade LiteLLM each time a new model supports a feature.
+
+   **Example of BAD** (hardcoded model checks):
+
+   ```python
+   @staticmethod
+   def _is_effort_supported_model(model: str) -> bool:
+       """Check if the model supports the output_config.effort parameter..."""
+       model_lower = model.lower()
+       if AnthropicConfig._is_claude_4_6_model(model):
+           return True
+       return any(
+           v in model_lower for v in ("opus-4-5", "opus_4_5", "opus-4.5", "opus_4.5")
+       )
+   ```
+
+   **Example of GOOD** (config-driven or helper that reads from config):
+
+   ```python
+   if (
+       "claude-3-7-sonnet" in model
+       or AnthropicConfig._is_claude_4_6_model(model)
+       or supports_reasoning(
+           model=model,
+           custom_llm_provider=self.custom_llm_provider,
+       )
+   ):
+       ...
+   ```
+
+   Using helpers like `supports_reasoning` (which read from `model_prices_and_context_window.json` / `get_model_info`) allows future model updates to "just work" without code changes.
+
+9. **Never close HTTP/SDK clients on cache eviction**: Do not add `close()`, `aclose()`, or `create_task(close_fn())` inside `LLMClientCache._remove_key()` or any cache eviction path. Evicted clients may still be held by in-flight requests; closing them causes `RuntimeError: Cannot send a request, as the client has been closed.` in production after the cache TTL (1 hour) expires. Connection cleanup is handled at shutdown by `close_litellm_async_clients()`. See PR #22247 for the full incident history.
 
 ## HELPFUL RESOURCES
 
@@ -187,4 +226,49 @@ When opening issues or pull requests, follow these templates:
 - Check similar provider implementations
 - Ensure comprehensive test coverage
 - Update documentation appropriately
-- Consider backward compatibility impact 
+- Consider backward compatibility impact
+
+## Cursor Cloud specific instructions
+
+### Environment
+
+- Poetry is installed in `~/.local/bin`; the update script ensures it is on `PATH`.
+- Python 3.12, Node 22 are pre-installed.
+- The virtual environment lives under `~/.cache/pypoetry/virtualenvs/`.
+
+### Running the proxy server
+
+Start the proxy with a config file:
+
+```bash
+poetry run litellm --config dev_config.yaml --port 4000
+```
+
+The proxy takes ~15-20 seconds to fully start (it runs Prisma migrations on boot). Wait for `/health` to return before sending requests. Without a PostgreSQL `DATABASE_URL`, the proxy connects to a default Neon dev database embedded in the `litellm-proxy-extras` package.
+
+### Running tests
+
+See `CLAUDE.md` and the `Makefile` for standard commands. Key notes:
+
+- `psycopg-binary` must be installed (`poetry run pip install psycopg-binary`) because the pytest-postgresql plugin requires it and the lock file only includes `psycopg` (no binary).
+- `openapi-core` must be installed (`poetry run pip install openapi-core`) for the OpenAPI compliance tests in `tests/test_litellm/interactions/`.
+- The `--timeout` pytest flag is NOT available; don't pass it.
+- Unit tests: `poetry run pytest tests/test_litellm/ -x -vv -n 4`
+- Black `--check` may report pre-existing formatting issues; this does not block test runs.
+- If `poetry install` fails with "pyproject.toml changed significantly since poetry.lock was last generated", run `poetry lock` first to regenerate the lock file.
+
+### Lint
+
+```bash
+cd litellm && poetry run ruff check .
+```
+
+Ruff is the primary fast linter. For the full lint suite (including mypy, black, circular imports), run `make lint` per `CLAUDE.md`.
+
+### UI Dashboard development
+
+- The UI is at `ui/litellm-dashboard/`. Run `npm run dev` from that directory for the Next.js dev server on port 3000.
+- The proxy at port 4000 serves a **pre-built** static UI from `litellm/proxy/_experimental/out/`. After making UI code changes, you must run `npm run build` in the dashboard directory and copy the output: `cp -r ui/litellm-dashboard/out/* litellm/proxy/_experimental/out/` for the proxy to serve the updated UI.
+- SVGs used as provider logos (loaded via `<img>` tags) must NOT use `fill="currentColor"` — replace with an explicit color like `#000000` or use the `-color` variant from lobehub icons, since CSS color inheritance does not work inside `<img>` elements.
+- Provider logos live in `ui/litellm-dashboard/public/assets/logos/` (source) and `litellm/proxy/_experimental/out/assets/logos/` (pre-built). Both locations must have the file for it to work in dev and proxy-served modes.
+- UI Vitest tests: `cd ui/litellm-dashboard && npx vitest run`

CLAUDE.md

@@ -91,19 +91,74 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components:
 - Async/await patterns throughout
 - Type hints required for all public APIs
 - **Avoid imports within methods** — place all imports at the top of the file (module-level). Inline imports inside functions/methods make dependencies harder to trace and hurt readability. The only exception is avoiding circular imports where absolutely necessary.
+- **Use dict spread for immutable copies** — prefer `{**original, "key": new_value}` over `dict(obj)` + mutation. The spread produces the final dict in one step and makes intent clear.
+- **Guard at resolution time** — when resolving an optional value through a fallback chain (`a or b or ""`), raise immediately if the resolved result being empty is an error. Don't pass empty strings or sentinel values downstream for the callee to deal with.
+- **Extract complex comprehensions to named helpers** — a set/dict comprehension that calls into the DB or manager (e.g. "which of these server IDs are OAuth2?") belongs in a named helper function, not inline in the caller.
+- **FastAPI parameter declarations** — mark required query/form params with `= Query(...)` / `= Form(...)` explicitly when other params in the same handler are optional. Mixing `str` (required) with `Optional[str] = None` in the same signature causes silent 422s when the required param is missing.
 
 ### Testing Strategy
 - Unit tests in `tests/test_litellm/`
 - Integration tests for each provider in `tests/llm_translation/`
 - Proxy tests in `tests/proxy_unit_tests/`
 - Load tests in `tests/load_tests/`
+- **Always add tests when adding new entity types or features** — if the existing test file covers other entity types, add corresponding tests for the new one
+- **Keep monkeypatch stubs in sync with real signatures** — when a function gains a new optional parameter, update every `fake_*` / `stub_*` in tests that patch it to also accept that kwarg (even as `**kwargs`). Stale stubs fail with `unexpected keyword argument` and mask real bugs.
+- **Test all branches of name→ID resolution** — when adding server/resource lookup that resolves names to UUIDs, test: (1) name resolves and UUID is allowed, (2) name resolves but UUID is not allowed, (3) name does not resolve at all. The silent-fallback path is where access-control bugs hide.
+
+### UI / Backend Consistency
+- When wiring a new UI entity type to an existing backend endpoint, verify the backend API contract (single value vs. array, required vs. optional params) and ensure the UI controls match — e.g., use a single-select dropdown when the backend accepts a single value, not a multi-select
+
+### MCP OAuth / OpenAPI Transport Mapping
+- `TRANSPORT.OPENAPI` is a UI-only concept. The backend only accepts `"http"`, `"sse"`, or `"stdio"`. Always map it to `"http"` before any API call (including pre-OAuth temp-session calls).
+- FastAPI validation errors return `detail` as an array of `{loc, msg, type}` objects. Error extractors must handle: array (map `.msg`), string, nested `{error: string}`, and fallback.
+- When an MCP server already has `authorization_url` stored, skip OAuth discovery (`_discovery_metadata`) — the server URL for OpenAPI MCPs is the spec file, not the API base, and fetching it causes timeouts.
+- `client_id` should be optional in the `/authorize` endpoint — if the server has a stored `client_id` in credentials, use that. Never require callers to re-supply it.
+
+### MCP Credential Storage
+- OAuth credentials and BYOK credentials share the `litellm_mcpusercredentials` table, distinguished by a `"type"` field in the JSON payload (`"oauth2"` vs plain string).
+- When deleting OAuth credentials, check type before deleting to avoid accidentally deleting a BYOK credential for the same `(user_id, server_id)` pair.
+- Always pass the raw `expires_at` timestamp to the client — never set it to `None` for expired credentials. Let the frontend compute the "Expired" display state from the timestamp.
+- Use `RecordNotFoundError` (not bare `except Exception`) when catching "already deleted" in credential delete endpoints.
+
+### Browser Storage Safety (UI)
+- Never write LiteLLM access tokens or API keys to `localStorage` — use `sessionStorage` only. `localStorage` survives browser close and is readable by any injected script (XSS).
+- Shared utility functions (e.g. `extractErrorMessage`) belong in `src/utils/` — never define them inline in hooks or duplicate them across files.
 
 ### Database Migrations
 - Prisma handles schema migrations
 - Migration files auto-generated with `prisma migrate dev`
 - Always test migrations against both PostgreSQL and SQLite
 
+### Proxy database access
+- **Do not write raw SQL** for proxy DB operations. Use Prisma model methods instead of `execute_raw` / `query_raw`.
+- Use the generated client: `prisma_client.db.<model>` (e.g. `litellm_tooltable`, `litellm_usertable`) with `.upsert()`, `.find_many()`, `.find_unique()`, `.update()`, `.update_many()` as appropriate. This avoids schema/client drift, keeps code testable with simple mocks, and matches patterns used in spend logs and other proxy code.
+- **No N+1 queries.** Never query the DB inside a loop. Batch-fetch with `{"in": ids}` and distribute in-memory.
+- **Batch writes.** Use `create_many`/`update_many`/`delete_many` instead of individual calls (these return counts only; `update_many`/`delete_many` no-op silently on missing rows). When multiple separate writes target the same table (e.g. in `batch_()`), order by primary key to avoid deadlocks.
+- **Push work to the DB.** Filter, sort, group, and aggregate in SQL, not Python. Verify Prisma generates the expected SQL — e.g. prefer `group_by` over `find_many(distinct=...)` which does client-side processing.
+- **Bound large result sets.** Prisma materializes full results in memory. For results over ~10 MB, paginate with `take`/`skip` or `cursor`/`take`, always with an explicit `order`. Prefer cursor-based pagination (`skip` is O(n)). Don't paginate naturally small result sets.
+- **Limit fetched columns on wide tables.** Use `select` to fetch only needed fields — returns a partial object, so downstream code must not access unselected fields.
+- **Check index coverage.** For new or modified queries, check `schema.prisma` for a supporting index. Prefer extending an existing index (e.g. `@@index([a])` → `@@index([a, b])`) over adding a new one, unless it's a `@@unique`. Only add indexes for large/frequent queries.
+- **Keep schema files in sync.** Apply schema changes to all `schema.prisma` copies (`schema.prisma`, `litellm/proxy/`, `litellm-proxy-extras/`, `litellm-js/spend-logs/` for SpendLogs) with a migration under `litellm-proxy-extras/litellm_proxy_extras/migrations/`.
+
+### Setup Wizard (`litellm/setup_wizard.py`)
+- The wizard is implemented as a single `SetupWizard` class with `@staticmethod` methods — keep it that way. No module-level functions except `run_setup_wizard()` (the public entrypoint) and pure helpers (color, ANSI).
+- Use `litellm.utils.check_valid_key(model, api_key)` for credential validation — never roll a custom completion call.
+- Do not hardcode provider env-key names or model lists that already exist in the codebase. Add a `test_model` field to each provider entry to drive `check_valid_key`; set it to `None` for providers that can't be validated with a single API key (Azure, Bedrock, Ollama).
+
 ### Enterprise Features
 - Enterprise-specific code in `enterprise/` directory
 - Optional features enabled via environment variables
-- Separate licensing and authentication for enterprise features
+- Separate licensing and authentication for enterprise features
+
+### HTTP Client Cache Safety
+- **Never close HTTP/SDK clients on cache eviction.** `LLMClientCache._remove_key()` must not call `close()`/`aclose()` on evicted clients — they may still be used by in-flight requests. Doing so causes `RuntimeError: Cannot send a request, as the client has been closed.` after the 1-hour TTL expires. Cleanup happens at shutdown via `close_litellm_async_clients()`.
+
+### Troubleshooting: DB schema out of sync after proxy restart
+`litellm-proxy-extras` runs `prisma migrate deploy` on startup using **its own** bundled migration files, which may lag behind schema changes in the current worktree. Symptoms: `Unknown column`, `Invalid prisma invocation`, or missing data on new fields.
+
+**Diagnose:** Run `\d "TableName"` in psql and compare against `schema.prisma` — missing columns confirm the issue.
+
+**Fix options:**
+1. **Create a Prisma migration** (permanent) — run `prisma migrate dev --name <description>` in the worktree. The generated file will be picked up by `prisma migrate deploy` on next startup.
+2. **Apply manually for local dev** — `psql -d litellm -c "ALTER TABLE ... ADD COLUMN IF NOT EXISTS ..."` after each proxy start. Fine for dev, not for production.
+3. **Update litellm-proxy-extras** — if the package is installed from PyPI, its migration directory must include the new file. Either update the package or run the migration manually until the next release ships it.

Dockerfile

@@ -39,7 +39,7 @@ RUN pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt
 # ensure pyjwt is used, not jwt
 RUN pip uninstall jwt -y
 RUN pip uninstall PyJWT -y
-RUN pip install PyJWT==2.9.0 --no-cache-dir
+RUN pip install PyJWT==2.12.0 --no-cache-dir
 
 # Runtime stage
 FROM $LITELLM_RUNTIME_IMAGE AS runtime
@@ -49,7 +49,7 @@ USER root
 
 # Install runtime dependencies (libsndfile needed for audio processing on ARM64)
 RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \
-    npm install -g npm@latest tar@7.5.8 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.1 diff@8.0.3 && \
+    npm install -g npm@latest tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
     # SECURITY FIX: npm bundles tar, glob, and brace-expansion at multiple nested
     # levels inside its dependency tree. `npm install -g <pkg>` only creates a
     # SEPARATE global package, it does NOT replace npm's internal copies.
@@ -70,7 +70,15 @@ RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile
     find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
     done && \
-    npm cache clean --force
+    # SECURITY FIX: patch npm's own package.json metadata so scanners see the
+    # actual installed versions instead of the stale declared dependencies.
+    find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
+        sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null && \
+    npm cache clean --force && \
+    # Remove the apk-tracked npm so its stale SBOM metadata (tar 7.5.9) is
+    # no longer visible to image scanners.  The globally installed npm@latest
+    # at /usr/local/lib/node_modules/npm/ remains fully functional.
+    { apk del --no-cache npm 2>/dev/null || true; }
 
 WORKDIR /app
 # Copy the current directory contents into the container at /app
@@ -96,6 +104,7 @@ RUN find /usr/lib -type f -path "*/tornado/test/*" -delete && \
 # npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
 # Patch every copy of tar, glob, and brace-expansion inside that tree.
 RUN GLOBAL="$(npm root -g)" && \
+    [ -n "$GLOBAL" ] || { echo "ERROR: npm root -g returned empty; aborting"; exit 1; } && \
     find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
     done && \

README.md

@@ -28,6 +28,9 @@
     <a href="https://www.litellm.ai/support">
         <img src="https://img.shields.io/static/v1?label=Chat%20on&message=Slack&color=black&logo=Slack&style=flat-square" alt="Slack">
     </a>
+    <a href="https://codspeed.io/BerriAI/litellm?utm_source=badge">
+        <img src="https://img.shields.io/endpoint?url=https://codspeed.io/badge.json" alt="CodSpeed"/>
+    </a>
 </h4>
 
 <img width="2688" height="1600" alt="Group 7154 (1)" src="https://github.com/user-attachments/assets/c5ee0412-6fb5-4fb6-ab5b-bafae4209ca6" />
@@ -203,7 +206,7 @@ curl -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
 {
   "mcpServers": {
     "LiteLLM": {
-      "url": "http://localhost:4000/mcp",
+      "url": "http://localhost:4000/mcp/",
       "headers": {
         "x-litellm-api-key": "Bearer sk-1234"
       }

ci_cd/security_scans.sh

@@ -11,7 +11,7 @@ echo "Starting security scans for LiteLLM..."
 install_trivy() {
     echo "Installing Trivy and required tools..."
     sudo apt-get update
-    sudo apt-get install -y wget apt-transport-https gnupg lsb-release jq curl
+    sudo apt-get install -y wget apt-transport-https gnupg lsb-release jq curl bsdmainutils
     wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | sudo apt-key add -
     echo "deb https://aquasecurity.github.io/trivy-repo/deb $(lsb_release -sc) main" | sudo tee -a /etc/apt/sources.list.d/trivy.list
     sudo apt-get update
@@ -161,6 +161,11 @@ run_grype_scans() {
         "GHSA-3ppc-4f35-3m26" # minimatch ReDoS via repeated wildcards - from nodejs_wheel bundled npm, not used in application runtime code
         "GHSA-83g3-92jg-28cx" # tar arbitrary file read/write via hardlink - from nodejs_wheel bundled npm, not used in application runtime code
         "CVE-2026-25639" # axios - full fix requires 1.x major version bump; pinned to >=0.30.2 to clear other axios CVEs, upgrade to 1.x in follow-up
+        "CVE-2026-2297" # Python 3.13 SourcelessFileLoader audit hook bypass - no fix available in base image
+        "GHSA-qffp-2rhf-9h96" # tar hardlink path traversal - from nodejs_wheel bundled npm, not used in application runtime code
+        "CVE-2026-2673" # OpenSSL 3.6.1 TLS 1.3 key exchange group negotiation issue - no fix available yet
+        "CVE-2026-3644" # Python 3.13 vulnerability - no fix available in base image
+        "CVE-2026-4224" # Python 3.13 Expat parser stack overflow in ElementDeclHandler - no fix available in base image
     )
 
     # Build JSON array of allowlisted CVE IDs for jq

deploy/charts/litellm-helm/templates/_helpers.tpl

@@ -61,6 +61,20 @@ Create the name of the service account to use
 {{- end }}
 {{- end }}
 
+{{/*
+Create the service account name used by migration jobs.
+When Helm hooks are enabled, pre-install/pre-upgrade hooks run before normal resources.
+If this chart is creating the ServiceAccount, it is not yet available for the hook job,
+so fall back to "default" (or an explicit override) to avoid a cyclic dependency.
+*/}}
+{{- define "litellm.migrationServiceAccountName" -}}
+{{- if and .Values.migrationJob.hooks.helm.enabled .Values.serviceAccount.create }}
+{{- default "default" .Values.migrationJob.serviceAccountName }}
+{{- else }}
+{{- include "litellm.serviceAccountName" . }}
+{{- end }}
+{{- end }}
+
 {{/*
 Get redis service name
 */}}

deploy/charts/litellm-helm/templates/deployment.yaml

@@ -13,9 +13,16 @@ spec:
   {{- if and (not .Values.keda.enabled) (not .Values.autoscaling.enabled) }}
   replicas: {{ .Values.replicaCount }}
   {{- end }}
+  {{- with .Values.strategy }}
+  strategy:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
   selector:
     matchLabels:
       {{- include "litellm.selectorLabels" . | nindent 6 }}
+  {{- if .Values.deploymentMinReadySeconds }}
+  minReadySeconds: {{ .Values.deploymentMinReadySeconds }}
+  {{- end }}
   template:
     metadata:
       annotations:

deploy/charts/litellm-helm/templates/migrations-job.yaml

@@ -34,7 +34,7 @@ spec:
       imagePullSecrets:
         {{- toYaml . | nindent 8 }}
       {{- end }}
-      serviceAccountName: {{ include "litellm.serviceAccountName" . }}
+      serviceAccountName: {{ include "litellm.migrationServiceAccountName" . }}
       {{- with .Values.migrationJob.extraInitContainers }}
       initContainers:
         {{- toYaml . | nindent 8 }}

deploy/charts/litellm-helm/tests/deployment_tests.yaml

@@ -306,3 +306,16 @@ tests:
       - equal:
           path: spec.template.spec.containers[0].resources
           value: {}
+  - it: should be able to set minReadySeconds
+    template: deployment.yaml
+    set:
+      deploymentMinReadySeconds: 5
+    asserts:
+      - equal:
+          path: spec.minReadySeconds
+          value: 5
+  - it: should have minReadySeconds absent when deploymentMinReadySeconds is not set
+    template: deployment.yaml
+    asserts:
+      - notExists:
+          path: spec.minReadySeconds

deploy/charts/litellm-helm/tests/migrations-job_tests.yaml

@@ -124,4 +124,67 @@ tests:
       - notContains:
           path: spec.template.spec.containers[0].env
           content:
-            name: DATABASE_URL
+            name: DATABASE_URL
+
+  - it: should use default service account for helm hooks when serviceAccount.create is true
+    template: migrations-job.yaml
+    set:
+      migrationJob:
+        enabled: true
+        hooks:
+          helm:
+            enabled: true
+      serviceAccount:
+        create: true
+    asserts:
+      - equal:
+          path: spec.template.spec.serviceAccountName
+          value: default
+
+  - it: should use migrationJob.serviceAccountName override for helm hooks when serviceAccount.create is true
+    template: migrations-job.yaml
+    set:
+      migrationJob:
+        enabled: true
+        serviceAccountName: migration-sa
+        hooks:
+          helm:
+            enabled: true
+      serviceAccount:
+        create: true
+    asserts:
+      - equal:
+          path: spec.template.spec.serviceAccountName
+          value: migration-sa
+
+  - it: should use chart service account when helm hooks are disabled
+    template: migrations-job.yaml
+    set:
+      migrationJob:
+        enabled: true
+        hooks:
+          helm:
+            enabled: false
+      serviceAccount:
+        create: true
+        name: my-custom-sa
+    asserts:
+      - equal:
+          path: spec.template.spec.serviceAccountName
+          value: my-custom-sa
+
+  - it: should use pre-existing service account when helm hooks are enabled but serviceAccount.create is false
+    template: migrations-job.yaml
+    set:
+      migrationJob:
+        enabled: true
+        hooks:
+          helm:
+            enabled: true
+      serviceAccount:
+        create: false
+        name: pre-existing-sa
+    asserts:
+      - equal:
+          path: spec.template.spec.serviceAccountName
+          value: pre-existing-sa

deploy/charts/litellm-helm/values.yaml

@@ -31,10 +31,20 @@ serviceAccount:
 # annotations for litellm deployment
 deploymentAnnotations: {}
 deploymentLabels: {}
+deploymentMinReadySeconds: 0
+
 # annotations for litellm pods
 podAnnotations: {}
 podLabels: {}
 
+# -- Deployment strategy configuration
+# Example:
+#   type: RollingUpdate
+#   rollingUpdate:
+#     maxUnavailable: 0
+#     maxSurge: 1
+strategy: {}
+
 terminationGracePeriodSeconds: 90
 topologySpreadConstraints:
   []
@@ -299,6 +309,10 @@ migrationJob:
   retries: 3 # Number of retries for the Job in case of failure
   backoffLimit: 4 # Backoff limit for Job restarts
   disableSchemaUpdate: false # Skip schema migrations for specific environments. When True, the job will exit with code 0.
+  # Optional service account for the migration job.
+  # Only used when migrationJob.hooks.helm.enabled=true and serviceAccount.create=true.
+  # In that case, pre-install/pre-upgrade hooks run before normal resources, so this defaults to "default".
+  serviceAccountName: ""
   annotations: {}
   ttlSecondsAfterFinished: 120
   resources: {}

dev_config.yaml

@@ -0,0 +1,13 @@
+model_list:
+  - model_name: fake-openai-endpoint
+    litellm_params:
+      model: openai/fake-model
+      api_key: fake-key
+      api_base: https://exampleopenaiendpoint-production.up.railway.app/
+
+general_settings:
+  master_key: sk-1234
+
+litellm_settings:
+  drop_params: True
+  telemetry: False

docker/Dockerfile.custom_ui

@@ -19,7 +19,7 @@ RUN apt-get update && apt-get upgrade -y \
         libgnutls30 \
         libc6 && \
     apt-get install -y nodejs npm && \
-    npm install -g npm@latest tar@7.5.8 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.1 diff@8.0.3 && \
+    npm install -g npm@latest tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
     GLOBAL="$(npm root -g)" && \
     find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
@@ -36,7 +36,10 @@ RUN apt-get update && apt-get upgrade -y \
     find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
     done && \
-    npm cache clean --force
+    find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
+        sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null && \
+    npm cache clean --force && \
+    apt-get purge -y npm
 
 # Copy the UI source into the container
 COPY ./ui/litellm-dashboard /app/ui/litellm-dashboard

docker/Dockerfile.database

@@ -50,7 +50,7 @@ USER root
 
 # Install runtime dependencies
 RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \
-    npm install -g npm@latest tar@7.5.8 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.1 diff@8.0.3 && \
+    npm install -g npm@latest tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
     GLOBAL="$(npm root -g)" && \
     find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
@@ -67,7 +67,10 @@ RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile
     find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
     done && \
-    npm cache clean --force
+    find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
+        sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null && \
+    npm cache clean --force && \
+    { apk del --no-cache npm 2>/dev/null || true; }
 
 WORKDIR /app
 # Copy the current directory contents into the container at /app
@@ -85,6 +88,7 @@ RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl
 # npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
 # Patch every copy of tar, glob, and brace-expansion inside that tree.
 RUN GLOBAL="$(npm root -g)" && \
+    [ -n "$GLOBAL" ] || { echo "ERROR: npm root -g returned empty; aborting"; exit 1; } && \
     find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
     done && \
@@ -108,7 +112,7 @@ RUN sed -i 's/\r$//' docker/install_auto_router.sh && chmod +x docker/install_au
 # ensure pyjwt is used, not jwt
 RUN pip uninstall jwt -y
 RUN pip uninstall PyJWT -y
-RUN pip install PyJWT==2.9.0 --no-cache-dir
+RUN pip install PyJWT==2.12.0 --no-cache-dir
 
 # Build Admin UI (runtime stage)
 # Convert Windows line endings to Unix and make executable

docker/Dockerfile.dev

@@ -31,7 +31,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
 # Fix JWT dependency conflicts early
 RUN pip uninstall jwt -y || true && \
     pip uninstall PyJWT -y || true && \
-    pip install PyJWT==2.9.0 --no-cache-dir
+    pip install PyJWT==2.12.0 --no-cache-dir
 
 # Copy only necessary files for build
 COPY pyproject.toml README.md schema.prisma poetry.lock ./
@@ -75,7 +75,7 @@ RUN apt-get update && apt-get upgrade -y \
     nodejs \
     npm \
     && rm -rf /var/lib/apt/lists/* \
-    && npm install -g npm@latest tar@7.5.8 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.1 diff@8.0.3 \
+    && npm install -g npm@latest tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \
     && GLOBAL="$(npm root -g)" \
     && find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
           rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
@@ -92,7 +92,10 @@ RUN apt-get update && apt-get upgrade -y \
     && find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
           rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
        done \
-    && npm cache clean --force
+    && find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
+        sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null \
+    && npm cache clean --force \
+    && apt-get purge -y npm
 
 WORKDIR /app
 
@@ -114,6 +117,7 @@ RUN pip install --no-cache-dir *.whl /wheels/* --no-index --find-links=/wheels/
 # npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
 # Patch every copy of tar, glob, and brace-expansion inside that tree.
 RUN GLOBAL="$(npm root -g)" && \
+    [ -n "$GLOBAL" ] || { echo "ERROR: npm root -g returned empty; aborting"; exit 1; } && \
     find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
     done && \

docker/Dockerfile.non_root

@@ -32,7 +32,7 @@ RUN for i in 1 2 3; do \
 # Cache Python dependencies
 COPY requirements.txt .
 RUN pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt \
-  && pip wheel --no-cache-dir --wheel-dir=/wheels/ "semantic_router==0.1.11" "aurelio-sdk==0.0.19" "PyJWT==2.9.0"
+  && pip wheel --no-cache-dir --wheel-dir=/wheels/ "semantic_router==0.1.11" "aurelio-sdk==0.0.19" "PyJWT==2.12.0"
 
 # Copy source after dependency layers
 COPY . .
@@ -106,7 +106,7 @@ RUN for i in 1 2 3; do \
     apk add --no-cache python3 py3-pip bash openssl tzdata nodejs npm supervisor && break || sleep 5; \
     done \
   && apk upgrade --no-cache nodejs \
-  && npm install -g npm@latest tar@7.5.8 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.1 diff@8.0.3 \
+  && npm install -g npm@latest tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \
   && GLOBAL="$(npm root -g)" \
   && find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
@@ -123,7 +123,10 @@ RUN for i in 1 2 3; do \
   && find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
      done \
-  && npm cache clean --force
+  && find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
+        sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null \
+  && npm cache clean --force \
+  && { apk del --no-cache npm 2>/dev/null || true; }
 
 # Copy artifacts from builder
 COPY --from=builder /app/requirements.txt /app/requirements.txt
@@ -169,6 +172,7 @@ RUN pip install --no-index --find-links=/wheels/ -r requirements.txt && \
 # npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
 # Patch every copy of tar, glob, and brace-expansion inside that tree.
 RUN GLOBAL="$(npm root -g)" && \
+    [ -n "$GLOBAL" ] || { echo "ERROR: npm root -g returned empty; aborting"; exit 1; } && \
     find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
     done && \
@@ -194,7 +198,7 @@ RUN sed -i 's/\r$//' docker/entrypoint.sh && \
     chown -R nobody:nogroup /app /var/lib/litellm/ui /var/lib/litellm/assets /nonexistent /.npm && \
     pip uninstall jwt -y || true && \
     pip uninstall PyJWT -y || true && \
-    pip install --no-index --find-links=/wheels/ PyJWT==2.10.1 --no-cache-dir && \
+    pip install --no-index --find-links=/wheels/ PyJWT==2.12.0 --no-cache-dir && \
     rm -rf /wheels && \
     PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \
     chown -R nobody:nogroup $PRISMA_PATH && \

docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md

@@ -3,18 +3,9 @@ slug: anthropic_advanced_features
 title: "Day 0 Support: Claude 4.5 Opus (+Advanced Features)"
 date: 2025-11-25T10:00:00
 authors:
-  - name: Sameer Kankute
-    title: SWE @ LiteLLM (LLM Translation)
-    url: https://www.linkedin.com/in/sameer-kankute/
-    image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
-  - name: Krrish Dholakia
-    title: "CEO, LiteLLM"
-    url: https://www.linkedin.com/in/krish-d/
-    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
-  - name: Ishaan Jaff
-    title: "CTO, LiteLLM"
-    url: https://www.linkedin.com/in/reffajnaahsi/
-    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+  - sameer
+  - krrish
+  - ishaan-alt
 description: "Guide to Claude Opus 4.5 and advanced features in LiteLLM: Tool Search, Programmatic Tool Calling, and Effort Parameter."
 tags: [anthropic, claude, tool search, programmatic tool calling, effort, advanced features]
 hide_table_of_contents: false
@@ -25,6 +16,8 @@ import TabItem from '@theme/TabItem';
 
 This guide covers Anthropic's latest model (Claude Opus 4.5) and its advanced features now available in LiteLLM: Tool Search, Programmatic Tool Calling, Tool Input Examples, and the Effort Parameter.
 
+{/* truncate */}
+
 ---
 
 | Feature | Supported Models |

docs/my-website/blog/anthropic_wildcard_model_access_incident/index.md

@@ -3,18 +3,9 @@ slug: anthropic-wildcard-model-access-incident
 title: "Incident Report: Wildcard Blocking New Models After Cost Map Reload"
 date: 2026-02-23T10:00:00
 authors:
-  - name: Sameer Kankute
-    title: SWE @ LiteLLM (LLM Translation)
-    url: https://www.linkedin.com/in/sameer-kankute/
-    image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
-  - name: Krrish Dholakia
-    title: "CEO, LiteLLM"
-    url: https://www.linkedin.com/in/krish-d/
-    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
-  - name: Ishaan Jaff
-    title: "CTO, LiteLLM"
-    url: https://www.linkedin.com/in/reffajnaahsi/
-    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+  - sameer
+  - krrish
+  - ishaan-alt
 tags: [incident-report, proxy, auth, model-access]
 hide_table_of_contents: false
 ---

docs/my-website/blog/authors.yml

@@ -4,6 +4,12 @@ litellm:
   url: https://github.com/BerriAI/litellm
   image_url: https://github.com/BerriAI.png
 
+sameer:
+  name: Sameer Kankute
+  title: SWE @ LiteLLM (LLM Translation)
+  url: https://www.linkedin.com/in/sameer-kankute/
+  image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
+
 krrish:
   name: Krrish Dholakia
   title: CEO, LiteLLM
@@ -22,3 +28,21 @@ ishaan-alt:
   title: CTO, LiteLLM
   url: https://www.linkedin.com/in/reffajnaahsi/
   image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+
+ryan:
+  name: Ryan Crabbe
+  title: Performance Engineer, LiteLLM
+  url: https://www.linkedin.com/in/ryan-crabbe-0b9687214
+  image_url: https://media.licdn.com/dms/image/v2/D5603AQHt1t9Z4BJ6Gw/profile-displayphoto-shrink_400_400/profile-displayphoto-shrink_400_400/0/1724453682340?e=1772064000&v=beta&t=VXdmr13rsNB05wyA2F1TENOB5UuDHUZ0FCHTolNyR5M
+
+alexsander:
+  name: Alexsander Hamir
+  title: Performance Engineer, LiteLLM
+  url: https://www.linkedin.com/in/alexsander-baptista/
+  image_url: https://github.com/AlexsanderHamir.png
+
+yuneng:
+  name: Yuneng Jiang
+  title: SWE @ LiteLLM (Full Stack)
+  url: https://www.linkedin.com/in/yuneng-david-jiang-455676139/
+  image_url: https://avatars.githubusercontent.com/u/171294688?v=4

docs/my-website/blog/claude_code_beta_headers/index.md

@@ -3,18 +3,9 @@ slug: claude-code-beta-headers-incident
 title: "Incident Report: Invalid beta headers with Claude Code"
 date: 2026-02-16T10:00:00
 authors:
-  - name: Sameer Kankute
-    title: SWE @ LiteLLM (LLM Translation)
-    url: https://www.linkedin.com/in/sameer-kankute/
-    image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
-  - name: Ishaan Jaff
-    title: "CTO, LiteLLM"
-    url: https://www.linkedin.com/in/reffajnaahsi/
-    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
-  - name: Krrish Dholakia
-    title: "CEO, LiteLLM"
-    url: https://www.linkedin.com/in/krish-d/
-    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
+  - sameer
+  - ishaan-alt
+  - krrish
 tags: [incident-report, anthropic, stability]
 hide_table_of_contents: false
 ---
@@ -173,5 +164,5 @@ curl -X POST "https://your-proxy-url/reload/anthropic_beta_headers" \
 
 ## Related documentation
 
-- [Managing Anthropic Beta Headers](../proxy/sync_anthropic_beta_headers.md) - Complete configuration guide
+- [Managing Anthropic Beta Headers](../../docs/proxy/sync_anthropic_beta_headers) - Complete configuration guide
 - [`anthropic_beta_headers_config.json`](https://github.com/BerriAI/litellm/blob/main/litellm/anthropic_beta_headers_config.json) - Current configuration file

docs/my-website/blog/claude_opus_4_6/index.md

@@ -3,18 +3,9 @@ slug: claude_opus_4_6
 title: "Day 0 Support: Claude Opus 4.6"
 date: 2026-02-05T10:00:00
 authors:
-  - name: Sameer Kankute
-    title: SWE @ LiteLLM (LLM Translation)
-    url: https://www.linkedin.com/in/sameer-kankute/
-    image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
-  - name: Ishaan Jaff
-    title: "CTO, LiteLLM"
-    url: https://www.linkedin.com/in/reffajnaahsi/
-    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
-  - name: Krrish Dholakia
-    title: "CEO, LiteLLM"
-    url: https://www.linkedin.com/in/krish-d/
-    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
+  - sameer
+  - ishaan-alt
+  - krrish
 description: "Day 0 support for Claude Opus 4.6 on LiteLLM AI Gateway - use across Anthropic, Azure, Vertex AI, and Bedrock."
 tags: [anthropic, claude, opus 4.6]
 hide_table_of_contents: false
@@ -25,6 +16,8 @@ import TabItem from '@theme/TabItem';
 
 LiteLLM now supports Claude Opus 4.6 on Day 0. Use it across Anthropic, Azure, Vertex AI, and Bedrock through the LiteLLM AI Gateway.
 
+{/* truncate */}
+
 ## Docker Image
 
 ```bash

docs/my-website/blog/claude_sonnet_4_6/index.md

@@ -3,14 +3,8 @@ slug: claude_sonnet_4_6
 title: "Day 0 Support: Claude Sonnet 4.6"
 date: 2026-02-17T10:00:00
 authors:
-  - name: Ishaan Jaff
-    title: "CTO, LiteLLM"
-    url: https://www.linkedin.com/in/reffajnaahsi/
-    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
-  - name: Krrish Dholakia
-    title: "CEO, LiteLLM"
-    url: https://www.linkedin.com/in/krish-d/
-    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
+  - ishaan-alt
+  - krrish
 description: "Day 0 support for Claude Sonnet 4.6 on LiteLLM AI Gateway - use across Anthropic, Azure, Vertex AI, and Bedrock."
 tags: [anthropic, claude, sonnet 4.6]
 hide_table_of_contents: false
@@ -21,6 +15,8 @@ import TabItem from '@theme/TabItem';
 
 LiteLLM now supports Claude Sonnet 4.6 on Day 0. Use it across Anthropic, Azure, Vertex AI, and Bedrock through the LiteLLM AI Gateway.
 
+{/* truncate */}
+
 ## Docker Image
 
 ```bash

docs/my-website/blog/fastapi_middleware_performance/index.mdx

@@ -3,18 +3,9 @@ slug: fastapi-middleware-performance
 title: "Your Middleware Could Be a Bottleneck"
 date: 2026-02-07T10:00:00
 authors:
-  - name: Krrish Dholakia
-    title: "CEO, LiteLLM"
-    url: https://www.linkedin.com/in/krish-d/
-    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
-  - name: Ishaan Jaff
-    title: "CTO, LiteLLM"
-    url: https://www.linkedin.com/in/reffajnaahsi/
-    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
-  - name: Ryan Crabbe
-    title: "Performance Engineer, LiteLLM"
-    url: https://www.linkedin.com/in/ryan-crabbe-0b9687214
-    image_url: https://media.licdn.com/dms/image/v2/D5603AQHt1t9Z4BJ6Gw/profile-displayphoto-shrink_400_400/profile-displayphoto-shrink_400_400/0/1724453682340?e=1772064000&v=beta&t=VXdmr13rsNB05wyA2F1TENOB5UuDHUZ0FCHTolNyR5M
+  - krrish
+  - ishaan-alt
+  - ryan
 description: "How we improved LiteLLM proxy latency and throughput by replacing a single middleware base class"
 tags: [performance, fastapi, middleware]
 hide_table_of_contents: false

docs/my-website/blog/gemin_3.1/index.md

@@ -3,18 +3,9 @@ slug: gemini_3_1_pro
 title: "DAY 0 Support: Gemini 3.1 Pro on LiteLLM"
 date: 2026-02-19T10:00:00
 authors:
-  - name: Sameer Kankute
-    title: SWE @ LiteLLM (LLM Translation)
-    url: https://www.linkedin.com/in/sameer-kankute/
-    image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
-  - name: Krrish Dholakia
-    title: "CEO, LiteLLM"
-    url: https://www.linkedin.com/in/krish-d/
-    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
-  - name: Ishaan Jaff
-    title: "CTO, LiteLLM"
-    url: https://www.linkedin.com/in/reffajnaahsi/
-    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+  - sameer
+  - krrish
+  - ishaan-alt
 description: "Guide to using Gemini 3.1 Pro on LiteLLM Proxy and SDK with day 0 support."
 tags: [gemini, day 0 support, llms]
 hide_table_of_contents: false
@@ -28,6 +19,8 @@ import TabItem from '@theme/TabItem';
 
 LiteLLM now supports `gemini-3.1-pro-preview` and all the new API changes along with it.
 
+{/* truncate */}
+
 ## Deploy this version
 
 <Tabs>
@@ -67,7 +60,7 @@ LiteLLM provides **full end-to-end support** for Gemini 3.1 Pro on:
 - ✅ `/v1/chat/completions` - OpenAI-compatible chat completions endpoint
 - ✅ `/v1/responses` - OpenAI Responses API endpoint (streaming and non-streaming)
 - ✅ [`/v1/messages`](../../docs/anthropic_unified) - Anthropic-compatible messages endpoint
-- ✅ `/v1/generateContent` – [Google Gemini API](../../docs/generateContent.md) compatible endpoint 
+- ✅ `/v1/generateContent` – [Google Gemini API](../../docs/generateContent) compatible endpoint 
 
 All endpoints support:
 - Streaming and non-streaming responses
@@ -147,4 +140,3 @@ curl -X POST http://localhost:4000/v1/chat/completions \
 | `high` | `high` |
 | `disable` | `minimal` |
 | `none` | `minimal` |
-

docs/my-website/blog/gemini_3/index.md

@@ -3,18 +3,9 @@ slug: gemini_3
 title: "DAY 0 Support: Gemini 3 on LiteLLM"
 date: 2025-11-19T10:00:00
 authors:
-  - name: Sameer Kankute
-    title: SWE @ LiteLLM (LLM Translation)
-    url: https://www.linkedin.com/in/sameer-kankute/
-    image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
-  - name: Krrish Dholakia
-    title: "CEO, LiteLLM"
-    url: https://www.linkedin.com/in/krish-d/
-    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
-  - name: Ishaan Jaff
-    title: "CTO, LiteLLM"
-    url: https://www.linkedin.com/in/reffajnaahsi/
-    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+  - sameer
+  - krrish
+  - ishaan-alt
 description: "Common questions and best practices for using gemini-3-pro-preview with LiteLLM Proxy and SDK."
 tags: [gemini, day 0 support, llms]
 hide_table_of_contents: false
@@ -29,6 +20,8 @@ This guide covers common questions and best practices for using `gemini-3-pro-pr
 
 :::
 
+{/* truncate */}
+
 ## Quick Start
 
 <Tabs>
@@ -976,8 +969,7 @@ messages.append(response.choices[0].message)  # ✅ Includes thought signatures
 
 ## Additional Resources
 
-- [Gemini Provider Documentation](../gemini.md)
-- [Thought Signatures Guide](../gemini.md#thought-signatures)
-- [Reasoning Content Documentation](../../reasoning_content.md)
-- [Function Calling Guide](../../function_calling.md)
-
+- [Gemini Provider Documentation](../../docs/providers/gemini)
+- [Thought Signatures Guide](../../docs/providers/gemini#thought-signatures)
+- [Reasoning Content Documentation](../../docs/reasoning_content)
+- [Function Calling Guide](../../docs/completion/function_call)

docs/my-website/blog/gemini_3_1_flash_lite/index.md

@@ -0,0 +1,168 @@
+---
+slug: gemini_3_1_flash_lite_preview
+title: "DAY 0 Support: Gemini 3.1 Flash Lite Preview on LiteLLM"
+date: 2026-03-03T08:00:00
+authors:
+  - sameer
+  - krrish
+  - ishaan-alt
+description: "Guide to using Gemini 3.1 Flash Lite Preview on LiteLLM Proxy and SDK with day 0 support."
+tags: [gemini, day 0 support, llms, supernova]
+hide_table_of_contents: false
+---
+
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Gemini 3.1 Flash Lite Preview Day 0 Support 
+
+LiteLLM now supports `gemini-3.1-flash-lite-preview` with full day 0 support!
+
+:::note
+If you only want cost tracking, you need no change in your current Litellm version. But if you want the support for new features introduced along with it like thinking levels, you will need to use v1.80.8-stable.1 or above.
+:::
+
+{/* truncate */}
+
+## Deploy this version
+
+<Tabs>
+<TabItem value="docker" label="Docker">
+
+``` showLineNumbers title="docker run litellm"
+docker run \
+-e STORE_MODEL_IN_DB=True \
+-p 4000:4000 \
+ghcr.io/berriai/litellm:main-v1.80.8-stable.1
+```
+
+</TabItem>
+
+<TabItem value="pip" label="Pip">
+
+``` showLineNumbers title="pip install litellm"
+pip install litellm==v1.80.8-stable.1
+```
+
+</TabItem>
+</Tabs>
+
+## What's New
+
+Supports all four thinking levels:
+- **MINIMAL**: Ultra-fast responses with minimal reasoning
+- **LOW**: Simple instruction following
+- **MEDIUM**: Balanced reasoning for complex tasks
+- **HIGH**: Maximum reasoning depth (dynamic)
+
+---
+
+## Quick Start
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+**Basic Usage**
+
+```python
+from litellm import completion
+
+response = completion(
+    model="gemini/gemini-3.1-flash-lite-preview",
+    messages=[{"role": "user", "content": "Extract key entities from this text: ..."}],
+)
+
+print(response.choices[0].message.content)
+```
+
+**With Thinking Levels**
+
+```python
+from litellm import completion
+
+# Use MEDIUM thinking for complex reasoning tasks
+response = completion(
+    model="gemini/gemini-3.1-flash-lite-preview",
+    messages=[{"role": "user", "content": "Analyze this dataset and identify patterns"}],
+    reasoning_effort="medium",  # low, medium , high
+)
+
+print(response.choices[0].message.content)
+```
+
+</TabItem>
+
+<TabItem value="proxy" label="PROXY">
+
+**1. Setup config.yaml**
+
+```yaml
+model_list:
+  - model_name: gemini-3.1-flash-lite
+    litellm_params:
+      model: gemini/gemini-3.1-flash-lite-preview
+      api_key: os.environ/GEMINI_API_KEY
+  
+  # Or use Vertex AI
+  - model_name: vertex-gemini-3.1-flash-lite
+    litellm_params:
+      model: vertex_ai/gemini-3.1-flash-lite-preview
+      vertex_project: your-project-id
+      vertex_location: us-central1
+```
+
+**2. Start proxy**
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+**3. Make requests**
+
+```bash
+curl -X POST http://localhost:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer <YOUR-LITELLM-KEY>" \
+  -d '{
+    "model": "gemini-3.1-flash-lite",
+    "messages": [{"role": "user", "content": "Extract structured data from this text"}],
+    "reasoning_effort": "low"
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+---
+
+## Supported Endpoints
+
+LiteLLM provides **full end-to-end support** for Gemini 3.1 Flash Lite Preview on:
+
+- ✅ `/v1/chat/completions` - OpenAI-compatible chat completions endpoint
+- ✅ `/v1/responses` - OpenAI Responses API endpoint (streaming and non-streaming)
+- ✅ [`/v1/messages`](../../docs/anthropic_unified) - Anthropic-compatible messages endpoint
+- ✅ `/v1/generateContent` – [Google Gemini API](../../docs/generateContent) compatible endpoint 
+
+All endpoints support:
+- Streaming and non-streaming responses
+- Function calling with thought signatures
+- Multi-turn conversations
+- All Gemini 3-specific features (thinking levels, thought signatures)
+- Full multimodal support (text, image, audio, video)
+
+---
+
+## `reasoning_effort` Mapping for Gemini 3.1
+
+LiteLLM automatically maps OpenAI's `reasoning_effort` parameter to Gemini's `thinkingLevel`:
+
+| reasoning_effort | thinking_level | Use Case |
+|------------------|----------------|----------|
+| `minimal` | `minimal` | Ultra-fast responses, simple queries |
+| `low` | `low` | Basic instruction following |
+| `medium` | `medium` | Balanced reasoning for moderate complexity |
+| `high` | `high` | Maximum reasoning depth, complex problems |
+| `disable` | `minimal` | Disable extended reasoning |
+| `none` | `minimal` | No extended reasoning |

docs/my-website/blog/gemini_3_flash/index.md

@@ -3,18 +3,9 @@ slug: gemini_3_flash
 title: "DAY 0 Support: Gemini 3 Flash on LiteLLM"
 date: 2025-12-17T10:00:00
 authors:
-  - name: Sameer Kankute
-    title: SWE @ LiteLLM (LLM Translation)
-    url: https://www.linkedin.com/in/sameer-kankute/
-    image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
-  - name: Krrish Dholakia
-    title: "CEO, LiteLLM"
-    url: https://www.linkedin.com/in/krish-d/
-    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
-  - name: Ishaan Jaff
-    title: "CTO, LiteLLM"
-    url: https://www.linkedin.com/in/reffajnaahsi/
-    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+  - sameer
+  - krrish
+  - ishaan-alt
 description: "Guide to using Gemini 3 Flash on LiteLLM Proxy and SDK with day 0 support."
 tags: [gemini, day 0 support, llms]
 hide_table_of_contents: false
@@ -32,6 +23,8 @@ LiteLLM now supports `gemini-3-flash-preview` and all the new API changes along
 If you only want cost tracking, you need no change in your current Litellm version. But if you want the support for new features introduced along with it like thinking levels, you will need to use v1.80.8-stable.1 or above.
 :::
 
+{/* truncate */}
+
 ## Deploy this version
 
 <Tabs>
@@ -80,7 +73,7 @@ LiteLLM provides **full end-to-end support** for Gemini 3 Flash on:
 - ✅ `/v1/chat/completions` - OpenAI-compatible chat completions endpoint
 - ✅ `/v1/responses` - OpenAI Responses API endpoint (streaming and non-streaming)
 - ✅ [`/v1/messages`](../../docs/anthropic_unified) - Anthropic-compatible messages endpoint
-- ✅ `/v1/generateContent` – [Google Gemini API](../../docs/generateContent.md) compatible endpoint 
+- ✅ `/v1/generateContent` – [Google Gemini API](../../docs/generateContent) compatible endpoint 
 All endpoints support:
 - Streaming and non-streaming responses
 - Function calling with thought signatures
@@ -252,4 +245,3 @@ If using this model via vertex_ai, keep the location as global as this is the on
 | `high` | `high` |
 | `disable` | `minimal` |
 | `none` | `minimal` |
-

docs/my-website/blog/gemini_embedding_2_multimodal/index.md

@@ -0,0 +1,168 @@
+---
+slug: gemini_embedding_2_multimodal
+title: "Gemini Embedding 2 Preview: Multimodal Embeddings on LiteLLM"
+date: 2025-03-11T10:00:00
+authors:
+  - sameer
+description: "Generate embeddings from text, images, audio, video, and PDFs with gemini-embedding-2-preview on LiteLLM via Gemini API and Vertex AI."
+tags: [gemini, embeddings, multimodal, vertex ai]
+hide_table_of_contents: false
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Gemini Embedding 2 Preview: Multimodal Embeddings
+
+LiteLLM now supports **multimodal embeddings** with `gemini-embedding-2-preview`—generating a single embedding from a mix of text, images, audio, video, and PDF content. Available via both the **Gemini API** (API key) and **Vertex AI** (GCP credentials).
+
+{/* truncate */}
+
+## Supported Input Types
+
+| Modality | Supported Formats | 
+|----------|-------------------|
+| **Text** | Plain text |
+| **Image** | PNG, JPEG | 
+| **Audio** | MP3, WAV | 
+| **Video** | MP4, MOV | 
+| **Documents** | PDF | 
+
+## Input Formats
+
+LiteLLM accepts three input formats for multimodal content:
+
+1. **Data URIs** – Base64-encoded inline: `data:image/png;base64,<encoded_data>`
+2. **GCS URLs** – Cloud Storage paths (Vertex AI): `gs://bucket/path/to/file.png`
+3. **Gemini File References** – Pre-uploaded files (Gemini API): `files/abc123`
+
+## Quick Start
+
+<Tabs>
+<TabItem value="gemini" label="Gemini API">
+
+```python
+from litellm import embedding
+import os
+
+os.environ["GEMINI_API_KEY"] = "your-api-key"
+
+# Text + Image (base64)
+response = embedding(
+    model="gemini/gemini-embedding-2-preview",
+    input=[
+        "The food was delicious and the waiter...",
+        "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAIAQMAAAD+wSzIAAAABlBMVEX///+/v7+jQ3Y5AAAADklEQVQI12P4AIX8EAgALgAD/aNpbtEAAAAASUVORK5CYII"
+    ],
+)
+print(response)
+```
+
+</TabItem>
+
+<TabItem value="vertex" label="Vertex AI">
+
+```python
+import litellm
+from litellm import embedding
+
+litellm.vertex_project = "your-project-id"
+litellm.vertex_location = "us-central1"
+
+# Text + Image (GCS URL)
+response = embedding(
+    model="vertex_ai/gemini-embedding-2-preview",
+    input=[
+        "Describe this image",
+        "gs://my-bucket/images/photo.png"
+    ],
+)
+print(response)
+```
+
+</TabItem>
+
+<TabItem value="proxy" label="LiteLLM Proxy">
+
+**1. Config (config.yaml)**
+
+```yaml
+model_list:
+  - model_name: gemini-embedding-2-preview
+    litellm_params:
+      model: gemini/gemini-embedding-2-preview
+      api_key: os.environ/GEMINI_API_KEY
+  - model_name: vertex-gemini-embedding-2-preview
+    litellm_params:
+      model: vertex_ai/gemini-embedding-2-preview
+      vertex_project: os.environ/VERTEXAI_PROJECT
+      vertex_location: os.environ/VERTEXAI_LOCATION
+
+general_settings:
+  master_key: sk-1234
+```
+
+**2. Start proxy**
+
+```bash
+litellm --config config.yaml
+```
+
+**3. Call embeddings**
+
+```bash
+curl -X POST http://localhost:4000/embeddings \
+  -H "Authorization: Bearer sk-1234" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "gemini-embedding-2-preview",
+    "input": [
+      "The food was delicious and the waiter...",
+      "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAIAQMAAAD+wSzIAAAABlBMVEX///+/v7+jQ3Y5AAAADklEQVQI12P4AIX8EAgALgAD/aNpbtEAAAAASUVORK5CYII"
+    ]
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+## Input Format Examples
+
+| Format | Example | Provider |
+|--------|---------|----------|
+| **Data URI** | `data:image/png;base64,...` | Gemini, Vertex AI |
+| **GCS URL** | `gs://bucket/path/image.png` | Vertex AI |
+| **File reference** | `files/abc123` | Gemini API only |
+
+### Supported MIME Types for Data URIs
+
+- **Images:** `image/png`, `image/jpeg`
+- **Audio:** `audio/mpeg`, `audio/wav`
+- **Video:** `video/mp4`, `video/quicktime`
+- **Documents:** `application/pdf`
+
+### GCS URL MIME Inference
+
+For Vertex AI, MIME types are inferred from file extensions:
+
+- `.png` → `image/png`
+- `.jpg` / `.jpeg` → `image/jpeg`
+- `.mp3` → `audio/mpeg`
+- `.wav` → `audio/wav`
+- `.mp4` → `video/mp4`
+- `.mov` → `video/quicktime`
+- `.pdf` → `application/pdf`
+
+## Optional Parameters
+
+| Parameter | Description | Maps to |
+|-----------|-------------|---------|
+| `dimensions` | Output embedding size | `outputDimensionality` |
+
+```python
+response = embedding(
+    model="gemini/gemini-embedding-2-preview",
+    input=["text to embed"],
+    dimensions=768,  # Optional: control output vector size
+)
+```

docs/my-website/blog/gpt_5_3_codex/index.md

@@ -0,0 +1,138 @@
+---
+slug: gpt_5_3_codex
+title: "Day 0 Support: GPT-5.3-Codex"
+date: 2026-02-24T10:00:00
+authors:
+  - sameer
+  - krrish
+  - ishaan-alt
+description: "Day 0 support for GPT-5.3-Codex on LiteLLM, including phase parameter handling for Responses API."
+tags: [openai, gpt-5.3-codex, codex, day 0 support]
+hide_table_of_contents: false
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+LiteLLM now supports GPT-5.3-Codex on Day 0, including support for the new assistant `phase` metadata on Responses API output items.
+
+{/* truncate */}
+
+## Why `phase` matters for GPT-5.3-Codex
+
+`phase` appears on assistant output items and helps distinguish preamble/commentary turns from final closeout responses.
+
+Reference: [Phase parameter docs](https://developers.openai.com/api/reference/overview)
+
+Supported values:
+- `null`
+- `"commentary"`
+- `"final_answer"`
+
+Important:
+- Persist assistant output items with `phase` exactly as returned.
+- Send those assistant items back on the next turn.
+- Do **not** add `phase` to user messages.
+
+## Docker Image
+
+```bash
+docker pull ghcr.io/berriai/litellm:v1.81.12-stable.gpt-5.3
+```
+
+## Usage 
+
+<Tabs>
+<TabItem value="proxy" label="LiteLLM Proxy">
+
+**1. Setup config.yaml**
+
+```yaml
+model_list:
+  - model_name: gpt-5.3-codex
+    litellm_params:
+      model: openai/gpt-5.3-codex
+```
+
+**2. Start the proxy**
+
+```bash
+docker run -d \
+  -p 4000:4000 \
+  -e ANTHROPIC_API_KEY=$OPENAI_API_KEY \
+  -v $(pwd)/config.yaml:/app/config.yaml \
+  ghcr.io/berriai/litellm:v1.81.12-stable.gpt-5.3 \
+  --config /app/config.yaml
+```
+
+
+**3. Test it**
+
+```bash
+curl -X POST "http://0.0.0.0:4000/v1/responses" \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer $LITELLM_KEY" \
+  -d '{
+    "model": "gpt-5.3-codex",
+    "input": "Write a Python script that checks if a number is prime."
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+## Python Example: Persist `phase` with OpenAI Client + LiteLLM Base URL
+
+```python
+from openai import OpenAI
+
+client = OpenAI(
+    base_url="http://0.0.0.0:4000/v1",  # LiteLLM Proxy
+    api_key="your-litellm-api-key",
+)
+
+items = []  # Persist this per conversation/thread
+
+
+def _item_get(item, key, default=None):
+    if isinstance(item, dict):
+        return item.get(key, default)
+    return getattr(item, key, default)
+
+
+def run_turn(user_text: str):
+    global items
+
+    # User message: no phase field
+    items.append(
+        {
+            "type": "message",
+            "role": "user",
+            "content": [{"type": "input_text", "text": user_text}],
+        }
+    )
+
+    resp = client.responses.create(
+        model="gpt-5.3-codex",
+        input=items,
+    )
+
+    # Persist assistant output items verbatim, including phase
+    for out_item in (resp.output or []):
+        items.append(out_item)
+
+    # Optional: inspect latest phase for UI/telemetry routing
+    latest_phase = None
+    for out_item in reversed(resp.output or []):
+        if _item_get(out_item, "type") == "output_item.done" and _item_get(out_item, "phase") is not None:
+            latest_phase = _item_get(out_item, "phase")
+            break
+
+    return resp, latest_phase
+```
+
+## Notes
+
+- Use `/v1/responses` for GPT Codex models.
+- Preserve full assistant output history for best multi-turn behavior.
+- If `phase` metadata is dropped during history reconstruction, output quality can degrade on long-running tasks.

docs/my-website/blog/gpt_5_4/index.md

@@ -0,0 +1,90 @@
+---
+slug: gpt_5_4
+title: "Day 0 Support: GPT-5.4"
+date: 2026-03-05T10:00:00
+authors:
+  - sameer
+  - krrish
+  - ishaan-alt
+description: "GPT-5.4 model support in LiteLLM"
+tags: [openai, gpt-5.4, completion]
+hide_table_of_contents: false
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+LiteLLM now supports fully GPT-5.4!
+
+{/* truncate */}
+
+## Docker Image
+
+```bash
+docker pull ghcr.io/berriai/litellm:v1.81.14-stable.gpt-5.4_patch
+```
+
+## Usage
+
+<Tabs>
+<TabItem value="proxy" label="LiteLLM Proxy">
+
+**1. Setup config.yaml**
+
+```yaml
+model_list:
+  - model_name: gpt-5.4
+    litellm_params:
+      model: openai/gpt-5.4
+      api_key: os.environ/OPENAI_API_KEY
+```
+
+**2. Start the proxy**
+
+```bash
+docker run -d \
+  -p 4000:4000 \
+  -e OPENAI_API_KEY=$OPENAI_API_KEY \
+  -v $(pwd)/config.yaml:/app/config.yaml \
+  ghcr.io/berriai/litellm:v1.81.14-stable.gpt-5.4_patch \
+  --config /app/config.yaml
+```
+
+**3. Test it**
+
+```bash
+curl -X POST "http://0.0.0.0:4000/chat/completions" \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer $LITELLM_KEY" \
+  -d '{
+    "model": "gpt-5.4",
+    "messages": [
+      {"role": "user", "content": "Write a Python function to check if a number is prime."}
+    ]
+  }'
+```
+
+</TabItem>
+<TabItem value="sdk" label="LiteLLM SDK">
+
+```python
+from litellm import completion
+
+response = completion(
+    model="openai/gpt-5.4",
+    messages=[
+        {"role": "user", "content": "Write a Python function to check if a number is prime."}
+    ],
+)
+
+print(response.choices[0].message.content)
+```
+
+</TabItem>
+</Tabs>
+
+## Notes
+
+- Restart your container to get the cost tracking for this model.
+- Use `/responses` for better model performance.
+- GPT-5.4 supports reasoning, function calling, vision, and tool-use — see the [OpenAI provider docs](../../docs/providers/openai) for advanced usage.

docs/my-website/blog/guardrail_logging_secret_exposure_incident/index.md

@@ -0,0 +1,78 @@
+---
+slug: guardrail-logging-secret-exposure-incident
+title: "Incident Report: Guardrail logging exposed secret headers in spend logs and traces"
+date: 2026-03-18T10:00:00
+authors:
+  - litellm
+tags: [incident-report, security, guardrails]
+hide_table_of_contents: false
+---
+
+**Date:** March 18, 2026
+**Duration:** Unknown
+**Severity:** High
+**Status:** Resolved
+
+## Summary
+
+When a custom guardrail returned the full LiteLLM request/data dictionary, the guardrail response logged by LiteLLM could include `secret_fields.raw_headers`, including plaintext `Authorization` headers containing API keys or other credentials.
+
+This information could then propagate to logging and observability surfaces that consume guardrail metadata, including:
+
+- **Spend logs in the LiteLLM UI:** visible to admins with access to spend-log data
+- **OpenTelemetry traces:** visible to anyone with access to the relevant telemetry backend
+
+LLM calls, proxy routing, and provider execution were not blocked by this bug. The impact was exposure of sensitive request headers in observability and logging paths.
+
+{/* truncate */}
+
+---
+
+## Background
+
+LiteLLM keeps internal request data (including request headers) for use during the call. That data is not meant to be written to logs or telemetry.
+
+When custom guardrails run, their outcomes are logged so they can appear in spend logs, OpenTelemetry traces, and other observability backends. If a guardrail returned the full request payload instead of a minimal result, that internal request data could be included in what was logged. Before the fix, the guardrail logging path did not strip that data before sending it to those systems.
+
+```mermaid
+flowchart TD
+    inboundRequest["1. Incoming proxy request"] --> storeSecrets["2. Store internal request data"]
+    storeSecrets --> guardrailRuns["3. Custom guardrail runs"]
+    guardrailRuns --> fullDataReturn["4. Guardrail returns full request payload"]
+    fullDataReturn --> loggingBuild["5. Build guardrail log payload"]
+    loggingBuild --> spendLogs["6a. Persist to spend logs / UI"]
+    loggingBuild --> otelTraces["6b. Attach to OTEL guardrail spans"]
+```
+
+---
+
+## Root Cause
+
+The root cause was incomplete sanitization in the guardrail logging path. When building the payload that gets sent to spend logs and traces, LiteLLM prepared guardrail responses for logging but did not strip internal request data (such as headers) from them. If a guardrail returned a response that included that data, it was passed through to the logging and observability systems unchanged.
+
+---
+
+## Impact
+
+This issue required all of the following:
+
+1. A custom guardrail returned the full LiteLLM request/data dictionary, or another response object containing `secret_fields`.
+2. LiteLLM logged that guardrail response through the standard guardrail logging path.
+3. An operator, admin, or telemetry consumer had access to the resulting logs or traces.
+
+When those conditions were met, sensitive values could become visible through:
+
+- **Spend logs / UI responses:** guardrail metadata could be included in spend-log payloads rendered in the admin UI.
+- **OpenTelemetry traces:** `guardrail_response` could be written as a span attribute on guardrail spans.
+- **Other downstream observability backends:** any integration consuming the same guardrail metadata could receive the leaked values.
+
+This was a logging and telemetry exposure bug. It did not let callers bypass auth, access other tenants directly, or change model behavior, but it could expose plaintext credentials to people with access to those observability systems.
+
+---
+
+## Guidance For Users
+
+- Upgrade to LiteLLM 1.82.3+.
+- If you operated custom guardrails that return the full request/data dict, review whether spend logs or telemetry traces were retained during the affected period.
+- Rotate any credentials that may have appeared in `Authorization` or other forwarded request headers in those systems.
+- Apply least-privilege access controls to spend-log views and telemetry backends that may contain request-derived metadata.

docs/my-website/blog/httpx_cache_eviction_incident/index.md

@@ -0,0 +1,126 @@
+---
+slug: httpx-cache-eviction-incident
+title: "Incident Report: Cache Eviction Closes In-Use httpx Clients"
+date: 2026-02-27T10:00:00
+authors:
+  - ryan
+  - ishaan-alt
+  - krrish
+tags: [incident-report, caching, stability]
+hide_table_of_contents: false
+---
+
+**Date:** February 27, 2026
+**Duration:** ~6 days (Feb 21 merge -> Feb 27 fix)
+**Severity:** High
+**Status:** Resolved
+
+> **Note:** This fix is available starting from LiteLLM `v1.81.14.rc.2` or higher.
+
+## Summary
+
+A change to improve Redis connection pool cleanup introduced a regression that closed **httpx clients** that were still actively being used by the proxy. The `LLMClientCache` (an in-memory TTL cache) stores both Redis clients *and* httpx clients under the same eviction policy. When a cache entry expired or was evicted, the new cleanup code called `aclose()`/`close()` on the evicted value which worked correctly for Redis clients, but destroyed httpx clients that other parts of the system still held references to and were actively using for LLM API calls.
+
+**Impact:** Any proxy instance that hit the cache TTL (default 10 minutes) or capacity limit (200 entries) would have its httpx clients closed out from under it, causing requests to LLM providers to fail with connection errors.
+
+{/* truncate */}
+
+---
+
+## Background
+
+`LLMClientCache` extends `InMemoryCache` and is used to cache SDK clients (OpenAI, Anthropic, etc.) to avoid re-creating them on every request. These clients are keyed by configuration + event loop ID. The cache has:
+
+- **Max size:** 200 entries
+- **Default TTL:** 10 minutes
+
+When the cache is full or entries expire, `InMemoryCache.evict_cache()` calls `_remove_key()` to drop entries.
+
+The cached values are a mix of:
+- **Redis/async Redis clients** — owned exclusively by the cache, safe to close on eviction
+- **httpx-backed SDK clients** (OpenAI, Anthropic, etc.) — shared references, still in use by router/model instances
+
+---
+
+## Root Cause
+
+[PR #21717](https://github.com/BerriAI/litellm/pull/21717) overrode `_remove_key()` in `LLMClientCache` to close async clients on eviction:
+
+<details>
+<summary>Problematic code added in PR #21717</summary>
+
+```python
+class LLMClientCache(InMemoryCache):
+    def _remove_key(self, key: str) -> None:
+        value = self.cache_dict.get(key)
+        super()._remove_key(key)
+        if value is not None:
+            close_fn = getattr(value, "aclose", None) or getattr(value, "close", None)
+            if close_fn and asyncio.iscoroutinefunction(close_fn):
+                try:
+                    asyncio.get_running_loop().create_task(close_fn())
+                except RuntimeError:
+                    pass
+            elif close_fn and callable(close_fn):
+                try:
+                    close_fn()
+                except Exception:
+                    pass
+```
+
+</details>
+
+The intent was correct for Redis clients — prevent connection pool leaks when cached Redis clients expire. But `LLMClientCache` also stores httpx-backed SDK clients (e.g., `AsyncOpenAI`, `AsyncAnthropic`). These clients:
+
+1. Have an `aclose()` method (inherited from httpx)
+2. Are still held by references elsewhere in the codebase (router, model instances)
+3. Were being closed without any check on whether they were still in use
+
+So when the cache evicted an entry, it would call `aclose()` on an httpx client that was still being used for active LLM requests → closed transport → connection errors.
+
+---
+
+## The Fix
+
+[PR #22247](https://github.com/BerriAI/litellm/pull/22247) removed the `_remove_key` override entirely:
+
+<details>
+<summary>The fix (PR #22247)</summary>
+
+```diff
+ class LLMClientCache(InMemoryCache):
+-    def _remove_key(self, key: str) -> None:
+-        """Close async clients before evicting them to prevent connection pool leaks."""
+-        value = self.cache_dict.get(key)
+-        super()._remove_key(key)
+-        if value is not None:
+-            close_fn = getattr(value, "aclose", None) or getattr(
+-                value, "close", None
+-            )
+-            ...
+-
+     def update_cache_key_with_event_loop(self, key):
+```
+
+</details>
+
+The eviction now simply drops the reference and lets Python's GC handle cleanup, which is safe because:
+- httpx clients that are still referenced elsewhere stay alive
+- Unreferenced clients get cleaned up by GC naturally
+
+The other improvements from PR #21717 were kept:
+- **`max_connections` respected for URL-based Redis configs**, previously silently dropped
+- **`disconnect()` now closes both sync and async Redis clients**, sync client was previously leaked
+- **Connection pool passthrough**, when a pool is provided with a URL config, it's used directly instead of creating a duplicate
+
+---
+
+## Remediation
+
+| Action | Status | Code |
+|--------|--------|------|
+| Remove `_remove_key` override that closes shared clients on eviction | ✅ Done | [PR #22247](https://github.com/BerriAI/litellm/pull/22247) |
+| Add e2e test: evicted client still usable (capacity) | ✅ Done | [PR #22313](https://github.com/BerriAI/litellm/pull/22313) |
+| Add e2e test: expired client still usable (TTL) | ✅ Done | [PR #22313](https://github.com/BerriAI/litellm/pull/22313) |
+
+The e2e tests go through `get_async_httpx_client()` the same code path the proxy uses in production and assert the client is still functional after eviction. These run in CI on every PR against `main`. If anyone modifies `LLMClientCache` eviction behavior, overrides `_remove_key`, or adds any form of client cleanup on eviction, these tests will fail regardless of the implementation approach.

docs/my-website/blog/litellm_observatory/index.md

@@ -3,18 +3,9 @@ slug: litellm-observatory
 title: "Improve release stability with 24 hour load tests"
 date: 2026-02-06T10:00:00
 authors:
-  - name: Alexsander Hamir
-    title: "Performance Engineer, LiteLLM"
-    url: https://www.linkedin.com/in/alexsander-baptista/
-    image_url: https://github.com/AlexsanderHamir.png
-  - name: Krrish Dholakia
-    title: "CEO, LiteLLM"
-    url: https://www.linkedin.com/in/krish-d/
-    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
-  - name: Ishaan Jaff
-    title: "CTO, LiteLLM"
-    url: https://www.linkedin.com/in/reffajnaahsi/
-    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+  - alexsander
+  - krrish
+  - ishaan-alt
 description: "How we built a long-running, release-validation system to catch regressions before they reach users."
 tags: [testing, observability, reliability, releases]
 hide_table_of_contents: false
@@ -28,6 +19,8 @@ As LiteLLM adoption has grown, so have expectations around reliability, performa
 
 This post introduces **LiteLLM Observatory**, a long-running release-validation system we built to catch regressions before they reach users.
 
+{/* truncate */}
+
 ---
 
 ## Why We Built the Observatory
@@ -133,4 +126,3 @@ Reliability is an ongoing investment.
 LiteLLM Observatory is one of several systems we’re building to continuously raise the bar on release quality and operational safety. As LiteLLM evolves, so will our validation tooling, informed by real-world usage and lessons learned.
 
 We’ll continue to share those improvements openly as we go.
-

docs/my-website/blog/minimax_m2_5/index.md

@@ -3,18 +3,9 @@ slug: minimax_m2_5
 title: "Day 0 Support: MiniMax-M2.5"
 date: 2026-02-12T10:00:00
 authors:
-  - name: Sameer Kankute
-    title: SWE @ LiteLLM (LLM Translation)
-    url: https://www.linkedin.com/in/sameer-kankute/
-    image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
-  - name: Krrish Dholakia
-    title: "CEO, LiteLLM"
-    url: https://www.linkedin.com/in/krish-d/
-    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
-  - name: Ishaan Jaff
-    title: "CTO, LiteLLM"
-    url: https://www.linkedin.com/in/reffajnaahsi/
-    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+  - sameer
+  - krrish
+  - ishaan-alt
 description: "Day 0 support for MiniMax-M2.5 on LiteLLM"
 tags: [minimax, M2.5, llm]
 hide_table_of_contents: false
@@ -25,6 +16,8 @@ import TabItem from '@theme/TabItem';
 
 LiteLLM now supports MiniMax-M2.5 on Day 0. Use it across OpenAI-compatible and Anthropic-compatible APIs through the LiteLLM AI Gateway.
 
+{/* truncate */}
+
 ## Supported Models
 
 LiteLLM supports the following MiniMax models:

docs/my-website/blog/model_cost_map_incident/index.md

@@ -3,10 +3,7 @@ slug: model-cost-map-incident
 title: "Incident Report: Invalid model cost map on main"
 date: 2026-02-10T10:00:00
 authors:
-  - name: Ishaan Jaffer
-    title: "CTO, LiteLLM"
-    url: https://www.linkedin.com/in/ishaanjaffer/
-    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+  - ishaan
 tags: [incident-report, stability]
 hide_table_of_contents: false
 ---

docs/my-website/blog/realtime_webrtc_http_endpoints/index.md

@@ -0,0 +1,111 @@
+---
+slug: realtime_webrtc_http_endpoints
+title: "Realtime WebRTC HTTP Endpoints"
+date: 2026-03-12T10:00:00
+authors:
+  - sameer
+  - krrish
+  - ishaan-alt
+description: "Use the LiteLLM proxy to route OpenAI-style WebRTC realtime via HTTP: client_secrets and SDP exchange."
+tags: [realtime, webrtc, proxy, openai]
+hide_table_of_contents: false
+---
+
+import WebRTCTester from '@site/src/components/WebRTCTester';
+
+Connect to the Realtime API via WebRTC from browser/mobile clients. LiteLLM handles auth and key management.
+
+{/* truncate */}
+
+## How it works
+
+![WebRTC flow: Browser, LiteLLM Proxy, and OpenAI/Azure](../../img/webrtc_flow.png)
+
+**Flow of generating ephemeral token**
+
+![Ephemeral token flow: Browser requests token, LiteLLM gets real token from OpenAI, returns encrypted token](../../img/ephemeral_token.png)
+
+
+## Proxy Setup
+
+```yaml
+model_list:
+  - model_name: gpt-4o-realtime
+    litellm_params:
+      model: openai/gpt-4o-realtime-preview-2024-12-17
+      api_key: os.environ/OPENAI_API_KEY
+    model_info:
+      mode: realtime
+```
+
+**Azure:** use `model: azure/gpt-4o-realtime-preview`, `api_key`, `api_base`.
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+## Try it live
+
+<WebRTCTester />
+
+## Client Usage
+
+**1. Get token** - `POST /v1/realtime/client_secrets` with LiteLLM API key and `{ model }`.
+
+**2. WebRTC handshake** - Create `RTCPeerConnection`, add mic track, create data channel `oai-events`, send SDP offer to `POST /v1/realtime/calls` with `Authorization: Bearer <encrypted_token>` and `Content-Type: application/sdp`.
+
+**3. Events** - Use the data channel for `session.update` and other events.
+
+<details>
+<summary>Full code example</summary>
+
+```javascript
+// 1. Token
+const r = await fetch("http://proxy:4000/v1/realtime/client_secrets", {
+  method: "POST",
+  headers: { "Authorization": "Bearer sk-litellm-key", "Content-Type": "application/json" },
+  body: JSON.stringify({ model: "gpt-4o-realtime" }),
+});
+const { client_secret } = await r.json();
+const token = client_secret.value;
+
+// 2. WebRTC
+const pc = new RTCPeerConnection();
+const audio = document.createElement("audio");
+audio.autoplay = true;
+pc.ontrack = (e) => (audio.srcObject = e.streams[0]);
+const ms = await navigator.mediaDevices.getUserMedia({ audio: true });
+pc.addTrack(ms.getTracks()[0]);
+const dc = pc.createDataChannel("oai-events");
+const offer = await pc.createOffer();
+await pc.setLocalDescription(offer);
+
+const sdpRes = await fetch("http://proxy:4000/v1/realtime/calls", {
+  method: "POST",
+  headers: { "Authorization": `Bearer ${token}`, "Content-Type": "application/sdp" },
+  body: offer.sdp,
+});
+await pc.setRemoteDescription({ type: "answer", sdp: await sdpRes.text() });
+
+// 3. Events
+dc.send(JSON.stringify({ type: "session.update", session: { instructions: "..." } }));
+```
+
+</details>
+
+## FAQ
+
+**Q: What do I do if I get a 401 Token expired error?**  
+A: Tokens are short-lived. Get a fresh token right before creating the WebRTC offer.
+
+**Q: Which key should I use for `/v1/realtime/calls`?**  
+A: Use the **encrypted token** from `client_secrets`, not your raw API key.
+
+**Q: Should I pass the `model` parameter when making the call?**  
+A: No, the encrypted token already encodes all routing information including model.
+
+**Q: How do I resolve Azure `api-version` errors?**  
+A: Set the correct `api_version` in `litellm_params` (or via the `AZURE_API_VERSION` environment variable), along with the right `api_base` and deployment values.
+
+**Q: What if I get no audio?**  
+A: Make sure you grant microphone permission, ensure `pc.ontrack` assigns the audio element with `autoplay` enabled, check your network/firewall for WebRTC traffic, and inspect the browser console for ICE or SDP errors.

docs/my-website/blog/responses_api_encrypted_content_incident/index.md

@@ -0,0 +1,312 @@
+---
+slug: responses-api-encrypted-content-incident
+title: "Incident Report: Encrypted Content Failures in Multi-Region Responses API Load Balancing"
+date: 2026-02-24T10:00:00
+authors:
+  - sameer
+  - krrish
+  - ishaan-alt
+tags: [incident-report, proxy, responses-api, load-balancing]
+hide_table_of_contents: false
+---
+
+**Date:** Feb 24, 2026  
+**Duration:** Ongoing (until fix deployed)  
+**Severity:** High (for users load balancing Responses API across different API keys)  
+**Status:** Resolved
+
+## Summary
+
+When load balancing OpenAI's Responses API across deployments with **different API keys** (e.g., different Azure regions or OpenAI organizations), follow-up requests containing encrypted content items (like `rs_...` reasoning items) would fail with:
+
+```json
+{
+  "error": {
+    "message": "The encrypted content for item rs_0d09d6e56879e76500699d6feee41c8197bd268aae76141f87 could not be verified. Reason: Encrypted content organization_id did not match the target organization.",
+    "type": "invalid_request_error",
+    "code": "invalid_encrypted_content"
+  }
+}
+```
+
+Encrypted content items are cryptographically tied to the API key's organization that created them. When the router load balanced a follow-up request to a deployment with a different API key, decryption failed.
+
+- **Responses API calls with encrypted content:** Complete failure when routed to wrong deployment
+- **Initial requests:** Unaffected — only follow-up requests containing encrypted items failed
+- **Other API endpoints:** No impact — chat completions, embeddings, etc. functioned normally
+
+{/* truncate */}
+
+---
+
+## Background
+
+OpenAI's Responses API can return encrypted "reasoning items" (with IDs like `rs_...`) that contain intermediate reasoning steps. These items are encrypted with the organization's key and can only be decrypted by the same organization's API key.
+
+When load balancing across deployments with different API keys, the existing affinity mechanisms were insufficient:
+
+- **`responses_api_deployment_check`**: Requires `previous_response_id` which some clients (like Codex) don't provide
+- **`deployment_affinity`**: Too broad — pins *all* requests from a user to one deployment, reducing effective quota by the number of users
+- **`session_affinity`**: Requires explicit session IDs and still reduces quota
+
+```mermaid
+flowchart TD
+    A["1. Initial request to Responses API
+    router.aresponses()"] --> B["2. Router load balances to Deployment A
+    (API Key 1, Azure East US)"]
+    B --> C["3. Response contains encrypted item
+    rs_abc123 (encrypted with Org 1 key)"]
+    C --> D["4. Follow-up request includes rs_abc123 in input"]
+    D --> E["5. Router load balances to Deployment B
+    (API Key 2, Azure West Europe)"]
+    E -->|"Different API key"| F["6. ❌ Deployment B cannot decrypt rs_abc123
+    Error: invalid_encrypted_content"]
+    
+    D -.->|"With encrypted_content_affinity"| G["5b. Router detects rs_abc123 was created by Deployment A"]
+    G --> H["6b. ✅ Routes to Deployment A (bypasses rate limits)
+    Request succeeds"]
+
+    style F fill:#f8d7da,stroke:#dc3545
+    style H fill:#d4edda,stroke:#28a745
+    style E fill:#fff3cd,stroke:#ffc107
+    style G fill:#d4edda,stroke:#28a745
+```
+
+---
+
+## Root Cause
+
+LiteLLM's router had no mechanism to track which deployment created specific encrypted content items and route follow-up requests accordingly. The router treated all deployments as interchangeable, leading to decryption failures when encrypted content crossed organizational boundaries.
+
+**The Problem Flow:**
+
+1. User calls `router.aresponses()` with model `gpt-5.1-codex`
+2. Router load balances to Deployment A (Azure East US, API Key 1)
+3. Response contains encrypted reasoning item `rs_abc123` (encrypted with Org 1's key)
+4. User makes follow-up request with `rs_abc123` in the input
+5. Router load balances to Deployment B (Azure West Europe, API Key 2)
+6. Deployment B tries to decrypt `rs_abc123` with Org 2's key → **fails**
+
+**Why Existing Solutions Didn't Work:**
+
+- **`previous_response_id`**: Not provided by all clients (e.g., Codex)
+- **`deployment_affinity`**: Pins *all* user requests to one deployment → reduces quota to 1/N where N = number of deployments
+- **`session_affinity`**: Requires explicit session management and still reduces quota
+
+**Timeline:**
+
+1. Users configured multi-region Responses API load balancing with different API keys
+2. Initial requests succeeded, but follow-up requests with encrypted content failed intermittently
+3. Error rate correlated with number of deployments (more deployments = higher chance of routing to wrong one)
+4. Investigation revealed encrypted content was organization-bound
+5. Existing affinity mechanisms deemed unsuitable (quota reduction, missing `previous_response_id`)
+6. New solution designed and implemented: `encrypted_content_affinity`
+
+---
+
+## The Fix
+
+Implemented a new `encrypted_content_affinity` pre-call check that intelligently tracks encrypted content and routes follow-up requests **only when necessary**.
+
+### Implementation
+
+**1. Encoding `model_id` into output items** ([`responses/utils.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/responses/utils.py))
+
+The same approach used for `previous_response_id` affinity — no cache needed. When a response contains output items with `encrypted_content`, LiteLLM encodes the originating deployment's `model_id` in **two places** for redundancy:
+
+1. **Into the item ID** (if present): `rs_abc123` → `encitem_{base64("litellm:model_id:{model_id};item_id:rs_abc123")}`
+2. **Into the encrypted_content itself**: Wraps the content with `litellm_enc:{base64("model_id:{model_id}")};{original_encrypted_content}`
+
+```python
+# Encoding item IDs (when present)
+def _build_encrypted_item_id(model_id: str, item_id: str) -> str:
+    assembled = f"litellm:model_id:{model_id};item_id:{item_id}"
+    encoded = base64.b64encode(assembled.encode("utf-8")).decode("utf-8")
+    return f"encitem_{encoded}"
+
+# Wrapping encrypted_content (always, for redundancy)
+def _wrap_encrypted_content_with_model_id(encrypted_content: str, model_id: str) -> str:
+    metadata = f"model_id:{model_id}"
+    encoded_metadata = base64.b64encode(metadata.encode("utf-8")).decode("utf-8")
+    return f"litellm_enc:{encoded_metadata};{encrypted_content}"
+```
+
+**Why wrap encrypted_content directly?** Some clients (like Codex) don't consistently send item IDs in follow-up requests, but they always send the `encrypted_content` itself. By embedding `model_id` into the content, affinity works even when IDs are missing.
+
+**Streaming responses:** The wrapping logic is applied to both:
+- Final response objects (non-streaming)
+- Individual streaming events (`response.output_item.added`, `response.output_item.done`)
+
+This ensures clients receiving streaming responses get wrapped content they can send back.
+
+Before forwarding to the upstream provider, LiteLLM restores the original item IDs and unwraps encrypted_content so the provider never sees the encoded form:
+
+```python
+# In responses/main.py — before calling the handler
+input = ResponsesAPIRequestUtils._restore_encrypted_content_item_ids_in_input(input)
+```
+
+**2. `EncryptedContentAffinityCheck` — routing only** ([`encrypted_content_affinity_check.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py))
+
+No `async_log_success_event` or cache lookups — the `model_id` is decoded directly from the item ID or encrypted_content:
+
+```python
+class EncryptedContentAffinityCheck(CustomLogger):
+    async def async_filter_deployments(self, model, healthy_deployments, ...):
+        """Extract model_id from input items (ID or encrypted_content) and pin to that deployment."""
+        for item in request_kwargs.get("input", []):
+            # Try to extract model_id from two sources:
+            model_id = self._extract_model_id_from_input(item)
+            
+            if model_id:
+                deployment = self._find_deployment_by_model_id(
+                    healthy_deployments, model_id
+                )
+                if deployment:
+                    request_kwargs["_encrypted_content_affinity_pinned"] = True
+                    return [deployment]
+        return healthy_deployments
+    
+    def _extract_model_id_from_input(self, item: dict) -> Optional[str]:
+        """Extract model_id from either encoded ID or wrapped encrypted_content."""
+        # 1. Try decoding from item ID (if present)
+        item_id = item.get("id", "")
+        if item_id:
+            decoded = ResponsesAPIRequestUtils._decode_encrypted_item_id(item_id)
+            if decoded:
+                return decoded["model_id"]
+        
+        # 2. Try unwrapping from encrypted_content (fallback for clients that omit IDs)
+        encrypted_content = item.get("encrypted_content", "")
+        if encrypted_content and encrypted_content.startswith("litellm_enc:"):
+            model_id, _ = ResponsesAPIRequestUtils._unwrap_encrypted_content_with_model_id(
+                encrypted_content
+            )
+            return model_id
+        
+        return None
+```
+
+**3. Rate Limit Bypass** ([`router.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/router.py))
+
+When encrypted content requires a specific deployment, RPM/TPM limits are bypassed (the request would fail on any other deployment anyway):
+
+```python
+# In async_get_available_deployment, after filtering healthy deployments:
+if (
+    request_kwargs.get("_encrypted_content_affinity_pinned")
+    and len(healthy_deployments) == 1
+):
+    return healthy_deployments[0]  # Bypass routing strategy (RPM/TPM checks)
+```
+
+**3. Configuration**
+
+```yaml
+router_settings:
+  routing_strategy: usage-based-routing-v2
+  enable_pre_call_checks: true
+  optional_pre_call_checks:
+    - encrypted_content_affinity
+  deployment_affinity_ttl_seconds: 86400  # 24 hours
+```
+
+### Key Benefits
+
+✅ **No quota reduction**: Only pins requests containing encrypted items  
+✅ **Bypasses rate limits**: When encrypted content requires a specific deployment, RPM/TPM limits don't block it  
+✅ **No `previous_response_id` required**: Works by encoding `model_id` directly into the item ID  
+✅ **No cache required**: `model_id` is decoded on-the-fly from the item ID — no Redis, no TTL  
+✅ **Globally safe**: Can be enabled for all models; non-Responses-API calls are unaffected  
+✅ **Surgical precision**: Normal requests continue to load balance freely
+
+---
+
+## Remediation
+
+| # | Action | Status | Code |
+|---|---|---|---|
+| 1 | Encode `model_id` into encrypted-content item IDs on response | ✅ Done | [`responses/utils.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/responses/utils.py) |
+| 2 | Restore original item IDs before forwarding to upstream provider | ✅ Done | [`responses/main.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/responses/main.py) |
+| 3 | `EncryptedContentAffinityCheck`: decode item IDs to route (no cache) | ✅ Done | [`encrypted_content_affinity_check.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py) |
+| 4 | Add `encrypted_content_affinity` to `OptionalPreCallChecks` type | ✅ Done | [`types/router.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/types/router.py) |
+| 5 | Implement rate limit bypass for affinity-pinned requests | ✅ Done | [`router.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/router.py) |
+| 6 | Unit tests: encoding/decoding utilities, routing, RPM bypass | ✅ Done | [`test_encrypted_content_affinity_check.py`](https://github.com/BerriAI/litellm/blob/main/litellm/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py) |
+| 7 | Documentation: Responses API guide, load balancing guide, config reference | ✅ Done | [Docs](https://docs.litellm.ai/docs/response_api#encrypted-content-affinity-multi-region-load-balancing) |
+| 8 | **[Mar 3]** Fix streaming events to wrap encrypted_content | ✅ Done | [`responses/streaming_iterator.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/responses/streaming_iterator.py) |
+
+---
+
+## Follow-up Fix: Streaming Responses (Mar 3, 2026)
+
+### The Issue
+
+After the initial fix was deployed, users reported that the `invalid_encrypted_content` error **still occurred** when using streaming responses with clients like Codex. Investigation revealed:
+
+- ✅ Non-streaming responses: `encrypted_content` was correctly wrapped with `litellm_enc:` prefix
+- ❌ Streaming responses: Individual `response.output_item.added` and `response.output_item.done` events contained **raw, unwrapped** `encrypted_content`
+
+Since Codex and other clients consume responses as streams, they received unwrapped content in these events and sent it back in follow-up requests, causing the affinity check to fail.
+
+### The Root Cause
+
+The `_update_encrypted_content_item_ids_in_response` function only modified the **final** response object, which is used for non-streaming responses. For streaming responses, individual chunks are processed by `ResponsesAPIStreamingIterator._process_chunk`, which was **not** applying the wrapping logic to streaming events.
+
+### The Fix
+
+Modified `litellm/litellm/responses/streaming_iterator.py` to wrap `encrypted_content` in streaming events:
+
+```python
+# In ResponsesAPIStreamingIterator._process_chunk
+if (
+    self.litellm_metadata
+    and self.litellm_metadata.get("encrypted_content_affinity_enabled")
+):
+    event_type = getattr(openai_responses_api_chunk, "type", None)
+    if event_type in (
+        ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED,
+        ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
+    ):
+        item = getattr(openai_responses_api_chunk, "item", None)
+        if item:
+            encrypted_content = getattr(item, "encrypted_content", None)
+            if encrypted_content and isinstance(encrypted_content, str):
+                model_id = (
+                    self.litellm_metadata.get("model_info", {}).get("id")
+                    if self.litellm_metadata
+                    else None
+                )
+                if model_id:
+                    wrapped_content = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id(
+                        encrypted_content, model_id
+                    )
+                    setattr(item, "encrypted_content", wrapped_content)
+```
+
+This ensures that **all** `encrypted_content` sent to clients (streaming or non-streaming) is wrapped with `model_id` metadata, enabling consistent affinity routing.
+
+---
+
+## Migration Guide
+
+### Before (Using `deployment_affinity`)
+
+```yaml
+router_settings:
+  optional_pre_call_checks:
+    - deployment_affinity  # ❌ Reduces quota by number of users
+```
+
+**Problem:** All requests from a user pin to one deployment, reducing effective quota to 1/N.
+
+### After (Using `encrypted_content_affinity`)
+
+```yaml
+router_settings:
+  optional_pre_call_checks:
+    - encrypted_content_affinity  # ✅ Only pins requests with encrypted content
+```
+
+**Benefit:** Normal requests load balance freely, only encrypted content requests pin when necessary.
+
+---

docs/my-website/blog/server_root_path/index.md

@@ -3,17 +3,9 @@ slug: server-root-path-incident
 title: "Incident Report: SERVER_ROOT_PATH regression broke UI routing"
 date: 2026-02-21T10:00:00
 authors:
-  - name: Yuneng Jiang
-    title: SWE @ LiteLLM (Full Stack)
-    url: https://www.linkedin.com/in/yunengjiang/
-  - name: Ishaan Jaff
-    title: "CTO, LiteLLM"
-    url: https://www.linkedin.com/in/reffajnaahsi/
-    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
-  - name: Krrish Dholakia
-    title: "CEO, LiteLLM"
-    url: https://www.linkedin.com/in/krish-d/
-    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
+  - yuneng
+  - ishaan-alt
+  - krrish
 tags: [incident-report, ui, stability]
 hide_table_of_contents: false
 ---

docs/my-website/blog/sub_millisecond_proxy_overhead/index.md

@@ -3,18 +3,9 @@ slug: sub-millisecond-proxy-overhead
 title: "Achieving Sub-Millisecond Proxy Overhead"
 date: 2026-02-02T10:00:00
 authors:
-  - name: Alexsander Hamir
-    title: "Performance Engineer, LiteLLM"
-    url: https://www.linkedin.com/in/alexsander-baptista/
-    image_url: https://github.com/AlexsanderHamir.png
-  - name: Krrish Dholakia
-    title: "CEO, LiteLLM"
-    url: https://www.linkedin.com/in/krish-d/
-    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
-  - name: Ishaan Jaff
-    title: "CTO, LiteLLM"
-    url: https://www.linkedin.com/in/reffajnaahsi/
-    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+  - alexsander
+  - krrish
+  - ishaan-alt
 description: "Our Q1 performance target and architectural direction for achieving sub-millisecond proxy overhead on modest hardware."
 tags: [performance, architecture]
 hide_table_of_contents: false
@@ -32,6 +23,8 @@ Proxy overhead refers to the latency introduced by LiteLLM itself, independent o
 
 To measure it, we run the same workload directly against the provider and through LiteLLM at identical QPS (for example, 1,000 QPS) and compare the latency delta. To reduce noise, the load generator, LiteLLM, and a mock LLM endpoint all run on the same machine, ensuring the difference reflects proxy overhead rather than network latency.
 
+{/* truncate */}
+
 ---
 
 ## Where We're Coming From

docs/my-website/blog/video_characters_litellm/index.md

@@ -0,0 +1,121 @@
+---
+slug: video_characters_api
+title: "New Video Characters, Edit and Extension API support"
+date: 2026-03-16T10:00:00
+authors:
+  - sameer
+  - krrish
+  - ishaan-alt
+description: "LiteLLM now supports creating, retrieving, and managing reusable video characters across multiple video generations."
+tags: [videos, characters, proxy, routing]
+hide_table_of_contents: false
+---
+
+LiteLLM now supoports videos character, edit and extension apis.
+
+{/* truncate */}
+
+## What's New
+
+Four new endpoints for video character operations:
+- **Create character** - Upload a video to create a reusable asset
+- **Get character** - Retrieve character metadata
+- **Edit video** - Modify generated videos
+- **Extend video** - Continue clips with character consistency
+
+**Available from:** LiteLLM v1.83.0+
+
+## Quick Example
+
+```python
+import litellm
+
+# Create character from video
+character = litellm.avideo_create_character(
+    name="Luna",
+    video=open("luna.mp4", "rb"),
+    custom_llm_provider="openai",
+    model="sora-2"
+)
+print(f"Character: {character.id}")
+
+# Use in generation
+video = litellm.avideo(
+    model="sora-2",
+    prompt="Luna dances through a magical forest.",
+    characters=[{"id": character.id}],
+    seconds="8"
+)
+
+# Get character info
+fetched = litellm.avideo_get_character(
+    character_id=character.id,
+    custom_llm_provider="openai"
+)
+
+# Edit with character preserved
+edited = litellm.avideo_edit(
+    video_id=video.id,
+    prompt="Add warm golden lighting"
+)
+
+# Extend sequence
+extended = litellm.avideo_extension(
+    video_id=video.id,
+    prompt="Luna waves goodbye",
+    seconds="5"
+)
+```
+
+## Via Proxy
+
+```bash
+# Create character
+curl -X POST "http://localhost:4000/v1/videos/characters" \
+  -H "Authorization: Bearer sk-litellm-key" \
+  -F "video=@luna.mp4" \
+  -F "name=Luna"
+
+# Get character
+curl -X GET "http://localhost:4000/v1/videos/characters/char_abc123def456" \
+  -H "Authorization: Bearer sk-litellm-key"
+
+# Edit video
+curl -X POST "http://localhost:4000/v1/videos/edits" \
+  -H "Authorization: Bearer sk-litellm-key" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "video": {"id": "video_xyz789"},
+    "prompt": "Add warm golden lighting and enhance colors"
+  }'
+
+# Extend video
+curl -X POST "http://localhost:4000/v1/videos/extensions" \
+  -H "Authorization: Bearer sk-litellm-key" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "video": {"id": "video_xyz789"},
+    "prompt": "Luna waves goodbye and walks into the sunset",
+    "seconds": "5"
+  }'
+```
+
+## Managed Character IDs
+
+LiteLLM automatically encodes provider and model metadata into character IDs:
+
+**What happens:**
+```
+Upload character "Luna" with model "sora-2" on OpenAI
+  ↓
+LiteLLM creates: char_abc123def456 (contains provider + model_id)
+  ↓
+When you reference it later, LiteLLM decodes automatically
+  ↓
+Router knows exactly which deployment to use
+```
+
+**Behind the scenes:**
+- Character ID format: `character_<base64_encoded_metadata>`
+- Metadata includes: provider, model_id, original_character_id
+- Transparent to you - just use the ID, LiteLLM handles routing

docs/my-website/blog/vllm_embeddings_incident/index.md

@@ -3,18 +3,9 @@ slug: vllm-embeddings-incident
 title: "Incident Report: vLLM Embeddings Broken by encoding_format Parameter"
 date: 2026-02-18T10:00:00
 authors:
-  - name: Sameer Kankute
-    title: SWE @ LiteLLM (LLM Translation)
-    url: https://www.linkedin.com/in/sameer-kankute/
-    image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
-  - name: Krrish Dholakia
-    title: "CEO, LiteLLM"
-    url: https://www.linkedin.com/in/krish-d/
-    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
-  - name: Ishaan Jaff
-    title: "CTO, LiteLLM"
-    url: https://www.linkedin.com/in/reffajnaahsi/
-    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+  - sameer
+  - krrish
+  - ishaan-alt
 tags: [incident-report, embeddings, vllm]
 hide_table_of_contents: false
 ---

docs/my-website/docs/a2a.md

@@ -20,6 +20,7 @@ Add A2A Agents on LiteLLM AI Gateway, Invoke agents in A2A Protocol, track reque
 | Logging | ✅ |
 | Load Balancing | ✅ |
 | Streaming | ✅ |
+| [Iteration Budgets](a2a_iteration_budgets) | ✅ |
 
 
 :::tip

docs/my-website/docs/a2a_agent_headers.md

@@ -0,0 +1,252 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# A2A Agent Authentication Headers
+
+Forward authentication credentials (Bearer tokens, API keys, etc.) from clients to backend A2A agents.
+
+## Overview
+
+When LiteLLM proxies a request to a backend A2A agent, the agent may require its own authentication headers. There are three ways to supply them:
+
+| Method | Who configures | How it works |
+|---|---|---|
+| **Static headers** | Admin (UI / API) | Always sent, regardless of client request |
+| **Forward client headers** | Admin (UI / API) | Header names to extract from client request and forward |
+| **Convention-based** | Client (no admin config) | Client sends `x-a2a-{agent_name}-{header}` — automatically routed |
+
+All three methods can be combined. **Static headers always win** on key conflicts.
+
+---
+
+## Method 1 — Static Headers
+
+Admin-configured headers that are always sent to the backend agent. Use this for server-to-server tokens or internal credentials that clients should never see or override.
+
+<Tabs>
+<TabItem value="ui" label="UI">
+
+1. Go to **Agents** in the LiteLLM dashboard.
+2. Create or edit an agent.
+3. Open the **Authentication Headers** panel.
+4. Under **Static Headers**, click **Add Static Header** and fill in the header name and value.
+
+</TabItem>
+<TabItem value="api" label="REST API">
+
+```bash
+curl -X POST http://localhost:4000/v1/agents \
+  -H "Authorization: Bearer sk-admin" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "agent_name": "my-agent",
+    "agent_card_params": { ... },
+    "static_headers": {
+      "Authorization": "Bearer internal-server-token",
+      "X-Internal-Service": "litellm-proxy"
+    }
+  }'
+```
+
+To update an existing agent:
+
+```bash
+curl -X PATCH http://localhost:4000/v1/agents/{agent_id} \
+  -H "Authorization: Bearer sk-admin" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "static_headers": {
+      "Authorization": "Bearer new-token"
+    }
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+**Client call — no special headers needed:**
+
+```bash
+curl -X POST http://localhost:4000/a2a/my-agent \
+  -H "Authorization: Bearer sk-client-key" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "jsonrpc": "2.0", "id": "1", "method": "message/send",
+    "params": { "message": { "role": "user", "parts": [{"kind": "text", "text": "Hello"}], "messageId": "msg-1" } }
+  }'
+```
+
+The backend agent receives `Authorization: Bearer internal-server-token` without the client ever knowing the value.
+
+---
+
+## Method 2 — Forward Client Headers
+
+Admin specifies a list of header **names**. When the client sends a request that includes those headers, LiteLLM extracts their values and forwards them to the backend agent. The client controls the values; the admin controls which headers are eligible to be forwarded.
+
+<Tabs>
+<TabItem value="ui" label="UI">
+
+1. Go to **Agents** in the LiteLLM dashboard.
+2. Create or edit an agent.
+3. Open the **Authentication Headers** panel.
+4. Under **Forward Client Headers**, type header names and press **Enter** (e.g. `x-api-key`, `Authorization`).
+
+</TabItem>
+<TabItem value="api" label="REST API">
+
+```bash
+curl -X POST http://localhost:4000/v1/agents \
+  -H "Authorization: Bearer sk-admin" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "agent_name": "my-agent",
+    "agent_card_params": { ... },
+    "extra_headers": ["x-api-key", "x-user-token"]
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+**Client call — include the forwarded headers:**
+
+```bash
+curl -X POST http://localhost:4000/a2a/my-agent \
+  -H "Authorization: Bearer sk-client-key" \
+  -H "x-api-key: user-secret-value" \
+  -H "Content-Type: application/json" \
+  -d '{ ... }'
+```
+
+The backend agent receives `x-api-key: user-secret-value`.
+
+:::note
+Header name matching is **case-insensitive**. If the client sends `X-API-Key` and `extra_headers` lists `x-api-key`, they match.
+:::
+
+---
+
+## Method 3 — Convention-Based Forwarding
+
+Clients can forward headers to a specific agent without any admin pre-configuration by using the naming convention:
+
+```
+x-a2a-{agent_name_or_id}-{header_name}: value
+```
+
+LiteLLM parses these headers automatically and routes them to the matching agent only.
+
+**Examples:**
+
+| Client header sent | Agent name/ID | Forwarded as |
+|---|---|---|
+| `x-a2a-my-agent-authorization: Bearer tok` | `my-agent` | `authorization: Bearer tok` |
+| `x-a2a-my-agent-x-api-key: secret` | `my-agent` | `x-api-key: secret` |
+| `x-a2a-abc123-authorization: Bearer tok` | agent ID `abc123` | `authorization: Bearer tok` |
+
+```bash
+curl -X POST http://localhost:4000/a2a/my-agent \
+  -H "Authorization: Bearer sk-client-key" \
+  -H "x-a2a-my-agent-authorization: Bearer agent-specific-token" \
+  -H "Content-Type: application/json" \
+  -d '{ ... }'
+```
+
+The `x-a2a-other-agent-authorization` header sent in the same request is **not** forwarded to `my-agent` — it is silently ignored.
+
+:::tip Matches both agent name and agent ID
+Both the human-readable name (e.g. `my-agent`) and the UUID (e.g. `abc123-...`) are valid. Use whichever is convenient for the client.
+:::
+
+---
+
+## Merge Precedence
+
+When multiple methods supply the same header name, **static headers win**:
+
+```
+dynamic (forwarded/convention)  →  merged  ←  static (overlays, wins)
+```
+
+Example:
+
+| Source | `Authorization` value |
+|---|---|
+| Client sends (via `extra_headers` or convention) | `Bearer client-token` |
+| Admin-configured `static_headers` | `Bearer server-token` |
+| **What the backend agent receives** | **`Bearer server-token`** |
+
+This ensures admin-controlled credentials cannot be overridden by client requests.
+
+---
+
+## Combining All Three Methods
+
+```bash
+# Register agent with static + forwarded headers
+curl -X POST http://localhost:4000/v1/agents \
+  -H "Authorization: Bearer sk-admin" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "agent_name": "my-agent",
+    "agent_card_params": { ... },
+    "static_headers": {
+      "X-Internal-Token": "secret123"
+    },
+    "extra_headers": ["x-user-id"]
+  }'
+
+# Client call using all three mechanisms
+curl -X POST http://localhost:4000/a2a/my-agent \
+  -H "Authorization: Bearer sk-client-key" \
+  -H "x-user-id: user-42" \
+  -H "x-a2a-my-agent-x-request-id: req-abc" \
+  -H "Content-Type: application/json" \
+  -d '{ ... }'
+```
+
+The backend agent receives:
+
+```
+X-Internal-Token: secret123          ← static header (always)
+x-user-id: user-42                   ← forwarded (in extra_headers)
+x-request-id: req-abc                ← convention-based (x-a2a-my-agent-*)
+X-LiteLLM-Trace-Id: <uuid>           ← LiteLLM internal
+X-LiteLLM-Agent-Id: <agent-id>       ← LiteLLM internal
+```
+
+---
+
+## Header Isolation
+
+Each agent invocation uses an isolated HTTP connection. Headers configured for agent A are **never** sent to agent B, even if both agents are running and receiving requests simultaneously.
+
+---
+
+## API Reference
+
+### `POST /v1/agents` / `PATCH /v1/agents/{agent_id}`
+
+| Field | Type | Description |
+|---|---|---|
+| `static_headers` | `object` | `{"Header-Name": "value"}` — always forwarded |
+| `extra_headers` | `string[]` | Header names to extract from client request and forward |
+
+### Agent Response
+
+Both fields are returned in `GET /v1/agents` and `GET /v1/agents/{agent_id}`:
+
+```json
+{
+  "agent_id": "...",
+  "agent_name": "my-agent",
+  "static_headers": { "X-Internal-Token": "secret123" },
+  "extra_headers": ["x-user-id"],
+  ...
+}
+```
+
+:::caution
+`static_headers` values are stored in the database and returned by the API. Treat them as you would any credential — do not store sensitive long-lived tokens here if your API is publicly accessible. Consider using short-lived tokens or environment-injected secrets instead.
+:::

docs/my-website/docs/a2a_iteration_budgets.md

@@ -0,0 +1,188 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Agent Iteration Budgets
+
+Control runaway costs from agentic loops with per-session iteration and budget caps.
+
+## Overview
+
+When agents run agentic loops, they can make unbounded LLM calls, causing unexpected costs. LiteLLM provides two controls:
+
+| Control | Description |
+|---------|-------------|
+| **Max Iterations** | Hard cap on the number of LLM calls per session |
+| **Max Budget Per Session** | Dollar cap per session (identified by `x-litellm-trace-id`) |
+
+Both controls require a `session_id` (sent via `x-litellm-trace-id` header or `metadata.session_id`) to track calls within a session.
+
+## Trace-ID Enforcement
+
+LiteLLM supports two independent trace-id flags, configured in `litellm_params` on the agent:
+
+| Flag | Description |
+|------|-------------|
+| `require_trace_id_on_calls_to_agent` | Requires callers invoking this agent to include `x-litellm-trace-id`. Use when the agent should only be called as a sub-agent with a trace context. Returns **400** if missing. |
+| `require_trace_id_on_calls_by_agent` | Requires all LLM/MCP calls made **by** this agent (via its virtual key) to include `x-litellm-trace-id`. This is what enables `max_iterations` and `max_budget_per_session` tracking. Returns **400** if missing. |
+
+## Configuring via UI
+
+When creating an agent in the LiteLLM Admin UI:
+
+1. Navigate to the **Agents** tab and click **Add Agent**
+2. In the **Agent Settings** step, expand the **Tracing** section
+3. Toggle **Require x-litellm-trace-id on calls BY this agent** to enable session tracking
+4. Set **Max Iterations** to cap the number of LLM calls per session
+5. Set **Max Budget Per Session ($)** to cap spend per session
+
+The trace-id flags are stored on the agent's `litellm_params`. Budget controls (`max_iterations`, `max_budget_per_session`) are stored in the virtual key's metadata.
+
+## Configuring via API
+
+Set trace-id enforcement on the agent itself:
+
+```bash
+curl -X POST 'http://localhost:4000/v1/agents' \
+  -H 'Authorization: Bearer sk-1234' \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "agent_name": "my-research-agent",
+    "agent_card_params": {
+      "name": "my-research-agent",
+      "description": "A research agent with budget controls",
+      "url": "http://my-agent:8080",
+      "version": "1.0.0"
+    },
+    "litellm_params": {
+      "require_trace_id_on_calls_to_agent": true,
+      "require_trace_id_on_calls_by_agent": true
+    }
+  }'
+```
+
+Budget controls are set on the agent's `litellm_params` (not on individual keys), so they apply across all keys for the agent:
+
+```bash
+curl -X POST 'http://localhost:4000/v1/agents' \
+  -H 'Authorization: Bearer sk-1234' \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "agent_name": "my-research-agent",
+    "agent_card_params": {
+      "name": "my-research-agent",
+      "description": "A research agent with budget controls",
+      "url": "http://my-agent:8080",
+      "version": "1.0.0"
+    },
+    "litellm_params": {
+      "require_trace_id_on_calls_by_agent": true,
+      "max_iterations": 25,
+      "max_budget_per_session": 5.00
+    }
+  }'
+```
+
+## How It Works
+
+### Session Tracking
+
+Callers identify their session by including a `session_id` in one of these ways:
+- **Header**: `x-litellm-trace-id: my-session-123`
+- **Metadata**: `{"metadata": {"session_id": "my-session-123"}}`
+
+### Max Iterations
+
+When `max_iterations` is set in agent `litellm_params`:
+- Each LLM call for a session increments a counter
+- When the counter exceeds `max_iterations`, the request receives a **429 Too Many Requests**
+- Counters expire after 1 hour by default (configurable via `LITELLM_MAX_ITERATIONS_TTL` env var)
+
+### Max Budget Per Session
+
+When `max_budget_per_session` is set in agent `litellm_params`:
+- After each successful LLM call, the response cost is accumulated for the session
+- Before each call, the accumulated spend is checked against the budget
+- When spend exceeds the budget, the request receives a **429 Too Many Requests**
+- Session spend counters expire after 1 hour by default (configurable via `LITELLM_MAX_BUDGET_PER_SESSION_TTL` env var)
+
+## Example
+
+Create an agent with max 25 iterations and a $5 budget cap:
+
+<Tabs>
+<TabItem value="ui" label="Via UI">
+
+1. Go to **Agents** → **Add Agent**
+2. Configure your agent (name, model, etc.)
+3. In **Agent Settings**, expand the **Tracing** section
+4. Toggle on **Require x-litellm-trace-id on calls BY this agent**
+5. Set **Max Iterations** to `25`
+6. Set **Max Budget Per Session** to `5.00`
+7. Proceed to create a new key for the agent
+8. Click **Create Agent**
+
+</TabItem>
+<TabItem value="api" label="Via API">
+
+```bash
+# 1. Create the agent with trace-id enforcement
+curl -X POST 'http://localhost:4000/v1/agents' \
+  -H 'Authorization: Bearer sk-1234' \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "agent_name": "my-research-agent",
+    "agent_card_params": {
+      "name": "my-research-agent",
+      "description": "A research agent with budget controls",
+      "url": "http://my-agent:8080",
+      "version": "1.0.0"
+    },
+    "litellm_params": {
+      "require_trace_id_on_calls_by_agent": true
+    }
+  }'
+
+# 2. Create a key for the agent
+curl -X POST 'http://localhost:4000/key/generate' \
+  -H 'Authorization: Bearer sk-1234' \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "agent_id": "<agent_id_from_step_1>",
+    "key_alias": "my-research-agent-key"
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+### Making Calls with Session Tracking
+
+```bash
+curl -X POST 'http://localhost:4000/chat/completions' \
+  -H 'Authorization: Bearer sk-agent-key-xxx' \
+  -H 'x-litellm-trace-id: session-abc-123' \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "model": "gpt-4o",
+    "messages": [{"role": "user", "content": "Hello"}]
+  }'
+```
+
+After 25 calls or $5 spent within this session, subsequent requests will receive:
+
+```json
+{
+  "error": {
+    "message": "Session budget exceeded for session session-abc-123. Current spend: $5.0032, max_budget_per_session: $5.00.",
+    "type": "budget_exceeded",
+    "code": 429
+  }
+}
+```
+
+## Environment Variables
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `LITELLM_MAX_ITERATIONS_TTL` | `3600` (1 hour) | TTL in seconds for session iteration counters |
+| `LITELLM_MAX_BUDGET_PER_SESSION_TTL` | `3600` (1 hour) | TTL in seconds for session budget counters |

docs/my-website/docs/adding_provider/generic_guardrail_api.md

@@ -244,6 +244,35 @@ litellm_settings:
           language: "en"
 ```
 
+### Static and dynamic headers
+
+You can send two kinds of headers to your guardrail endpoint:
+
+- **Static headers** (`headers`): A key/value map sent with **every** request to your guardrail. Use this for fixed values (e.g. API keys, `X-Service-Name`). Configure in `litellm_params`:
+
+  ```yaml
+  litellm_params:
+    guardrail: generic_guardrail_api
+    api_base: https://your-guardrail-api.com
+    headers:
+      X-Service-Name: "my-app"
+      X-API-Key: "secret"
+  ```
+
+- **Dynamic headers** (`extra_headers`): A list of **header names** that are forwarded from the **client request** to your guardrail. Only headers in this list (plus a small default allowlist such as `x-litellm-*`) have their values sent; others are sent as `[present]`. Use this to pass through client-provided headers (e.g. `x-request-id`, `x-correlation-id`). Configure in `litellm_params`:
+
+  ```yaml
+  litellm_params:
+    guardrail: generic_guardrail_api
+    api_base: https://your-guardrail-api.com
+    extra_headers:
+      - x-request-id
+      - x-correlation-id
+      - x-custom-auth
+  ```
+
+This mirrors the [MCP static and extra headers](/docs/mcp#forwarding-custom-headers-to-mcp-servers) behavior.
+
 ### Example: Pillar Security
 
 [Pillar Security](https://pillar.security) uses the Generic Guardrail API to provide comprehensive AI security scanning including prompt injection protection, PII/PCI detection, secret detection, and content moderation.

docs/my-website/docs/anthropic_count_tokens.md

@@ -138,6 +138,7 @@ The `/v1/messages/count_tokens` endpoint automatically routes to the appropriate
 | Provider | Token Counting Method |
 |----------|----------------------|
 | Anthropic | [Anthropic Token Counting API](https://docs.anthropic.com/en/docs/build-with-claude/token-counting) |
+| OpenAI | [OpenAI Responses API `/input_tokens`](https://platform.openai.com/docs/api-reference/responses/input-tokens) — see [Token Counting](./count_tokens.md) |
 | Vertex AI (Claude) | Vertex AI Partner Models Token Counter |
 | Bedrock (Claude) | AWS Bedrock CountTokens API |
 | Gemini | Google AI Studio countTokens API |

docs/my-website/docs/anthropic_unified/messages_to_responses_mapping.md

@@ -0,0 +1,120 @@
+# v1/messages → /responses Parameter Mapping
+
+When you send a request to `/v1/messages` targeting an OpenAI or Azure model, LiteLLM internally routes it through the OpenAI Responses API. This page documents exactly how every parameter gets translated in both directions.
+
+The transformation lives in `litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py`.
+
+
+## Request: Anthropic → Responses API
+
+### Top-level parameters
+
+| Anthropic (`/v1/messages`) | Responses API | Notes |
+|---|---|---|
+| `model` | `model` | Passed through as-is |
+| `messages` | `input` | Structurally transformed — see the messages section below |
+| `system` (string) | `instructions` | Passed as a plain string |
+| `system` (list of content blocks) | `instructions` | Text blocks are joined with `\n`; non-text blocks are ignored |
+| `max_tokens` | `max_output_tokens` | Renamed |
+| `temperature` | `temperature` | Passed through as-is |
+| `top_p` | `top_p` | Passed through as-is |
+| `tools` | `tools` | Format-translated — see the tools section below |
+| `tool_choice` | `tool_choice` | Type-remapped — see the tool_choice section below |
+| `thinking` | `reasoning` | Budget tokens mapped to effort level — see the thinking section below |
+| `output_format` or `output_config.format` | `text` | Wrapped as `{"format": {"type": "json_schema", "name": "structured_output", "schema": ..., "strict": true}}` |
+| `context_management` | `context_management` | Converted from Anthropic dict to OpenAI array format — see the context_management section below |
+| `metadata.user_id` | `user` | Extracted from the metadata object and truncated to 64 characters |
+| `stop_sequences` | ❌ Not mapped | Dropped silently |
+| `top_k` | ❌ Not mapped | Dropped silently |
+| `speed` | ❌ Not mapped | Only used to set Anthropic beta headers on the native path |
+
+
+### How messages get converted
+
+Each Anthropic message is expanded into one or more Responses API input items. The key difference is that `tool_result` and `tool_use` blocks become **top-level items** in the input array rather than being nested inside a message.
+
+| Anthropic message | Responses API input item |
+|---|---|
+| `user` role, string content | `{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "..."}]}` |
+| `user` role, `{"type": "text"}` block | `{"type": "input_text", "text": "..."}` inside a user message |
+| `user` role, `{"type": "image", "source": {"type": "base64"}}` | `{"type": "input_image", "image_url": "data:<media_type>;base64,<data>"}` inside a user message |
+| `user` role, `{"type": "image", "source": {"type": "url"}}` | `{"type": "input_image", "image_url": "<url>"}` inside a user message |
+| `user` role, `{"type": "tool_result"}` block | Top-level `{"type": "function_call_output", "call_id": "...", "output": "..."}` — pulled out of the message entirely |
+| `assistant` role, string content | `{"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "..."}]}` |
+| `assistant` role, `{"type": "text"}` block | `{"type": "output_text", "text": "..."}` inside an assistant message |
+| `assistant` role, `{"type": "tool_use"}` block | Top-level `{"type": "function_call", "call_id": "<id>", "name": "...", "arguments": "<JSON string>"}` — pulled out of the message entirely |
+| `assistant` role, `{"type": "thinking"}` block | `{"type": "output_text", "text": "<thinking text>"}` inside an assistant message |
+
+
+### tools
+
+| Anthropic tool | Responses API tool |
+|---|---|
+| Any tool where `type` starts with `"web_search"` or `name == "web_search"` | `{"type": "web_search_preview"}` |
+| All other tools | `{"type": "function", "name": "...", "description": "...", "parameters": <input_schema>}` |
+
+
+### tool_choice
+
+| Anthropic `tool_choice.type` | Responses API `tool_choice` |
+|---|---|
+| `"auto"` | `{"type": "auto"}` |
+| `"any"` | `{"type": "required"}` |
+| `"tool"` | `{"type": "function", "name": "<tool name>"}` |
+
+
+### thinking → reasoning
+
+The `budget_tokens` value is mapped to a string effort level. `summary` is always set to `"detailed"`.
+
+| `thinking.budget_tokens` | `reasoning.effort` |
+|---|---|
+| >= 10000 | `"high"` |
+| >= 5000 | `"medium"` |
+| >= 2000 | `"low"` |
+| < 2000 | `"minimal"` |
+
+If `thinking.type` is anything other than `"enabled"`, the `reasoning` field is not sent at all.
+
+
+### context_management
+
+Anthropic uses a nested dict with an `edits` array. OpenAI uses a flat array of compaction objects.
+
+```
+Anthropic input:
+{
+  "edits": [
+    {
+      "type": "compact_20260112",
+      "trigger": {"type": "input_tokens", "value": 150000}
+    }
+  ]
+}
+
+Responses API output:
+[
+  {"type": "compaction", "compact_threshold": 150000}
+]
+```
+
+
+## Response: Responses API → Anthropic
+
+When the Responses API reply comes back, LiteLLM converts it into an Anthropic `AnthropicMessagesResponse`.
+
+| Responses API field | Anthropic response field | Notes |
+|---|---|---|
+| `response.id` | `id` | |
+| `response.model` | `model` | Falls back to `"unknown-model"` if missing |
+| `ResponseReasoningItem` — `summary[*].text` | `content` block `{"type": "thinking", "thinking": "..."}` | Each non-empty summary text becomes a thinking block |
+| `ResponseOutputMessage` — `content[*]` where `type == "output_text"` | `content` block `{"type": "text", "text": "..."}` | |
+| `ResponseFunctionToolCall` — `{call_id, name, arguments}` | `content` block `{"type": "tool_use", "id": "...", "name": "...", "input": {...}}` | `arguments` is JSON-parsed back into a dict |
+| Any `function_call` present in output | `stop_reason: "tool_use"` | |
+| `response.status == "incomplete"` | `stop_reason: "max_tokens"` | Takes precedence over the default |
+| Everything else | `stop_reason: "end_turn"` | Default |
+| `response.usage.input_tokens` | `usage.input_tokens` | |
+| `response.usage.output_tokens` | `usage.output_tokens` | |
+| *(hardcoded)* | `type: "message"` | Always set |
+| *(hardcoded)* | `role: "assistant"` | Always set |
+| *(hardcoded)* | `stop_sequence: null` | Always null on this path |

docs/my-website/docs/apply_guardrail.md

@@ -11,6 +11,7 @@ This endpoint supports various guardrail types including:
 - **Presidio** - PII detection and masking
 - **Bedrock** - AWS Bedrock guardrails for content moderation
 - **Lakera** - AI safety guardrails
+- **PANW Prisma AIRS** - Threat detection, DLP, and policy enforcement
 - **Custom guardrails** - User-defined guardrails
 
 ## Configuration

docs/my-website/docs/audio_transcription.md

@@ -13,7 +13,7 @@ import TabItem from '@theme/TabItem';
 | Fallbacks | ✅ | Works between supported models |
 | Loadbalancing | ✅ | Works between supported models |
 | Guardrails | ✅ | Applies to output transcribed text (non-streaming only) |
-| Supported Providers | `openai`, `azure`, `vertex_ai`, `gemini`, `deepgram`, `groq`, `fireworks_ai`, `ovhcloud` | |
+| Supported Providers | `openai`, `azure`, `vertex_ai`, `gemini`, `deepgram`, `groq`, `fireworks_ai`, `ovhcloud`, `mistral` | |
 
 ## Quick Start
 
@@ -126,6 +126,7 @@ transcript = client.audio.transcriptions.create(
 - [Fireworks AI](./providers/fireworks_ai.md#audio-transcription)
 - [Groq](./providers/groq.md#speech-to-text---whisper)
 - [Deepgram](./providers/deepgram.md)
+- [Mistral (Voxtral)](./providers/mistral.md#audio-transcription)
 - [OVHcloud AI Endpoints](./providers/ovhcloud.md)
 
 ---

docs/my-website/docs/completion/output.md

@@ -51,6 +51,28 @@ Here's what an example response looks like
 }
 ```
 
+## Native Finish Reason
+
+LiteLLM maps all provider-specific `finish_reason` values to OpenAI-compatible values (`stop`, `length`, `tool_calls`, `function_call`, `content_filter`). When the original provider value differs from the mapped value, it is preserved in `provider_specific_fields["native_finish_reason"]`.
+
+This is useful for agent loops that need to distinguish between different stop conditions (e.g., Gemini's `MALFORMED_FUNCTION_CALL` vs a normal `stop`).
+
+```python
+response = completion(model="gemini/gemini-2.0-flash", messages=messages)
+
+choice = response.choices[0]
+print(choice.finish_reason)  # "stop" (OpenAI-compatible)
+
+# Access the original provider value when it differs:
+if hasattr(choice, "provider_specific_fields") and choice.provider_specific_fields:
+    native = choice.provider_specific_fields.get("native_finish_reason")
+    if native == "MALFORMED_FUNCTION_CALL":
+        # Handle malformed function call differently from a normal stop
+        pass
+```
+
+When the provider already returns an OpenAI-compatible value (e.g., `stop`), `native_finish_reason` is not set.
+
 ## Additional Attributes
 
 You can also access information like latency. 

docs/my-website/docs/completion/prompt_caching.md

@@ -63,7 +63,6 @@ for _ in range(2):
                     }
                 ],
             },
-            # marked for caching with the cache_control parameter, so that this checkpoint can read from the previous cache.
             {
                 "role": "user",
                 "content": [
@@ -77,7 +76,6 @@ for _ in range(2):
                 "role": "assistant",
                 "content": "Certainly! the key terms and conditions are the following: the contract is 1 year long for $10/mo",
             },
-            # The final turn is marked with cache-control, for continuing in followups.
             {
                 "role": "user",
                 "content": [
@@ -112,16 +110,16 @@ model_list:
         api_key: os.environ/OPENAI_API_KEY
 ```
 
-2. Start proxy 
+2. Start proxy
 
 ```bash
 litellm --config /path/to/config.yaml
 ```
 
-3. Test it! 
+3. Test it!
 
 ```python
-from openai import OpenAI 
+from openai import OpenAI
 import os
 
 client = OpenAI(
@@ -144,7 +142,6 @@ for _ in range(2):
                     }
                 ],
             },
-            # marked for caching with the cache_control parameter, so that this checkpoint can read from the previous cache.
             {
                 "role": "user",
                 "content": [
@@ -158,7 +155,6 @@ for _ in range(2):
                 "role": "assistant",
                 "content": "Certainly! the key terms and conditions are the following: the contract is 1 year long for $10/mo",
             },
-            # The final turn is marked with cache-control, for continuing in followups.
             {
                 "role": "user",
                 "content": [
@@ -183,6 +179,78 @@ assert response.usage.prompt_tokens_details.cached_tokens > 0
 </TabItem>
 </Tabs>
 
+### OpenAI `prompt_cache_key` and `prompt_cache_retention`
+
+OpenAI prompt caching is [**automatic**](https://platform.openai.com/docs/guides/prompt-caching) — no `cache_control` message annotations are needed. Any request with 1024+ prompt tokens is eligible for caching.
+
+OpenAI also supports two optional parameters for more control over caching behavior:
+
+- **`prompt_cache_key`** (string) — A routing hint that improves cache hit rates for requests sharing long common prefixes. Requests with the same cache key are routed to the same backend, increasing the likelihood of a cache hit.
+- **`prompt_cache_retention`** (`"in_memory"` or `"24h"`) — Controls cache TTL. Default is `"in_memory"` (5–10 min). Set to `"24h"` for extended caching that offloads KV tensors to GPU-local storage.
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+from litellm import completion
+import os
+
+os.environ["OPENAI_API_KEY"] = ""
+
+response = completion(
+    model="gpt-4o",
+    messages=[
+        {
+            "role": "system",
+            "content": "You are an AI assistant tasked with analyzing legal documents. "
+            + "Here is the full text of a complex legal agreement " * 400,
+        },
+        {
+            "role": "user",
+            "content": "What are the key terms and conditions?",
+        },
+    ],
+    prompt_cache_key="legal-doc-analysis",
+    prompt_cache_retention="24h",
+)
+print(response.usage)
+```
+
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+```python
+from openai import OpenAI
+
+client = OpenAI(
+    api_key="LITELLM_PROXY_KEY",
+    base_url="LITELLM_PROXY_BASE",
+)
+
+response = client.chat.completions.create(
+    model="gpt-4o",
+    messages=[
+        {
+            "role": "system",
+            "content": "You are an AI assistant tasked with analyzing legal documents. "
+            + "Here is the full text of a complex legal agreement " * 400,
+        },
+        {
+            "role": "user",
+            "content": "What are the key terms and conditions?",
+        },
+    ],
+    extra_body={
+        "prompt_cache_key": "legal-doc-analysis",
+        "prompt_cache_retention": "24h",
+    },
+)
+print(response.usage)
+```
+
+</TabItem>
+</Tabs>
+
 ### Anthropic Example 
 
 Anthropic charges for cache writes. 

docs/my-website/docs/completion/web_fetch.md

@@ -115,6 +115,11 @@ print(response)
 
 Web fetch is available on the following Anthropic API models:
 
+- `claude-opus-4-6` (Claude Opus 4.6)
+- `claude-sonnet-4-6` (Claude Sonnet 4.6)
+- `claude-opus-4-5` (Claude Opus 4.5)
+- `claude-sonnet-4-5` (Claude Sonnet 4.5)
+- `claude-haiku-4-5` (Claude Haiku 4.5)
 - `claude-opus-4-1-20250805` (Claude Opus 4.1)
 - `claude-opus-4-20250514` (Claude Opus 4)
 - `claude-sonnet-4-20250514` (Claude Sonnet 4)

docs/my-website/docs/contributing/adding_openai_compatible_providers.md

@@ -80,6 +80,36 @@ That's it! The provider is now available.
 }
 ```
 
+## Responses API Support
+
+If your provider also supports the OpenAI Responses API (`/v1/responses`), add `supported_endpoints`:
+
+```json
+{
+  "your_provider": {
+    "base_url": "https://api.yourprovider.com/v1",
+    "api_key_env": "YOUR_PROVIDER_API_KEY",
+    "supported_endpoints": ["/v1/chat/completions", "/v1/responses"]
+  }
+}
+```
+
+This enables `litellm.responses()` with zero additional code:
+
+```python
+import litellm
+
+response = litellm.responses(
+    model="your_provider/model-name",
+    input="Hello, what can you do?",
+)
+print(response.output)
+```
+
+If `supported_endpoints` is omitted, it defaults to `[]`. Chat completions is always enabled for JSON providers regardless of this field.
+
+The provider inherits all request/response handling from OpenAI's Responses API — streaming, tools, and all standard parameters work out of the box.
+
 ## Usage
 
 ```python
@@ -89,11 +119,17 @@ import os
 # Set your API key
 os.environ["YOUR_PROVIDER_API_KEY"] = "your-key-here"
 
-# Use the provider
+# Chat completions
 response = litellm.completion(
     model="your_provider/model-name",
     messages=[{"role": "user", "content": "Hello"}],
 )
+
+# Responses API (if supported_endpoints includes "/v1/responses")
+response = litellm.responses(
+    model="your_provider/model-name",
+    input="Hello",
+)
 ```
 
 ## When to Use Python Instead
@@ -105,7 +141,9 @@ Use a Python config class if you need:
 - Provider-specific streaming logic
 - Advanced tool calling modifications
 
-For these cases, create a config class in `litellm/llms/your_provider/chat/transformation.py` that inherits from `OpenAIGPTConfig` or `OpenAILikeChatConfig`.
+For chat completions, create a config class in `litellm/llms/your_provider/chat/transformation.py` that inherits from `OpenAIGPTConfig` or `OpenAILikeChatConfig`.
+
+For responses API with small overrides, inherit from `OpenAIResponsesAPIConfig` and override only what's needed. See `litellm/llms/perplexity/responses/transformation.py` for a minimal example (~40 lines vs 400+).
 
 ## Testing
 

docs/my-website/docs/count_tokens.md

@@ -0,0 +1,189 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Token Counting
+
+## Overview
+
+LiteLLM provides exact token counting by calling provider-specific token counting APIs. This gives you accurate token counts before sending requests, helping with cost estimation and context window management.
+
+| Feature | Details |
+|---------|---------|
+| SDK Method | `litellm.acount_tokens()` |
+| Proxy Endpoints | `/v1/messages/count_tokens` (Anthropic format), `/v1/responses/input_tokens` (OpenAI format) |
+| Fallback | Local tiktoken-based counting for unsupported providers |
+
+## Supported Providers
+
+| Provider | Token Counting API | Format |
+|----------|-------------------|--------|
+| OpenAI | [Responses API `/input_tokens`](https://platform.openai.com/docs/api-reference/responses/input-tokens) | OpenAI Responses |
+| Anthropic | [Messages `/count_tokens`](https://docs.anthropic.com/en/docs/build-with-claude/token-counting) | Anthropic Messages |
+| Vertex AI (Claude) | Vertex AI Partner Models Token Counter | Anthropic Messages |
+| Bedrock (Claude) | AWS Bedrock CountTokens API | Anthropic Messages |
+| Gemini | Google AI Studio countTokens API | Anthropic Messages |
+| Vertex AI (Gemini) | Vertex AI countTokens API | Anthropic Messages |
+| Other providers | Local tiktoken fallback | N/A |
+
+## SDK Usage
+
+### Basic Usage
+
+```python
+import asyncio
+import litellm
+
+async def main():
+    # OpenAI
+    result = await litellm.acount_tokens(
+        model="openai/gpt-4o",
+        messages=[{"role": "user", "content": "Hello, how are you?"}],
+    )
+    print(f"Token count: {result.total_tokens}")
+    print(f"Tokenizer: {result.tokenizer_type}")  # "openai_api"
+
+    # Anthropic
+    result = await litellm.acount_tokens(
+        model="anthropic/claude-3-5-sonnet-20241022",
+        messages=[{"role": "user", "content": "Hello, how are you?"}],
+    )
+    print(f"Token count: {result.total_tokens}")
+    print(f"Tokenizer: {result.tokenizer_type}")  # "anthropic_api"
+
+asyncio.run(main())
+```
+
+### With Tools and System Message
+
+```python
+import asyncio
+import litellm
+
+async def main():
+    result = await litellm.acount_tokens(
+        model="openai/gpt-4o",
+        messages=[{"role": "user", "content": "What's the weather in Paris?"}],
+        tools=[{
+            "type": "function",
+            "function": {
+                "name": "get_weather",
+                "description": "Get weather for a city",
+                "parameters": {
+                    "type": "object",
+                    "properties": {"city": {"type": "string"}},
+                },
+            },
+        }],
+        system="You are a helpful weather assistant.",
+    )
+    print(f"Token count (with tools): {result.total_tokens}")
+
+asyncio.run(main())
+```
+
+### Response Format
+
+`litellm.acount_tokens()` returns a `TokenCountResponse`:
+
+```python
+TokenCountResponse(
+    total_tokens=15,           # Token count
+    request_model="openai/gpt-4o",  # Model requested
+    model_used="gpt-4o",      # Model used for counting
+    tokenizer_type="openai_api",    # "openai_api", "anthropic_api", "local_tokenizer"
+    original_response={"input_tokens": 15},  # Raw API response
+    error=False,               # True if counting failed
+    error_message=None,        # Error details if failed
+)
+```
+
+### Fallback Behavior
+
+If a provider doesn't support a token counting API, or if the API key is missing, `acount_tokens()` automatically falls back to local tiktoken-based counting:
+
+```python
+# Unsupported provider → automatic fallback
+result = await litellm.acount_tokens(
+    model="together_ai/meta-llama/Llama-3-8b-chat-hf",
+    messages=[{"role": "user", "content": "Hello"}],
+)
+print(result.tokenizer_type)  # "local_tokenizer"
+```
+
+## Proxy Usage
+
+### OpenAI Format — `/v1/responses/input_tokens`
+
+<Tabs>
+<TabItem value="curl" label="curl">
+
+```bash
+curl -X POST "http://localhost:4000/v1/responses/input_tokens" \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer sk-1234" \
+  -d '{
+    "model": "gpt-4o",
+    "input": "Hello, how are you?"
+  }'
+```
+
+</TabItem>
+<TabItem value="python" label="Python (httpx)">
+
+```python
+import httpx
+
+response = httpx.post(
+    "http://localhost:4000/v1/responses/input_tokens",
+    headers={
+        "Content-Type": "application/json",
+        "Authorization": "Bearer sk-1234"
+    },
+    json={
+        "model": "gpt-4o",
+        "input": "Hello, how are you?"
+    }
+)
+
+print(response.json())
+# {"input_tokens": 7}
+```
+
+</TabItem>
+</Tabs>
+
+**Response:**
+```json
+{"input_tokens": 7}
+```
+
+### Anthropic Format — `/v1/messages/count_tokens`
+
+See [Anthropic Token Counting](./anthropic_count_tokens.md) for full documentation.
+
+```bash
+curl -X POST "http://localhost:4000/v1/messages/count_tokens" \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer sk-1234" \
+  -d '{
+    "model": "claude-3-5-sonnet-20241022",
+    "messages": [
+      {"role": "user", "content": "Hello, how are you?"}
+    ]
+  }'
+```
+
+## Proxy Configuration
+
+```yaml
+model_list:
+  - model_name: gpt-4o
+    litellm_params:
+      model: openai/gpt-4o
+      api_key: os.environ/OPENAI_API_KEY
+
+  - model_name: claude-3-5-sonnet
+    litellm_params:
+      model: anthropic/claude-3-5-sonnet-20241022
+      api_key: os.environ/ANTHROPIC_API_KEY
+```

docs/my-website/docs/embedding/supported_embedding.md

@@ -514,6 +514,57 @@ All models listed [here](https://ai.google.dev/gemini-api/docs/models/gemini) ar
 | Model Name         | Function Call                                         |
 | :---               | :---                                                  |
 | text-embedding-004 | `embedding(model="gemini/text-embedding-004", input)` |
+| gemini-embedding-2-preview | `embedding(model="gemini/gemini-embedding-2-preview", input)` | [Multimodal docs](#gemini-embedding-2-preview-multimodal) |
+
+### Gemini Embedding 2 Preview (Multimodal)
+
+`gemini-embedding-2-preview` supports **multimodal embeddings**—text, images, audio, video, and PDF in a single request. See [blog post](/blog/gemini_embedding_2_multimodal) for details.
+
+**Input formats:**
+- **Data URIs:** `data:image/png;base64,<encoded_data>`
+- **Gemini file references:** `files/abc123` (pre-uploaded via Gemini Files API)
+
+**Supported MIME types:** `image/png`, `image/jpeg`, `audio/mpeg`, `audio/wav`, `video/mp4`, `video/quicktime`, `application/pdf`
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+from litellm import embedding
+import os
+os.environ["GEMINI_API_KEY"] = ""
+
+# Text + Image (base64)
+response = embedding(
+    model="gemini/gemini-embedding-2-preview",
+    input=[
+        "The food was delicious and the waiter...",
+        "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAIAQMAAAD+wSzIAAAABlBMVEX///+/v7+jQ3Y5AAAADklEQVQI12P4AIX8EAgALgAD/aNpbtEAAAAASUVORK5CYII"
+    ],
+)
+print(response)
+```
+
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+```bash
+curl -X POST http://localhost:4000/embeddings \
+  -H "Authorization: Bearer sk-1234" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "gemini-embedding-2-preview",
+    "input": [
+      "The food was delicious and the waiter...",
+      "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAIAQMAAAD+wSzIAAAABlBMVEX///+/v7+jQ3Y5AAAADklEQVQI12P4AIX8EAgALgAD/aNpbtEAAAAASUVORK5CYII"
+    ]
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+**Optional:** `dimensions` maps to Gemini's `outputDimensionality`.
 
 
 ## Vertex AI Embedding Models

docs/my-website/docs/files_endpoints.md

@@ -326,4 +326,10 @@ print("file content=", content.text)
 
 ### [Bedrock](./providers/bedrock_batches#4-retrieve-batch-results)
 
+### [Anthropic](./providers/anthropic#files-api)
+
+:::note
+Anthropic Files API has a different purpose than OpenAI's. It's **not** for Batches or Fine-tuning—it's for uploading files once and referencing them by `file_id` in multiple messages, avoiding re-uploads. File API operations are free — file content used in Messages requests is priced as input tokens.
+:::
+
 ## [Swagger API Reference](https://litellm-api.up.railway.app/#/files)

docs/my-website/docs/generateContent.md

@@ -15,6 +15,7 @@ Use LiteLLM to call Google AI's generateContent endpoints for text generation, m
 | Streaming | ✅ | |
 | Fallbacks | ✅ | between supported models |
 | Loadbalancing | ✅ | between supported models |
+| Metadata Tracking | ✅ | passes trace ID, metadata to observability callbacks (e.g. S3, Langfuse) |
 
 ## Usage 
 ---

docs/my-website/docs/guides/index.md

@@ -0,0 +1,78 @@
+---
+title: Guides
+sidebar_label: Overview
+---
+
+import NavigationCards from '@site/src/components/NavigationCards';
+
+**Guides** are focused references organized by the job you are trying to do with LiteLLM: make requests, use tools, handle media, manage context, or operate the gateway safely.
+
+> New to LiteLLM or not sure whether you need the SDK or Gateway path first? Start at [Learn →](/docs/learn)
+
+---
+
+## Build With LiteLLM
+
+<NavigationCards
+columns={3}
+items={[
+  {
+    icon: "⚡",
+    title: "Core Requests",
+    description: "Streaming, batching, structured outputs, and reasoning behavior.",
+    to: "/docs/guides/core_request_response_patterns",
+  },
+  {
+    icon: "🛠️",
+    title: "Tool Calling",
+    description: "Function calling, web tools, interception patterns, computer use, code interpreter, and tool-call hygiene.",
+    to: "/docs/guides/tools_integrations",
+  },
+  {
+    icon: "🖼️",
+    title: "Multimodal I/O",
+    description: "Vision, audio, PDFs, image generation, and video generation.",
+    to: "/docs/guides/multimodal_io",
+  },
+  {
+    icon: "📚",
+    title: "Retrieval & Knowledge",
+    description: "Vector stores, file search, citations, and knowledge-base routing.",
+    to: "/docs/guides/retrieval_knowledge",
+  },
+  {
+    icon: "🧠",
+    title: "Prompts & Context",
+    description: "Prompt caching, trimming, formatting, assistant prefill, and predicted outputs.",
+    to: "/docs/guides/prompts_context",
+  },
+]}
+/>
+
+---
+
+## Operate & Extend
+
+<NavigationCards
+columns={3}
+items={[
+  {
+    icon: "🎛️",
+    title: "Compatibility & Extensibility",
+    description: "Provider-specific params, model aliases, fine-tuned models, and adapters.",
+    to: "/docs/guides/compatibility_extensibility",
+  },
+  {
+    icon: "🧪",
+    title: "Reliability, Testing & Spend",
+    description: "Retries, fallbacks, mock responses, and budget controls.",
+    to: "/docs/guides/reliability_testing_spend",
+  },
+  {
+    icon: "🔒",
+    title: "Security & Network",
+    description: "SSL, custom CA bundles, HTTP proxy settings, and per-service verification.",
+    to: "/docs/guides/security_network",
+  },
+]}
+/>

docs/my-website/docs/image_edits.md

@@ -16,7 +16,7 @@ LiteLLM provides image editing functionality that maps to OpenAI's `/images/edit
 | Supported operations | Create image edits | Single and multiple images supported |
 | Supported LiteLLM SDK Versions | 1.63.8+ | Gemini support requires 1.79.3+ |
 | Supported LiteLLM Proxy Versions | 1.71.1+ | Gemini support requires 1.79.3+ |
-| Supported LLM providers | **OpenAI**, **Gemini (Google AI Studio)**, **Vertex AI**, **Stability AI**, **AWS Bedrock (Stability)** | Gemini supports the new `gemini-2.5-flash-image` family. Vertex AI supports both Gemini and Imagen models. Stability AI and Bedrock Stability support various image editing operations. |
+| Supported LLM providers | **OpenAI**, **Gemini (Google AI Studio)**, **Vertex AI**, **OpenRouter**, **Stability AI**, **AWS Bedrock (Stability)**, **Black Forest Labs** | Gemini supports the new `gemini-2.5-flash-image` family. Vertex AI supports both Gemini and Imagen models. OpenRouter routes image edits through chat completions. Stability AI and Bedrock Stability support various image editing operations. Black Forest Labs supports FLUX Kontext models. |
 
  #### ⚡️See all supported models and providers at [models.litellm.ai](https://models.litellm.ai/)
 
@@ -199,6 +199,63 @@ for idx, image_obj in enumerate(response.data):
 
 </TabItem>
 
+<TabItem value="bfl" label="Black Forest Labs">
+
+#### Basic Image Edit
+```python showLineNumbers title="Black Forest Labs Image Edit"
+import os
+import litellm
+
+os.environ["BFL_API_KEY"] = "your-api-key"
+
+response = litellm.image_edit(
+    model="black_forest_labs/flux-kontext-pro",
+    image=open("original_image.png", "rb"),
+    prompt="Add a green leaf to the scene",
+)
+
+print(response.data[0].url)
+```
+
+#### Inpainting with Mask
+```python showLineNumbers title="Black Forest Labs Inpainting"
+import os
+import litellm
+
+os.environ["BFL_API_KEY"] = "your-api-key"
+
+# Use flux-pro-1.0-fill for inpainting
+response = litellm.image_edit(
+    model="black_forest_labs/flux-pro-1.0-fill",
+    image=open("original_image.png", "rb"),
+    mask=open("mask_image.png", "rb"),
+    prompt="Replace with a garden",
+)
+
+print(response.data[0].url)
+```
+
+#### Outpainting (Expand)
+```python showLineNumbers title="Black Forest Labs Outpainting"
+import os
+import litellm
+
+os.environ["BFL_API_KEY"] = "your-api-key"
+
+# Use flux-pro-1.0-expand to extend image borders
+response = litellm.image_edit(
+    model="black_forest_labs/flux-pro-1.0-expand",
+    image=open("original_image.png", "rb"),
+    prompt="Continue the scene with mountains",
+    top=256,
+    bottom=256,
+)
+
+print(response.data[0].url)
+```
+
+</TabItem>
+
 <TabItem value="vertex_ai" label="Vertex AI">
 
 #### Basic Image Edit (Gemini)
@@ -244,6 +301,47 @@ response = litellm.image_edit(
 print(response)
 ```
 
+</TabItem>
+
+<TabItem value="openrouter" label="OpenRouter">
+
+#### Basic Image Edit
+```python showLineNumbers title="OpenRouter Image Edit"
+import os
+from litellm import image_edit
+
+os.environ["OPENROUTER_API_KEY"] = "your-api-key"
+
+response = image_edit(
+    model="openrouter/google/gemini-2.5-flash-image",
+    image=open("original_image.png", "rb"),
+    prompt="Add aurora borealis to the night sky",
+)
+
+print(response)
+```
+
+#### Multiple Images Edit
+```python showLineNumbers title="OpenRouter Multiple Images Edit"
+import os
+from litellm import image_edit
+
+os.environ["OPENROUTER_API_KEY"] = "your-api-key"
+
+response = image_edit(
+    model="openrouter/google/gemini-2.5-flash-image",
+    image=[
+        open("scene.png", "rb"),
+        open("style_reference.png", "rb"),
+    ],
+    prompt="Blend the reference style into the scene",
+    size="1536x1024",   # mapped to aspect_ratio 3:2
+    quality="high",      # mapped to image_size 4K
+)
+
+print(response)
+```
+
 </TabItem>
 </Tabs>
 
@@ -351,6 +449,35 @@ curl -X POST "http://0.0.0.0:4000/v1/images/edits" \
 
 </TabItem>
 
+<TabItem value="bfl" label="Black Forest Labs">
+
+1. Add Black Forest Labs image edit models to your `config.yaml`:
+```yaml showLineNumbers title="Black Forest Labs Proxy Configuration"
+model_list:
+  - model_name: bfl-kontext-pro
+    litellm_params:
+      model: black_forest_labs/flux-kontext-pro
+      api_key: os.environ/BFL_API_KEY
+    model_info:
+      mode: image_edit
+```
+
+2. Start the LiteLLM proxy server:
+```bash showLineNumbers title="Start LiteLLM Proxy Server"
+litellm --config /path/to/config.yaml
+```
+
+3. Make an image edit request:
+```bash showLineNumbers title="Black Forest Labs Proxy Image Edit"
+curl -X POST "http://0.0.0.0:4000/v1/images/edits" \
+  -H "Authorization: Bearer <YOUR-LITELLM-KEY>" \
+  -F "model=bfl-kontext-pro" \
+  -F "image=@original_image.png" \
+  -F "prompt=Add a sunset in the background"
+```
+
+</TabItem>
+
 <TabItem value="vertex_ai" label="Vertex AI">
 
 1. Add Vertex AI image edit models to your `config.yaml`:
@@ -398,6 +525,34 @@ curl -X POST "http://0.0.0.0:4000/v1/images/edits" \
   -F "size=1024x1024"
 ```
 
+</TabItem>
+
+<TabItem value="openrouter" label="OpenRouter">
+
+1. Add the OpenRouter image edit model to your `config.yaml`:
+```yaml showLineNumbers title="OpenRouter Proxy Configuration"
+model_list:
+  - model_name: openrouter-image-edit
+    litellm_params:
+      model: openrouter/google/gemini-2.5-flash-image
+      api_key: os.environ/OPENROUTER_API_KEY
+```
+
+2. Start the LiteLLM proxy server:
+```bash showLineNumbers title="Start LiteLLM Proxy Server"
+litellm --config /path/to/config.yaml
+```
+
+3. Make an image edit request:
+```bash showLineNumbers title="OpenRouter Proxy Image Edit"
+curl -X POST "http://0.0.0.0:4000/v1/images/edits" \
+  -H "Authorization: Bearer <YOUR-LITELLM-KEY>" \
+  -F "model=openrouter-image-edit" \
+  -F "image=@original_image.png" \
+  -F "prompt=Make the sky a vibrant purple sunset" \
+  -F "size=1024x1024"
+```
+
 </TabItem>
 </Tabs>
 

docs/my-website/docs/image_generation.md

@@ -15,7 +15,7 @@ import TabItem from '@theme/TabItem';
 | Fallbacks | ✅ | Works between supported models |
 | Loadbalancing | ✅ | Works between supported models |
 | Guardrails | ✅ | Applies to input prompts (non-streaming only) |
-| Supported Providers | OpenAI, Azure, Google AI Studio, Vertex AI, AWS Bedrock, Recraft, OpenRouter, Xinference, Nscale | |
+| Supported Providers | OpenAI, Azure, Google AI Studio, Vertex AI, AWS Bedrock, Black Forest Labs, Recraft, OpenRouter, Xinference, Nscale | |
 
 ## Quick Start
 

docs/my-website/docs/integrations/index.md

@@ -1,18 +1,336 @@
-# Integrations
+---
+title: Integrations
+sidebar_label: Overview
+---
+
+import NavigationCards from '@site/src/components/NavigationCards';
 
 This section covers integrations with various tools and services that can be used with LiteLLM (either Proxy or SDK).
 
-## AI Agent Frameworks
-- **[Letta](./letta.md)** - Build stateful LLM agents with persistent memory using LiteLLM Proxy
+---
 
-## Development Tools
-- **[OpenWebUI](../tutorials/openweb_ui.md)** - Self-hosted ChatGPT-style interface
+## Observability
 
-## Observability & Monitoring
-- **[Langfuse](../observability/langfuse_integration.md)** - LLM observability and analytics
-- **[Prometheus](../proxy/prometheus.md)** - Metrics collection and monitoring
-- **[PagerDuty](../proxy/pagerduty.md)** - Incident response and alerting
-- **[Datadog](../observability/datadog.md)**
+Track, debug, and analyze LLM calls with observability platforms.
 
+<NavigationCards
+columns={3}
+items={[
+  {
+    icon: "🪢",
+    title: "Langfuse",
+    description: "LLM observability and analytics.",
+    to: "/docs/observability/langfuse_integration",
+  },
+  {
+    icon: "🐶",
+    title: "Datadog",
+    description: "Metrics, traces, and dashboards.",
+    to: "/docs/observability/datadog",
+  },
+  {
+    icon: "📡",
+    title: "OpenTelemetry",
+    description: "Vendor-neutral tracing.",
+    to: "/docs/observability/opentelemetry_integration",
+  },
+  {
+    icon: "🔗",
+    title: "LangSmith",
+    description: "LLM debugging and evaluation.",
+    to: "/docs/observability/langsmith_integration",
+  },
+  {
+    icon: "🔥",
+    title: "Arize / Phoenix",
+    description: "ML observability and evaluation.",
+    to: "/docs/observability/arize_integration",
+  },
+  {
+    icon: "🌀",
+    title: "Helicone",
+    description: "LLM request logging and analytics.",
+    to: "/docs/observability/helicone_integration",
+  },
+  {
+    icon: "📊",
+    title: "MLflow",
+    description: "Experiment tracking.",
+    to: "/docs/observability/mlflow",
+  },
+  {
+    icon: "🏋️",
+    title: "Weights & Biases",
+    description: "ML experiment tracking.",
+    to: "/docs/observability/wandb_integration",
+  },
+  {
+    icon: "📉",
+    title: "PostHog",
+    description: "Product analytics.",
+    to: "/docs/observability/posthog_integration",
+  },
+]}
+/>
 
-Click into each section to learn more about the integrations.
+[View all observability integrations →](/docs/integrations/observability_integrations)
+
+---
+
+## Alerting & Monitoring
+
+Set up alerts, metrics collection, and infrastructure monitoring.
+
+<NavigationCards
+columns={2}
+items={[
+  {
+    icon: "📈",
+    title: "Prometheus",
+    description: "Metrics collection and monitoring.",
+    to: "../proxy/prometheus",
+  },
+  {
+    icon: "🚨",
+    title: "PagerDuty",
+    description: "Incident response and alerting.",
+    to: "../proxy/pagerduty",
+  },
+  {
+    icon: "🔔",
+    title: "Alerting",
+    description: "Slack, Teams, and webhook alerts.",
+    to: "../proxy/alerting",
+  },
+  {
+    icon: "🔍",
+    title: "Pyroscope",
+    description: "Continuous profiling.",
+    to: "../proxy/pyroscope_profiling",
+  },
+]}
+/>
+
+---
+
+## Guardrail Providers
+
+Add safety and content filtering to LLM calls.
+
+<NavigationCards
+columns={3}
+items={[
+  {
+    icon: "🛡️",
+    title: "Lakera AI",
+    description: "Prompt injection detection.",
+    to: "/docs/proxy/guardrails/lakera_ai",
+  },
+  {
+    icon: "☁️",
+    title: "Azure Content Safety",
+    description: "Content moderation.",
+    to: "/docs/proxy/guardrails/azure_content_guardrail",
+  },
+  {
+    icon: "🛏️",
+    title: "Bedrock Guardrails",
+    description: "AWS Bedrock safety.",
+    to: "/docs/proxy/guardrails/bedrock",
+  },
+  {
+    icon: "🤖",
+    title: "OpenAI Moderation",
+    description: "OpenAI content policy.",
+    to: "/docs/proxy/guardrails/openai_moderation",
+  },
+  {
+    icon: "🔐",
+    title: "Secret Detection",
+    description: "Prevent credential leaks.",
+    to: "/docs/proxy/guardrails/secret_detection",
+  },
+  {
+    icon: "🕵️",
+    title: "PII Masking",
+    description: "Mask sensitive data.",
+    to: "/docs/proxy/guardrails/pii_masking_v2",
+  },
+]}
+/>
+
+[View all guardrail providers →](/docs/guardrail_providers)
+
+---
+
+## Policies
+
+Define and enforce usage policies across your LLM deployment.
+
+<NavigationCards
+columns={3}
+items={[
+  {
+    icon: "📋",
+    title: "Guardrail Policies",
+    description: "Policy-based guardrail rules.",
+    to: "../proxy/guardrails/guardrail_policies",
+  },
+  {
+    icon: "🔀",
+    title: "Policy Flow Builder",
+    description: "Visual policy configuration.",
+    to: "../proxy/guardrails/policy_flow_builder",
+  },
+  {
+    icon: "📄",
+    title: "Policy Templates",
+    description: "Pre-built policy templates.",
+    to: "../proxy/guardrails/policy_templates",
+  },
+]}
+/>
+
+---
+
+## AI Tools
+
+Connect LiteLLM to AI-powered coding and productivity tools.
+
+<NavigationCards
+columns={3}
+items={[
+  {
+    icon: "💬",
+    title: "OpenWebUI",
+    description: "Self-hosted ChatGPT-style interface.",
+    to: "../tutorials/openweb_ui",
+  },
+  {
+    icon: "🤖",
+    title: "Claude Code",
+    description: "Use LiteLLM with Claude Code.",
+    to: "../tutorials/claude_responses_api",
+  },
+  {
+    icon: "🖱️",
+    title: "Cursor",
+    description: "AI code editor integration.",
+    to: "../tutorials/cursor_integration",
+  },
+  {
+    icon: "🐙",
+    title: "GitHub Copilot",
+    description: "GitHub Copilot integration.",
+    to: "../tutorials/github_copilot_integration",
+  },
+  {
+    icon: "💻",
+    title: "OpenCode",
+    description: "Open source coding assistant.",
+    to: "../tutorials/opencode_integration",
+  },
+  {
+    icon: "🔧",
+    title: "Retool Assist",
+    description: "Retool AI assistant.",
+    to: "../tutorials/retool_assist",
+  },
+]}
+/>
+
+---
+
+## Agent SDKs
+
+Use LiteLLM with agent frameworks and SDKs.
+
+<NavigationCards
+columns={3}
+items={[
+  {
+    icon: "🤖",
+    title: "OpenAI Agents SDK",
+    description: "Build agents with OpenAI's SDK.",
+    to: "../tutorials/openai_agents_sdk",
+  },
+  {
+    icon: "🧠",
+    title: "Claude Agent SDK",
+    description: "Build agents with Anthropic's SDK.",
+    to: "../tutorials/claude_agent_sdk",
+  },
+  {
+    icon: "🌐",
+    title: "Google ADK",
+    description: "Google Agent Development Kit.",
+    to: "../tutorials/google_adk",
+  },
+  {
+    icon: "🚀",
+    title: "CopilotKit",
+    description: "In-app AI copilots.",
+    to: "../tutorials/copilotkit_sdk",
+  },
+  {
+    icon: "🧬",
+    title: "Letta",
+    description: "Build stateful LLM agents with persistent memory.",
+    to: "./letta",
+  },
+  {
+    icon: "🎙️",
+    title: "LiveKit",
+    description: "Real-time voice and video AI agents.",
+    to: "../tutorials/livekit_xai_realtime",
+  },
+]}
+/>
+
+---
+
+## Prompt Management
+
+Manage, version, and deploy prompts.
+
+<NavigationCards
+columns={3}
+items={[
+  {
+    icon: "📝",
+    title: "LiteLLM Prompt Management",
+    description: "Built-in prompt management.",
+    to: "../proxy/litellm_prompt_management",
+  },
+  {
+    icon: "🔌",
+    title: "Custom Prompt Management",
+    description: "Bring your own prompt store.",
+    to: "../proxy/custom_prompt_management",
+  },
+  {
+    icon: "🔥",
+    title: "Arize Phoenix Prompts",
+    description: "Prompt management with Phoenix.",
+    to: "../proxy/arize_phoenix_prompts",
+  },
+]}
+/>
+
+---
+
+## Manage with AI Agents
+
+Use AI agents to manage your LiteLLM deployment — create users, teams, keys, models, and more via natural language.
+
+<NavigationCards
+columns={1}
+items={[
+  {
+    icon: "🤖",
+    title: "LiteLLM Skills",
+    description: "Manage LiteLLM via Claude Code — create keys, teams, models, and more using natural language commands.",
+    to: "../tutorials/claude_code_skills",
+  },
+]}
+/>

docs/my-website/docs/integrations/letta.md

@@ -920,9 +920,9 @@ for model in models:
 
 ## Resources
 
-- [Letta Documentation](https://docs.letta.ai/)
-- [LiteLLM Proxy Documentation](../proxy/quick_start.md)
-- [LiteLLM SDK Documentation](../completion/input.md)
-- [Function Calling Guide](../completion/function_call.md)
-- [Observability Setup](../observability/langfuse_integration.md)
-- [Router Configuration](../routing.md)
+- [Letta Documentation](https://docs.letta.com/)
+- [LiteLLM Proxy Documentation](/docs/simple_proxy)
+- [LiteLLM SDK Documentation](/docs/#litellm-python-sdk)
+- [Function Calling Guide](/docs/completion/function_call)
+- [Observability Setup](/docs/integrations/observability_integrations)
+- [Router Configuration](/docs/routing)

docs/my-website/docs/integrations/observability_index.md

@@ -0,0 +1,28 @@
+---
+title: Observability
+sidebar_label: Overview
+slug: observability_integrations
+---
+
+Track, debug, and analyze LLM calls with observability platforms.
+
+import NavigationCards from '@site/src/components/NavigationCards';
+
+## Observability Integrations
+
+<NavigationCards
+columns={3}
+items={[
+  { icon: "🪢", title: "Langfuse", description: "LLM observability and analytics.", to: "/docs/observability/langfuse_integration" },
+  { icon: "🐶", title: "Datadog", description: "Metrics, traces, and dashboards.", to: "/docs/observability/datadog" },
+  { icon: "📡", title: "OpenTelemetry", description: "Vendor-neutral tracing.", to: "/docs/observability/opentelemetry_integration" },
+  { icon: "🔗", title: "LangSmith", description: "LLM debugging and evaluation.", to: "/docs/observability/langsmith_integration" },
+  { icon: "🔥", title: "Arize / Phoenix", description: "ML observability and evaluation.", to: "/docs/observability/arize_integration" },
+  { icon: "🌀", title: "Helicone", description: "LLM request logging and analytics.", to: "/docs/observability/helicone_integration" },
+  { icon: "📊", title: "MLflow", description: "Experiment tracking.", to: "/docs/observability/mlflow" },
+  { icon: "🏋️", title: "Weights & Biases", description: "ML experiment tracking.", to: "/docs/observability/wandb_integration" },
+  { icon: "📉", title: "PostHog", description: "Product analytics.", to: "/docs/observability/posthog_integration" },
+]}
+/>
+
+[View all observability integrations →](/docs/observability/callbacks)

docs/my-website/docs/integrations/websearch_interception.md

@@ -375,7 +375,7 @@ search_tools:
 - [Search Providers](../search/index.md) - Detailed search provider setup
 - [Claude Code WebSearch](../tutorials/claude_code_websearch.md) - Using with Claude Code
 - [Tool Calling](../completion/function_call.md) - General tool calling documentation
-- [Callbacks](./custom_callback.md) - Custom callback documentation
+- [Callbacks](../observability/custom_callback.md) - Custom callback documentation
 
 ## Technical Details
 

docs/my-website/docs/learn/gateway_quickstart.md

@@ -0,0 +1,174 @@
+---
+title: Gateway Quickstart
+sidebar_label: Gateway Quickstart
+description: Start LiteLLM Gateway, add models and keys, then connect applications and SDKs to one shared endpoint.
+---
+
+import NavigationCards from '@site/src/components/NavigationCards';
+
+Use this path if you need one shared OpenAI-compatible endpoint for a team or platform.
+
+If you need a Docker or database-first setup, use the [Docker + Database tutorial](/docs/proxy/docker_quick_start). Otherwise, use the steps below to get to a working request fast.
+
+## 1. Install The Gateway
+
+```bash
+pip install 'litellm[proxy]'
+```
+
+## 2. Set One Provider Key
+
+```bash
+export OPENAI_API_KEY="your-api-key"
+```
+
+## 3. Create `config.yaml`
+
+```yaml
+model_list:
+  - model_name: gpt-4o-mini
+    litellm_params:
+      model: openai/gpt-4o-mini
+      api_key: os.environ/OPENAI_API_KEY
+
+general_settings:
+  master_key: sk-1234
+```
+
+## 4. Start The Gateway
+
+```bash
+litellm --config config.yaml
+```
+
+You should see the proxy start on `http://0.0.0.0:4000`.
+
+## 5. Send Your First Request
+
+```bash
+curl -X POST 'http://0.0.0.0:4000/chat/completions' \
+  -H 'Content-Type: application/json' \
+  -H 'Authorization: Bearer sk-1234' \
+  -d '{
+    "model": "gpt-4o-mini",
+    "messages": [
+      {"role": "user", "content": "Hello from LiteLLM Gateway"}
+    ]
+  }'
+```
+
+## 6. Check The Response
+
+If the request succeeds, the proxy returns `200 OK` with an OpenAI-style response.
+
+The assistant text will be in:
+
+```json
+choices[0].message.content
+```
+
+If your gateway is routing to OpenAI, a real response can look like this:
+
+```json
+{
+  "id": "chatcmpl-abc123",
+  "created": 1677858242,
+  "model": "gpt-4o-mini-2024-07-18",
+  "object": "chat.completion",
+  "system_fingerprint": "fp_406d6473f8",
+  "choices": [
+    {
+      "finish_reason": "stop",
+      "index": 0,
+      "message": {
+        "role": "assistant",
+        "content": "Hello! How can I assist you today?",
+        "tool_calls": null,
+        "function_call": null,
+        "annotations": []
+      }
+    }
+  ],
+  "usage": {
+    "completion_tokens": 9,
+    "prompt_tokens": 13,
+    "total_tokens": 22,
+    "completion_tokens_details": {
+      "accepted_prediction_tokens": 0,
+      "audio_tokens": 0,
+      "reasoning_tokens": 0,
+      "rejected_prediction_tokens": 0
+    },
+    "prompt_tokens_details": {
+      "audio_tokens": 0,
+      "cached_tokens": 0
+    }
+  },
+  "service_tier": "default"
+}
+```
+
+`id`, `created`, the resolved model version, token counts, and message text will vary by request. Other providers may return a smaller or slightly different set of fields, but `choices[0].message.content` is the main field to read.
+
+## 7. Add Keys And The UI
+
+If you need virtual keys, spend tracking, or the admin UI, add a database next.
+
+- Add `database_url` under `general_settings`
+- Use [Virtual keys](/docs/proxy/virtual_keys) for key creation and budgets
+- Use [Admin UI](/docs/proxy/ui) to manage models and keys
+- Use the [Docker + Database tutorial](/docs/proxy/docker_quick_start) if you want a fuller setup
+
+## 8. Pick Your Next Step
+
+<NavigationCards
+columns={3}
+items={[
+  {
+    icon: "🖥️",
+    title: "Make LLM Requests",
+    description: "Point LiteLLM or OpenAI-compatible clients to the gateway.",
+    to: "/docs/proxy/user_keys",
+  },
+  {
+    icon: "🎛️",
+    title: "Model Config",
+    description: "Add more models and gateway settings.",
+    to: "/docs/proxy/configs",
+  },
+  {
+    icon: "🔑",
+    title: "Virtual Keys",
+    description: "Create keys, budgets, and access controls.",
+    to: "/docs/proxy/virtual_keys",
+  },
+  {
+    icon: "📈",
+    title: "Add Logging",
+    description: "Capture logs, spend, and traces.",
+    to: "/docs/proxy/logging",
+  },
+  {
+    icon: "🔀",
+    title: "Load Balance",
+    description: "Route across deployments, regions, or providers.",
+    to: "/docs/proxy/load_balancing",
+  },
+  {
+    icon: "🛡️",
+    title: "Add Guardrails",
+    description: "Add safety checks and policy enforcement.",
+    to: "/docs/proxy/guardrails/quick_start",
+  },
+  {
+    icon: "📊",
+    title: "Reliability",
+    description: "Configure retries, fallbacks, and timeouts.",
+    to: "/docs/proxy/reliability",
+  },
+]}
+/>
+
+## When To Use The SDK Path Instead
+
+If you only need to call models from one application and do not need centralized auth or shared infrastructure, start with the [SDK Quickstart](/docs/learn/sdk_quickstart) instead.

docs/my-website/docs/learn/index.md

@@ -0,0 +1,117 @@
+---
+title: Learn LiteLLM
+sidebar_label: Learn
+slug: /learn
+---
+
+import NavigationCards from '@site/src/components/NavigationCards';
+
+LiteLLM gives you one OpenAI-compatible interface for 100+ LLM providers. Start with the path that matches your setup.
+
+---
+
+## Start Here
+
+Pick one path first.
+
+<NavigationCards
+columns={2}
+items={[
+  {
+    icon: "🐍",
+    title: "SDK Quickstart",
+    description: "Use LiteLLM directly in application code.",
+    listDescription: [
+      "Install",
+      "First request",
+      "Next SDK features",
+    ],
+    to: "/docs/learn/sdk_quickstart",
+  },
+  {
+    icon: "🖥️",
+    title: "Gateway Quickstart",
+    description: "Run LiteLLM as a shared gateway.",
+    listDescription: [
+      "Start proxy",
+      "Add models and keys",
+      "Connect clients",
+    ],
+    to: "/docs/learn/gateway_quickstart",
+  },
+]}
+/>
+
+---
+
+## Common Tasks
+
+Jump to a specific task.
+
+<NavigationCards
+columns={3}
+items={[
+  {
+    icon: "⚡",
+    title: "Stream Responses",
+    description: "Return tokens as they are generated.",
+    to: "/docs/guides/core_request_response_patterns",
+  },
+  {
+    icon: "🧰",
+    title: "Use Tools",
+    description: "Add function calling to your app.",
+    to: "/docs/guides/tools_integrations",
+  },
+  {
+    icon: "🔀",
+    title: "Add Routing",
+    description: "Retries, fallbacks, and load balancing.",
+    to: "/docs/routing-load-balancing",
+  },
+  {
+    icon: "🔑",
+    title: "Set Up Keys",
+    description: "Gateway auth, virtual keys, and access control.",
+    to: "/docs/proxy/virtual_keys",
+  },
+  {
+    icon: "📈",
+    title: "Add Logging",
+    description: "Capture request logs and spend data.",
+    to: "/docs/proxy/logging",
+  },
+  {
+    icon: "🌐",
+    title: "Choose A Provider",
+    description: "Find provider-specific auth and params.",
+    to: "/docs/providers",
+  },
+]}
+/>
+
+---
+
+## Docs Map
+
+Use these when you already know the type of doc you want.
+
+<NavigationCards
+columns={2}
+items={[
+  {
+    icon: "📚",
+    title: "Guides",
+    description: "Feature reference.",
+    to: "/docs/guides",
+  },
+  {
+    icon: "🛠️",
+    title: "Tutorials",
+    description: "Step-by-step integrations.",
+    to: "/docs/tutorials",
+  },
+]}
+/>
+
+Not sure where to start? Use [SDK Quickstart](/docs/learn/sdk_quickstart) for app code or [Gateway Quickstart](/docs/learn/gateway_quickstart) for shared infrastructure.

docs/my-website/docs/learn/sdk_quickstart.md

@@ -0,0 +1,174 @@
+---
+title: SDK Quickstart
+sidebar_label: SDK Quickstart
+description: Make your first LiteLLM SDK call, then jump to the right docs for the next feature you need.
+---
+
+import NavigationCards from '@site/src/components/NavigationCards';
+
+Use this path if you are integrating LiteLLM directly into application code.
+
+## 1. Install LiteLLM
+
+```bash
+pip install litellm
+```
+
+## 2. Set Provider Credentials
+
+Start with one provider and set its environment variables.
+
+- OpenAI: `OPENAI_API_KEY`
+- Anthropic: `ANTHROPIC_API_KEY`
+- Azure OpenAI: `AZURE_API_KEY`, `AZURE_API_BASE`, `AZURE_API_VERSION`
+- Bedrock: standard AWS credentials
+- Vertex AI: `VERTEXAI_PROJECT`, `VERTEXAI_LOCATION`
+
+If you have not picked a provider yet, browse [all supported providers](/docs/providers).
+
+## 3. Make Your First Call
+
+```python
+from litellm import completion
+import os
+
+os.environ["OPENAI_API_KEY"] = "your-api-key"
+
+response = completion(
+    model="openai/gpt-4o",
+    messages=[{"role": "user", "content": "Hello, how are you?"}],
+)
+
+print(response.choices[0].message.content)
+```
+
+## 4. Check The Response
+
+The line below:
+
+```python
+print(response.choices[0].message.content)
+```
+
+prints the assistant text, for example:
+
+```text
+Hello! I'm doing well, thanks for asking.
+```
+
+If you print the full object with:
+
+```python
+print(response)
+```
+
+you will see a Python `ModelResponse(...)` object. For an OpenAI-backed model, it can look like this:
+
+```python
+ModelResponse(
+    id='chatcmpl-abc123',
+    created=1773782130,
+    model='gpt-4o-2024-08-06',
+    object='chat.completion',
+    system_fingerprint='fp_4ff89bf575',
+    choices=[
+        Choices(
+            finish_reason='stop',
+            index=0,
+            message=Message(
+                content="Hello! I'm just a program, but I'm here to help you. How can I assist you today?",
+                role='assistant',
+                tool_calls=None,
+                function_call=None,
+                provider_specific_fields={'refusal': None},
+                annotations=[]
+            ),
+            provider_specific_fields={}
+        )
+    ],
+    usage=Usage(
+        completion_tokens=21,
+        prompt_tokens=13,
+        total_tokens=34,
+        completion_tokens_details=CompletionTokensDetailsWrapper(...),
+        prompt_tokens_details=PromptTokensDetailsWrapper(...)
+    ),
+    service_tier='default'
+)
+```
+
+The same response follows an OpenAI-style shape. Conceptually, it looks like this:
+
+```json
+{
+  "id": "chatcmpl-abc123",
+  "object": "chat.completion",
+  "created": 1677858242,
+  "model": "gpt-4o",
+  "choices": [
+    {
+      "index": 0,
+      "message": {
+        "role": "assistant",
+        "content": "Hello! I'm doing well, thanks for asking."
+      },
+      "finish_reason": "stop"
+    }
+  ],
+  "usage": {
+    "prompt_tokens": 13,
+    "completion_tokens": 12,
+    "total_tokens": 25
+  }
+}
+```
+
+`id`, `created`, token counts, and message text will vary by request.
+
+If you call an OpenAI-backed model, you may also see extra fields such as `system_fingerprint`, `service_tier`, `tool_calls`, `function_call`, `annotations`, `provider_specific_fields`, and detailed token usage. For the full output reference, see [completion output](/docs/completion/output).
+
+Need more provider examples? See the main [Getting Started](/docs/#quick-start) page.
+
+## 5. Pick Your Next Step
+
+<NavigationCards
+columns={3}
+items={[
+{
+icon: "⚡",
+title: "Stream Responses",
+description: "Receive tokens incrementally with stream=True.",
+to: "/docs/completion/stream",
+},
+{
+icon: "🧰",
+title: "Use Tools",
+description: "Add function calling in a provider-agnostic way.",
+to: "/docs/completion/function_call",
+},
+{
+icon: "📦",
+title: "Return JSON",
+description: "Constrain responses to structured JSON output.",
+to: "/docs/completion/json_mode",
+},
+{
+icon: "🔀",
+title: "Add Routing",
+description: "Use retries, fallbacks, and load balancing in app code.",
+to: "/docs/routing",
+},
+{
+icon: "🌐",
+title: "Choose A Provider",
+description: "Find provider-specific auth, model naming, and params.",
+to: "/docs/providers",
+},
+]}
+/>
+
+## When To Use Gateway Instead
+
+Use LiteLLM Gateway if you need centralized auth, virtual keys, spend tracking, shared logging, or one OpenAI-compatible endpoint for multiple apps.
+
+[Go to Gateway Quickstart →](/docs/learn/gateway_quickstart)

docs/my-website/docs/load_test_advanced.md

@@ -11,7 +11,7 @@ Tutorial on how to get to 1K+ RPS with LiteLLM Proxy on locust
     - [Github releases](https://github.com/BerriAI/litellm/releases)
     - [litellm docker containers](https://github.com/BerriAI/litellm/pkgs/container/litellm)
     - [litellm database docker container](https://github.com/BerriAI/litellm/pkgs/container/litellm-database)
-- [ ] Ensure you're following **ALL** [best practices for production](./proxy/production_setup.md)
+- [ ] Ensure you're following **ALL** [best practices for production](./proxy/prod.md)
 - [ ] Locust - Ensure you're Locust instance can create 1K+ requests per second
     - 👉 You can use our **[maintained locust instance here](https://locust-load-tester-production.up.railway.app/)**
     - If you're self hosting locust
@@ -222,4 +222,4 @@ class MyUser(HttpUser):
     def on_start(self):
         self.api_key = os.getenv('API_KEY', 'sk-1234')
         self.client.headers.update({'Authorization': f'Bearer {self.api_key}'})
-```
+```

docs/my-website/docs/mcp.md

@@ -133,6 +133,21 @@ LiteLLM attempts [OAuth 2.0 Authorization Server Discovery](https://datatracker.
 
 <br/>
 
+### AWS SigV4 Authentication
+
+For MCP servers hosted on [AWS Bedrock AgentCore](https://docs.aws.amazon.com/bedrock/latest/userguide/agentcore.html), select **AWS SigV4** as the authentication type. LiteLLM will sign every outgoing MCP request with your AWS credentials using [Signature Version 4](https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html).
+
+<Image
+  img={require('../img/mcp_aws_sigv4_ui.png')}
+  style={{width: '80%', display: 'block', margin: '0'}}
+/>
+
+Fill in your AWS region, service name (defaults to `bedrock-agentcore`), and optionally your AWS access key and secret. If credentials are omitted, LiteLLM falls back to the boto3 credential chain (IAM roles, environment variables, etc.).
+
+[**See full SigV4 setup guide**](./mcp_aws_sigv4.md)
+
+<br/>
+
 ### Static Headers
 
 Sometimes your MCP server needs specific headers on every request. Maybe it's an API key, maybe it's a custom header the server expects. Instead of configuring auth, you can just set them directly.
@@ -217,6 +232,7 @@ mcp_servers:
   | `bearer_token` | `Authorization: Bearer <auth_value>` |
   | `basic` | `Authorization: Basic <auth_value>` |
   | `authorization` | `Authorization: <auth_value>` |
+  | `aws_sigv4` | Per-request AWS SigV4 signature ([details](./mcp_aws_sigv4.md)) |
 
 - **Extra Headers**: Optional list of additional header names that should be forwarded from client to the MCP server
 - **Static Headers**: Optional map of header key/value pairs to include every request to the MCP server.
@@ -257,6 +273,16 @@ mcp_servers:
     auth_type: "authorization"
     auth_value: "Token example123"  # headers={"Authorization": "Token example123"}
 
+  # AWS SigV4 for Bedrock AgentCore MCP servers
+  agentcore_mcp:
+    url: "https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes/<url-encoded-ARN>/invocations"
+    transport: "http"
+    auth_type: "aws_sigv4"
+    aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
+    aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
+    aws_region_name: us-east-1
+    aws_service_name: bedrock-agentcore
+
   # Example with extra headers forwarding
   github_mcp:
     url: "https://api.githubcopilot.com/mcp"
@@ -336,175 +362,9 @@ litellm_settings:
 
 ## Converting OpenAPI Specs to MCP Servers
 
-LiteLLM can automatically convert OpenAPI specifications into MCP servers, allowing you to expose any REST API as MCP tools. This is useful when you have existing APIs with OpenAPI/Swagger documentation and want to make them available as MCP tools.
+LiteLLM can convert OpenAPI specifications into MCP servers, exposing any REST API as MCP tools without writing custom server code.
 
-**Benefits:**
-
-- **Rapid Integration**: Convert existing APIs to MCP tools without writing custom MCP server code
-- **Automatic Tool Generation**: LiteLLM automatically generates MCP tools from your OpenAPI spec
-- **Unified Interface**: Use the same MCP interface for both native MCP servers and OpenAPI-based APIs
-- **Easy Testing**: Test and iterate on API integrations quickly
-
-**Configuration:**
-
-Add your OpenAPI-based MCP server to your `config.yaml`:
-
-```yaml title="config.yaml - OpenAPI to MCP" showLineNumbers
-model_list:
-  - model_name: gpt-4o
-    litellm_params:
-      model: openai/gpt-4o
-      api_key: sk-xxxxxxx
-
-mcp_servers:
-  # OpenAPI Spec Example - Petstore API
-  petstore_mcp:
-    url: "https://petstore.swagger.io/v2"
-    spec_path: "/path/to/openapi.json"
-    auth_type: "none"
-  
-  # OpenAPI Spec with API Key Authentication
-  my_api_mcp:
-    url: "http://0.0.0.0:8090"
-    spec_path: "/path/to/openapi.json"
-    auth_type: "api_key"
-    auth_value: "your-api-key-here"
-  
-  # OpenAPI Spec with Bearer Token
-  secured_api_mcp:
-    url: "https://api.example.com"
-    spec_path: "/path/to/openapi.json" 
-    auth_type: "bearer_token"
-    auth_value: "your-bearer-token"
-```
-
-**Configuration Parameters:**
-
-| Parameter | Required | Description |
-|-----------|----------|-------------|
-| `url` | Yes | The base URL of your API endpoint |
-| `spec_path` | Yes | Path or URL to your OpenAPI specification file (JSON or YAML) |
-| `auth_type` | No | Authentication type: `none`, `api_key`, `bearer_token`, `basic`, `authorization` |
-| `auth_value` | No | Authentication value (required if `auth_type` is set) |
-| `authorization_url` | No | For `auth_type: oauth2`. Optional override; if omitted LiteLLM auto-discovers it. |
-| `token_url` | No | For `auth_type: oauth2`. Optional override; if omitted LiteLLM auto-discovers it. |
-| `registration_url` | No | For `auth_type: oauth2`. Optional override; if omitted LiteLLM auto-discovers it. |
-| `scopes` | No | For `auth_type: oauth2`. Optional override; if omitted LiteLLM uses the scopes advertised by the server. |
-| `description` | No | Optional description for the MCP server |
-| `allowed_tools` | No | List of specific tools to allow (see [MCP Tool Filtering](#mcp-tool-filtering)) |
-| `disallowed_tools` | No | List of specific tools to block (see [MCP Tool Filtering](#mcp-tool-filtering)) |
-
-### Usage Example
-
-Once configured, you can use the OpenAPI-based MCP server just like any other MCP server:
-
-<Tabs>
-<TabItem value="fastmcp" label="Python FastMCP">
-
-```python title="Using OpenAPI-based MCP Server" showLineNumbers
-from fastmcp import Client
-import asyncio
-
-# Standard MCP configuration
-config = {
-    "mcpServers": {
-        "petstore": {
-            "url": "http://localhost:4000/petstore_mcp/mcp",
-            "headers": {
-                "x-litellm-api-key": "Bearer sk-1234"
-            }
-        }
-    }
-}
-
-# Create a client that connects to the server
-client = Client(config)
-
-async def main():
-    async with client:
-        # List available tools generated from OpenAPI spec
-        tools = await client.list_tools()
-        print(f"Available tools: {[tool.name for tool in tools]}")
-
-        # Example: Get a pet by ID (from Petstore API)
-        response = await client.call_tool(
-            name="getpetbyid", 
-            arguments={"petId": "1"}
-        )
-        print(f"Response:\n{response}\n")
-
-        # Example: Find pets by status
-        response = await client.call_tool(
-            name="findpetsbystatus", 
-            arguments={"status": "available"}
-        )
-        print(f"Response:\n{response}\n")
-
-if __name__ == "__main__":
-    asyncio.run(main())
-```
-
-</TabItem>
-
-<TabItem value="cursor" label="Cursor IDE">
-
-```json title="Cursor MCP Configuration for OpenAPI Server" showLineNumbers
-{
-  "mcpServers": {
-    "Petstore": {
-      "url": "http://localhost:4000/petstore_mcp/mcp",
-      "headers": {
-        "x-litellm-api-key": "Bearer $LITELLM_API_KEY"
-      }
-    }
-  }
-}
-```
-
-</TabItem>
-
-<TabItem value="openai" label="OpenAI Responses API">
-
-```bash title="Using OpenAPI MCP Server with OpenAI" showLineNumbers
-curl --location 'https://api.openai.com/v1/responses' \
---header 'Content-Type: application/json' \
---header "Authorization: Bearer $OPENAI_API_KEY" \
---data '{
-    "model": "gpt-4o",
-    "tools": [
-        {
-            "type": "mcp",
-            "server_label": "petstore",
-            "server_url": "http://localhost:4000/petstore_mcp/mcp",
-            "require_approval": "never",
-            "headers": {
-                "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY"
-            }
-        }
-    ],
-    "input": "Find all available pets in the petstore",
-    "tool_choice": "required"
-}'
-```
-
-</TabItem>
-</Tabs>
-
-**How It Works**
-
-1. **Spec Loading**: LiteLLM loads your OpenAPI specification from the provided `spec_path`
-2. **Tool Generation**: Each API endpoint in the spec becomes an MCP tool
-3. **Parameter Mapping**: OpenAPI parameters are automatically mapped to MCP tool parameters
-4. **Request Handling**: When a tool is called, LiteLLM converts the MCP request to the appropriate HTTP request
-5. **Response Translation**: API responses are converted back to MCP format
-
-**OpenAPI Spec Requirements**
-
-Your OpenAPI specification should follow standard OpenAPI/Swagger conventions:
-- **Supported versions**: OpenAPI 3.0.x, OpenAPI 3.1.x, Swagger 2.0
-- **Required fields**: `paths`, `info` sections should be properly defined
-- **Operation IDs**: Each operation should have a unique `operationId` (this becomes the tool name)
-- **Parameters**: Request parameters should be properly documented with types and descriptions
+See the **[MCP from OpenAPI Specs guide](./mcp_openapi.md)** for full setup, usage examples, and how to override tool names and descriptions.
 
 ## MCP OAuth
 
@@ -641,7 +501,7 @@ import asyncio
 config = {
     "mcpServers": {
         "mcp_group": {
-            "url": "http://localhost:4000/mcp",
+            "url": "http://localhost:4000/mcp/",
             "headers": {
                 "x-mcp-servers": "dev_group", # assume this gives access to github, zapier and deepwiki
                 "x-litellm-api-key": "Bearer sk-1234",
@@ -870,6 +730,63 @@ asyncio.run(main())
 
 [Learn more about customer management →](./proxy/customers)
 
+## Calling the Proxy's /v1/responses Endpoint
+
+When calling your LiteLLM Proxy's `/v1/responses` endpoint to use MCP tools, **always use `server_url: "litellm_proxy"`** in the tools array. This tells the proxy to use its configured MCP servers.
+
+:::important Do not use the full proxy URL
+Using `server_url: "https://your-proxy.com/mcp"` is incorrect when the request is already going to the proxy. The proxy needs the literal value `litellm_proxy` to route to its configured MCP servers.
+:::
+
+```bash title="Correct: Using litellm_proxy" showLineNumbers
+curl --location 'https://your-proxy.com/v1/responses' \
+--header 'Content-Type: application/json' \
+--header "Authorization: Bearer $LITELLM_API_KEY" \
+--data '{
+    "model": "gpt-4",
+    "tools": [
+        {
+            "type": "mcp",
+            "server_label": "litellm",
+            "server_url": "litellm_proxy",
+            "require_approval": "never"
+        }
+    ],
+    "input": "Run available tools",
+    "tool_choice": "required"
+}'
+```
+
+### Sending Custom Headers to MCP Servers
+
+To pass custom headers (e.g., API keys, auth tokens) to specific MCP servers, use either:
+
+**Option 1: Request headers** – Add `x-mcp-{server_alias}-{header_name}` to your request headers. The proxy forwards these to the matching MCP server.
+
+```bash
+# Send Authorization header to the "weather2" MCP server
+--header 'x-mcp-weather2-authorization: Bearer your-token'
+
+# Send custom header to the "github" MCP server  
+--header 'x-mcp-github-x-api-key: your-api-key'
+```
+
+**Option 2: Headers in tool config** – Include a `headers` object in the tool definition. These are merged with request headers.
+
+```json
+{
+    "type": "mcp",
+    "server_label": "litellm",
+    "server_url": "litellm_proxy",
+    "require_approval": "never",
+    "headers": {
+        "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY",
+        "x-mcp-servers": "Zapier_MCP,dev-group",
+        "x-mcp-weather2-authorization": "Bearer your-weather-api-token"
+    }
+}
+```
+
 ## Using your MCP with client side credentials
 
 Use this if you want to pass a client side authentication token to LiteLLM to then pass to your MCP to auth to your MCP.

docs/my-website/docs/mcp_aws_sigv4.md

@@ -0,0 +1,181 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import Image from '@theme/IdealImage';
+
+# MCP - AWS SigV4 Auth
+
+Use AWS SigV4 authentication to connect LiteLLM to MCP servers hosted on [AWS Bedrock AgentCore](https://docs.aws.amazon.com/bedrock/latest/userguide/agentcore.html).
+
+## Why SigV4?
+
+AWS services authenticate requests using [Signature Version 4](https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html) — a per-request signing protocol that includes the request body in the cryptographic signature. This is fundamentally different from static-header auth types (`api_key`, `bearer_token`, etc.) which send the same header on every request.
+
+LiteLLM's `aws_sigv4` auth type handles this automatically: every outgoing MCP request is signed with your AWS credentials before it's sent.
+
+## Quick Start
+
+<Tabs>
+<TabItem value="ui" label="LiteLLM UI">
+
+1. Navigate to **MCP Servers** and click **Add New MCP Server**
+2. Set the transport to **Streamable HTTP**
+3. Select **AWS SigV4** as the authentication type
+4. Fill in your AWS credentials:
+
+<Image
+  img={require('../img/mcp_aws_sigv4_ui.png')}
+  style={{width: '80%', display: 'block', margin: '0'}}
+/>
+
+<br/>
+
+| Field | Required | Description |
+|-------|----------|-------------|
+| **AWS Region** | Yes | AWS region for SigV4 signing (e.g., `us-east-1`) |
+| **AWS Service Name** | No | Defaults to `bedrock-agentcore` |
+| **AWS Access Key ID** | No | Falls back to boto3 credential chain if blank |
+| **AWS Secret Access Key** | No | Required if Access Key ID is provided |
+| **AWS Session Token** | No | Only needed for temporary STS credentials |
+
+Once created, LiteLLM will sign every outgoing MCP request with SigV4. The server's tools appear automatically in the MCP Tools list.
+
+**Editing credentials:** When editing an existing SigV4 server, leave credential fields blank to keep the current values. Only fields you fill in will be updated.
+
+</TabItem>
+<TabItem value="config" label="config.yaml">
+
+### 1. Set AWS credentials
+
+```bash
+export AWS_ACCESS_KEY_ID="AKIA..."
+export AWS_SECRET_ACCESS_KEY="..."
+export AWS_REGION_NAME="us-east-1"
+```
+
+### 2. Add your AgentCore MCP server to config.yaml
+
+```yaml title="config.yaml" showLineNumbers
+model_list:
+  - model_name: gpt-4o
+    litellm_params:
+      model: openai/gpt-4o
+      api_key: os.environ/OPENAI_API_KEY
+
+mcp_servers:
+  my_agentcore_mcp:
+    url: "https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes/<url-encoded-ARN>/invocations"
+    transport: "http"
+    auth_type: "aws_sigv4"
+    aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
+    aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
+    aws_region_name: "us-east-1"
+    aws_service_name: "bedrock-agentcore"
+```
+
+:::info URL encoding
+
+The AgentCore runtime ARN must be URL-encoded in the `url` field. For example:
+
+```
+arn:aws:bedrock-agentcore:us-east-1:123456789012:runtime/my-mcp-server
+```
+
+becomes:
+
+```
+arn%3Aaws%3Abedrock-agentcore%3Aus-east-1%3A123456789012%3Aruntime%2Fmy-mcp-server
+```
+
+:::
+
+### 3. Start the proxy
+
+```bash
+litellm --config config.yaml
+```
+
+</TabItem>
+</Tabs>
+
+## Use the MCP tools
+
+Once configured, your AgentCore MCP tools are available through LiteLLM like any other MCP server:
+
+```bash title="List available tools"
+curl http://localhost:4000/mcp-rest/tools/list \
+  -H "Authorization: Bearer sk-1234"
+```
+
+```bash title="Call a tool"
+curl http://localhost:4000/mcp-rest/tools/call \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer sk-1234" \
+  -d '{
+    "name": "my_agentcore_mcp_your_tool_name",
+    "arguments": {"key": "value"}
+  }'
+```
+
+## Config Reference
+
+| Field | Required | Description |
+|-------|----------|-------------|
+| `url` | Yes | AgentCore MCP server URL (with URL-encoded ARN) |
+| `transport` | Yes | Must be `"http"` |
+| `auth_type` | Yes | Must be `"aws_sigv4"` |
+| `aws_access_key_id` | No | AWS access key. Supports `os.environ/VAR_NAME`. Falls back to boto3 credential chain if omitted |
+| `aws_secret_access_key` | No | AWS secret key. Supports `os.environ/VAR_NAME`. Falls back to boto3 credential chain if omitted |
+| `aws_region_name` | Yes | AWS region (e.g., `us-east-1`) |
+| `aws_service_name` | No | AWS service name for signing. Defaults to `bedrock-agentcore` |
+| `aws_session_token` | No | AWS session token for temporary credentials. Supports `os.environ/VAR_NAME` |
+
+## How It Works
+
+LiteLLM uses an `httpx.Auth` subclass (`MCPSigV4Auth`) that hooks into the HTTP request lifecycle:
+
+1. For every outgoing MCP request, the auth handler computes a SHA-256 hash of the request body
+2. It creates a SigV4 signature using your AWS credentials, the request URL, headers, and body hash
+3. The signed `Authorization` and `x-amz-date` headers are added to the request
+4. AWS validates the signature and processes the MCP request
+
+This happens transparently — no manual token management required.
+
+## Using Temporary Credentials (STS)
+
+If you use AWS STS temporary credentials (e.g., from IAM roles or SSO), include the session token:
+
+```yaml title="config.yaml with STS credentials" showLineNumbers
+mcp_servers:
+  my_agentcore_mcp:
+    url: "https://bedrock-agentcore.us-east-1.amazonaws.com/runtimes/<url-encoded-ARN>/invocations"
+    transport: "http"
+    auth_type: "aws_sigv4"
+    aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
+    aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
+    aws_session_token: os.environ/AWS_SESSION_TOKEN
+    aws_region_name: "us-east-1"
+    aws_service_name: "bedrock-agentcore"
+```
+
+## Troubleshooting
+
+### 403 Forbidden from AWS
+
+- Verify your AWS credentials are valid and not expired
+- Check that `aws_region_name` matches the region in your AgentCore URL
+- Ensure `aws_service_name` is set to `bedrock-agentcore`
+- If using STS credentials, confirm `aws_session_token` is set and not expired
+
+### Health check errors on startup
+
+SigV4-authenticated MCP servers skip the standard health check on proxy startup. This is expected — the proxy will still sign requests correctly when tools are invoked.
+
+### "botocore not found" error
+
+Install the `botocore` package:
+
+```bash
+pip install botocore
+```
+
+`botocore` is used for SigV4 credential handling and is required when using `aws_sigv4` auth.

docs/my-website/docs/mcp_control.md

@@ -323,7 +323,7 @@ curl --location '<your-litellm-proxy-base-url>/v1/responses' \
         {
             "type": "mcp",
             "server_label": "litellm",
-            "server_url": "<your-litellm-proxy-base-url>/dev_group/mcp",
+            "server_url": "litellm_proxy",
             "require_approval": "never",
             "headers": {
                 "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY"
@@ -335,7 +335,7 @@ curl --location '<your-litellm-proxy-base-url>/v1/responses' \
 }'
 ```
 
-This example uses URL namespacing to access all servers in the "dev_group" access group.
+This example uses the `x-mcp-servers` header to access all servers in the "dev_group" access group. Use `server_url: "litellm_proxy"` when calling the proxy's `/v1/responses` endpoint—do not use the full proxy URL.
 
 </TabItem>
 
@@ -423,7 +423,7 @@ curl --location '<your-litellm-proxy-base-url>/v1/responses' \
         {
             "type": "mcp",
             "server_label": "litellm",
-            "server_url": "<your-litellm-proxy-base-url>/mcp/",
+            "server_url": "litellm_proxy",
             "require_approval": "never",
             "headers": {
                 "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY",
@@ -436,7 +436,7 @@ curl --location '<your-litellm-proxy-base-url>/v1/responses' \
 }'
 ```
 
-This configuration restricts the request to only use tools from the specified MCP servers.
+This configuration restricts the request to only use tools from the specified MCP servers. Use `server_url: "litellm_proxy"` when calling the proxy's `/v1/responses` endpoint.
 
 </TabItem>
 

docs/my-website/docs/mcp_guardrail.md

@@ -86,4 +86,5 @@ MCP guardrails work with all LiteLLM-supported guardrail providers:
 - **Lakera**: Content moderation
 - **Aporia**: Custom guardrails
 - **Noma**: Noma Security
+- **PANW Prisma AIRS**: Prisma AIRS guardrails
 - **Custom**: Your own guardrail implementations

docs/my-website/docs/mcp_openapi.md

@@ -0,0 +1,226 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import Image from '@theme/IdealImage';
+
+# MCP from OpenAPI Specs
+
+LiteLLM can convert any OpenAPI/Swagger spec into an MCP server — no custom MCP server code required.
+
+## Step 1 — Add the MCP Server
+
+Add your OpenAPI-based server in `config.yaml`:
+
+```yaml title="config.yaml" showLineNumbers
+mcp_servers:
+  petstore_mcp:
+    url: "https://petstore.swagger.io/v2"
+    spec_path: "/path/to/openapi.json"
+    auth_type: "none"
+
+  my_api_mcp:
+    url: "http://0.0.0.0:8090"
+    spec_path: "/path/to/openapi.json"
+    auth_type: "api_key"
+    auth_value: "your-api-key-here"
+
+  secured_api_mcp:
+    url: "https://api.example.com"
+    spec_path: "/path/to/openapi.json"
+    auth_type: "bearer_token"
+    auth_value: "your-bearer-token"
+```
+
+Or from the UI: go to **MCP Servers → Add New MCP Server**, fill in the URL and spec path, and LiteLLM will fetch the spec and load all endpoints as tools.
+
+**Configuration parameters:**
+
+| Parameter | Required | Description |
+|-----------|----------|-------------|
+| `url` | Yes | Base URL of your API |
+| `spec_path` | Yes | Path or URL to your OpenAPI spec (JSON or YAML) |
+| `auth_type` | No | `none`, `api_key`, `bearer_token`, `basic`, `authorization`, `oauth2` |
+| `auth_value` | No | Auth value (required if `auth_type` is set) |
+| `description` | No | Optional description |
+| `allowed_tools` | No | Allowlist of specific tools |
+| `disallowed_tools` | No | Blocklist of specific tools |
+
+**Supported spec versions:** OpenAPI 3.0.x, 3.1.x, Swagger 2.0. Each operation's `operationId` becomes the tool name — make sure they're unique.
+
+Once tools are loaded, you'll see them in the Tool Configuration section:
+
+<Image
+  img={require('../img/mcp_openapi_tools_loaded.png')}
+  style={{width: '80%', display: 'block', margin: '0'}}
+/>
+
+<br/>
+
+## Step 2 — Optionally Override Tool Names and Descriptions
+
+By default, tool names and descriptions come from the `operationId` and description fields in your spec. You can rename or rewrite them so MCP clients see something cleaner — without touching the upstream spec.
+
+### From the UI
+
+Each tool card has a pencil icon. Click it to open the inline editor:
+
+<Image
+  img={require('../img/mcp_openapi_tool_edit_panel.png')}
+  style={{width: '80%', display: 'block', margin: '0'}}
+/>
+
+<br/>
+
+- **Display Name** — overrides the name MCP clients see
+- **Description** — overrides the description MCP clients see
+- Leave a field blank to keep the original from the spec
+
+After setting overrides, a purple **Custom name** badge appears on the tool card:
+
+<Image
+  img={require('../img/mcp_openapi_custom_name_badge.png')}
+  style={{width: '80%', display: 'block', margin: '0'}}
+/>
+
+<br/>
+
+### From the API
+
+Pass `tool_name_to_display_name` and `tool_name_to_description` in the create or update request:
+
+```bash title="Create server with tool name overrides" showLineNumbers
+curl -X POST http://localhost:4000/v1/mcp/server \
+  -H "Authorization: Bearer $LITELLM_MASTER_KEY" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "name": "petstore_mcp",
+    "url": "https://petstore.swagger.io/v2",
+    "spec_path": "/path/to/openapi.json",
+    "tool_name_to_display_name": {
+      "getPetById": "Get Pet",
+      "findPetsByStatus": "List Available Pets"
+    },
+    "tool_name_to_description": {
+      "getPetById": "Look up a pet by its ID",
+      "findPetsByStatus": "Returns all pets matching a given status (available, pending, sold)"
+    }
+  }'
+```
+
+```bash title="Update overrides on an existing server" showLineNumbers
+curl -X PUT http://localhost:4000/v1/mcp/server/{server_id} \
+  -H "Authorization: Bearer $LITELLM_MASTER_KEY" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "tool_name_to_display_name": {
+      "getPetById": "Get Pet"
+    },
+    "tool_name_to_description": {
+      "getPetById": "Look up a pet by its ID"
+    }
+  }'
+```
+
+The map key is the **original `operationId`** from the spec — not the prefixed tool name. LiteLLM strips the server prefix before doing the lookup.
+
+For example, if your server is `petstore_mcp`, the tool is exposed as `petstore_mcp-getPetById`. The map key is still `getPetById`.
+
+**Before and after:**
+
+```
+# Without overrides
+Tool: "petstore_mcp-getPetById"
+Description: "Returns a single pet"
+
+Tool: "petstore_mcp-findPetsByStatus"
+Description: "Finds Pets by status"
+
+# After overrides
+Tool: "Get Pet"
+Description: "Look up a pet by its ID"
+
+Tool: "List Available Pets"
+Description: "Returns all pets matching a given status (available, pending, sold)"
+```
+
+## Using the Server
+
+<Tabs>
+<TabItem value="fastmcp" label="Python FastMCP">
+
+```python title="Using OpenAPI-based MCP Server" showLineNumbers
+from fastmcp import Client
+import asyncio
+
+config = {
+    "mcpServers": {
+        "petstore": {
+            "url": "http://localhost:4000/petstore_mcp/mcp",
+            "headers": {
+                "x-litellm-api-key": "Bearer sk-1234"
+            }
+        }
+    }
+}
+
+client = Client(config)
+
+async def main():
+    async with client:
+        tools = await client.list_tools()
+        print(f"Available tools: {[tool.name for tool in tools]}")
+
+        response = await client.call_tool(
+            name="Get Pet",        # overridden name
+            arguments={"petId": "1"}
+        )
+        print(f"Response: {response}")
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+</TabItem>
+
+<TabItem value="cursor" label="Cursor IDE">
+
+```json title="Cursor MCP Configuration" showLineNumbers
+{
+  "mcpServers": {
+    "Petstore": {
+      "url": "http://localhost:4000/petstore_mcp/mcp",
+      "headers": {
+        "x-litellm-api-key": "Bearer $LITELLM_API_KEY"
+      }
+    }
+  }
+}
+```
+
+</TabItem>
+
+<TabItem value="openai" label="OpenAI Responses API">
+
+```bash title="Using OpenAPI MCP Server with OpenAI" showLineNumbers
+curl --location 'https://api.openai.com/v1/responses' \
+--header 'Content-Type: application/json' \
+--header "Authorization: Bearer $OPENAI_API_KEY" \
+--data '{
+    "model": "gpt-4o",
+    "tools": [
+        {
+            "type": "mcp",
+            "server_label": "petstore",
+            "server_url": "http://localhost:4000/petstore_mcp/mcp",
+            "require_approval": "never",
+            "headers": {
+                "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY"
+            }
+        }
+    ],
+    "input": "Find all available pets",
+    "tool_choice": "required"
+}'
+```
+
+</TabItem>
+</Tabs>

docs/my-website/docs/mcp_zero_trust.md

@@ -0,0 +1,294 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# MCP Zero Trust Auth (JWT Signer)
+
+![Zero Trust MCP Gateway](/img/mcp_zero_trust_gateway.png)
+
+MCP servers have no built-in way to verify that a request actually came through LiteLLM. Without this guardrail, any client that can reach your MCP server directly can call tools — bypassing your access controls entirely.
+
+`MCPJWTSigner` fixes this. It signs every outbound tool call with a short-lived RS256 JWT. Your MCP server verifies the signature against LiteLLM's public key. Requests that didn't go through LiteLLM have no valid signature and are rejected.
+
+---
+
+## Basic setup
+
+Add the guardrail to your config and point your MCP server at LiteLLM's JWKS endpoint. Every tool call gets a signed JWT automatically — no changes needed on the client side.
+
+```yaml title="config.yaml"
+mcp_servers:
+  - server_name: weather
+    url: http://localhost:8000/mcp
+    transport: http
+
+guardrails:
+  - guardrail_name: mcp-jwt-signer
+    litellm_params:
+      guardrail: mcp_jwt_signer
+      mode: pre_mcp_call
+      default_on: true
+      issuer: "https://my-litellm.example.com"  # defaults to request base URL
+      audience: "mcp"                            # default: "mcp"
+      ttl_seconds: 300                           # default: 300
+```
+
+**Bring your own signing key** — recommended for production. Auto-generated keys are lost on restart.
+
+```bash
+export MCP_JWT_SIGNING_KEY="-----BEGIN RSA PRIVATE KEY-----\n..."
+# or point to a file
+export MCP_JWT_SIGNING_KEY="file:///secrets/mcp-signing-key.pem"
+```
+
+**Build a verified MCP server with [FastMCP](https://gofastmcp.com):**
+
+```python title="weather_server.py"
+from fastmcp import FastMCP, Context
+from fastmcp.server.auth.providers.jwt import JWTVerifier
+
+auth = JWTVerifier(
+    jwks_uri="https://my-litellm.example.com/.well-known/jwks.json",
+    issuer="https://my-litellm.example.com",
+    audience="mcp",
+    algorithm="RS256",
+)
+
+mcp = FastMCP("weather-server", auth=auth)
+
+@mcp.tool()
+async def get_weather(city: str, ctx: Context) -> str:
+    caller = ctx.client_id  # JWT `sub` — the verified user identity
+    return f"Weather in {city}: sunny, 72°F (requested by {caller})"
+
+if __name__ == "__main__":
+    mcp.run(transport="http", host="0.0.0.0", port=8000)
+```
+
+FastMCP fetches the JWKS automatically and re-fetches when the signing key changes.
+
+LiteLLM publishes OIDC discovery so MCP servers find the key without any manual configuration:
+
+```
+GET /.well-known/openid-configuration  →  { "jwks_uri": "https://<litellm>/.well-known/jwks.json" }
+GET /.well-known/jwks.json             →  { "keys": [{ "kty": "RSA", "alg": "RS256", ... }] }
+```
+
+> **Read further only if you need to:** thread a corporate IdP identity into the JWT, enforce specific claims on callers, add custom metadata, use AWS Bedrock AgentCore Gateway, or debug JWT rejections.
+
+---
+
+## Thread IdP identity into MCP JWTs
+
+By default the outbound JWT `sub` is LiteLLM's internal `user_id`. If your users authenticate with Okta, Azure AD, or another IdP, the MCP server sees a LiteLLM-internal ID — not the user's email or employee ID.
+
+With verify+re-sign, LiteLLM validates the incoming IdP token first, then builds the outbound JWT using the real identity claims from that token. The MCP server gets the user's actual identity without ever having to trust the original IdP directly.
+
+```yaml title="config.yaml"
+guardrails:
+  - guardrail_name: mcp-jwt-signer
+    litellm_params:
+      guardrail: mcp_jwt_signer
+      mode: pre_mcp_call
+      default_on: true
+      issuer: "https://my-litellm.example.com"
+
+      # Validate the incoming Bearer token against the IdP
+      access_token_discovery_uri: "https://login.microsoftonline.com/{tenant}/v2.0/.well-known/openid-configuration"
+      verify_issuer: "https://login.microsoftonline.com/{tenant}/v2.0"
+      verify_audience: "api://my-app"
+
+      # Which claim to use for `sub` in the outbound JWT — first non-empty value wins
+      end_user_claim_sources:
+        - "token:sub"       # from the verified incoming JWT
+        - "token:email"     # fallback to email
+        - "litellm:user_id" # last resort: LiteLLM's internal user_id
+```
+
+If the incoming token is **opaque** (not a JWT — some IdPs issue these), add an introspection endpoint. LiteLLM will POST the token to it (RFC 7662) and use the returned claims:
+
+```yaml
+      token_introspection_endpoint: "https://idp.example.com/oauth2/introspect"
+```
+
+**Supported `end_user_claim_sources` values:**
+
+| Source | Resolves to |
+|--------|-------------|
+| `token:<claim>` | Any claim from the verified incoming JWT (e.g. `token:sub`, `token:email`, `token:oid`) |
+| `litellm:user_id` | LiteLLM's internal user ID |
+| `litellm:email` | User email from LiteLLM auth context |
+| `litellm:end_user_id` | End-user ID if set separately |
+| `litellm:team_id` | Team ID from LiteLLM auth context |
+
+---
+
+## Block callers missing required attributes
+
+Some MCP servers expose sensitive operations that should only be reachable by verified employees — not service accounts, not external API keys. You can enforce this at the LiteLLM layer so the MCP server never receives the request at all.
+
+`required_claims` rejects with `403` if the incoming token is missing any listed claim. `optional_claims` forwards claims that are useful but not mandatory.
+
+```yaml title="config.yaml"
+guardrails:
+  - guardrail_name: mcp-jwt-signer
+    litellm_params:
+      guardrail: mcp_jwt_signer
+      mode: pre_mcp_call
+      default_on: true
+
+      access_token_discovery_uri: "https://idp.example.com/.well-known/openid-configuration"
+
+      # Service accounts without `employee_id` are blocked before the tool runs
+      required_claims:
+        - "sub"
+        - "employee_id"
+
+      # Forward these into the outbound JWT when present — skipped silently if absent
+      optional_claims:
+        - "groups"
+        - "department"
+```
+
+**What the client sees when blocked:**
+```json
+HTTP 403
+{ "error": "MCPJWTSigner: incoming token is missing required claims: ['employee_id']. Configure the IdP to include these claims." }
+```
+
+---
+
+## Add custom metadata to every JWT
+
+Your MCP server may need context that LiteLLM doesn't carry natively — which deployment sent the request, a tenant ID, an environment tag. Use claim operations to inject, override, or strip claims from the outbound JWT.
+
+```yaml title="config.yaml"
+guardrails:
+  - guardrail_name: mcp-jwt-signer
+    litellm_params:
+      guardrail: mcp_jwt_signer
+      mode: pre_mcp_call
+      default_on: true
+
+      # add: insert only when the key is not already in the JWT
+      add_claims:
+        deployment_id: "prod-us-east-1"
+        tenant_id: "acme-corp"
+
+      # set: always override — even if the claim came from the incoming token
+      set_claims:
+        env: "production"
+
+      # remove: strip claims the MCP server shouldn't see
+      remove_claims:
+        - "nbf"   # some validators reject nbf; remove it if yours does
+```
+
+Operations run in order — `add_claims` → `set_claims` → `remove_claims`. `set_claims` always wins over `add_claims`; `remove_claims` beats both.
+
+---
+
+## AWS Bedrock AgentCore Gateway
+
+Bedrock AgentCore Gateway uses two separate JWTs: one to authenticate the transport connection and another to authorize tool calls. They need different `aud` values and TTLs — a single JWT won't work for both.
+
+LiteLLM can issue both in one hook and inject them into separate headers:
+
+```yaml title="config.yaml"
+guardrails:
+  - guardrail_name: mcp-jwt-signer
+    litellm_params:
+      guardrail: mcp_jwt_signer
+      mode: pre_mcp_call
+      default_on: true
+      issuer: "https://my-litellm.example.com"
+      audience: "mcp-resource"   # for the MCP resource layer
+      ttl_seconds: 300
+
+      # Second JWT for the transport channel — same sub/act/scope, different aud + TTL
+      channel_token_audience: "bedrock-agentcore-gateway"
+      channel_token_ttl: 60      # transport tokens should be short-lived
+```
+
+LiteLLM injects two headers on every tool call:
+- `Authorization: Bearer <resource-token>` — audience `mcp-resource`, TTL 300s
+- `x-mcp-channel-token: Bearer <channel-token>` — audience `bedrock-agentcore-gateway`, TTL 60s
+
+Both tokens are signed with the same LiteLLM key, so your MCP server only needs to trust one JWKS endpoint.
+
+---
+
+## Control which scopes go into the JWT
+
+By default LiteLLM generates least-privilege scopes per request:
+- Tool call → `mcp:tools/call mcp:tools/{name}:call`
+- List tools → `mcp:tools/call mcp:tools/list`
+
+If your MCP server does its own scope enforcement and needs a specific format, set `allowed_scopes` to replace auto-generation entirely:
+
+```yaml title="config.yaml"
+guardrails:
+  - guardrail_name: mcp-jwt-signer
+    litellm_params:
+      guardrail: mcp_jwt_signer
+      mode: pre_mcp_call
+      default_on: true
+
+      allowed_scopes:
+        - "mcp:tools/call"
+        - "mcp:tools/list"
+        - "mcp:admin"
+```
+
+Every JWT carries exactly those scopes regardless of which tool is being called.
+
+---
+
+## Debug JWT rejections
+
+Your MCP server is returning 401 and you're not sure what's in the JWT. Enable `debug_headers` and LiteLLM adds a `x-litellm-mcp-debug` response header with the key claims that were signed:
+
+```yaml title="config.yaml"
+guardrails:
+  - guardrail_name: mcp-jwt-signer
+    litellm_params:
+      guardrail: mcp_jwt_signer
+      mode: pre_mcp_call
+      default_on: true
+      debug_headers: true
+```
+
+Response header:
+```
+x-litellm-mcp-debug: v=1; kid=a3f1b2c4d5e6f708; sub=alice@corp.com; iss=https://my-litellm.example.com; exp=1712345678; scope=mcp:tools/call mcp:tools/get_weather:call
+```
+
+Check that `kid` matches what the MCP server fetched from JWKS, `iss`/`aud` match your server's expected values, and `exp` hasn't passed. Disable in production — the header leaks claim metadata.
+
+---
+
+## JWT claims reference
+
+| Claim | Value |
+|-------|-------|
+| `iss` | `issuer` config value (or request base URL) |
+| `aud` | `audience` config value (default: `"mcp"`) |
+| `sub` | Resolved via `end_user_claim_sources` (default: `user_id` → api-key hash → `"litellm-proxy"`) |
+| `act.sub` | `team_id` → `org_id` → `"litellm-proxy"` (RFC 8693 delegation) |
+| `email` | `user_email` from LiteLLM auth context (when available) |
+| `scope` | Auto-generated per tool call, or `allowed_scopes` when set |
+| `iat`, `exp`, `nbf` | Standard timing claims (RFC 7519) |
+
+---
+
+## Limitations
+
+- **OpenAPI-backed MCP servers** (`spec_path` set) do not support JWT injection. LiteLLM logs a warning and skips the header. Use SSE/HTTP transport servers to get full JWT injection.
+- The keypair is **in-memory by default** and rotated on each restart unless `MCP_JWT_SIGNING_KEY` is set. FastMCP's `JWTVerifier` handles key rotation transparently via JWKS key ID matching.
+
+---
+
+## Related
+
+- [MCP Guardrails](./mcp_guardrail) — PII masking and blocking for MCP calls
+- [MCP OAuth](./mcp_oauth) — upstream OAuth2 for MCP server access
+- [MCP AWS SigV4](./mcp_aws_sigv4) — AWS-signed requests to MCP servers

docs/my-website/docs/observability/datadog.md

@@ -7,6 +7,7 @@ import TabItem from '@theme/TabItem';
 LiteLLM Supports logging to the following Datdog Integrations:
 - `datadog` [Datadog Logs](https://docs.datadoghq.com/logs/)
 - `datadog_llm_observability` [Datadog LLM Observability](https://www.datadoghq.com/product/llm-observability/)
+- `datadog_metrics` [Datadog Custom Metrics](#datadog-custom-metrics)
 - `datadog_cost_management` [Datadog Cloud Cost Management](#datadog-cloud-cost-management)
 - `ddtrace-run` [Datadog Tracing](#datadog-tracing)
 
@@ -168,6 +169,65 @@ On the Datadog LLM Observability page, you should see that both input messages a
 <Image img={require('../../img/dd_llm_obs.png')} />
 
 
+## Datadog Custom Metrics
+
+| Feature | Details |
+|---------|---------|
+| **What is logged** | Latency metrics, request counts by status code |
+| **Events** | Success + Failure |
+| **Product Link** | [Datadog Metrics](https://docs.datadoghq.com/metrics/) |
+
+Publishes the following metrics to Datadog via the `/api/v2/series` endpoint:
+
+| Metric | Type | Description |
+|--------|------|-------------|
+| `litellm.request.total_latency` | Gauge | End-to-end request latency (seconds) |
+| `litellm.llm_api.latency` | Gauge | Time spent waiting for the LLM provider response (seconds) |
+| `litellm.llm_api.request_count` | Count | Request count, tagged with status code |
+
+Using `total_latency` and `llm_api.latency`, you can derive **internal latency** = `total_latency - llm_api.latency`.
+
+All metrics include the following tags: `env`, `service`, `version`, `HOSTNAME`, `POD_NAME`, `provider`, `model_name`, `model_group`, `team`, `status_code`.
+
+**Step 1**: Create a `config.yaml` file
+
+```yaml
+model_list:
+  - model_name: gpt-3.5-turbo
+    litellm_params:
+      model: gpt-3.5-turbo
+litellm_settings:
+  success_callback: ["datadog_metrics"]
+  failure_callback: ["datadog_metrics"]
+```
+
+**Step 2**: Set required env variables
+
+```shell
+DD_API_KEY="your-api-key"
+DD_SITE="us5.datadoghq.com"  # your datadog site
+```
+
+**Step 3**: Start the proxy and make a test request
+
+```shell
+litellm --config config.yaml
+```
+
+```shell
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+    --header 'Content-Type: application/json' \
+    --header 'Authorization: Bearer sk-1234' \
+    --data '{
+    "model": "gpt-3.5-turbo",
+    "messages": [{"role": "user", "content": "hello"}]
+}'
+```
+
+**Step 4**: View metrics in Datadog Metrics Explorer
+
+Navigate to **Metrics > Explorer** in Datadog and search for `litellm.request.total_latency`, `litellm.llm_api.latency`, or `litellm.llm_api.request_count`.
+
 ## Datadog Cloud Cost Management
 
 | Feature | Details |

docs/my-website/docs/observability/langfuse_otel_integration.md

@@ -83,6 +83,9 @@ os.environ["LANGFUSE_OTEL_HOST"] = "https://cloud.langfuse.com"  # EU region
 # Or use self-hosted instance
 # os.environ["LANGFUSE_OTEL_HOST"] = "https://my-langfuse.company.com"
 
+# Optional: Ignore otel context propagation to prevent parent-child relationships with spans from other providers
+# os.environ["OTEL_IGNORE_CONTEXT_PROPAGATION"] = "true"
+
 litellm.callbacks = ["langfuse_otel"]
 ```
 
@@ -124,6 +127,9 @@ export LANGFUSE_PUBLIC_KEY="pk-lf-..."
 export LANGFUSE_SECRET_KEY="sk-lf-..."
 export LANGFUSE_OTEL_HOST="https://us.cloud.langfuse.com"  # Default US region
 # export LANGFUSE_OTEL_HOST="https://otel.my-langfuse.company.com"  # custom OTEL endpoint
+
+# Optional: Ignore otel context propagation to prevent parent-child relationships with spans from other providers
+# export OTEL_IGNORE_CONTEXT_PROPAGATION="true"
 ```
 
 2. Setup config.yaml

docs/my-website/docs/observability/vantage.md

@@ -0,0 +1,148 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Vantage Integration
+
+LiteLLM can export proxy spend data to [Vantage](https://vantage.sh) as [FOCUS 1.2](https://focus.finops.org/) formatted cost reports. This lets you visualize LLM spend alongside your cloud infrastructure costs in the Vantage dashboard.
+
+## Overview
+
+| Property | Details |
+|----------|---------|
+| Destination | Export LiteLLM usage data to Vantage Custom Provider |
+| Data format | FOCUS CSV (automatically transformed from LiteLLM spend data) |
+| Supported operations | Manual export, automatic scheduled export (hourly/daily/interval) |
+| Authentication | Vantage API key + Custom Provider token |
+
+## Prerequisites
+
+You need two credentials from the [Vantage console](https://console.vantage.sh):
+
+1. **API Key** — Go to **Settings → API Access Tokens** → Create a token with **Write** scope. The token looks like `vntg_tkn_...`.
+2. **Custom Provider Token** — Go to **Settings → Integrations** → Create a **Custom Provider** integration → Copy the Provider ID (looks like `accss_crdntl_...`).
+
+## Setup via API
+
+The recommended setup uses the proxy admin endpoints. No config file changes needed.
+
+### 1. Initialize credentials
+
+```bash
+curl -X POST http://localhost:4000/vantage/init \
+  -H "Authorization: Bearer $LITELLM_ADMIN_KEY" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "api_key": "vntg_tkn_YOUR_VANTAGE_API_KEY",
+    "integration_token": "accss_crdntl_YOUR_PROVIDER_TOKEN"
+  }'
+```
+
+Credentials are encrypted and stored in the proxy database.
+
+### 2. Preview data (dry run)
+
+```bash
+curl -X POST http://localhost:4000/vantage/dry-run \
+  -H "Authorization: Bearer $LITELLM_ADMIN_KEY" \
+  -H "Content-Type: application/json" \
+  -d '{"limit": 10}'
+```
+
+This returns FOCUS-transformed data without sending anything to Vantage. Use it to verify the pipeline works and inspect the data mapping.
+
+### 3. Export to Vantage
+
+```bash
+curl -X POST http://localhost:4000/vantage/export \
+  -H "Authorization: Bearer $LITELLM_ADMIN_KEY" \
+  -H "Content-Type: application/json" \
+  -d '{}'
+```
+
+Optional parameters:
+- `limit` — Max number of records to export
+- `start_time_utc` / `end_time_utc` — Filter by time range (must be provided together)
+
+### 4. Verify in Vantage
+
+Go to **Settings → Integrations → your Custom Provider → Import Costs** tab to see uploaded CSVs. Once the status changes from "Importing and Processing" to "Stable", costs appear in **Cost Reporting → All Resources**.
+
+## Setup via Environment Variables
+
+For automatic scheduled exports, configure via environment variables and proxy config:
+
+### Environment variables
+
+| Variable | Required | Description |
+|----------|----------|-------------|
+| `VANTAGE_API_KEY` | Yes | Vantage API access token |
+| `VANTAGE_INTEGRATION_TOKEN` | Yes | Custom Provider token from Vantage dashboard |
+| `VANTAGE_BASE_URL` | No | API URL override (default: `https://api.vantage.sh`) |
+| `VANTAGE_EXPORT_FREQUENCY` | No | `hourly` (default), `daily`, or `interval` |
+| `VANTAGE_EXPORT_INTERVAL_SECONDS` | No | Seconds between exports when frequency is `interval` |
+
+### Proxy config
+
+```yaml
+model_list:
+  - model_name: gpt-4o
+    litellm_params:
+      model: openai/gpt-4o
+      api_key: sk-your-key
+
+litellm_settings:
+  callbacks: ["vantage"]
+```
+
+```bash
+export VANTAGE_API_KEY="vntg_tkn_..."
+export VANTAGE_INTEGRATION_TOKEN="accss_crdntl_..."
+litellm --config /path/to/config.yaml
+```
+
+The proxy registers a background job that exports data on the configured schedule.
+
+## API Endpoints
+
+All endpoints require admin authentication.
+
+| Method | Endpoint | Description |
+|--------|----------|-------------|
+| `POST` | `/vantage/init` | Store Vantage credentials (encrypted) |
+| `GET` | `/vantage/settings` | View current config (credentials masked) |
+| `PUT` | `/vantage/settings` | Update credentials or base URL |
+| `POST` | `/vantage/dry-run` | Preview FOCUS data without uploading |
+| `POST` | `/vantage/export` | Upload cost data to Vantage |
+| `DELETE` | `/vantage/delete` | Remove credentials and stop scheduled exports |
+
+## FOCUS Field Mapping
+
+LiteLLM spend data is transformed into the FOCUS 1.2 schema:
+
+| LiteLLM Field | FOCUS Column | Description |
+|---------------|-------------|-------------|
+| `spend` | BilledCost, EffectiveCost | Cost of the usage |
+| `model` | ChargeDescription, ResourceId | Model identifier |
+| `model_group` | ServiceName | Model group / deployment |
+| `custom_llm_provider` | ProviderName, PublisherName | Provider (openai, anthropic, etc.) |
+| `api_key` | BillingAccountId | Hashed API key |
+| `api_key_alias` | BillingAccountName | Human-readable key alias |
+| `team_id` | SubAccountId | Team identifier |
+| `team_alias` | SubAccountName | Team name |
+
+Additional metadata (user_id, model_group, etc.) is included in the `Tags` column as JSON.
+
+## Upload Limits
+
+Vantage enforces per-upload limits. LiteLLM handles these automatically:
+
+- **10,000 rows** per upload — large exports are split into batches
+- **2 MB** per upload — oversized batches are further split by size
+- **Unsupported columns** are stripped before upload
+
+## Related Links
+
+- [Vantage](https://vantage.sh)
+- [Vantage Custom Providers](https://docs.vantage.sh/connecting_custom_providers)
+- [FOCUS Specification](https://focus.finops.org/)
+- [Focus Export (S3/Parquet)](./focus.md)

docs/my-website/docs/ocr.md

@@ -61,6 +61,52 @@ async def test_async_ocr():
 asyncio.run(test_async_ocr())
 ```
 
+### Using Local Files
+
+LiteLLM can read local files directly — no manual base64 encoding needed:
+
+```python
+from litellm import ocr
+
+# OCR with a local PDF file path
+response = ocr(
+    model="mistral/mistral-ocr-latest",
+    document={
+        "type": "file",
+        "file": "/path/to/document.pdf"
+    }
+)
+
+# OCR with a file object
+response = ocr(
+    model="mistral/mistral-ocr-latest",
+    document={
+        "type": "file",
+        "file": open("document.pdf", "rb")
+    }
+)
+
+# OCR with raw bytes
+with open("document.pdf", "rb") as f:
+    pdf_bytes = f.read()
+
+response = ocr(
+    model="mistral/mistral-ocr-latest",
+    document={
+        "type": "file",
+        "file": pdf_bytes,
+        "mime_type": "application/pdf"  # recommended for raw bytes (auto-detected from extension for file paths)
+    }
+)
+```
+
+The `file` field accepts:
+- **File path** (`str` or `pathlib.Path`) — LiteLLM reads the file and detects the MIME type from the extension
+- **File object** (binary file-like object) — e.g. `open("doc.pdf", "rb")`
+- **Raw bytes** (`bytes`) — use `mime_type` to specify the content type
+
+LiteLLM automatically converts file inputs to base64 data URIs internally, so all providers work seamlessly.
+
 ### Using Base64 Encoded Documents
 
 ```python
@@ -121,7 +167,7 @@ litellm --config /path/to/config.yaml
 # RUNNING on http://0.0.0.0:4000
 ```
 
-Test request
+**Test request — JSON body**
 
 ```bash
 curl http://0.0.0.0:4000/v1/ocr \
@@ -136,6 +182,27 @@ curl http://0.0.0.0:4000/v1/ocr \
   }'
 ```
 
+**Test request — multipart file upload**
+
+Upload a file directly using multipart form data. No need to base64-encode the file yourself.
+
+```bash
+curl http://0.0.0.0:4000/v1/ocr \
+  -H "Authorization: Bearer sk-1234" \
+  -F "model=mistral-ocr" \
+  -F "file=@/path/to/document.pdf"
+```
+
+You can also pass optional parameters as additional form fields:
+
+```bash
+curl http://0.0.0.0:4000/v1/ocr \
+  -H "Authorization: Bearer sk-1234" \
+  -F "model=mistral-ocr" \
+  -F "file=@screenshot.png" \
+  -F 'pages=[0,1,2]' \
+  -F "include_image_base64=true"
+```
 
 ## **Request/Response Format**
 
@@ -168,10 +235,12 @@ See the [official Mistral OCR documentation](https://docs.mistral.ai/capabilitie
 | Parameter | Type | Required | Description |
 |-----------|------|----------|-------------|
 | `model` | string | Yes | The OCR model to use (e.g., `"mistral/mistral-ocr-latest"`) |
-| `document` | object | Yes | Document to process. Must contain `type` and URL field |
-| `document.type` | string | Yes | Either `"document_url"` for PDFs/docs or `"image_url"` for images |
-| `document.document_url` | string | Conditional | URL to the document (required if `type` is `"document_url"`) |
-| `document.image_url` | string | Conditional | URL to the image (required if `type` is `"image_url"`) |
+| `document` | object | Yes | Document to process. Must contain `type` and the corresponding field |
+| `document.type` | string | Yes | `"document_url"` for PDFs/docs, `"image_url"` for images, or `"file"` for local files |
+| `document.document_url` | string | Conditional | URL or data URI to the document (required if `type` is `"document_url"`) |
+| `document.image_url` | string | Conditional | URL or data URI to the image (required if `type` is `"image_url"`) |
+| `document.file` | string/bytes/file | Conditional | File path, bytes, or file-like object (required if `type` is `"file"`) |
+| `document.mime_type` | string | No | Explicit MIME type for file inputs (auto-detected from extension if not provided) |
 | `pages` | array | No | List of specific page indices to process (0-indexed) |
 | `include_image_base64` | boolean | No | Whether to include extracted images as base64 strings |
 | `image_limit` | integer | No | Maximum number of images to return |
@@ -179,7 +248,7 @@ See the [official Mistral OCR documentation](https://docs.mistral.ai/capabilitie
 
 #### Document Format Examples
 
-**For PDFs and documents:**
+**For PDFs and documents (URL):**
 ```json
 {
   "type": "document_url",
@@ -187,7 +256,7 @@ See the [official Mistral OCR documentation](https://docs.mistral.ai/capabilitie
 }
 ```
 
-**For images:**
+**For images (URL):**
 ```json
 {
   "type": "image_url",
@@ -203,6 +272,21 @@ See the [official Mistral OCR documentation](https://docs.mistral.ai/capabilitie
 }
 ```
 
+**For local files (SDK):**
+```python
+{"type": "file", "file": "/path/to/document.pdf"}
+{"type": "file", "file": open("image.png", "rb")}
+{"type": "file", "file": pdf_bytes, "mime_type": "application/pdf"}
+```
+
+**For file uploads (Proxy — multipart form):**
+```bash
+curl http://0.0.0.0:4000/v1/ocr \
+  -H "Authorization: Bearer sk-1234" \
+  -F "model=mistral-ocr" \
+  -F "file=@document.pdf"
+```
+
 ### Response Format
 
 The response follows Mistral's OCR format with the following structure: