fix: merge main, fix summary in Responses API path, resolve conflicts

- Fix translate_thinking_to_reasoning in responses_adapters/transformation.py
  to make summary opt-in (was hardcoded to "detailed")
- Update e2e test to mock litellm.responses (new OpenAI routing path)
- Add tests for Responses API adapter summary preservation
- Resolve merge conflict in test file

.circleci/config.yml

@@ -21,9 +21,7 @@ commands:
       - run:
           name: "Install local version of litellm-enterprise"
           command: |
-            cd enterprise
-            python -m pip install -e .
-            cd ..
+            pip install --force-reinstall --no-deps -e enterprise/
   setup_litellm_test_deps:
     steps:
       - checkout
@@ -1183,7 +1181,7 @@ jobs:
           command: |
             pwd
             ls
-            python -m pytest tests/proxy_unit_tests --ignore=tests/proxy_unit_tests/test_key_generate_prisma.py --ignore=tests/proxy_unit_tests/test_auth_checks.py --ignore=tests/proxy_unit_tests/test_user_api_key_auth.py --cov=litellm --cov-report=xml --junitxml=test-results/junit-part2.xml --durations=10 -n 8 --timeout=300 -vv --log-cli-level=INFO
+            python -m pytest tests/proxy_unit_tests --ignore=tests/proxy_unit_tests/test_key_generate_prisma.py --ignore=tests/proxy_unit_tests/test_auth_checks.py --ignore=tests/proxy_unit_tests/test_user_api_key_auth.py --cov=litellm --cov-report=xml --junitxml=test-results/junit-part2.xml --durations=10 -n 4 --timeout=300 -vv --log-cli-level=INFO
           no_output_timeout: 120m
       - run:
           name: Rename the coverage files
@@ -1458,6 +1456,7 @@ jobs:
             pip install "respx==0.22.0"
             pip install "pydantic==2.10.2"
             pip install "boto3==1.36.0"
+            pip install "semantic_router==0.1.10"
       # Run pytest and generate JUnit XML report
       - run:
           name: Run tests
@@ -1700,7 +1699,7 @@ jobs:
           command: |
             prisma generate
             export PYTHONUNBUFFERED=1
-            python -m pytest tests/test_litellm/proxy --ignore=tests/test_litellm/proxy/guardrails --ignore=tests/test_litellm/proxy/management_endpoints --ignore=tests/test_litellm/proxy/_experimental --ignore=tests/test_litellm/proxy/client --ignore=tests/test_litellm/proxy/auth --cov=litellm --cov-report=xml --junitxml=test-results/junit-proxy-part2.xml --durations=10 -n 8 --maxfail=5 --timeout=60 -vv --log-cli-level=WARNING -r A
+            python -m pytest tests/test_litellm/proxy --ignore=tests/test_litellm/proxy/guardrails --ignore=tests/test_litellm/proxy/management_endpoints --ignore=tests/test_litellm/proxy/_experimental --ignore=tests/test_litellm/proxy/client --ignore=tests/test_litellm/proxy/auth --cov=litellm --cov-report=xml --junitxml=test-results/junit-proxy-part2.xml --durations=10 -n 4 --maxfail=5 --timeout=120 -vv --log-cli-level=WARNING -r A
           no_output_timeout: 60m
       - run:
           name: Rename the coverage files
@@ -3887,7 +3886,7 @@ jobs:
           command: |
             cd ~/project
             # Check pyproject.toml
-            CURRENT_VERSION=$(python -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['dependencies']['litellm-proxy-extras'].split('\"')[1])")
+            CURRENT_VERSION=$(python -c "import toml; dep = toml.load('pyproject.toml')['tool']['poetry']['dependencies']['litellm-proxy-extras']; print(dep['version'] if isinstance(dep, dict) else dep)")
             if [ "$CURRENT_VERSION" != "$NEW_VERSION" ]; then
               echo "Error: Version in pyproject.toml ($CURRENT_VERSION) doesn't match new version ($NEW_VERSION)"
               exit 1
@@ -4101,6 +4100,63 @@ jobs:
           path: playwright-report
           destination: playwright-report
 
+  prisma_schema_sync:
+    machine:
+      image: ubuntu-2204:2023.10.1
+    resource_class: xlarge
+    working_directory: ~/project
+    steps:
+      - checkout
+      - setup_google_dns
+      - attach_workspace:
+          at: ~/project
+      - run:
+          name: Load Docker Database Image
+          command: |
+            gunzip -c litellm-docker-database.tar.gz | docker load
+            docker images | grep litellm-docker-database
+      - run:
+          name: Install Neon CLI
+          command: |
+            npm i -g neonctl
+      - run:
+          name: Install curl and dockerize
+          command: |
+            sudo apt-get update
+            sudo apt-get install -y curl
+            sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
+            sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
+            sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
+      - run:
+          name: Sync schema on base e2e database
+          command: |
+            BASE_DATABASE_URL=$(neon connection-string \
+              --project-id $NEON_PROJECT_ID \
+              --api-key $NEON_API_KEY \
+              --branch br-fancy-paper-ad1olsb3 \
+              --database-name yuneng-trial-db \
+              --role neondb_owner)
+            docker run -d \
+              -p 4000:4000 \
+              -e DATABASE_URL=$BASE_DATABASE_URL \
+              -e LITELLM_MASTER_KEY="sk-1234" \
+              --name schema-sync \
+              -v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \
+              litellm-docker-database:ci \
+              --config /app/config.yaml \
+              --port 4000 \
+              --use_prisma_db_push
+      - run:
+          name: Start outputting logs
+          command: docker logs -f schema-sync
+          background: true
+      - run:
+          name: Wait for proxy to be ready (schema sync complete)
+          command: dockerize -wait http://localhost:4000 -timeout 5m
+      - run:
+          name: Stop schema sync container
+          command: docker stop schema-sync
+
   test_nonroot_image:
     machine:
       image: ubuntu-2204:2023.10.1
@@ -4299,6 +4355,15 @@ workflows:
               only:
                 - main
                 - /litellm_.*/
+      - prisma_schema_sync:
+          context: e2e_ui_tests
+          requires:
+            - build_docker_database_image
+          filters:
+            branches:
+              only:
+                - main
+                - /litellm_.*/
       - e2e_ui_testing:
           name: e2e_ui_testing_chromium
           browser: chromium
@@ -4306,6 +4371,7 @@ workflows:
           requires:
             - ui_build
             - build_docker_database_image
+            - prisma_schema_sync
           filters:
             branches:
               only:
@@ -4318,6 +4384,7 @@ workflows:
           requires:
             - ui_build
             - build_docker_database_image
+            - prisma_schema_sync
           filters:
             branches:
               only:

.claude/settings.json

@@ -0,0 +1,36 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(git show:*)",
+      "Bash(git worktree add:*)",
+      "Read(//Users/krrishdholakia/Documents/litellm/**)",
+      "Read(//Users/krrishdholakia/Documents/litellm-claude-code-guardrails/litellm/types/**)",
+      "Read(//Users/krrishdholakia/Documents/litellm-claude-code-guardrails/**)",
+      "Read(//Users/krrishdholakia/Documents/litellm-claude-code-guardrails/litellm/**)",
+      "Bash(python:*)",
+      "Bash(python -c \"\nimport sys; sys.path.insert\\(0, ''.''\\)\nfrom litellm.proxy.guardrails.guardrail_hooks.claude_code.guardrail import ClaudeCodeGuardrail, HOSTED_TOOL_PREFIXES\nprint\\(''HOSTED_TOOL_PREFIXES:'', HOSTED_TOOL_PREFIXES\\)\nprint\\(''ClaudeCodeGuardrail imported OK''\\)\n\")",
+      "Read(//Users/krrishdholakia/Documents/litellm-mcp-jwt-groups/litellm/proxy/**)",
+      "Read(//Users/krrishdholakia/Documents/litellm-mcp-jwt-groups/**)",
+      "Bash(poetry run pytest:*)",
+      "Bash(git add:*)",
+      "Bash(git commit:*)",
+      "Bash(poetry run python:*)",
+      "Bash(poetry run pip:*)",
+      "Bash(git reset:*)",
+      "Bash(git cherry-pick:*)",
+      "Bash(git checkout:*)",
+      "Read(//Users/krrishdholakia/Documents/litellm/litellm/proxy/guardrails/guardrail_hooks/**)",
+      "Read(//Users/krrishdholakia/Documents/**)",
+      "Bash(git -C /Users/krrishdholakia/Documents/litellm-mcp-user-permissions worktree list)",
+      "Bash(ls:*)"
+    ],
+    "additionalDirectories": [
+      "/Users/krrishdholakia/Documents/litellm-mcp-group-plan/plan",
+      "/Users/krrishdholakia/Documents/litellm-claude-code-guardrails/litellm/proxy/guardrails/guardrail_hooks/claude_code",
+      "/Users/krrishdholakia/Documents/litellm-claude-code-guardrails/litellm/types",
+      "/Users/krrishdholakia/Documents/litellm-claude-code-guardrails",
+      "/Users/krrishdholakia/Documents/litellm-mcp-jwt-groups/litellm/proxy",
+      "/Users/krrishdholakia/Documents/litellm-mcp-jwt-groups/tests/test_litellm/proxy/auth"
+    ]
+  }
+}

.github/ISSUE_TEMPLATE/config.yml

@@ -1,7 +1,7 @@
 blank_issues_enabled: true
 contact_links:
   - name: Schedule Demo
-    url: https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat
+    url: https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions
     about: Speak directly with Krrish and Ishaan, the founders, to discuss issues, share feedback, or explore improvements for LiteLLM
   - name: Discord
     url: https://discord.com/invite/wuPM9dRgDw

.github/codeql/codeql-config.yml

@@ -0,0 +1,15 @@
+name: "LiteLLM CodeQL config"
+
+# Exclude queries that produce result sets > 2 GiB on this codebase,
+# causing 49+ minute runs that fail and block CI resources.
+query-filters:
+  - exclude:
+      id: py/clear-text-logging-sensitive-data  # CWE-312/CleartextLogging.ql — result set > 2 GiB
+  - exclude:
+      id: py/polynomial-redos                   # CWE-730/PolynomialReDoS.ql — result set > 2 GiB
+
+paths-ignore:
+  - tests
+  - docs
+  - "**/*.md"
+  - litellm/proxy/_experimental/out

.github/observatory/litellm_config.yaml

@@ -0,0 +1,19 @@
+# LiteLLM Observatory Test Configuration
+# This config is used by CI to spin up a temporary LiteLLM instance
+# for running observatory tests against RC/stable releases.
+#
+# Add model definitions for the providers you want to test.
+# Provider API keys are injected via environment variables in CI.
+
+model_list:
+  - model_name: gpt-4o
+    litellm_params:
+      model: azure/gpt-4o
+      api_key: os.environ/AZURE_API_KEY
+      api_base: os.environ/AZURE_API_BASE
+
+  - model_name: gpt-4o-mini
+    litellm_params:
+      model: azure/gpt-4o-mini
+      api_key: os.environ/AZURE_API_KEY
+      api_base: os.environ/AZURE_API_BASE

.github/pull_request_template.md

@@ -6,7 +6,7 @@
 
 **Please complete all items before asking a LiteLLM maintainer to review your PR**
 
-- [ ] I have Added testing in the [`tests/litellm/`](https://github.com/BerriAI/litellm/tree/main/tests/litellm) directory, **Adding at least 1 test is a hard requirement** - [see details](https://docs.litellm.ai/docs/extras/contributing_code)
+- [ ] I have Added testing in the [`tests/test_litellm/`](https://github.com/BerriAI/litellm/tree/main/tests/test_litellm) directory, **Adding at least 1 test is a hard requirement** - [see details](https://docs.litellm.ai/docs/extras/contributing_code)
 - [ ] My PR passes all unit tests on [`make test-unit`](https://docs.litellm.ai/docs/extras/contributing_code)
 - [ ] My PR's scope is as isolated as possible, it only solves 1 specific problem
 - [ ] I have requested a Greptile review by commenting `@greptileai` and received a **Confidence Score of at least 4/5** before requesting a maintainer review

.github/scripts/close_duplicate_issues.py

@@ -0,0 +1,208 @@
+#!/usr/bin/env python3
+"""
+Detect and close duplicate GitHub issues using title similarity.
+
+Modes:
+  --scan           Compare all open issues against each other (batch)
+  --issue-number N Check a single issue against older open issues
+
+Requires the `gh` CLI to be authenticated.
+"""
+
+import argparse
+import difflib
+import json
+import re
+import subprocess
+import sys
+
+
+def normalize_title(title: str) -> str:
+    """Strip common prefixes, lowercase, and collapse whitespace."""
+    title = re.sub(
+        r"^\[?(bug|feature request|enhancement|question|docs)[:\]]?\s*",
+        "",
+        title,
+        flags=re.IGNORECASE,
+    )
+    return " ".join(title.lower().split())
+
+
+def gh(*args: str) -> str:
+    """Run a gh CLI command and return stdout."""
+    result = subprocess.run(
+        ["gh", *args],
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    return result.stdout
+
+
+def fetch_open_issues(repo: str | None) -> list[dict]:
+    """Fetch all open issues (excluding PRs) via gh api --paginate."""
+    if repo:
+        endpoint = f"repos/{repo}/issues?state=open&per_page=100&sort=created&direction=asc"
+    else:
+        endpoint = "repos/{owner}/{repo}/issues?state=open&per_page=100&sort=created&direction=asc"
+    cmd = ["api", "--paginate", endpoint]
+
+    raw = gh(*cmd)
+    # gh --paginate concatenates JSON arrays, so we may get multiple arrays
+    issues = []
+    for line in raw.strip().splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        parsed = json.loads(line)
+        if isinstance(parsed, list):
+            issues.extend(parsed)
+        else:
+            issues.append(parsed)
+
+    # Filter out pull requests (they also appear in the issues endpoint)
+    return [i for i in issues if "pull_request" not in i]
+
+
+def close_as_duplicate(
+    issue_number: int, duplicate_of: int, repo: str | None, dry_run: bool
+) -> None:
+    """Close an issue as duplicate of another, adding a comment and label."""
+    repo_args = ["--repo", repo] if repo else []
+
+    if dry_run:
+        print(f"  [DRY RUN] Would close #{issue_number} as duplicate of #{duplicate_of}")
+        return
+
+    # Add comment
+    comment_body = (
+        f"Closing as duplicate of #{duplicate_of}.\n\n"
+        "If you believe this is not a duplicate, please reopen and add context "
+        "explaining how this differs."
+    )
+    gh("issue", "comment", str(issue_number), "--body", comment_body, *repo_args)
+
+    # Add label
+    gh("issue", "edit", str(issue_number), "--add-label", "duplicate", *repo_args)
+
+    # Close with not_planned reason
+    gh(
+        "api",
+        f"repos/{repo or '{owner}/{repo}'}/issues/{issue_number}",
+        "-X",
+        "PATCH",
+        "-f",
+        "state=closed",
+        "-f",
+        "state_reason=not_planned",
+    )
+
+    print(f"  Closed #{issue_number} as duplicate of #{duplicate_of}")
+
+
+def find_duplicate(
+    issue: dict, candidates: list[dict], threshold: float
+) -> dict | None:
+    """Return the first candidate whose normalized title is above threshold."""
+    norm = normalize_title(issue["title"])
+    for candidate in candidates:
+        if candidate["number"] == issue["number"]:
+            continue
+        cand_norm = normalize_title(candidate["title"])
+        ratio = difflib.SequenceMatcher(None, norm, cand_norm).ratio()
+        if ratio >= threshold:
+            return candidate
+    return None
+
+
+def scan_all(issues: list[dict], threshold: float, repo: str | None, dry_run: bool) -> int:
+    """Compare every issue against all older issues. Returns count of duplicates found."""
+    # Sort oldest first
+    issues.sort(key=lambda i: i["number"])
+    closed_count = 0
+
+    for idx, issue in enumerate(issues):
+        older = issues[:idx]
+        if not older:
+            continue
+        dup = find_duplicate(issue, older, threshold)
+        if dup:
+            ratio = difflib.SequenceMatcher(
+                None,
+                normalize_title(issue["title"]),
+                normalize_title(dup["title"]),
+            ).ratio()
+            print(
+                f"#{issue['number']}: \"{issue['title']}\"\n"
+                f"  -> duplicate of #{dup['number']}: \"{dup['title']}\" "
+                f"({ratio:.0%} similar)"
+            )
+            close_as_duplicate(issue["number"], dup["number"], repo, dry_run)
+            closed_count += 1
+
+    return closed_count
+
+
+def check_single(
+    issue_number: int, issues: list[dict], threshold: float, repo: str | None, dry_run: bool
+) -> bool:
+    """Check a single issue against all older open issues. Returns True if duplicate found."""
+    target = None
+    for i in issues:
+        if i["number"] == issue_number:
+            target = i
+            break
+
+    if target is None:
+        print(f"Issue #{issue_number} not found among open issues.")
+        return False
+
+    older = [i for i in issues if i["number"] < issue_number]
+    dup = find_duplicate(target, older, threshold)
+    if dup:
+        ratio = difflib.SequenceMatcher(
+            None,
+            normalize_title(target["title"]),
+            normalize_title(dup["title"]),
+        ).ratio()
+        print(
+            f"#{target['number']}: \"{target['title']}\"\n"
+            f"  -> duplicate of #{dup['number']}: \"{dup['title']}\" "
+            f"({ratio:.0%} similar)"
+        )
+        close_as_duplicate(issue_number, dup["number"], repo, dry_run)
+        return True
+
+    print(f"#{issue_number}: no duplicate found above threshold {threshold}")
+    return False
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Detect and close duplicate GitHub issues")
+    mode = parser.add_mutually_exclusive_group(required=True)
+    mode.add_argument("--scan", action="store_true", help="Scan all open issues")
+    mode.add_argument("--issue-number", type=int, help="Check a single issue number")
+    parser.add_argument("--threshold", type=float, default=0.85, help="Similarity threshold (0-1)")
+    parser.add_argument("--close", action="store_true", help="Actually close duplicates (default is dry-run)")
+    parser.add_argument("--repo", type=str, help="Repository (owner/repo). Auto-detected if omitted.")
+    args = parser.parse_args()
+
+    dry_run = not args.close
+
+    if dry_run:
+        print("=== DRY RUN MODE (pass --close to actually close issues) ===\n")
+
+    print("Fetching open issues...")
+    issues = fetch_open_issues(args.repo)
+    print(f"Found {len(issues)} open issues.\n")
+
+    if args.scan:
+        count = scan_all(issues, args.threshold, args.repo, dry_run)
+        print(f"\nTotal duplicates {'found' if dry_run else 'closed'}: {count}")
+    else:
+        found = check_single(args.issue_number, issues, args.threshold, args.repo, dry_run)
+        sys.exit(0 if found else 0)  # Always exit 0; finding no dup is not an error
+
+
+if __name__ == "__main__":
+    main()

.github/workflows/auto_update_price_and_context_window.yml

@@ -7,6 +7,7 @@ on:
 
 jobs:
   auto_update_price_and_context_window:
+    if: github.repository == 'BerriAI/litellm'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3

.github/workflows/check_duplicate_issues.yml

@@ -20,10 +20,33 @@ jobs:
           reaction: eyes
           comment: |
             **⚠️ Potential duplicate detected**
-            
+
             This issue appears similar to existing issue(s):
             {{#issues}}
             - [#{{number}}]({{html_url}}) - {{title}} ({{accuracy}}% similar)
             {{/issues}}
-            
+
             Please review the linked issue(s) to see if they address your concern. If this is not a duplicate, please provide additional context to help us understand the difference.
+
+      - name: Checkout close script
+        if: github.event.action == 'opened'
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: .github/scripts
+
+      - name: Set up Python
+        if: github.event.action == 'opened'
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Auto-close if high-confidence duplicate
+        if: github.event.action == 'opened'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          python3 .github/scripts/close_duplicate_issues.py \
+            --issue-number ${{ github.event.issue.number }} \
+            --repo ${{ github.repository }} \
+            --threshold 0.85 \
+            --close

.github/workflows/codeql.yml

@@ -0,0 +1,54 @@
+name: "CodeQL"
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  schedule:
+    # Run weekly on Sundays at 04:00 UTC
+    - cron: "0 4 * * 0"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  analyze:
+    name: Analyze (${{ matrix.language }})
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    permissions:
+      security-events: write
+      packages: read
+      actions: read
+      contents: read
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - language: actions
+            build-mode: none
+          - language: javascript-typescript
+            build-mode: none
+          - language: python
+            build-mode: none
+          - language: ruby
+            build-mode: none
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v3
+        with:
+          languages: ${{ matrix.language }}
+          build-mode: ${{ matrix.build-mode }}
+          config-file: ./.github/codeql/codeql-config.yml
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v3
+        with:
+          category: "/language:${{ matrix.language }}"

.github/workflows/ghcr_deploy.yml

@@ -299,6 +299,15 @@ jobs:
             ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-spend_logs:main-stable', env.REGISTRY) || '' }}
           platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
 
+  run-observatory-tests:
+    if: github.event.inputs.release_type == 'rc' || github.event.inputs.release_type == 'stable'
+    needs: [docker-hub-deploy]
+    uses: ./.github/workflows/run_observatory_tests.yml
+    with:
+      tag: ${{ github.event.inputs.tag }}
+      commit_hash: ${{ github.event.inputs.commit_hash }}
+    secrets: inherit
+
   build-and-push-helm-chart:
     if: github.event.inputs.release_type  != 'dev'
     needs: [docker-hub-deploy, build-and-push-image, build-and-push-image-database]

.github/workflows/interpret_load_test.py

@@ -123,7 +123,7 @@ if __name__ == "__main__":
         + docker_run_command
         + "\n\n"
         + "### Don't want to maintain your internal proxy? get in touch 🎉"
-        + "\nHosted Proxy Alpha: https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat"
+        + "\nHosted Proxy Alpha: https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions"
         + "\n\n"
         + "## Load Test LiteLLM Proxy Results"
         + "\n\n"

.github/workflows/publish_enterprise.yml

@@ -0,0 +1,74 @@
+name: Publish litellm-enterprise to PyPI
+
+on:
+  workflow_dispatch:
+    inputs:
+      bump:
+        description: "Version bump type"
+        required: true
+        default: "patch"
+        type: choice
+        options:
+          - patch
+          - minor
+          - major
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    if: github.repository == 'BerriAI/litellm'
+    permissions:
+      contents: write
+    defaults:
+      run:
+        working-directory: enterprise
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install Poetry
+        run: pip install poetry
+
+      - name: Bump version
+        id: bump
+        run: |
+          OLD=$(poetry version -s)
+          poetry version ${{ github.event.inputs.bump }}
+          NEW=$(poetry version -s)
+          echo "old=$OLD" >> $GITHUB_OUTPUT
+          echo "new=$NEW" >> $GITHUB_OUTPUT
+
+      - name: Update version refs in root pyproject.toml and requirements.txt
+        run: |
+          OLD=${{ steps.bump.outputs.old }}
+          NEW=${{ steps.bump.outputs.new }}
+          sed -i "s/litellm-enterprise = {version = \"${OLD}\"/litellm-enterprise = {version = \"${NEW}\"/" ../pyproject.toml
+          sed -i "s/litellm-enterprise==${OLD}/litellm-enterprise==${NEW}/" ../requirements.txt
+
+      - name: Update poetry.lock
+        working-directory: .
+        run: poetry lock
+
+      - name: Build
+        run: poetry build
+
+      - name: Commit version bump
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          cd ..
+          git add enterprise/pyproject.toml pyproject.toml requirements.txt poetry.lock
+          git commit -m "bump: litellm-enterprise ${{ steps.bump.outputs.old }} → ${{ steps.bump.outputs.new }}"
+          git push
+
+      - name: Publish to PyPI
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_ENTERPRISE }}
+        run: |
+          pip install twine
+          twine upload dist/litellm_enterprise-${{ steps.bump.outputs.new }}*

.github/workflows/publish_proxy_extras.yml

@@ -0,0 +1,74 @@
+name: Publish litellm-proxy-extras to PyPI
+
+on:
+  workflow_dispatch:
+    inputs:
+      bump:
+        description: "Version bump type"
+        required: true
+        default: "patch"
+        type: choice
+        options:
+          - patch
+          - minor
+          - major
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    if: github.repository == 'BerriAI/litellm'
+    permissions:
+      contents: write
+    defaults:
+      run:
+        working-directory: litellm-proxy-extras
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install Poetry
+        run: pip install poetry
+
+      - name: Bump version
+        id: bump
+        run: |
+          OLD=$(poetry version -s)
+          poetry version ${{ github.event.inputs.bump }}
+          NEW=$(poetry version -s)
+          echo "old=$OLD" >> $GITHUB_OUTPUT
+          echo "new=$NEW" >> $GITHUB_OUTPUT
+
+      - name: Update version refs in root pyproject.toml and requirements.txt
+        run: |
+          OLD=${{ steps.bump.outputs.old }}
+          NEW=${{ steps.bump.outputs.new }}
+          sed -i "s/litellm-proxy-extras = {version = \"${OLD}\"/litellm-proxy-extras = {version = \"${NEW}\"/" ../pyproject.toml
+          sed -i "s/litellm-proxy-extras==${OLD}/litellm-proxy-extras==${NEW}/" ../requirements.txt
+
+      - name: Update poetry.lock
+        working-directory: .
+        run: poetry lock
+
+      - name: Build
+        run: poetry build
+
+      - name: Commit version bump
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          cd ..
+          git add litellm-proxy-extras/pyproject.toml pyproject.toml requirements.txt poetry.lock
+          git commit -m "bump: litellm-proxy-extras ${{ steps.bump.outputs.old }} → ${{ steps.bump.outputs.new }}"
+          git push
+
+      - name: Publish to PyPI
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_PUBLISH_PASSWORD }}
+        run: |
+          pip install twine
+          twine upload dist/litellm_proxy_extras-${{ steps.bump.outputs.new }}*

.github/workflows/regenerate-poetry-lock.yml

@@ -0,0 +1,80 @@
+name: Regenerate poetry.lock
+
+# Runs whenever pyproject.toml is merged into main (the most common cause of
+# the "pyproject.toml changed significantly since poetry.lock was last generated"
+# CI failure).  Can also be triggered manually.
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - pyproject.toml
+  workflow_dispatch:
+
+permissions:
+  contents: write       # needed to push the auto/regenerate-poetry-lock-* branch
+  pull-requests: write  # needed to open the PR and enable auto-merge
+
+jobs:
+  regenerate-lock:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install Poetry
+        run: pip install poetry
+
+      - name: Regenerate poetry.lock
+        run: poetry lock
+
+      - name: Check whether poetry.lock actually changed
+        id: diff
+        run: |
+          if git diff --quiet poetry.lock; then
+            echo "changed=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Open PR with the refreshed lock file
+        if: steps.diff.outputs.changed == 'true'
+        id: open-pr
+        run: |
+          BRANCH="auto/regenerate-poetry-lock-$(date +'%Y%m%d%H%M%S')"
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git checkout -b "$BRANCH"
+          git add poetry.lock
+          git commit -m "chore: regenerate poetry.lock to match pyproject.toml"
+          git push -f origin "$BRANCH"
+
+          cat > /tmp/pr-body.md << 'BODY'
+          Automated regeneration of `poetry.lock` after `pyproject.toml` was updated on `main`.
+
+          Fixes the recurring CI failure:
+          ```
+          pyproject.toml changed significantly since poetry.lock was last generated.
+          Run `poetry lock` to fix the lock file.
+          ```
+          BODY
+
+          PR_URL=$(gh pr create \
+            --title "chore: regenerate poetry.lock to match pyproject.toml" \
+            --body-file /tmp/pr-body.md \
+            --head "$BRANCH" \
+            --base main)
+          echo "pr_url=$PR_URL" >> "$GITHUB_OUTPUT"
+        env:
+          GH_TOKEN: ${{ github.token }}
+
+      - name: Enable auto-merge
+        if: steps.diff.outputs.changed == 'true'
+        run: |
+          gh pr merge "${{ steps.open-pr.outputs.pr_url }}" --auto --squash
+        env:
+          GH_TOKEN: ${{ github.token }}

.github/workflows/run_observatory_tests.yml

@@ -0,0 +1,225 @@
+name: Run Observatory Tests
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: "Docker image tag to test (e.g. v1.61.0.rc1)"
+        required: true
+        type: string
+      commit_hash:
+        description: "Commit hash (defaults to HEAD of current branch)"
+        required: false
+        type: string
+  workflow_call:
+    inputs:
+      tag:
+        description: "Docker image tag to test"
+        required: true
+        type: string
+      commit_hash:
+        description: "Commit hash of the release"
+        required: true
+        type: string
+
+permissions:
+  contents: read
+
+env:
+  LITELLM_MASTER_KEY: ${{ secrets.LITELLM_MASTER_KEY_STAGING }}
+
+jobs:
+  observatory-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Validate tag input
+        env:
+          TAG: ${{ inputs.tag }}
+        run: |
+          if [[ ! "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+ ]]; then
+            echo "Invalid tag format: $TAG (expected vX.Y.Z...)"
+            exit 1
+          fi
+
+      - name: Start LiteLLM container
+        env:
+          TAG: ${{ inputs.tag }}
+          AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
+          AZURE_API_BASE: ${{ secrets.AZURE_API_BASE }}
+        run: |
+          docker run -d \
+            --name litellm-rc \
+            -p 4000:4000 \
+            -v "${{ github.workspace }}/.github/observatory/litellm_config.yaml:/app/config.yaml" \
+            -e LITELLM_MASTER_KEY="${LITELLM_MASTER_KEY}" \
+            -e AZURE_API_KEY="${AZURE_API_KEY}" \
+            -e AZURE_API_BASE="${AZURE_API_BASE}" \
+            "litellm/litellm:${TAG}" \
+            --config /app/config.yaml --port 4000
+
+      - name: Wait for LiteLLM health check
+        run: |
+          echo "Waiting for LiteLLM to be ready..."
+          for i in $(seq 1 30); do
+            if curl -s -f http://localhost:4000/health/liveliness > /dev/null 2>&1; then
+              echo "LiteLLM is healthy"
+              exit 0
+            fi
+            echo "Attempt $i/30 - not ready yet, waiting 10s..."
+            sleep 10
+          done
+          echo "LiteLLM failed to start within 5 minutes"
+          docker logs litellm-rc
+          exit 1
+
+      - name: Start cloudflared tunnel
+        run: |
+          # Install cloudflared
+          curl -sL https://github.com/cloudflare/cloudflared/releases/download/2025.2.1/cloudflared-linux-amd64 -o /usr/local/bin/cloudflared
+          chmod +x /usr/local/bin/cloudflared
+
+          # Start a quick tunnel (no account needed) and capture the URL
+          cloudflared tunnel --url http://localhost:4000 --no-autoupdate > /tmp/cloudflared.log 2>&1 &
+          CLOUDFLARED_PID=$!
+          echo "CLOUDFLARED_PID=$CLOUDFLARED_PID" >> $GITHUB_ENV
+
+          # Wait for tunnel URL to appear in logs
+          echo "Waiting for tunnel URL..."
+          for i in $(seq 1 30); do
+            TUNNEL_URL=$(grep -oP 'https://[a-z0-9-]+\.trycloudflare\.com' /tmp/cloudflared.log | head -1 || true)
+            if [ -n "$TUNNEL_URL" ]; then
+              echo "Tunnel URL: $TUNNEL_URL"
+              echo "TUNNEL_URL=$TUNNEL_URL" >> $GITHUB_ENV
+              exit 0
+            fi
+            sleep 2
+          done
+          echo "Failed to get tunnel URL"
+          cat /tmp/cloudflared.log
+          exit 1
+
+      - name: Verify tunnel connectivity
+        run: |
+          echo "Testing tunnel at ${{ env.TUNNEL_URL }}..."
+          # Quick tunnels need time for DNS propagation; retry to avoid
+          # transient NXDOMAIN (curl exit code 6) on first attempt.
+          for i in $(seq 1 10); do
+            if curl -sf "${{ env.TUNNEL_URL }}/health/liveliness" > /dev/null 2>&1; then
+              echo "Tunnel is working (attempt $i)"
+              exit 0
+            fi
+            echo "Attempt $i/10 - tunnel not routable yet, waiting 5s..."
+            sleep 5
+          done
+          echo "Tunnel failed to become reachable after 50s"
+          cat /tmp/cloudflared.log
+          exit 1
+
+      - name: Trigger observatory test run
+        id: trigger
+        env:
+          OBSERVATORY_URL: ${{ secrets.OBSERVATORY_URL }}
+          OBSERVATORY_API_KEY: ${{ secrets.OBSERVATORY_API_KEY }}
+        run: |
+          PAYLOAD=$(jq -n \
+            --arg url "${TUNNEL_URL}" \
+            --arg key "${LITELLM_MASTER_KEY}" \
+            '{
+              deployment_url: $url,
+              api_key: $key,
+              test_suite: "TestOAIAzureRelease",
+              models: ["gpt-4o-mini", "gpt-4o"]
+            }')
+          RESPONSE=$(curl -s -w "\n%{http_code}" -X POST "${OBSERVATORY_URL}/run-test" \
+            -H "Content-Type: application/json" \
+            -H "X-LiteLLM-Observatory-API-Key: ${OBSERVATORY_API_KEY}" \
+            -d "$PAYLOAD")
+          HTTP_CODE=$(echo "$RESPONSE" | tail -1)
+          BODY=$(echo "$RESPONSE" | head -n -1)
+          echo "Response ($HTTP_CODE): $BODY"
+          if [ "$HTTP_CODE" -ge 400 ]; then
+            echo "Failed to trigger test run"
+            exit 1
+          fi
+
+          # Extract request_id for polling this specific run
+          REQUEST_ID=$(echo "$BODY" | jq -r '.results.request_id')
+          if [ -z "$REQUEST_ID" ] || [ "$REQUEST_ID" = "null" ]; then
+            echo "Failed to extract request_id from response"
+            exit 1
+          fi
+          echo "Request ID: $REQUEST_ID"
+          echo "request_id=$REQUEST_ID" >> $GITHUB_OUTPUT
+
+      - name: Poll for test completion
+        id: poll
+        env:
+          OBSERVATORY_URL: ${{ secrets.OBSERVATORY_URL }}
+          OBSERVATORY_API_KEY: ${{ secrets.OBSERVATORY_API_KEY }}
+          REQUEST_ID: ${{ steps.trigger.outputs.request_id }}
+        run: |
+          TIMEOUT=900  # 15 minutes
+          INTERVAL=30
+          ELAPSED=0
+          while [ $ELAPSED -lt $TIMEOUT ]; do
+            STATUS=$(curl -s "${OBSERVATORY_URL}/run-status/${REQUEST_ID}" \
+              -H "X-LiteLLM-Observatory-API-Key: ${OBSERVATORY_API_KEY}")
+            RUN_STATUS=$(echo "$STATUS" | jq -r '.status')
+            echo "Run status (${ELAPSED}s elapsed): $RUN_STATUS"
+
+            if [ "$RUN_STATUS" = "completed" ] || [ "$RUN_STATUS" = "failed" ]; then
+              echo "Test finished with status: $RUN_STATUS"
+              echo "$STATUS" > /tmp/observatory_result.json
+              exit 0
+            fi
+
+            sleep $INTERVAL
+            ELAPSED=$((ELAPSED + INTERVAL))
+          done
+          echo "Timed out waiting for test to complete after ${TIMEOUT}s"
+          exit 1
+
+      - name: Verify test results
+        run: |
+          RESULT=$(cat /tmp/observatory_result.json)
+          echo "Full result: $RESULT"
+
+          STATUS=$(echo "$RESULT" | jq -r '.status')
+          TEST_PASSED=$(echo "$RESULT" | jq -r '.result.test_passed // false')
+          FAILURE_RATE=$(echo "$RESULT" | jq -r '.result.failure_rate // "N/A"')
+          ERROR=$(echo "$RESULT" | jq -r '.error // empty')
+
+          echo "Status: $STATUS"
+          echo "Test passed: $TEST_PASSED"
+          echo "Failure rate: $FAILURE_RATE"
+
+          if [ -n "$ERROR" ]; then
+            echo "Error: $ERROR"
+          fi
+
+          if [ "$STATUS" = "failed" ]; then
+            echo "Test run failed"
+            exit 1
+          fi
+
+          if [ "$TEST_PASSED" != "true" ]; then
+            echo "Tests did not pass (failure rate: $FAILURE_RATE)"
+            exit 1
+          fi
+
+          echo "All tests passed!"
+
+      - name: Print LiteLLM logs on failure
+        if: failure()
+        run: |
+          docker logs litellm-rc 2>/dev/null || true
+          cat /tmp/cloudflared.log 2>/dev/null || true
+
+      - name: Cleanup
+        if: always()
+        run: |
+          kill "${{ env.CLOUDFLARED_PID }}" 2>/dev/null || true
+          docker rm -f litellm-rc 2>/dev/null || true

.github/workflows/scan_duplicate_issues.yml

@@ -0,0 +1,47 @@
+name: Scan Duplicate Issues (One-Time)
+
+on:
+  workflow_dispatch:
+    inputs:
+      threshold:
+        description: "Similarity threshold (0-1)"
+        required: false
+        default: "0.85"
+      close:
+        description: "Actually close duplicates (false = dry run)"
+        required: false
+        type: boolean
+        default: false
+
+jobs:
+  scan:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      contents: read
+    steps:
+      - name: Checkout scripts
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: .github/scripts
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Scan for duplicate issues
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          INPUT_THRESHOLD: ${{ inputs.threshold }}
+          INPUT_CLOSE: ${{ inputs.close }}
+        run: |
+          CLOSE_FLAG=""
+          if [ "$INPUT_CLOSE" = "true" ]; then
+            CLOSE_FLAG="--close"
+          fi
+          python3 .github/scripts/close_duplicate_issues.py \
+            --scan \
+            --repo ${{ github.repository }} \
+            --threshold "$INPUT_THRESHOLD" \
+            $CLOSE_FLAG

.github/workflows/test-linting.yml

@@ -74,3 +74,35 @@ jobs:
     - name: Check import safety
       run: |
         poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
+
+  secret-scan:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      contents: read
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.12'
+
+    - name: Run secret scan test
+      run: |
+        pip install pytest
+        pytest tests/litellm/test_no_hardcoded_secrets.py -v
+
+    - name: Run ggshield secret scan
+      env:
+        GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }}
+      run: |
+        if [ -n "$GITGUARDIAN_API_KEY" ]; then
+          pip install ggshield
+          ggshield secret scan repo .
+        else
+          echo "GITGUARDIAN_API_KEY not set, skipping ggshield scan"
+        fi

.github/workflows/test-litellm-matrix.yml

@@ -12,44 +12,107 @@ concurrency:
 jobs:
   test:
     runs-on: ubuntu-latest
-    timeout-minutes: 15
+    timeout-minutes: 20  # Increased from 15 to 20
     strategy:
       fail-fast: false
       matrix:
         test-group:
           # tests/test_litellm split by subdirectory (~560 files total)
-          - name: "llms"
-            path: "tests/test_litellm/llms"
-            workers: 4
+          # Vertex AI tests separated for better isolation (prevent auth/env pollution)
+          - name: "llms-vertex"
+            path: "tests/test_litellm/llms/vertex_ai"
+            workers: 1
+            reruns: 2
+          - name: "llms-other"
+            path: "tests/test_litellm/llms --ignore=tests/test_litellm/llms/vertex_ai"
+            workers: 2
+            reruns: 2
           # tests/test_litellm/proxy split by subdirectory (~180 files total)
           - name: "proxy-guardrails"
             path: "tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/management_helpers"
-            workers: 4
+            workers: 2
+            reruns: 2
           - name: "proxy-core"
             path: "tests/test_litellm/proxy/auth tests/test_litellm/proxy/client tests/test_litellm/proxy/db tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine"
-            workers: 4
+            workers: 2
+            reruns: 2
           - name: "proxy-misc"
             path: "tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py"
-            workers: 4
+            workers: 2
+            reruns: 2
           - name: "integrations"
             path: "tests/test_litellm/integrations"
-            workers: 4
+            workers: 2
+            reruns: 3  # Integration tests tend to be flakier
           - name: "core-utils"
             path: "tests/test_litellm/litellm_core_utils"
             workers: 2
-          - name: "other"
-            path: "tests/test_litellm/caching tests/test_litellm/responses tests/test_litellm/secret_managers tests/test_litellm/vector_stores tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/enterprise tests/test_litellm/experimental_mcp_client tests/test_litellm/google_genai tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/router_strategy tests/test_litellm/router_utils tests/test_litellm/types"
-            workers: 4
+            reruns: 1
+          - name: "other-1"
+            # responses (5942) + caching (1723) + types (819) ≈ 8.5k lines
+            path: "tests/test_litellm/responses tests/test_litellm/caching tests/test_litellm/types"
+            workers: 2
+            reruns: 2
+          - name: "other-2"
+            # enterprise (3062) + google_genai (2511) + router_utils (1982) ≈ 7.6k lines
+            path: "tests/test_litellm/enterprise tests/test_litellm/google_genai tests/test_litellm/router_utils"
+            workers: 2
+            reruns: 2
+          - name: "other-3"
+            # remaining dirs ≈ 8.0k lines
+            path: "tests/test_litellm/router_strategy tests/test_litellm/secret_managers tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/experimental_mcp_client tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/vector_stores"
+            workers: 2
+            reruns: 2
           - name: "root"
             path: "tests/test_litellm/test_*.py"
-            workers: 4
+            workers: 2
+            reruns: 2
           # tests/proxy_unit_tests split alphabetically (~48 files total)
-          - name: "proxy-unit-a"
-            path: "tests/proxy_unit_tests/test_[a-o]*.py"
+          - name: "proxy-unit-a1"
+            # test_[a-j]*.py: jwt (1564) + auth_checks (978) + google_gemini (478) + e2e_pod_lock (437) + rest
+            path: "tests/proxy_unit_tests/test_[a-j]*.py"
             workers: 2
-          - name: "proxy-unit-b"
-            path: "tests/proxy_unit_tests/test_[p-z]*.py"
+            reruns: 1
+          - name: "proxy-unit-a2"
+            # test_[k-o]*.py: key_generate_prisma (4346) + key_generate_dynamodb + models_fallback
+            path: "tests/proxy_unit_tests/test_[k-o]*.py"
             workers: 2
+            reruns: 1
+          - name: "proxy-unit-b1"
+            # lighter config/utility proxy tests (prisma, project, prompt, proxy_[c-r]*)
+            path: "tests/proxy_unit_tests/test_prisma*.py tests/proxy_unit_tests/test_project*.py tests/proxy_unit_tests/test_prompt*.py tests/proxy_unit_tests/test_proxy_[c-r]*.py"
+            workers: 2
+            reruns: 1
+          - name: "proxy-unit-b2"
+            # proxy_server.py alone (2750 lines) - isolated to avoid blocking smaller tests
+            path: "tests/proxy_unit_tests/test_proxy_server.py"
+            workers: 2
+            reruns: 1
+          - name: "proxy-unit-b3"
+            # proxy_server_* (618) + proxy_setting_guardrails (71) - smaller server-related tests
+            path: "tests/proxy_unit_tests/test_proxy_server_*.py tests/proxy_unit_tests/test_proxy_setting_guardrails.py"
+            workers: 2
+            reruns: 1
+          - name: "proxy-unit-b4"
+            # proxy_utils.py alone (2339 lines) - isolated to avoid blocking token counter
+            path: "tests/proxy_unit_tests/test_proxy_utils.py"
+            workers: 2
+            reruns: 1
+          - name: "proxy-unit-b5"
+            # proxy_token_counter (1279) - runs independently from utils
+            path: "tests/proxy_unit_tests/test_proxy_token_counter.py"
+            workers: 2
+            reruns: 1
+          - name: "proxy-unit-b6"
+            # test_[r-t]*.py: response_polling (1399) + search_api_logging (202) + server_root (64) + skills_db (261) + realtime_cache (62)
+            path: "tests/proxy_unit_tests/test_[r-t]*.py"
+            workers: 2
+            reruns: 1
+          - name: "proxy-unit-b7"
+            # test_[u-z]*.py: user_api_key_auth (1136) + zero_cost (590) + update_spend (305) + unit_test_* (206) + ui_path (157)
+            path: "tests/proxy_unit_tests/test_[u-z]*.py"
+            workers: 2
+            reruns: 1
 
     name: test (${{ matrix.test-group.name }})
 
@@ -79,12 +142,17 @@ jobs:
         run: |
           poetry config virtualenvs.in-project true
           poetry install --with dev,proxy-dev --extras "proxy semantic-router"
-          poetry run pip install pytest-retry==1.6.3 pytest-xdist google-genai==1.22.0 \
+          # pytest-rerunfailures and pytest-xdist are in pyproject.toml dev dependencies
+          poetry run pip install google-genai==1.22.0 \
             google-cloud-aiplatform>=1.38 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core
 
       - name: Setup litellm-enterprise
         run: |
-          cd enterprise && poetry run pip install -e . && cd ..
+          poetry run pip install --force-reinstall --no-deps -e enterprise/
+
+      - name: Generate Prisma client
+        run: |
+          poetry run prisma generate --schema litellm/proxy/schema.prisma
 
       - name: Run tests - ${{ matrix.test-group.name }}
         run: |
@@ -92,4 +160,7 @@ jobs:
             --tb=short -vv \
             --maxfail=10 \
             -n ${{ matrix.test-group.workers }} \
+            --reruns ${{ matrix.test-group.reruns }} \
+            --reruns-delay 1 \
+            --dist=loadscope \
             --durations=20

.github/workflows/test-litellm.yml

@@ -42,9 +42,7 @@ jobs:
         poetry run pip install "openapi-core"
     - name: Setup litellm-enterprise as local package
       run: |
-        cd enterprise
-        poetry run pip install -e .
-        cd ..
+        poetry run pip install --force-reinstall --no-deps -e enterprise/
     - name: Run tests
       run: |
         poetry run pytest tests/test_litellm --tb=short -vv --maxfail=10 -n 4 --durations=50

.github/workflows/test-mcp.yml

@@ -40,9 +40,7 @@ jobs:
 
     - name: Setup litellm-enterprise as local package
       run: |
-        cd enterprise
-        python -m pip install -e .
-        cd ..
+        poetry run pip install --force-reinstall --no-deps -e enterprise/
 
     - name: Run MCP tests
       run: |

.github/workflows/test_server_root_path.yml

@@ -26,7 +26,7 @@ jobs:
         uses: docker/build-push-action@v5
         with:
           context: .
-          file: ./docker/Dockerfile.database
+          file: ./docker/Dockerfile.non_root
           tags: litellm-test:${{ github.sha }}
           load: true
           cache-from: type=gha

.gitignore

@@ -89,6 +89,7 @@ tests/test_custom_dir/*
 test.py
 
 litellm_config.yaml
+!.github/observatory/litellm_config.yaml
 .cursor
 .vscode/launch.json
 litellm/proxy/to_delete_loadtest_work/*

AGENTS.md

@@ -174,6 +174,40 @@ When opening issues or pull requests, follow these templates:
 3. **Rate Limits**: Respect provider rate limits in tests
 4. **Memory Usage**: Be mindful of memory usage in streaming scenarios
 5. **Dependencies**: Keep dependencies minimal and well-justified
+6. **UI/Backend Contract Mismatch**: When adding a new entity type to the UI, always check whether the backend endpoint accepts a single value or an array. Match the UI control accordingly (single-select vs. multi-select) to avoid silently dropping user selections
+7. **Missing Tests for New Entity Types**: When adding a new entity type (e.g., in `EntityUsage`, `UsageViewSelect`), always add corresponding tests in the existing test files and update any icon/component mocks
+
+8. **Do not hardcode model-specific flags**: Put model-specific capability flags in `model_prices_and_context_window.json` and read them via `get_model_info` (or existing helpers like `supports_reasoning`). This prevents users from needing to upgrade LiteLLM each time a new model supports a feature.
+
+   **Example of BAD** (hardcoded model checks):
+
+   ```python
+   @staticmethod
+   def _is_effort_supported_model(model: str) -> bool:
+       """Check if the model supports the output_config.effort parameter..."""
+       model_lower = model.lower()
+       if AnthropicConfig._is_claude_4_6_model(model):
+           return True
+       return any(
+           v in model_lower for v in ("opus-4-5", "opus_4_5", "opus-4.5", "opus_4.5")
+       )
+   ```
+
+   **Example of GOOD** (config-driven or helper that reads from config):
+
+   ```python
+   if (
+       "claude-3-7-sonnet" in model
+       or AnthropicConfig._is_claude_4_6_model(model)
+       or supports_reasoning(
+           model=model,
+           custom_llm_provider=self.custom_llm_provider,
+       )
+   ):
+       ...
+   ```
+
+   Using helpers like `supports_reasoning` (which read from `model_prices_and_context_window.json` / `get_model_info`) allows future model updates to "just work" without code changes.
 
 ## HELPFUL RESOURCES
 
@@ -187,4 +221,47 @@ When opening issues or pull requests, follow these templates:
 - Check similar provider implementations
 - Ensure comprehensive test coverage
 - Update documentation appropriately
-- Consider backward compatibility impact 
+- Consider backward compatibility impact
+
+## Cursor Cloud specific instructions
+
+### Environment
+
+- Poetry is installed in `~/.local/bin`; the update script ensures it is on `PATH`.
+- Python 3.12, Node 22 are pre-installed.
+- The virtual environment lives under `~/.cache/pypoetry/virtualenvs/`.
+
+### Running the proxy server
+
+Start the proxy with a config file:
+
+```bash
+poetry run litellm --config dev_config.yaml --port 4000
+```
+
+The proxy takes ~15-20 seconds to fully start (it runs Prisma migrations on boot). Wait for `/health` to return before sending requests. Without a PostgreSQL `DATABASE_URL`, the proxy connects to a default Neon dev database embedded in the `litellm-proxy-extras` package.
+
+### Running tests
+
+See `CLAUDE.md` and the `Makefile` for standard commands. Key notes:
+
+- `psycopg-binary` must be installed (`poetry run pip install psycopg-binary`) because the pytest-postgresql plugin requires it and the lock file only includes `psycopg` (no binary).
+- The `--timeout` pytest flag is NOT available; don't pass it.
+- Unit tests: `poetry run pytest tests/test_litellm/ -x -vv -n 4`
+- Black `--check` may report pre-existing formatting issues; this does not block test runs.
+
+### Lint
+
+```bash
+cd litellm && poetry run ruff check .
+```
+
+Ruff is the primary fast linter. For the full lint suite (including mypy, black, circular imports), run `make lint` per `CLAUDE.md`.
+
+### UI Dashboard development
+
+- The UI is at `ui/litellm-dashboard/`. Run `npm run dev` from that directory for the Next.js dev server on port 3000.
+- The proxy at port 4000 serves a **pre-built** static UI from `litellm/proxy/_experimental/out/`. After making UI code changes, you must run `npm run build` in the dashboard directory and copy the output: `cp -r ui/litellm-dashboard/out/* litellm/proxy/_experimental/out/` for the proxy to serve the updated UI.
+- SVGs used as provider logos (loaded via `<img>` tags) must NOT use `fill="currentColor"` — replace with an explicit color like `#000000` or use the `-color` variant from lobehub icons, since CSS color inheritance does not work inside `<img>` elements.
+- Provider logos live in `ui/litellm-dashboard/public/assets/logos/` (source) and `litellm/proxy/_experimental/out/assets/logos/` (pre-built). Both locations must have the file for it to work in dev and proxy-served modes.
+- UI Vitest tests: `cd ui/litellm-dashboard && npx vitest run`

CLAUDE.md

@@ -97,6 +97,10 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components:
 - Integration tests for each provider in `tests/llm_translation/`
 - Proxy tests in `tests/proxy_unit_tests/`
 - Load tests in `tests/load_tests/`
+- **Always add tests when adding new entity types or features** — if the existing test file covers other entity types, add corresponding tests for the new one
+
+### UI / Backend Consistency
+- When wiring a new UI entity type to an existing backend endpoint, verify the backend API contract (single value vs. array, required vs. optional params) and ensure the UI controls match — e.g., use a single-select dropdown when the backend accepts a single value, not a multi-select
 
 ### Database Migrations
 - Prisma handles schema migrations

Dockerfile

@@ -49,7 +49,7 @@ USER root
 
 # Install runtime dependencies (libsndfile needed for audio processing on ARM64)
 RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \
-    npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 && \
+    npm install -g npm@latest tar@7.5.8 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
     # SECURITY FIX: npm bundles tar, glob, and brace-expansion at multiple nested
     # levels inside its dependency tree. `npm install -g <pkg>` only creates a
     # SEPARATE global package, it does NOT replace npm's internal copies.
@@ -64,6 +64,12 @@ RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile
     find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
     done && \
+    find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
+    done && \
+    find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
+    done && \
     npm cache clean --force
 
 WORKDIR /app
@@ -90,14 +96,20 @@ RUN find /usr/lib -type f -path "*/tornado/test/*" -delete && \
 # npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
 # Patch every copy of tar, glob, and brace-expansion inside that tree.
 RUN GLOBAL="$(npm root -g)" && \
-    find /usr/lib -path "*/nodejs_wheel/*/node_modules/tar" -type d | while read d; do \
+    find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
     done && \
-    find /usr/lib -path "*/nodejs_wheel/*/node_modules/glob" -type d | while read d; do \
+    find /usr/lib -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
     done && \
-    find /usr/lib -path "*/nodejs_wheel/*/node_modules/@isaacs/brace-expansion" -type d | while read d; do \
+    find /usr/lib -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
+    done && \
+    find /usr/lib -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
+    done && \
+    find /usr/lib -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
     done
 
 # Install semantic_router and aurelio-sdk using script

README.md

@@ -203,7 +203,7 @@ curl -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
 {
   "mcpServers": {
     "LiteLLM": {
-      "url": "http://localhost:4000/mcp",
+      "url": "http://localhost:4000/mcp/",
       "headers": {
         "x-litellm-api-key": "Bearer sk-1234"
       }
@@ -399,7 +399,7 @@ Support for more providers. Missing a provider or LLM Platform, raise a [feature
 # Enterprise
 For companies that need better security, user management and professional support
 
-[Talk to founders](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
+[Talk to founders](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
 
 This covers:
 - ✅ **Features under the [LiteLLM Commercial License](https://docs.litellm.ai/docs/proxy/enterprise):**

ci_cd/security_scans.sh

@@ -158,6 +158,9 @@ run_grype_scans() {
         "CVE-2025-11468" # No fix available yet
         "CVE-2026-1299" # Python 3.13 email module header injection - not applicable, LiteLLM doesn't use BytesGenerator for email serialization
         "CVE-2026-0775" # npm cli incorrect permission assignment - no fix available yet, npm is only used at build/prisma-generate time
+        "GHSA-3ppc-4f35-3m26" # minimatch ReDoS via repeated wildcards - from nodejs_wheel bundled npm, not used in application runtime code
+        "GHSA-83g3-92jg-28cx" # tar arbitrary file read/write via hardlink - from nodejs_wheel bundled npm, not used in application runtime code
+        "CVE-2026-25639" # axios - full fix requires 1.x major version bump; pinned to >=0.30.2 to clear other axios CVEs, upgrade to 1.x in follow-up
     )
 
     # Build JSON array of allowlisted CVE IDs for jq

cookbook/benchmark/readme.md

@@ -178,4 +178,4 @@ Benchmark Results for 'When will BerriAI IPO?':
 ```
 
 ## Support 
-**🤝 Schedule a 1-on-1 Session:** Book a [1-on-1 session](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) with Krrish and Ishaan, the founders, to discuss any issues, provide feedback, or explore how we can improve LiteLLM for you.
+**🤝 Schedule a 1-on-1 Session:** Book a [1-on-1 session](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) with Krrish and Ishaan, the founders, to discuss any issues, provide feedback, or explore how we can improve LiteLLM for you.

cookbook/gollem_go_agent_framework/README.md

@@ -0,0 +1,119 @@
+# Gollem Go Agent Framework with LiteLLM
+
+A working example showing how to use [gollem](https://github.com/fugue-labs/gollem), a production-grade Go agent framework, with LiteLLM as a proxy gateway. This lets Go developers access 100+ LLM providers through a single proxy while keeping compile-time type safety for tools and structured output.
+
+## Quick Start
+
+### 1. Start LiteLLM Proxy
+
+```bash
+# Simple start with a single model
+litellm --model gpt-4o
+
+# Or with the example config for multi-provider access
+litellm --config proxy_config.yaml
+```
+
+### 2. Run the examples
+
+```bash
+# Install Go dependencies
+go mod tidy
+
+# Basic agent
+go run ./basic
+
+# Agent with type-safe tools
+go run ./tools
+
+# Streaming responses
+go run ./streaming
+```
+
+## Configuration
+
+The included `proxy_config.yaml` sets up three providers through LiteLLM:
+
+```yaml
+model_list:
+  - model_name: gpt-4o          # OpenAI
+  - model_name: claude-sonnet    # Anthropic
+  - model_name: gemini-pro       # Google Vertex AI
+```
+
+Switch providers in Go by changing a single string — no code changes needed:
+
+```go
+model := openai.NewLiteLLM("http://localhost:4000",
+    openai.WithModel("gpt-4o"),        // OpenAI
+    // openai.WithModel("claude-sonnet"),  // Anthropic
+    // openai.WithModel("gemini-pro"),     // Google
+)
+```
+
+## Examples
+
+### `basic/` — Basic Agent
+
+Connects gollem to LiteLLM and runs a simple prompt. Demonstrates the `NewLiteLLM` constructor and basic agent creation.
+
+### `tools/` — Type-Safe Tools
+
+Shows gollem's compile-time type-safe tool framework working through LiteLLM's tool-use passthrough. The tool parameters are Go structs with JSON tags — the schema is generated automatically at compile time.
+
+### `streaming/` — Streaming Responses
+
+Real-time token streaming using Go 1.23+ range-over-function iterators, proxied through LiteLLM's SSE passthrough.
+
+## How It Works
+
+Gollem's `openai.NewLiteLLM()` constructor creates an OpenAI-compatible provider pointed at your LiteLLM proxy. Since LiteLLM speaks the OpenAI API protocol, everything works out of the box:
+
+- **Chat completions** — standard request/response
+- **Tool use** — LiteLLM passes tool definitions and calls through transparently
+- **Streaming** — Server-Sent Events proxied through LiteLLM
+- **Structured output** — JSON schema response format works with supporting models
+
+```
+Go App (gollem) → LiteLLM Proxy → OpenAI / Anthropic / Google / ...
+```
+
+## Why Use This?
+
+- **Type-safe Go**: Compile-time type checking for tools, structured output, and agent configuration — no runtime surprises
+- **Single proxy, many models**: Switch between OpenAI, Anthropic, Google, and 100+ other providers by changing a model name string
+- **Zero-dependency core**: gollem's core has no external dependencies — just stdlib
+- **Single binary deployment**: `go build` produces one binary, no pip/venv/Docker needed
+- **Cost tracking & rate limiting**: LiteLLM handles cost tracking, rate limits, and fallbacks at the proxy layer
+
+## Environment Variables
+
+```bash
+# Required for providers you want to use (set in LiteLLM config or env)
+export OPENAI_API_KEY="sk-..."
+export ANTHROPIC_API_KEY="sk-ant-..."
+
+# Optional: point to a non-default LiteLLM proxy
+export LITELLM_PROXY_URL="http://localhost:4000"
+```
+
+## Troubleshooting
+
+**Connection errors?**
+- Make sure LiteLLM is running: `litellm --model gpt-4o`
+- Check the URL is correct (default: `http://localhost:4000`)
+
+**Model not found?**
+- Verify the model name matches what's configured in LiteLLM
+- Run `curl http://localhost:4000/models` to see available models
+
+**Tool calls not working?**
+- Ensure the underlying model supports tool use (GPT-4o, Claude, Gemini)
+- Check LiteLLM logs for any provider-specific errors
+
+## Learn More
+
+- [gollem GitHub](https://github.com/fugue-labs/gollem)
+- [gollem API Reference](https://pkg.go.dev/github.com/fugue-labs/gollem/core)
+- [LiteLLM Proxy Docs](https://docs.litellm.ai/docs/simple_proxy)
+- [LiteLLM Supported Models](https://docs.litellm.ai/docs/providers)

cookbook/gollem_go_agent_framework/basic/main.go

@@ -0,0 +1,41 @@
+// Basic gollem agent connected to a LiteLLM proxy.
+//
+// Usage:
+//
+//	litellm --model gpt-4o   # start proxy in another terminal
+//	go run ./basic
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/fugue-labs/gollem/core"
+	"github.com/fugue-labs/gollem/provider/openai"
+)
+
+func main() {
+	proxyURL := "http://localhost:4000"
+	if u := os.Getenv("LITELLM_PROXY_URL"); u != "" {
+		proxyURL = u
+	}
+
+	// Connect to LiteLLM proxy. NewLiteLLM creates an OpenAI-compatible
+	// provider pointed at the given URL.
+	model := openai.NewLiteLLM(proxyURL,
+		openai.WithModel("gpt-4o"), // any model name configured in LiteLLM
+	)
+
+	// Create and run a simple agent.
+	agent := core.NewAgent[string](model,
+		core.WithSystemPrompt[string]("You are a helpful assistant. Be concise."),
+	)
+
+	result, err := agent.Run(context.Background(), "Explain quantum computing in two sentences.")
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Println(result.Output)
+}

cookbook/gollem_go_agent_framework/go.mod

@@ -0,0 +1,5 @@
+module github.com/BerriAI/litellm/cookbook/gollem_go_agent_framework
+
+go 1.25.1
+
+require github.com/fugue-labs/gollem v0.1.0

cookbook/gollem_go_agent_framework/go.sum

@@ -0,0 +1,2 @@
+github.com/fugue-labs/gollem v0.1.0 h1:QexYnvkb44QZFEljgAePqMIGZjgsbk0Y5GJ2jYYgfa8=
+github.com/fugue-labs/gollem v0.1.0/go.mod h1:htW1YO81uysSKVOkYJtxhGCFrzm+36HBFxEWuECoHKQ=

cookbook/gollem_go_agent_framework/proxy_config.yaml

@@ -0,0 +1,16 @@
+model_list:
+  - model_name: gpt-4o
+    litellm_params:
+      model: openai/gpt-4o
+      api_key: os.environ/OPENAI_API_KEY
+
+  - model_name: claude-sonnet
+    litellm_params:
+      model: anthropic/claude-sonnet-4-20250514
+      api_key: os.environ/ANTHROPIC_API_KEY
+
+  - model_name: gemini-pro
+    litellm_params:
+      model: vertex_ai/gemini-2.0-flash
+      vertex_project: my-project
+      vertex_location: us-central1

cookbook/gollem_go_agent_framework/streaming/main.go

@@ -0,0 +1,56 @@
+// Streaming responses from gollem through LiteLLM.
+//
+// Uses Go 1.23+ range-over-function iterators for real-time token
+// streaming via LiteLLM's SSE passthrough.
+//
+// Usage:
+//
+//	litellm --model gpt-4o
+//	go run ./streaming
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/fugue-labs/gollem/core"
+	"github.com/fugue-labs/gollem/provider/openai"
+)
+
+func main() {
+	proxyURL := "http://localhost:4000"
+	if u := os.Getenv("LITELLM_PROXY_URL"); u != "" {
+		proxyURL = u
+	}
+
+	model := openai.NewLiteLLM(proxyURL,
+		openai.WithModel("gpt-4o"),
+	)
+
+	agent := core.NewAgent[string](model)
+
+	// RunStream returns a streaming result that yields tokens as they arrive.
+	stream, err := agent.RunStream(context.Background(), "Write a haiku about distributed systems")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// StreamText yields text chunks in real-time.
+	// The boolean argument controls whether deltas (true) or accumulated
+	// text (false) is returned.
+	fmt.Print("Response: ")
+	for text, err := range stream.StreamText(true) {
+		if err != nil {
+			log.Fatal(err)
+		}
+		fmt.Print(text)
+	}
+	fmt.Println()
+
+	// After streaming completes, the final response is available.
+	resp := stream.Response()
+	fmt.Printf("\nTokens used: input=%d, output=%d\n",
+		resp.Usage.InputTokens, resp.Usage.OutputTokens)
+}

cookbook/gollem_go_agent_framework/tools/main.go

@@ -0,0 +1,64 @@
+// Gollem agent with type-safe tools through LiteLLM.
+//
+// The tool parameters are Go structs — gollem generates the JSON schema
+// automatically at compile time. LiteLLM passes tool definitions through
+// transparently to the underlying provider.
+//
+// Usage:
+//
+//	litellm --model gpt-4o
+//	go run ./tools
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/fugue-labs/gollem/core"
+	"github.com/fugue-labs/gollem/provider/openai"
+)
+
+// WeatherParams defines the tool's input schema via struct tags.
+// The JSON schema is generated at compile time — no runtime reflection needed.
+type WeatherParams struct {
+	City string `json:"city" description:"City name to get weather for"`
+	Unit string `json:"unit,omitempty" description:"Temperature unit: celsius or fahrenheit"`
+}
+
+func main() {
+	proxyURL := "http://localhost:4000"
+	if u := os.Getenv("LITELLM_PROXY_URL"); u != "" {
+		proxyURL = u
+	}
+
+	model := openai.NewLiteLLM(proxyURL,
+		openai.WithModel("gpt-4o"),
+	)
+
+	// Define a type-safe tool. The function signature enforces correct types.
+	weatherTool := core.FuncTool[WeatherParams](
+		"get_weather",
+		"Get current weather for a city",
+		func(ctx context.Context, p WeatherParams) (string, error) {
+			unit := p.Unit
+			if unit == "" {
+				unit = "fahrenheit"
+			}
+			// In production, call a real weather API here.
+			return fmt.Sprintf("Weather in %s: 72°F (22°C), sunny", p.City), nil
+		},
+	)
+
+	agent := core.NewAgent[string](model,
+		core.WithTools[string](weatherTool),
+		core.WithSystemPrompt[string]("You are a helpful weather assistant. Use the get_weather tool to answer weather questions."),
+	)
+
+	result, err := agent.Run(context.Background(), "What's the weather like in San Francisco and Tokyo?")
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Println(result.Output)
+}

cookbook/mock_prompt_management_server/README.md

@@ -0,0 +1,293 @@
+# Mock Prompt Management Server
+
+A reference implementation of the [LiteLLM Generic Prompt Management API](https://docs.litellm.ai/docs/adding_provider/generic_prompt_management_api).
+
+This FastAPI server demonstrates how to build a prompt management API that integrates with LiteLLM without requiring a PR to the LiteLLM repository.
+
+## Quick Start
+
+### 1. Install Dependencies
+
+```bash
+pip install fastapi uvicorn pydantic
+```
+
+### 2. Start the Server
+
+```bash
+python mock_prompt_management_server.py
+```
+
+The server will start on `http://localhost:8080`
+
+### 3. Test the Endpoint
+
+```bash
+# Get a prompt
+curl "http://localhost:8080/beta/litellm_prompt_management?prompt_id=hello-world-prompt"
+
+# Get a prompt with authentication
+curl "http://localhost:8080/beta/litellm_prompt_management?prompt_id=hello-world-prompt" \
+  -H "Authorization: Bearer test-token-12345"
+
+# List all prompts
+curl "http://localhost:8080/prompts"
+
+# Get prompt variables
+curl "http://localhost:8080/prompts/hello-world-prompt/variables"
+```
+
+## Using with LiteLLM
+
+### Configuration
+
+Create a `config.yaml` file:
+
+```yaml
+model_list:
+  - model_name: gpt-3.5-turbo
+    litellm_params:
+      model: openai/gpt-3.5-turbo
+      api_key: os.environ/OPENAI_API_KEY
+
+prompts:
+  - prompt_id: "hello-world-prompt"
+    litellm_params:
+      prompt_integration: "generic_prompt_management"
+      api_base: http://localhost:8080
+      api_key: test-token-12345
+```
+
+### Start LiteLLM Proxy
+
+```bash
+litellm --config config.yaml
+```
+
+### Make a Request
+
+```bash
+curl http://0.0.0.0:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer sk-1234" \
+  -d '{
+    "model": "gpt-3.5-turbo",
+    "prompt_id": "hello-world-prompt",
+    "prompt_variables": {
+      "domain": "data science",
+      "task": "analyzing customer behavior"
+    },
+    "messages": [
+      {"role": "user", "content": "Please help me get started"}
+    ]
+  }'
+```
+
+## Available Prompts
+
+The server includes several example prompts:
+
+| Prompt ID | Description | Variables |
+|-----------|-------------|-----------|
+| `hello-world-prompt` | Basic helpful assistant | `domain`, `task` |
+| `code-review-prompt` | Code review assistant | `years_experience`, `language`, `code` |
+| `customer-support-prompt` | Customer support agent | `company_name`, `customer_message` |
+| `data-analysis-prompt` | Data analysis expert | `analysis_type`, `dataset_name`, `data` |
+| `creative-writing-prompt` | Creative writing assistant | `genre`, `length`, `topic` |
+
+## Authentication
+
+The server supports optional Bearer token authentication. Valid tokens for testing:
+
+- `test-token-12345`
+- `dev-token-67890`
+- `prod-token-abcdef`
+
+If no `Authorization` header is provided, requests are allowed (for testing purposes).
+
+## API Endpoints
+
+### LiteLLM Spec Endpoints
+
+#### `GET /beta/litellm_prompt_management`
+
+Get a prompt by ID (required by LiteLLM).
+
+**Query Parameters:**
+- `prompt_id` (required): The prompt ID
+- `project_name` (optional): Project filter
+- `slug` (optional): Slug filter
+- `version` (optional): Version filter
+
+**Response:**
+```json
+{
+  "prompt_id": "hello-world-prompt",
+  "prompt_template": [
+    {
+      "role": "system",
+      "content": "You are a helpful assistant specialized in {domain}."
+    },
+    {
+      "role": "user",
+      "content": "Help me with: {task}"
+    }
+  ],
+  "prompt_template_model": "gpt-4",
+  "prompt_template_optional_params": {
+    "temperature": 0.7,
+    "max_tokens": 500
+  }
+}
+```
+
+### Convenience Endpoints (Not in LiteLLM Spec)
+
+#### `GET /health`
+
+Health check endpoint.
+
+#### `GET /prompts`
+
+List all available prompts.
+
+#### `GET /prompts/{prompt_id}/variables`
+
+Get all variables used in a prompt template.
+
+#### `POST /prompts`
+
+Create a new prompt (in-memory only, for testing).
+
+## Example: Full Integration Test
+
+### 1. Start the Mock Server
+
+```bash
+python mock_prompt_management_server.py
+```
+
+### 2. Test with Python
+
+```python
+from litellm import completion
+
+# The completion will:
+# 1. Fetch the prompt from your API
+# 2. Replace {domain} with "machine learning"
+# 3. Replace {task} with "building a recommendation system"
+# 4. Merge with your messages
+# 5. Use the model and params from the prompt
+
+response = completion(
+    model="gpt-4",
+    prompt_id="hello-world-prompt",
+    prompt_variables={
+        "domain": "machine learning",
+        "task": "building a recommendation system"
+    },
+    messages=[
+        {"role": "user", "content": "I have user behavior data from the past year."}
+    ],
+    # Configure the generic prompt manager
+    generic_prompt_config={
+        "api_base": "http://localhost:8080",
+        "api_key": "test-token-12345",
+    }
+)
+
+print(response.choices[0].message.content)
+```
+
+## Customization
+
+### Adding New Prompts
+
+Edit the `PROMPTS_DB` dictionary in `mock_prompt_management_server.py`:
+
+```python
+PROMPTS_DB = {
+    "my-custom-prompt": {
+        "prompt_id": "my-custom-prompt",
+        "prompt_template": [
+            {
+                "role": "system",
+                "content": "You are a {role}."
+            },
+            {
+                "role": "user",
+                "content": "{user_input}"
+            }
+        ],
+        "prompt_template_model": "gpt-4",
+        "prompt_template_optional_params": {
+            "temperature": 0.8,
+            "max_tokens": 1000
+        }
+    }
+}
+```
+
+### Using a Database
+
+Replace the `PROMPTS_DB` dictionary with database queries:
+
+```python
+@app.get("/beta/litellm_prompt_management")
+async def get_prompt(prompt_id: str):
+    # Fetch from database
+    prompt = await db.prompts.find_one({"prompt_id": prompt_id})
+    
+    if not prompt:
+        raise HTTPException(status_code=404, detail="Prompt not found")
+    
+    return PromptResponse(**prompt)
+```
+
+### Adding Access Control
+
+Use the custom query parameters for access control:
+
+```python
+@app.get("/beta/litellm_prompt_management")
+async def get_prompt(
+    prompt_id: str,
+    project_name: Optional[str] = None,
+    user_id: Optional[str] = None,
+    authorization: Optional[str] = Header(None)
+):
+    token = verify_api_key(authorization)
+    
+    # Check if user has access to this project
+    if not has_project_access(token, project_name):
+        raise HTTPException(status_code=403, detail="Access denied")
+    
+    # Fetch and return prompt
+    ...
+```
+
+## Production Considerations
+
+Before deploying to production:
+
+1. **Use a real database** instead of in-memory storage
+2. **Implement proper authentication** with JWT tokens or API keys
+3. **Add rate limiting** to prevent abuse
+4. **Use HTTPS** for encrypted communication
+5. **Add logging and monitoring** for observability
+6. **Implement caching** for frequently accessed prompts
+7. **Add versioning** for prompt management
+8. **Implement access control** based on teams/users
+9. **Add input validation** for all parameters
+10. **Use environment variables** for configuration
+
+## Related Documentation
+
+- [Generic Prompt Management API Documentation](https://docs.litellm.ai/docs/adding_provider/generic_prompt_management_api)
+- [LiteLLM Prompt Management](https://docs.litellm.ai/docs/proxy/prompt_management)
+- [Generic Guardrail API](https://docs.litellm.ai/docs/adding_provider/generic_guardrail_api)
+
+## Questions?
+
+This is a reference implementation for the LiteLLM Generic Prompt Management API. For questions or issues, please open an issue on the [LiteLLM GitHub repository](https://github.com/BerriAI/litellm).
+

cookbook/mock_prompt_management_server/mock_prompt_management_server.py

@@ -0,0 +1,390 @@
+#!/usr/bin/env python3
+"""
+Mock Prompt Management API Server
+
+This is a FastAPI server that implements the LiteLLM Generic Prompt Management API
+for testing and demonstration purposes.
+
+Usage:
+    python mock_prompt_management_server.py
+
+The server will start on http://localhost:8080
+
+Test the endpoint:
+    curl "http://localhost:8080/beta/litellm_prompt_management?prompt_id=hello-world-prompt"
+"""
+
+import os
+import json
+from typing import Any, Dict, List, Optional
+
+from fastapi import FastAPI, HTTPException, Header, Query, status
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel, Field
+
+# ============================================================================
+# Response Models
+# ============================================================================
+
+
+class MessageContent(BaseModel):
+    """A single message in the prompt template"""
+
+    role: str = Field(..., description="Message role (system, user, assistant)")
+    content: str = Field(
+        ..., description="Message content with optional {variable} placeholders"
+    )
+
+
+class PromptResponse(BaseModel):
+    """Response format for the prompt management API"""
+
+    prompt_id: str = Field(..., description="The ID of the prompt")
+    prompt_template: List[MessageContent] = Field(
+        ..., description="Array of messages in OpenAI format"
+    )
+    prompt_template_model: Optional[str] = Field(
+        None, description="Optional model to use for this prompt"
+    )
+    prompt_template_optional_params: Optional[Dict[str, Any]] = Field(
+        None, description="Optional parameters like temperature, max_tokens, etc."
+    )
+
+
+# ============================================================================
+# Mock Prompt Database
+# ============================================================================
+
+PROMPTS_DB = {
+    "hello-world-prompt": {
+        "prompt_id": "hello-world-prompt",
+        "prompt_template": [
+            {
+                "role": "system",
+                "content": "You are a helpful assistant specialized in {domain}.",
+            },
+            {"role": "user", "content": "Help me with: {task}"},
+        ],
+        "prompt_template_model": "gpt-4",
+        "prompt_template_optional_params": {"temperature": 0.7, "max_tokens": 500},
+    },
+    "code-review-prompt": {
+        "prompt_id": "code-review-prompt",
+        "prompt_template": [
+            {
+                "role": "system",
+                "content": "You are an expert code reviewer with {years_experience} years of experience in {language}.",
+            },
+            {
+                "role": "user",
+                "content": "Please review the following code for bugs, security issues, and best practices:\n\n{code}",
+            },
+        ],
+        "prompt_template_model": "gpt-4-turbo",
+        "prompt_template_optional_params": {
+            "temperature": 0.3,
+            "max_tokens": 1500,
+        },
+    },
+    "customer-support-prompt": {
+        "prompt_id": "customer-support-prompt",
+        "prompt_template": [
+            {
+                "role": "system",
+                "content": "You are a friendly customer support agent for {company_name}. Always be professional, empathetic, and solution-oriented.",
+            },
+            {
+                "role": "user",
+                "content": "Customer inquiry: {customer_message}",
+            },
+        ],
+        "prompt_template_model": "gpt-3.5-turbo",
+        "prompt_template_optional_params": {
+            "temperature": 0.8,
+            "max_tokens": 800,
+            "top_p": 0.9,
+        },
+    },
+    "data-analysis-prompt": {
+        "prompt_id": "data-analysis-prompt",
+        "prompt_template": [
+            {
+                "role": "system",
+                "content": "You are a data scientist expert in {analysis_type} analysis.",
+            },
+            {
+                "role": "user",
+                "content": "Analyze the following data and provide insights:\n\nDataset: {dataset_name}\nData: {data}",
+            },
+        ],
+        "prompt_template_model": "gpt-4",
+        "prompt_template_optional_params": {
+            "temperature": 0.5,
+            "max_tokens": 2000,
+        },
+    },
+    "creative-writing-prompt": {
+        "prompt_id": "creative-writing-prompt",
+        "prompt_template": [
+            {
+                "role": "system",
+                "content": "You are a creative writer specializing in {genre} fiction.",
+            },
+            {
+                "role": "user",
+                "content": "Write a {length} story about: {topic}",
+            },
+        ],
+        "prompt_template_model": "gpt-4",
+        "prompt_template_optional_params": {
+            "temperature": 0.9,
+            "max_tokens": 3000,
+            "top_p": 0.95,
+        },
+    },
+}
+
+# Valid API tokens for authentication (in production, use a secure token store)
+VALID_API_TOKENS = {
+    "test-token-12345",
+    "dev-token-67890",
+    "prod-token-abcdef",
+}
+
+# ============================================================================
+# FastAPI App
+# ============================================================================
+
+app = FastAPI(
+    title="Mock Prompt Management API",
+    description="A mock server implementing the LiteLLM Generic Prompt Management API",
+    version="1.0.0",
+)
+
+
+def verify_api_key(authorization: Optional[str] = Header(None)) -> bool:
+    """
+    Verify the API key from the Authorization header.
+
+    Args:
+        authorization: Authorization header (Bearer token)
+
+    Returns:
+        True if valid, raises HTTPException if invalid
+    """
+    if authorization is None:
+        # Allow requests without authentication for testing
+        return True
+
+    # Extract token from "Bearer <token>"
+    if not authorization.startswith("Bearer "):
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Invalid authorization header format. Expected 'Bearer <token>'",
+        )
+
+    token = authorization.replace("Bearer ", "").strip()
+
+    if token not in VALID_API_TOKENS:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Invalid API key",
+        )
+
+    return True
+
+
+@app.get("/beta/litellm_prompt_management", response_model=PromptResponse)
+async def get_prompt(
+    prompt_id: str = Query(..., description="The ID of the prompt to fetch"),
+    project_name: Optional[str] = Query(
+        None, description="Optional project name filter"
+    ),
+    slug: Optional[str] = Query(None, description="Optional slug filter"),
+    version: Optional[str] = Query(None, description="Optional version filter"),
+    authorization: Optional[str] = Header(None),
+) -> PromptResponse:
+    """
+    Get a prompt by ID with optional filtering.
+
+    This endpoint implements the LiteLLM Generic Prompt Management API specification.
+
+    Args:
+        prompt_id: The ID of the prompt to fetch
+        project_name: Optional project name for filtering
+        slug: Optional slug for filtering
+        version: Optional version for filtering
+        authorization: Optional Bearer token for authentication
+
+    Returns:
+        PromptResponse with the prompt template and configuration
+
+    Raises:
+        HTTPException: 401 if authentication fails, 404 if prompt not found
+    """
+    # Verify authentication
+    verify_api_key(authorization)
+
+    # Log the request parameters (useful for debugging)
+    print(f"Fetching prompt: {prompt_id}")
+    if project_name:
+        print(f"  Project: {project_name}")
+    if slug:
+        print(f"  Slug: {slug}")
+    if version:
+        print(f"  Version: {version}")
+
+    # Check if prompt exists
+    if prompt_id not in PROMPTS_DB:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Prompt '{prompt_id}' not found. Available prompts: {list(PROMPTS_DB.keys())}",
+        )
+
+    # Get the prompt from the database
+    prompt_data = PROMPTS_DB[prompt_id]
+
+    # Optional: Apply filtering based on project_name, slug, or version
+    # In a real implementation, you might use these to filter prompts by access control
+    # or to fetch specific versions from your database
+
+    return PromptResponse(**prompt_data)
+
+
+@app.get("/health")
+async def health_check():
+    """Health check endpoint"""
+    return {
+        "status": "healthy",
+        "service": "mock-prompt-management-api",
+        "version": "1.0.0",
+    }
+
+
+@app.get("/prompts")
+async def list_prompts(authorization: Optional[str] = Header(None)):
+    """
+    List all available prompts.
+
+    This is a convenience endpoint (not part of the LiteLLM spec) for
+    discovering available prompts.
+    """
+    # Verify authentication
+    verify_api_key(authorization)
+
+    prompts_list = [
+        {
+            "prompt_id": pid,
+            "model": p.get("prompt_template_model"),
+            "has_variables": any(
+                "{" in msg.get("content", "") for msg in p.get("prompt_template", [])
+            ),
+        }
+        for pid, p in PROMPTS_DB.items()
+    ]
+
+    return {"prompts": prompts_list, "total": len(prompts_list)}
+
+
+@app.get("/prompts/{prompt_id}/variables")
+async def get_prompt_variables(
+    prompt_id: str, authorization: Optional[str] = Header(None)
+):
+    """
+    Get all variables in a prompt template.
+
+    This is a convenience endpoint (not part of the LiteLLM spec) for
+    discovering what variables a prompt expects.
+    """
+    # Verify authentication
+    verify_api_key(authorization)
+
+    if prompt_id not in PROMPTS_DB:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Prompt '{prompt_id}' not found",
+        )
+
+    prompt_data = PROMPTS_DB[prompt_id]
+    variables = set()
+
+    # Extract variables from the prompt template
+    import re
+
+    for message in prompt_data["prompt_template"]:
+        content = message.get("content", "")
+        # Find all {variable} patterns
+        found_vars = re.findall(r"\{(\w+)\}", content)
+        variables.update(found_vars)
+
+    return {
+        "prompt_id": prompt_id,
+        "variables": sorted(list(variables)),
+        "example_usage": {
+            "prompt_id": prompt_id,
+            "prompt_variables": {var: f"<{var}_value>" for var in variables},
+        },
+    }
+
+
+@app.post("/prompts")
+async def create_prompt(
+    prompt: PromptResponse, authorization: Optional[str] = Header(None)
+):
+    """
+    Create a new prompt (convenience endpoint for testing).
+
+    This is NOT part of the LiteLLM spec - it's just for testing purposes.
+    """
+    # Verify authentication
+    verify_api_key(authorization)
+
+    if prompt.prompt_id in PROMPTS_DB:
+        raise HTTPException(
+            status_code=status.HTTP_409_CONFLICT,
+            detail=f"Prompt '{prompt.prompt_id}' already exists",
+        )
+
+    PROMPTS_DB[prompt.prompt_id] = prompt.dict()
+
+    return {
+        "status": "created",
+        "prompt_id": prompt.prompt_id,
+        "message": "Prompt created successfully (in-memory only)",
+    }
+
+
+# ============================================================================
+# Main
+# ============================================================================
+
+if __name__ == "__main__":
+    import uvicorn
+
+    print("=" * 70)
+    print("Mock Prompt Management API Server")
+    print("=" * 70)
+    print(f"\nStarting server on http://localhost:8080")
+    print(f"\nAvailable prompts: {len(PROMPTS_DB)}")
+    for prompt_id in PROMPTS_DB.keys():
+        print(f"  - {prompt_id}")
+    print(f"\nValid API tokens: {len(VALID_API_TOKENS)}")
+    print("  - test-token-12345")
+    print("  - dev-token-67890")
+    print("  - prod-token-abcdef")
+    print("\nEndpoints:")
+    print("  GET  /beta/litellm_prompt_management?prompt_id=<id>  (LiteLLM spec)")
+    print("  GET  /health                                          (health check)")
+    print("  GET  /prompts                                         (list all prompts)")
+    print(
+        "  GET  /prompts/{id}/variables                          (get prompt variables)"
+    )
+    print("  POST /prompts                                         (create prompt)")
+    print("\nExample usage:")
+    print(
+        '  curl "http://localhost:8080/beta/litellm_prompt_management?prompt_id=hello-world-prompt"'
+    )
+    print("\nPress CTRL+C to stop the server")
+    print("=" * 70)
+
+    uvicorn.run(app, host="0.0.0.0", port=8080, log_level="info")

deploy/charts/litellm-helm/README.md

@@ -36,6 +36,10 @@ If `db.useStackgresOperator` is used (not yet implemented):
 | `serviceAccount.create`     | Whether or not to create a Kubernetes Service Account for this deployment. The default is `false` because LiteLLM has no need to access the Kubernetes API.                                                                                   | `false`                   |
 | `service.type`              | Kubernetes Service type (e.g. `LoadBalancer`, `ClusterIP`, etc.)                                                                                                                                                                              | `ClusterIP`               |
 | `service.port`              | TCP port that the Kubernetes Service will listen on. Also the TCP port within the Pod that the proxy will listen on.                                                                                                                          | `4000`                    |
+| `livenessProbe.*`           | Liveness probe settings for the LiteLLM container (`path`, `periodSeconds`, `timeoutSeconds`, thresholds, and initial delay).                                                                                                                  | See `values.yaml`         |
+| `readinessProbe.*`          | Readiness probe settings for the LiteLLM container (`path`, `periodSeconds`, `timeoutSeconds`, thresholds, and initial delay).                                                                                                                 | See `values.yaml`         |
+| `startupProbe.*`            | Startup probe settings for the LiteLLM container (`path`, `periodSeconds`, `timeoutSeconds`, thresholds, and initial delay).                                                                                                                   | See `values.yaml`         |
+| `resources.*`               | CPU/memory requests and limits for the LiteLLM container.                                                                                                                                                                                       | `{}`                      |
 | `service.loadBalancerClass` | Optional LoadBalancer implementation class (only used when `service.type` is `LoadBalancer`)                                                                                                                                                  | `""`                      |
 | `ingress.labels`            | Additional labels for the Ingress resource                                                                                                                                                                                                    | `{}`                      |
 | `ingress.*`                 | See [values.yaml](./values.yaml) for example settings                                                                                                                                                                                         | N/A                       |

deploy/charts/litellm-helm/templates/configmap-litellm.yaml

@@ -6,4 +6,4 @@ metadata:
 data:
   config.yaml: |
 {{ .Values.proxy_config | toYaml | indent 6 }}
-{{- end }}
+{{- end }}

deploy/charts/litellm-helm/templates/deployment.yaml

@@ -158,18 +158,31 @@ spec:
             {{- end }}
           livenessProbe:
             httpGet:
-              path: /health/liveliness
+              path: {{ .Values.livenessProbe.path | quote }}
               port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
+            initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
+            periodSeconds: {{ .Values.livenessProbe.periodSeconds }}
+            timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }}
+            successThreshold: {{ .Values.livenessProbe.successThreshold }}
+            failureThreshold: {{ .Values.livenessProbe.failureThreshold }}
           readinessProbe:
             httpGet:
-              path: /health/readiness
+              path: {{ .Values.readinessProbe.path | quote }}
               port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
+            initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
+            periodSeconds: {{ .Values.readinessProbe.periodSeconds }}
+            timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }}
+            successThreshold: {{ .Values.readinessProbe.successThreshold }}
+            failureThreshold: {{ .Values.readinessProbe.failureThreshold }}
           startupProbe:
             httpGet:
-              path: /health/readiness
+              path: {{ .Values.startupProbe.path | quote }}
               port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
-            failureThreshold: 30
-            periodSeconds: 10
+            initialDelaySeconds: {{ .Values.startupProbe.initialDelaySeconds }}
+            periodSeconds: {{ .Values.startupProbe.periodSeconds }}
+            timeoutSeconds: {{ .Values.startupProbe.timeoutSeconds }}
+            successThreshold: {{ .Values.startupProbe.successThreshold }}
+            failureThreshold: {{ .Values.startupProbe.failureThreshold }}
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
           volumeMounts:
@@ -235,4 +248,4 @@ spec:
       {{- if .Values.topologySpreadConstraints }}
       topologySpreadConstraints:
         {{- toYaml .Values.topologySpreadConstraints | nindent 8 }}
-      {{- end }}
+      {{- end }}

deploy/charts/litellm-helm/tests/deployment_tests.yaml

@@ -159,4 +159,150 @@ tests:
           value: -c
       - equal:
           path: spec.template.spec.containers[0].lifecycle.preStop.exec.command[2]
-          value: echo "Container stopping"
+          value: echo "Container stopping"
+  - it: should render background health check settings from proxy_config.general_settings
+    template: configmap-litellm.yaml
+    set:
+      proxy_config.general_settings.background_health_checks: true
+      proxy_config.general_settings.health_check_interval: 240
+      proxy_config.general_settings.health_check_concurrency: 16
+      proxy_config.general_settings.health_check_details: false
+    asserts:
+      - matchRegex:
+          path: data["config.yaml"]
+          pattern: '(?m)^\s*background_health_checks:\s*true$'
+      - matchRegex:
+          path: data["config.yaml"]
+          pattern: '(?m)^\s*health_check_interval:\s*240$'
+      - matchRegex:
+          path: data["config.yaml"]
+          pattern: '(?m)^\s*health_check_concurrency:\s*16$'
+      - matchRegex:
+          path: data["config.yaml"]
+          pattern: '(?m)^\s*health_check_details:\s*false$'
+  - it: should allow overriding liveness, readiness, and startup probes
+    template: deployment.yaml
+    set:
+      livenessProbe:
+        path: /custom/livez
+        initialDelaySeconds: 5
+        periodSeconds: 15
+        timeoutSeconds: 5
+        successThreshold: 1
+        failureThreshold: 5
+      readinessProbe:
+        path: /custom/readyz
+        initialDelaySeconds: 10
+        periodSeconds: 20
+        timeoutSeconds: 6
+        successThreshold: 1
+        failureThreshold: 6
+      startupProbe:
+        path: /custom/startupz
+        initialDelaySeconds: 15
+        periodSeconds: 25
+        timeoutSeconds: 7
+        successThreshold: 1
+        failureThreshold: 40
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.httpGet.path
+          value: /custom/livez
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.timeoutSeconds
+          value: 5
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.httpGet.path
+          value: /custom/readyz
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.timeoutSeconds
+          value: 6
+      - equal:
+          path: spec.template.spec.containers[0].startupProbe.httpGet.path
+          value: /custom/startupz
+      - equal:
+          path: spec.template.spec.containers[0].startupProbe.failureThreshold
+          value: 40
+  - it: should render container resources from values
+    template: deployment.yaml
+    set:
+      resources:
+        limits:
+          cpu: 500m
+          memory: 2Gi
+        requests:
+          cpu: 250m
+          memory: 1Gi
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].resources.limits.cpu
+          value: 500m
+      - equal:
+          path: spec.template.spec.containers[0].resources.limits.memory
+          value: 2Gi
+      - equal:
+          path: spec.template.spec.containers[0].resources.requests.cpu
+          value: 250m
+      - equal:
+          path: spec.template.spec.containers[0].resources.requests.memory
+          value: 1Gi
+  - it: should keep default probes and empty resources unchanged
+    template: deployment.yaml
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.httpGet.path
+          value: /health/liveliness
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.initialDelaySeconds
+          value: 0
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.periodSeconds
+          value: 10
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.timeoutSeconds
+          value: 1
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.successThreshold
+          value: 1
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.failureThreshold
+          value: 3
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.httpGet.path
+          value: /health/readiness
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.initialDelaySeconds
+          value: 0
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.periodSeconds
+          value: 10
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.timeoutSeconds
+          value: 1
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.successThreshold
+          value: 1
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.failureThreshold
+          value: 3
+      - equal:
+          path: spec.template.spec.containers[0].startupProbe.httpGet.path
+          value: /health/readiness
+      - equal:
+          path: spec.template.spec.containers[0].startupProbe.initialDelaySeconds
+          value: 0
+      - equal:
+          path: spec.template.spec.containers[0].startupProbe.periodSeconds
+          value: 10
+      - equal:
+          path: spec.template.spec.containers[0].startupProbe.timeoutSeconds
+          value: 1
+      - equal:
+          path: spec.template.spec.containers[0].startupProbe.successThreshold
+          value: 1
+      - equal:
+          path: spec.template.spec.containers[0].startupProbe.failureThreshold
+          value: 30
+      - equal:
+          path: spec.template.spec.containers[0].resources
+          value: {}

deploy/charts/litellm-helm/values.yaml

@@ -84,6 +84,31 @@ service:
 separateHealthApp: false
 separateHealthPort: 8081
 
+# Probe tuning for proxy container
+livenessProbe:
+  path: /health/liveliness
+  initialDelaySeconds: 0
+  periodSeconds: 10
+  timeoutSeconds: 1
+  successThreshold: 1
+  failureThreshold: 3
+
+readinessProbe:
+  path: /health/readiness
+  initialDelaySeconds: 0
+  periodSeconds: 10
+  timeoutSeconds: 1
+  successThreshold: 1
+  failureThreshold: 3
+
+startupProbe:
+  path: /health/readiness
+  initialDelaySeconds: 0
+  periodSeconds: 10
+  timeoutSeconds: 1
+  successThreshold: 1
+  failureThreshold: 30
+
 ingress:
   enabled: false
   className: "nginx"

docker/Dockerfile.custom_ui

@@ -5,8 +5,21 @@ FROM ghcr.io/berriai/litellm:litellm_fwd_server_root_path-dev
 WORKDIR /app
 
 # Install Node.js and npm (adjust version as needed)
-RUN apt-get update && apt-get install -y nodejs npm && \
-    npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 && \
+RUN apt-get update && apt-get upgrade -y \
+        libxml2 \
+        libexpat1 \
+        openssl \
+        libssl3 \
+        git \
+        libkrb5-3 \
+        libglib2.0-0 \
+        wget \
+        libaom3 \
+        libxslt1.1 \
+        libgnutls30 \
+        libc6 && \
+    apt-get install -y nodejs npm && \
+    npm install -g npm@latest tar@7.5.8 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.1 diff@8.0.3 && \
     GLOBAL="$(npm root -g)" && \
     find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
@@ -17,6 +30,12 @@ RUN apt-get update && apt-get install -y nodejs npm && \
     find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
     done && \
+    find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
+    done && \
+    find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
+    done && \
     npm cache clean --force
 
 # Copy the UI source into the container

docker/Dockerfile.database

@@ -50,7 +50,7 @@ USER root
 
 # Install runtime dependencies
 RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \
-    npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 && \
+    npm install -g npm@latest tar@7.5.8 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.1 diff@8.0.3 && \
     GLOBAL="$(npm root -g)" && \
     find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
@@ -61,6 +61,12 @@ RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile
     find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
     done && \
+    find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
+    done && \
+    find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
+    done && \
     npm cache clean --force
 
 WORKDIR /app
@@ -79,14 +85,20 @@ RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl
 # npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
 # Patch every copy of tar, glob, and brace-expansion inside that tree.
 RUN GLOBAL="$(npm root -g)" && \
-    find /usr/lib -path "*/nodejs_wheel/*/node_modules/tar" -type d | while read d; do \
+    find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
     done && \
-    find /usr/lib -path "*/nodejs_wheel/*/node_modules/glob" -type d | while read d; do \
+    find /usr/lib -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
     done && \
-    find /usr/lib -path "*/nodejs_wheel/*/node_modules/@isaacs/brace-expansion" -type d | while read d; do \
+    find /usr/lib -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
+    done && \
+    find /usr/lib -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
+    done && \
+    find /usr/lib -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
     done
 
 # Install semantic_router and aurelio-sdk using script

docker/Dockerfile.dev

@@ -56,13 +56,26 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
 USER root
 
 # Install only runtime dependencies
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    libssl3 \
+RUN apt-get update && apt-get upgrade -y \
+        libxml2 \
+        libexpat1 \
+        openssl \
+        libssl3 \
+        git \
+        libkrb5-3 \
+        libglib2.0-0 \
+        wget \
+        libaom3 \
+        libxslt1.1 \
+        libgnutls30 \
+        libc6 \
+    && apt-get install -y --no-install-recommends \
+        libssl3 \
     libatomic1 \
     nodejs \
     npm \
     && rm -rf /var/lib/apt/lists/* \
-    && npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 \
+    && npm install -g npm@latest tar@7.5.8 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.1 diff@8.0.3 \
     && GLOBAL="$(npm root -g)" \
     && find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
           rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
@@ -73,6 +86,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     && find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
           rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
        done \
+    && find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
+          rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
+       done \
+    && find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
+          rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
+       done \
     && npm cache clean --force
 
 WORKDIR /app
@@ -95,14 +114,20 @@ RUN pip install --no-cache-dir *.whl /wheels/* --no-index --find-links=/wheels/
 # npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
 # Patch every copy of tar, glob, and brace-expansion inside that tree.
 RUN GLOBAL="$(npm root -g)" && \
-    find /usr/lib -path "*/nodejs_wheel/*/node_modules/tar" -type d | while read d; do \
+    find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
     done && \
-    find /usr/lib -path "*/nodejs_wheel/*/node_modules/glob" -type d | while read d; do \
+    find /usr/lib -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
     done && \
-    find /usr/lib -path "*/nodejs_wheel/*/node_modules/@isaacs/brace-expansion" -type d | while read d; do \
+    find /usr/lib -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
+    done && \
+    find /usr/lib -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
+    done && \
+    find /usr/lib -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
     done
 
 # Generate prisma client and set permissions

docker/Dockerfile.non_root

@@ -80,7 +80,7 @@ ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
     XDG_CACHE_HOME=/app/.cache \
     PATH="/usr/lib/python3.13/site-packages/nodejs/bin:${PATH}"
 
-RUN pip install --no-cache-dir prisma==0.11.0 nodejs-wheel-binaries==24.12.0 \
+RUN pip install --no-cache-dir prisma==0.11.0 nodejs-wheel-binaries==24.13.1 \
     && mkdir -p /app/.cache/npm
 
 RUN NPM_CONFIG_CACHE=/app/.cache/npm \
@@ -105,7 +105,8 @@ RUN for i in 1 2 3; do \
   && for i in 1 2 3; do \
     apk add --no-cache python3 py3-pip bash openssl tzdata nodejs npm supervisor && break || sleep 5; \
     done \
-  && npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 \
+  && apk upgrade --no-cache nodejs \
+  && npm install -g npm@latest tar@7.5.8 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.1 diff@8.0.3 \
   && GLOBAL="$(npm root -g)" \
   && find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
@@ -116,6 +117,12 @@ RUN for i in 1 2 3; do \
   && find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
      done \
+  && find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
+     done \
+  && find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
+     done \
   && npm cache clean --force
 
 # Copy artifacts from builder
@@ -162,14 +169,20 @@ RUN pip install --no-index --find-links=/wheels/ -r requirements.txt && \
 # npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
 # Patch every copy of tar, glob, and brace-expansion inside that tree.
 RUN GLOBAL="$(npm root -g)" && \
-    find /usr/lib -path "*/nodejs_wheel/*/node_modules/tar" -type d | while read d; do \
+    find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
     done && \
-    find /usr/lib -path "*/nodejs_wheel/*/node_modules/glob" -type d | while read d; do \
+    find /usr/lib -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
     done && \
-    find /usr/lib -path "*/nodejs_wheel/*/node_modules/@isaacs/brace-expansion" -type d | while read d; do \
+    find /usr/lib -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
+    done && \
+    find /usr/lib -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
+    done && \
+    find /usr/lib -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
     done
 
 # Permissions, cleanup, and Prisma prep

docs/my-website/blog/anthropic_wildcard_model_access_incident/index.md

@@ -0,0 +1,147 @@
+---
+slug: anthropic-wildcard-model-access-incident
+title: "Incident Report: Wildcard Blocking New Models After Cost Map Reload"
+date: 2026-02-23T10:00:00
+authors:
+  - name: Sameer Kankute
+    title: SWE @ LiteLLM (LLM Translation)
+    url: https://www.linkedin.com/in/sameer-kankute/
+    image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
+  - name: Krrish Dholakia
+    title: "CEO, LiteLLM"
+    url: https://www.linkedin.com/in/krish-d/
+    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
+  - name: Ishaan Jaff
+    title: "CTO, LiteLLM"
+    url: https://www.linkedin.com/in/reffajnaahsi/
+    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+tags: [incident-report, proxy, auth, model-access]
+hide_table_of_contents: false
+---
+
+**Date:** Feb 23, 2026  
+**Duration:** ~3 hours  
+**Severity:** High (for users with provider wildcard access rules)  
+**Status:** Resolved
+
+## Summary
+
+When a new Anthropic model (e.g. `claude-sonnet-4-6`) was added to the LiteLLM model cost map and a cost map reload was triggered, requests to the new model were rejected with:
+
+```
+key not allowed to access model. This key can only access models=['anthropic/*']. Tried to access claude-sonnet-4-6.
+```
+
+The reload updated `litellm.model_cost` correctly but never re-ran `add_known_models()`, so `litellm.anthropic_models` (the in-memory set used by the wildcard resolver) remained stale. The new model was invisible to the `anthropic/*` wildcard even though the cost map knew about it.
+
+- **LLM calls:** All requests to newly-added Anthropic models were blocked with a 401.
+- **Existing models:** Unaffected — only models missing from the stale provider set were impacted.
+- **Other providers:** Same bug class existed for any provider wildcard (e.g. `openai/*`, `gemini/*`).
+
+{/* truncate */}
+
+---
+
+## Background
+
+LiteLLM supports provider-level wildcard access rules. When an admin configures a key or team with `models=['anthropic/*']`, any model whose provider resolves to `anthropic` should be allowed. The resolution happens in `_model_custom_llm_provider_matches_wildcard_pattern`:
+
+```mermaid
+flowchart TD
+    A["1. Request arrives for claude-sonnet-4-6"] --> B["2. Auth check: can this key call this model?
+    proxy/auth/auth_checks.py"]
+    B --> C["3. Key has models=['anthropic/*']
+    → wildcard match attempted"]
+    C --> D["4. get_llm_provider('claude-sonnet-4-6')
+    checks litellm.anthropic_models set"]
+    D -->|"model IN set"| E["5a. ✅ Provider = 'anthropic'
+    → 'anthropic/claude-sonnet-4-6' matches 'anthropic/*'"]
+    D -->|"model NOT IN set"| F["5b. ❌ Provider unknown
+    → exception raised → wildcard returns False"]
+    E --> G["6. Request allowed"]
+    F --> H["6. 401: key not allowed to access model"]
+
+    style E fill:#d4edda,stroke:#28a745
+    style F fill:#f8d7da,stroke:#dc3545
+    style H fill:#f8d7da,stroke:#dc3545
+    style D fill:#fff3cd,stroke:#ffc107
+```
+
+`litellm.anthropic_models` is a Python `set` populated at import time by `add_known_models()`. It is the source `get_llm_provider()` consults to map a bare model name like `claude-sonnet-4-6` to the provider string `"anthropic"`.
+
+---
+
+## Root Cause
+
+`add_known_models()` is called **once** at module import time. Both reload paths in `proxy_server.py` updated `litellm.model_cost` with the fresh map but never called `add_known_models()` again:
+
+```python
+# Before the fix — both reload paths looked like this:
+new_model_cost_map = get_model_cost_map(url=model_cost_map_url)
+litellm.model_cost = new_model_cost_map          # ✅ cost map updated
+_invalidate_model_cost_lowercase_map()           # ✅ cache cleared
+# ❌ add_known_models() never called
+#    → litellm.anthropic_models still has the old set
+#    → new model not in the set
+#    → get_llm_provider() raises for the new model
+#    → wildcard match returns False
+#    → 401 for every request to the new model
+```
+
+The gap existed in two places:
+1. `_check_and_reload_model_cost_map` — the periodic automatic reload (every 10 s)
+2. The `/reload/model_cost_map` admin endpoint — the manual reload
+
+**Timeline:**
+
+1. New model (`claude-sonnet-4-6`) added to `model_prices_and_context_window.json`
+2. Admin triggers cost map reload via UI → `litellm.model_cost` updated
+3. Users with `anthropic/*` wildcard keys attempt requests to `claude-sonnet-4-6`
+4. `get_llm_provider('claude-sonnet-4-6')` raises → wildcard returns False → 401
+5. Admin reloads cost map again — same result (root cause not addressed)
+6. ~3 hours of investigation → root cause identified → fix deployed
+
+---
+
+## The Fix
+
+After each reload, `add_known_models()` is called with the freshly fetched map passed explicitly. Passing the map directly (rather than relying on the module-level reference) removes any ambiguity about which dict is iterated:
+
+```python
+# After the fix — both reload paths now do:
+new_model_cost_map = get_model_cost_map(url=model_cost_map_url)
+litellm.model_cost = new_model_cost_map
+_invalidate_model_cost_lowercase_map()
+litellm.add_known_models(model_cost_map=new_model_cost_map)  # ✅ sets repopulated
+```
+
+`add_known_models()` was also updated to accept an optional explicit map so callers cannot accidentally iterate a stale module-level reference:
+
+```python
+# Before
+def add_known_models():
+    for key, value in model_cost.items():   # reads module global — ambiguous after reload
+        ...
+
+# After
+def add_known_models(model_cost_map: Optional[Dict] = None):
+    _map = model_cost_map if model_cost_map is not None else model_cost
+    for key, value in _map.items():         # always iterates the map you just fetched
+        ...
+```
+
+After the fix, the provider sets (`anthropic_models`, `open_ai_chat_completion_models`, etc.) are always consistent with `litellm.model_cost` immediately after every reload. New models become accessible via wildcard rules without any proxy restart.
+
+---
+
+## Remediation
+
+| # | Action | Status | Code |
+|---|---|---|---|
+| 1 | Call `add_known_models(model_cost_map=...)` in the periodic reload path | ✅ Done | [`proxy_server.py#L4393`](https://github.com/BerriAI/litellm/blob/main/litellm/proxy/proxy_server.py#L4393) |
+| 2 | Call `add_known_models(model_cost_map=...)` in the `/reload/model_cost_map` endpoint | ✅ Done | [`proxy_server.py#L11904`](https://github.com/BerriAI/litellm/blob/main/litellm/proxy/proxy_server.py#L11904) |
+| 3 | Update `add_known_models()` to accept an explicit map parameter | ✅ Done | [`__init__.py#L617`](https://github.com/BerriAI/litellm/blob/main/litellm/__init__.py#L617) |
+| 4 | Regression test: `add_known_models(model_cost_map=...)` populates provider sets | ✅ Done | [`test_auth_checks.py`](https://github.com/BerriAI/litellm/blob/main/tests/proxy_unit_tests/test_auth_checks.py) |
+| 5 | Regression test: `anthropic/*` wildcard grants/denies access correctly after reload | ✅ Done | [`test_auth_checks.py`](https://github.com/BerriAI/litellm/blob/main/tests/proxy_unit_tests/test_auth_checks.py) |
+
+---

docs/my-website/blog/claude_code_beta_headers/index.md

@@ -24,6 +24,8 @@ hide_table_of_contents: false
 **Severity:** High
 **Status:** Resolved
 
+> **Note:** This fix will be available starting from `v1.81.13-nightly` or higher of LiteLLM.
+
 ## Summary
 
 Claude Code began sending unsupported Anthropic beta headers to non-Anthropic providers (Bedrock, Azure AI, Vertex AI), causing `invalid beta flag` errors. LiteLLM was forwarding all beta headers without provider-specific validation. Users experienced request failures when routing Claude Code requests through LiteLLM to these providers.

docs/my-website/blog/claude_sonnet_4_6/index.md

@@ -0,0 +1,283 @@
+---
+slug: claude_sonnet_4_6
+title: "Day 0 Support: Claude Sonnet 4.6"
+date: 2026-02-17T10:00:00
+authors:
+  - name: Ishaan Jaff
+    title: "CTO, LiteLLM"
+    url: https://www.linkedin.com/in/reffajnaahsi/
+    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+  - name: Krrish Dholakia
+    title: "CEO, LiteLLM"
+    url: https://www.linkedin.com/in/krish-d/
+    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
+description: "Day 0 support for Claude Sonnet 4.6 on LiteLLM AI Gateway - use across Anthropic, Azure, Vertex AI, and Bedrock."
+tags: [anthropic, claude, sonnet 4.6]
+hide_table_of_contents: false
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+LiteLLM now supports Claude Sonnet 4.6 on Day 0. Use it across Anthropic, Azure, Vertex AI, and Bedrock through the LiteLLM AI Gateway.
+
+## Docker Image
+
+```bash
+docker pull ghcr.io/berriai/litellm:v1.81.3-stable.sonnet-4-6
+```
+
+## Usage - Anthropic
+
+<Tabs>
+<TabItem value="proxy" label="LiteLLM Proxy">
+
+**1. Setup config.yaml**
+
+```yaml
+model_list:
+  - model_name: claude-sonnet-4-6
+    litellm_params:
+      model: anthropic/claude-sonnet-4-6
+      api_key: os.environ/ANTHROPIC_API_KEY
+```
+
+**2. Start the proxy**
+
+```bash
+docker run -d \
+  -p 4000:4000 \
+  -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
+  -v $(pwd)/config.yaml:/app/config.yaml \
+  ghcr.io/berriai/litellm:v1.81.3-stable.sonnet-4-6 \
+  --config /app/config.yaml
+```
+
+**3. Test it!**
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+--header 'Content-Type: application/json' \
+--header 'Authorization: Bearer $LITELLM_KEY' \
+--data '{
+  "model": "claude-sonnet-4-6",
+  "messages": [
+    {
+      "role": "user",
+      "content": "what llm are you"
+    }
+  ]
+}'
+```
+
+</TabItem>
+
+<TabItem value="sdk" label="LiteLLM SDK">
+
+```python
+from litellm import completion
+
+response = completion(
+    model="anthropic/claude-sonnet-4-6",
+    messages=[{"role": "user", "content": "what llm are you"}]
+)
+print(response.choices[0].message.content)
+```
+
+</TabItem>
+</Tabs>
+
+## Usage - Azure
+
+<Tabs>
+<TabItem value="proxy" label="LiteLLM Proxy">
+
+**1. Setup config.yaml**
+
+```yaml
+model_list:
+  - model_name: claude-sonnet-4-6
+    litellm_params:
+      model: azure_ai/claude-sonnet-4-6
+      api_key: os.environ/AZURE_AI_API_KEY
+      api_base: os.environ/AZURE_AI_API_BASE  # https://<resource>.services.ai.azure.com
+```
+
+**2. Start the proxy**
+
+```bash
+docker run -d \
+  -p 4000:4000 \
+  -e AZURE_AI_API_KEY=$AZURE_AI_API_KEY \
+  -e AZURE_AI_API_BASE=$AZURE_AI_API_BASE \
+  -v $(pwd)/config.yaml:/app/config.yaml \
+  ghcr.io/berriai/litellm:v1.81.3-stable.sonnet-4-6 \
+  --config /app/config.yaml
+```
+
+**3. Test it!**
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+--header 'Content-Type: application/json' \
+--header 'Authorization: Bearer $LITELLM_KEY' \
+--data '{
+  "model": "claude-sonnet-4-6",
+  "messages": [
+    {
+      "role": "user",
+      "content": "what llm are you"
+    }
+  ]
+}'
+```
+
+</TabItem>
+
+<TabItem value="sdk" label="LiteLLM SDK">
+
+```python
+from litellm import completion
+
+response = completion(
+    model="azure_ai/claude-sonnet-4-6",
+    api_key="your-azure-api-key",
+    api_base="https://<resource>.services.ai.azure.com",
+    messages=[{"role": "user", "content": "what llm are you"}]
+)
+print(response.choices[0].message.content)
+```
+
+</TabItem>
+</Tabs>
+
+## Usage - Vertex AI
+
+<Tabs>
+<TabItem value="proxy" label="LiteLLM Proxy">
+
+**1. Setup config.yaml**
+
+```yaml
+model_list:
+  - model_name: claude-sonnet-4-6
+    litellm_params:
+      model: vertex_ai/claude-sonnet-4-6
+      vertex_project: os.environ/VERTEX_PROJECT
+      vertex_location: us-east5
+```
+
+**2. Start the proxy**
+
+```bash
+docker run -d \
+  -p 4000:4000 \
+  -e VERTEX_PROJECT=$VERTEX_PROJECT \
+  -e GOOGLE_APPLICATION_CREDENTIALS=/app/credentials.json \
+  -v $(pwd)/config.yaml:/app/config.yaml \
+  -v $(pwd)/credentials.json:/app/credentials.json \
+  ghcr.io/berriai/litellm:v1.81.3-stable.sonnet-4-6 \
+  --config /app/config.yaml
+```
+
+**3. Test it!**
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+--header 'Content-Type: application/json' \
+--header 'Authorization: Bearer $LITELLM_KEY' \
+--data '{
+  "model": "claude-sonnet-4-6",
+  "messages": [
+    {
+      "role": "user",
+      "content": "what llm are you"
+    }
+  ]
+}'
+```
+
+</TabItem>
+
+<TabItem value="sdk" label="LiteLLM SDK">
+
+```python
+from litellm import completion
+
+response = completion(
+    model="vertex_ai/claude-sonnet-4-6",
+    vertex_project="your-project-id",
+    vertex_location="us-east5",
+    messages=[{"role": "user", "content": "what llm are you"}]
+)
+print(response.choices[0].message.content)
+```
+
+</TabItem>
+</Tabs>
+
+## Usage - Bedrock
+
+<Tabs>
+<TabItem value="proxy" label="LiteLLM Proxy">
+
+**1. Setup config.yaml**
+
+```yaml
+model_list:
+  - model_name: claude-sonnet-4-6
+    litellm_params:
+      model: bedrock/anthropic.claude-sonnet-4-6-v1
+      aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
+      aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
+      aws_region_name: us-east-1
+```
+
+**2. Start the proxy**
+
+```bash
+docker run -d \
+  -p 4000:4000 \
+  -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
+  -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
+  -v $(pwd)/config.yaml:/app/config.yaml \
+  ghcr.io/berriai/litellm:v1.81.3-stable.sonnet-4-6 \
+  --config /app/config.yaml
+```
+
+**3. Test it!**
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+--header 'Content-Type: application/json' \
+--header 'Authorization: Bearer $LITELLM_KEY' \
+--data '{
+  "model": "claude-sonnet-4-6",
+  "messages": [
+    {
+      "role": "user",
+      "content": "what llm are you"
+    }
+  ]
+}'
+```
+
+</TabItem>
+
+<TabItem value="sdk" label="LiteLLM SDK">
+
+```python
+from litellm import completion
+
+response = completion(
+    model="bedrock/anthropic.claude-sonnet-4-6-v1",
+    aws_access_key_id="your-access-key",
+    aws_secret_access_key="your-secret-key",
+    aws_region_name="us-east-1",
+    messages=[{"role": "user", "content": "what llm are you"}]
+)
+print(response.choices[0].message.content)
+```
+
+</TabItem>
+</Tabs>

docs/my-website/blog/gemin_3.1/index.md

@@ -0,0 +1,150 @@
+---
+slug: gemini_3_1_pro
+title: "DAY 0 Support: Gemini 3.1 Pro on LiteLLM"
+date: 2026-02-19T10:00:00
+authors:
+  - name: Sameer Kankute
+    title: SWE @ LiteLLM (LLM Translation)
+    url: https://www.linkedin.com/in/sameer-kankute/
+    image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
+  - name: Krrish Dholakia
+    title: "CEO, LiteLLM"
+    url: https://www.linkedin.com/in/krish-d/
+    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
+  - name: Ishaan Jaff
+    title: "CTO, LiteLLM"
+    url: https://www.linkedin.com/in/reffajnaahsi/
+    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+description: "Guide to using Gemini 3.1 Pro on LiteLLM Proxy and SDK with day 0 support."
+tags: [gemini, day 0 support, llms]
+hide_table_of_contents: false
+---
+
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Gemini 3.1 Pro Day 0 Support 
+
+LiteLLM now supports `gemini-3.1-pro-preview` and all the new API changes along with it.
+
+## Deploy this version
+
+<Tabs>
+<TabItem value="docker" label="Docker">
+
+``` showLineNumbers title="docker run litellm"
+docker run \
+-e STORE_MODEL_IN_DB=True \
+-p 4000:4000 \
+ghcr.io/berriai/litellm:main-v1.81.9-stable.gemini.3.1-pro
+```
+
+</TabItem>
+
+<TabItem value="pip" label="Pip">
+
+``` showLineNumbers title="pip install litellm"
+pip install litellm==v1.81.9-stable.gemini.3.1-pro
+```
+
+</TabItem>
+</Tabs>
+
+## What's New
+
+### 1. New Thinking Levels: `thinkingLevel` with MINIMAL & MEDIUM
+
+Gemini 3.1 Pro introduces support for **medium** thinking level
+
+LiteLLM automatically maps the OpenAI `reasoning_effort` parameter to Gemini's `thinkingLevel`, so you can use familiar `reasoning_effort` values (`minimal`, `low`, `medium`, `high`) without changing your code!
+
+---
+## Supported Endpoints
+
+LiteLLM provides **full end-to-end support** for Gemini 3.1 Pro on:
+
+- ✅ `/v1/chat/completions` - OpenAI-compatible chat completions endpoint
+- ✅ `/v1/responses` - OpenAI Responses API endpoint (streaming and non-streaming)
+- ✅ [`/v1/messages`](../../docs/anthropic_unified) - Anthropic-compatible messages endpoint
+- ✅ `/v1/generateContent` – [Google Gemini API](../../docs/generateContent.md) compatible endpoint 
+
+All endpoints support:
+- Streaming and non-streaming responses
+- Function calling with thought signatures
+- Multi-turn conversations
+- All Gemini 3-specific features
+- Conversion of provider specific thinking related param to thinkingLevel
+
+## Quick Start
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+**Basic Usage with MEDIUM thinking (NEW)**
+
+```python
+from litellm import completion
+
+# No need to make any changes to your code as we map openai reasoning param to thinkingLevel
+response = completion(
+    model="gemini/gemini-3.1-pro-preview",
+    messages=[{"role": "user", "content": "Solve this complex math problem: 25 * 4 + 10"}],
+    reasoning_effort="medium",  # NEW: MEDIUM thinking level
+)
+
+print(response.choices[0].message.content)
+```
+
+</TabItem>
+
+<TabItem value="proxy" label="PROXY">
+
+**1. Setup config.yaml**
+
+```yaml
+model_list:
+  - model_name: gemini-3.1-pro-preview
+    litellm_params:
+      model: gemini/gemini-3.1-pro-preview
+      api_key: os.environ/GEMINI_API_KEY
+  - model_name: vertex-gemini-3.1-pro-preview
+    litellm_params:
+      model: vertex_ai/gemini-3.1-pro-preview
+```
+
+**2. Start proxy**
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+**3. Call with MEDIUM thinking**
+
+```bash
+curl -X POST http://localhost:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer <YOUR-LITELLM-KEY>" \
+  -d '{
+    "model": "gemini-3.1-pro-preview",
+    "messages": [{"role": "user", "content": "Complex reasoning task"}],
+    "reasoning_effort": "medium"
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+---
+
+## `reasoning_effort` Mapping for Gemini 3+
+
+| reasoning_effort | thinking_level | 
+|------------------|----------------|
+| `minimal` | `minimal` |
+| `low` | `low` |
+| `medium` | `medium` |
+| `high` | `high` |
+| `disable` | `minimal` |
+| `none` | `minimal` |
+

docs/my-website/blog/gemini_3_1_flash_lite/index.md

@@ -0,0 +1,175 @@
+---
+slug: gemini_3_1_flash_lite_preview
+title: "DAY 0 Support: Gemini 3.1 Flash Lite Preview on LiteLLM"
+date: 2026-03-03T08:00:00
+authors:
+  - name: Sameer Kankute
+    title: SWE @ LiteLLM (LLM Translation)
+    url: https://www.linkedin.com/in/sameer-kankute/
+    image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
+  - name: Krrish Dholakia
+    title: "CEO, LiteLLM"
+    url: https://www.linkedin.com/in/krish-d/
+    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
+  - name: Ishaan Jaff
+    title: "CTO, LiteLLM"
+    url: https://www.linkedin.com/in/reffajnaahsi/
+    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+description: "Guide to using Gemini 3.1 Flash Lite Preview on LiteLLM Proxy and SDK with day 0 support."
+tags: [gemini, day 0 support, llms, supernova]
+hide_table_of_contents: false
+---
+
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Gemini 3.1 Flash Lite Preview Day 0 Support 
+
+LiteLLM now supports `gemini-3.1-flash-lite-preview` with full day 0 support!
+
+:::note
+If you only want cost tracking, you need no change in your current Litellm version. But if you want the support for new features introduced along with it like thinking levels, you will need to use v1.80.8-stable.1 or above.
+:::
+
+## Deploy this version
+
+<Tabs>
+<TabItem value="docker" label="Docker">
+
+``` showLineNumbers title="docker run litellm"
+docker run \
+-e STORE_MODEL_IN_DB=True \
+-p 4000:4000 \
+ghcr.io/berriai/litellm:main-v1.80.8-stable.1
+```
+
+</TabItem>
+
+<TabItem value="pip" label="Pip">
+
+``` showLineNumbers title="pip install litellm"
+pip install litellm==v1.80.8-stable.1
+```
+
+</TabItem>
+</Tabs>
+
+## What's New
+
+Supports all four thinking levels:
+- **MINIMAL**: Ultra-fast responses with minimal reasoning
+- **LOW**: Simple instruction following
+- **MEDIUM**: Balanced reasoning for complex tasks
+- **HIGH**: Maximum reasoning depth (dynamic)
+
+---
+
+## Quick Start
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+**Basic Usage**
+
+```python
+from litellm import completion
+
+response = completion(
+    model="gemini/gemini-3.1-flash-lite-preview",
+    messages=[{"role": "user", "content": "Extract key entities from this text: ..."}],
+)
+
+print(response.choices[0].message.content)
+```
+
+**With Thinking Levels**
+
+```python
+from litellm import completion
+
+# Use MEDIUM thinking for complex reasoning tasks
+response = completion(
+    model="gemini/gemini-3.1-flash-lite-preview",
+    messages=[{"role": "user", "content": "Analyze this dataset and identify patterns"}],
+    reasoning_effort="medium",  # low, medium , high
+)
+
+print(response.choices[0].message.content)
+```
+
+</TabItem>
+
+<TabItem value="proxy" label="PROXY">
+
+**1. Setup config.yaml**
+
+```yaml
+model_list:
+  - model_name: gemini-3.1-flash-lite
+    litellm_params:
+      model: gemini/gemini-3.1-flash-lite-preview
+      api_key: os.environ/GEMINI_API_KEY
+  
+  # Or use Vertex AI
+  - model_name: vertex-gemini-3.1-flash-lite
+    litellm_params:
+      model: vertex_ai/gemini-3.1-flash-lite-preview
+      vertex_project: your-project-id
+      vertex_location: us-central1
+```
+
+**2. Start proxy**
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+**3. Make requests**
+
+```bash
+curl -X POST http://localhost:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer <YOUR-LITELLM-KEY>" \
+  -d '{
+    "model": "gemini-3.1-flash-lite",
+    "messages": [{"role": "user", "content": "Extract structured data from this text"}],
+    "reasoning_effort": "low"
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+---
+
+## Supported Endpoints
+
+LiteLLM provides **full end-to-end support** for Gemini 3.1 Flash Lite Preview on:
+
+- ✅ `/v1/chat/completions` - OpenAI-compatible chat completions endpoint
+- ✅ `/v1/responses` - OpenAI Responses API endpoint (streaming and non-streaming)
+- ✅ [`/v1/messages`](../../docs/anthropic_unified) - Anthropic-compatible messages endpoint
+- ✅ `/v1/generateContent` – [Google Gemini API](../../docs/generateContent.md) compatible endpoint 
+
+All endpoints support:
+- Streaming and non-streaming responses
+- Function calling with thought signatures
+- Multi-turn conversations
+- All Gemini 3-specific features (thinking levels, thought signatures)
+- Full multimodal support (text, image, audio, video)
+
+---
+
+## `reasoning_effort` Mapping for Gemini 3.1
+
+LiteLLM automatically maps OpenAI's `reasoning_effort` parameter to Gemini's `thinkingLevel`:
+
+| reasoning_effort | thinking_level | Use Case |
+|------------------|----------------|----------|
+| `minimal` | `minimal` | Ultra-fast responses, simple queries |
+| `low` | `low` | Basic instruction following |
+| `medium` | `medium` | Balanced reasoning for moderate complexity |
+| `high` | `high` | Maximum reasoning depth, complex problems |
+| `disable` | `minimal` | Disable extended reasoning |
+| `none` | `minimal` | No extended reasoning |

docs/my-website/blog/gpt_5_3_codex/index.md

@@ -0,0 +1,145 @@
+---
+slug: gpt_5_3_codex
+title: "Day 0 Support: GPT-5.3-Codex"
+date: 2026-02-24T10:00:00
+authors:
+  - name: Sameer Kankute
+    title: SWE @ LiteLLM (LLM Translation)
+    url: https://www.linkedin.com/in/sameer-kankute/
+    image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
+  - name: Krrish Dholakia
+    title: "CEO, LiteLLM"
+    url: https://www.linkedin.com/in/krish-d/
+    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
+  - name: Ishaan Jaff
+    title: "CTO, LiteLLM"
+    url: https://www.linkedin.com/in/reffajnaahsi/
+    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+description: "Day 0 support for GPT-5.3-Codex on LiteLLM, including phase parameter handling for Responses API."
+tags: [openai, gpt-5.3-codex, codex, day 0 support]
+hide_table_of_contents: false
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+LiteLLM now supports GPT-5.3-Codex on Day 0, including support for the new assistant `phase` metadata on Responses API output items.
+
+## Why `phase` matters for GPT-5.3-Codex
+
+`phase` appears on assistant output items and helps distinguish preamble/commentary turns from final closeout responses.
+
+Reference: [Phase parameter docs](https://developers.openai.com/api/reference/overview)
+
+Supported values:
+- `null`
+- `"commentary"`
+- `"final_answer"`
+
+Important:
+- Persist assistant output items with `phase` exactly as returned.
+- Send those assistant items back on the next turn.
+- Do **not** add `phase` to user messages.
+
+## Docker Image
+
+```bash
+docker pull ghcr.io/berriai/litellm:v1.81.12-stable.gpt-5.3
+```
+
+## Usage 
+
+<Tabs>
+<TabItem value="proxy" label="LiteLLM Proxy">
+
+**1. Setup config.yaml**
+
+```yaml
+model_list:
+  - model_name: gpt-5.3-codex
+    litellm_params:
+      model: openai/gpt-5.3-codex
+```
+
+**2. Start the proxy**
+
+```bash
+docker run -d \
+  -p 4000:4000 \
+  -e ANTHROPIC_API_KEY=$OPENAI_API_KEY \
+  -v $(pwd)/config.yaml:/app/config.yaml \
+  ghcr.io/berriai/litellm:v1.81.12-stable.gpt-5.3 \
+  --config /app/config.yaml
+```
+
+
+**3. Test it**
+
+```bash
+curl -X POST "http://0.0.0.0:4000/v1/responses" \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer $LITELLM_KEY" \
+  -d '{
+    "model": "gpt-5.3-codex",
+    "input": "Write a Python script that checks if a number is prime."
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+## Python Example: Persist `phase` with OpenAI Client + LiteLLM Base URL
+
+```python
+from openai import OpenAI
+
+client = OpenAI(
+    base_url="http://0.0.0.0:4000/v1",  # LiteLLM Proxy
+    api_key="your-litellm-api-key",
+)
+
+items = []  # Persist this per conversation/thread
+
+
+def _item_get(item, key, default=None):
+    if isinstance(item, dict):
+        return item.get(key, default)
+    return getattr(item, key, default)
+
+
+def run_turn(user_text: str):
+    global items
+
+    # User message: no phase field
+    items.append(
+        {
+            "type": "message",
+            "role": "user",
+            "content": [{"type": "input_text", "text": user_text}],
+        }
+    )
+
+    resp = client.responses.create(
+        model="gpt-5.3-codex",
+        input=items,
+    )
+
+    # Persist assistant output items verbatim, including phase
+    for out_item in (resp.output or []):
+        items.append(out_item)
+
+    # Optional: inspect latest phase for UI/telemetry routing
+    latest_phase = None
+    for out_item in reversed(resp.output or []):
+        if _item_get(out_item, "type") == "output_item.done" and _item_get(out_item, "phase") is not None:
+            latest_phase = _item_get(out_item, "phase")
+            break
+
+    return resp, latest_phase
+```
+
+## Notes
+
+- Use `/v1/responses` for GPT Codex models.
+- Preserve full assistant output history for best multi-turn behavior.
+- If `phase` metadata is dropped during history reconstruction, output quality can degrade on long-running tasks.

docs/my-website/blog/httpx_cache_eviction_incident/index.md

@@ -0,0 +1,132 @@
+---
+slug: httpx-cache-eviction-incident
+title: "Incident Report: Cache Eviction Closes In-Use httpx Clients"
+date: 2026-02-27T10:00:00
+authors:
+  - name: Ryan Crabbe
+    title: Performance Engineer, LiteLLM
+    url: https://www.linkedin.com/in/ryan-crabbe-0b9687214
+  - name: Ishaan Jaff
+    title: "CTO, LiteLLM"
+    url: https://www.linkedin.com/in/reffajnaahsi/
+    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+  - name: Krrish Dholakia
+    title: "CEO, LiteLLM"
+    url: https://www.linkedin.com/in/krish-d/
+    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
+tags: [incident-report, caching, stability]
+hide_table_of_contents: false
+---
+
+**Date:** February 27, 2026
+**Duration:** ~6 days (Feb 21 merge -> Feb 27 fix)
+**Severity:** High
+**Status:** Resolved
+
+> **Note:** This fix is available starting from LiteLLM `v1.81.14.rc.2` or higher.
+
+## Summary
+
+A change to improve Redis connection pool cleanup introduced a regression that closed **httpx clients** that were still actively being used by the proxy. The `LLMClientCache` (an in-memory TTL cache) stores both Redis clients *and* httpx clients under the same eviction policy. When a cache entry expired or was evicted, the new cleanup code called `aclose()`/`close()` on the evicted value which worked correctly for Redis clients, but destroyed httpx clients that other parts of the system still held references to and were actively using for LLM API calls.
+
+**Impact:** Any proxy instance that hit the cache TTL (default 10 minutes) or capacity limit (200 entries) would have its httpx clients closed out from under it, causing requests to LLM providers to fail with connection errors.
+
+---
+
+## Background
+
+`LLMClientCache` extends `InMemoryCache` and is used to cache SDK clients (OpenAI, Anthropic, etc.) to avoid re-creating them on every request. These clients are keyed by configuration + event loop ID. The cache has:
+
+- **Max size:** 200 entries
+- **Default TTL:** 10 minutes
+
+When the cache is full or entries expire, `InMemoryCache.evict_cache()` calls `_remove_key()` to drop entries.
+
+The cached values are a mix of:
+- **Redis/async Redis clients** — owned exclusively by the cache, safe to close on eviction
+- **httpx-backed SDK clients** (OpenAI, Anthropic, etc.) — shared references, still in use by router/model instances
+
+---
+
+## Root Cause
+
+[PR #21717](https://github.com/BerriAI/litellm/pull/21717) overrode `_remove_key()` in `LLMClientCache` to close async clients on eviction:
+
+<details>
+<summary>Problematic code added in PR #21717</summary>
+
+```python
+class LLMClientCache(InMemoryCache):
+    def _remove_key(self, key: str) -> None:
+        value = self.cache_dict.get(key)
+        super()._remove_key(key)
+        if value is not None:
+            close_fn = getattr(value, "aclose", None) or getattr(value, "close", None)
+            if close_fn and asyncio.iscoroutinefunction(close_fn):
+                try:
+                    asyncio.get_running_loop().create_task(close_fn())
+                except RuntimeError:
+                    pass
+            elif close_fn and callable(close_fn):
+                try:
+                    close_fn()
+                except Exception:
+                    pass
+```
+
+</details>
+
+The intent was correct for Redis clients — prevent connection pool leaks when cached Redis clients expire. But `LLMClientCache` also stores httpx-backed SDK clients (e.g., `AsyncOpenAI`, `AsyncAnthropic`). These clients:
+
+1. Have an `aclose()` method (inherited from httpx)
+2. Are still held by references elsewhere in the codebase (router, model instances)
+3. Were being closed without any check on whether they were still in use
+
+So when the cache evicted an entry, it would call `aclose()` on an httpx client that was still being used for active LLM requests → closed transport → connection errors.
+
+---
+
+## The Fix
+
+[PR #22247](https://github.com/BerriAI/litellm/pull/22247) removed the `_remove_key` override entirely:
+
+<details>
+<summary>The fix (PR #22247)</summary>
+
+```diff
+ class LLMClientCache(InMemoryCache):
+-    def _remove_key(self, key: str) -> None:
+-        """Close async clients before evicting them to prevent connection pool leaks."""
+-        value = self.cache_dict.get(key)
+-        super()._remove_key(key)
+-        if value is not None:
+-            close_fn = getattr(value, "aclose", None) or getattr(
+-                value, "close", None
+-            )
+-            ...
+-
+     def update_cache_key_with_event_loop(self, key):
+```
+
+</details>
+
+The eviction now simply drops the reference and lets Python's GC handle cleanup, which is safe because:
+- httpx clients that are still referenced elsewhere stay alive
+- Unreferenced clients get cleaned up by GC naturally
+
+The other improvements from PR #21717 were kept:
+- **`max_connections` respected for URL-based Redis configs**, previously silently dropped
+- **`disconnect()` now closes both sync and async Redis clients**, sync client was previously leaked
+- **Connection pool passthrough**, when a pool is provided with a URL config, it's used directly instead of creating a duplicate
+
+---
+
+## Remediation
+
+| Action | Status | Code |
+|--------|--------|------|
+| Remove `_remove_key` override that closes shared clients on eviction | ✅ Done | [PR #22247](https://github.com/BerriAI/litellm/pull/22247) |
+| Add e2e test: evicted client still usable (capacity) | ✅ Done | [PR #22313](https://github.com/BerriAI/litellm/pull/22313) |
+| Add e2e test: expired client still usable (TTL) | ✅ Done | [PR #22313](https://github.com/BerriAI/litellm/pull/22313) |
+
+The e2e tests go through `get_async_httpx_client()` the same code path the proxy uses in production and assert the client is still functional after eviction. These run in CI on every PR against `main`. If anyone modifies `LLMClientCache` eviction behavior, overrides `_remove_key`, or adds any form of client cleanup on eviction, these tests will fail regardless of the implementation approach.

docs/my-website/blog/responses_api_encrypted_content_incident/index.md

@@ -0,0 +1,321 @@
+---
+slug: responses-api-encrypted-content-incident
+title: "Incident Report: Encrypted Content Failures in Multi-Region Responses API Load Balancing"
+date: 2026-02-24T10:00:00
+authors:
+  - name: Sameer Kankute
+    title: SWE @ LiteLLM (LLM Translation)
+    url: https://www.linkedin.com/in/sameer-kankute/
+    image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
+  - name: Krrish Dholakia
+    title: "CEO, LiteLLM"
+    url: https://www.linkedin.com/in/krish-d/
+    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
+  - name: Ishaan Jaff
+    title: "CTO, LiteLLM"
+    url: https://www.linkedin.com/in/reffajnaahsi/
+    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+tags: [incident-report, proxy, responses-api, load-balancing]
+hide_table_of_contents: false
+---
+
+**Date:** Feb 24, 2026  
+**Duration:** Ongoing (until fix deployed)  
+**Severity:** High (for users load balancing Responses API across different API keys)  
+**Status:** Resolved
+
+## Summary
+
+When load balancing OpenAI's Responses API across deployments with **different API keys** (e.g., different Azure regions or OpenAI organizations), follow-up requests containing encrypted content items (like `rs_...` reasoning items) would fail with:
+
+```json
+{
+  "error": {
+    "message": "The encrypted content for item rs_0d09d6e56879e76500699d6feee41c8197bd268aae76141f87 could not be verified. Reason: Encrypted content organization_id did not match the target organization.",
+    "type": "invalid_request_error",
+    "code": "invalid_encrypted_content"
+  }
+}
+```
+
+Encrypted content items are cryptographically tied to the API key's organization that created them. When the router load balanced a follow-up request to a deployment with a different API key, decryption failed.
+
+- **Responses API calls with encrypted content:** Complete failure when routed to wrong deployment
+- **Initial requests:** Unaffected — only follow-up requests containing encrypted items failed
+- **Other API endpoints:** No impact — chat completions, embeddings, etc. functioned normally
+
+{/* truncate */}
+
+---
+
+## Background
+
+OpenAI's Responses API can return encrypted "reasoning items" (with IDs like `rs_...`) that contain intermediate reasoning steps. These items are encrypted with the organization's key and can only be decrypted by the same organization's API key.
+
+When load balancing across deployments with different API keys, the existing affinity mechanisms were insufficient:
+
+- **`responses_api_deployment_check`**: Requires `previous_response_id` which some clients (like Codex) don't provide
+- **`deployment_affinity`**: Too broad — pins *all* requests from a user to one deployment, reducing effective quota by the number of users
+- **`session_affinity`**: Requires explicit session IDs and still reduces quota
+
+```mermaid
+flowchart TD
+    A["1. Initial request to Responses API
+    router.aresponses()"] --> B["2. Router load balances to Deployment A
+    (API Key 1, Azure East US)"]
+    B --> C["3. Response contains encrypted item
+    rs_abc123 (encrypted with Org 1 key)"]
+    C --> D["4. Follow-up request includes rs_abc123 in input"]
+    D --> E["5. Router load balances to Deployment B
+    (API Key 2, Azure West Europe)"]
+    E -->|"Different API key"| F["6. ❌ Deployment B cannot decrypt rs_abc123
+    Error: invalid_encrypted_content"]
+    
+    D -.->|"With encrypted_content_affinity"| G["5b. Router detects rs_abc123 was created by Deployment A"]
+    G --> H["6b. ✅ Routes to Deployment A (bypasses rate limits)
+    Request succeeds"]
+
+    style F fill:#f8d7da,stroke:#dc3545
+    style H fill:#d4edda,stroke:#28a745
+    style E fill:#fff3cd,stroke:#ffc107
+    style G fill:#d4edda,stroke:#28a745
+```
+
+---
+
+## Root Cause
+
+LiteLLM's router had no mechanism to track which deployment created specific encrypted content items and route follow-up requests accordingly. The router treated all deployments as interchangeable, leading to decryption failures when encrypted content crossed organizational boundaries.
+
+**The Problem Flow:**
+
+1. User calls `router.aresponses()` with model `gpt-5.1-codex`
+2. Router load balances to Deployment A (Azure East US, API Key 1)
+3. Response contains encrypted reasoning item `rs_abc123` (encrypted with Org 1's key)
+4. User makes follow-up request with `rs_abc123` in the input
+5. Router load balances to Deployment B (Azure West Europe, API Key 2)
+6. Deployment B tries to decrypt `rs_abc123` with Org 2's key → **fails**
+
+**Why Existing Solutions Didn't Work:**
+
+- **`previous_response_id`**: Not provided by all clients (e.g., Codex)
+- **`deployment_affinity`**: Pins *all* user requests to one deployment → reduces quota to 1/N where N = number of deployments
+- **`session_affinity`**: Requires explicit session management and still reduces quota
+
+**Timeline:**
+
+1. Users configured multi-region Responses API load balancing with different API keys
+2. Initial requests succeeded, but follow-up requests with encrypted content failed intermittently
+3. Error rate correlated with number of deployments (more deployments = higher chance of routing to wrong one)
+4. Investigation revealed encrypted content was organization-bound
+5. Existing affinity mechanisms deemed unsuitable (quota reduction, missing `previous_response_id`)
+6. New solution designed and implemented: `encrypted_content_affinity`
+
+---
+
+## The Fix
+
+Implemented a new `encrypted_content_affinity` pre-call check that intelligently tracks encrypted content and routes follow-up requests **only when necessary**.
+
+### Implementation
+
+**1. Encoding `model_id` into output items** ([`responses/utils.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/responses/utils.py))
+
+The same approach used for `previous_response_id` affinity — no cache needed. When a response contains output items with `encrypted_content`, LiteLLM encodes the originating deployment's `model_id` in **two places** for redundancy:
+
+1. **Into the item ID** (if present): `rs_abc123` → `encitem_{base64("litellm:model_id:{model_id};item_id:rs_abc123")}`
+2. **Into the encrypted_content itself**: Wraps the content with `litellm_enc:{base64("model_id:{model_id}")};{original_encrypted_content}`
+
+```python
+# Encoding item IDs (when present)
+def _build_encrypted_item_id(model_id: str, item_id: str) -> str:
+    assembled = f"litellm:model_id:{model_id};item_id:{item_id}"
+    encoded = base64.b64encode(assembled.encode("utf-8")).decode("utf-8")
+    return f"encitem_{encoded}"
+
+# Wrapping encrypted_content (always, for redundancy)
+def _wrap_encrypted_content_with_model_id(encrypted_content: str, model_id: str) -> str:
+    metadata = f"model_id:{model_id}"
+    encoded_metadata = base64.b64encode(metadata.encode("utf-8")).decode("utf-8")
+    return f"litellm_enc:{encoded_metadata};{encrypted_content}"
+```
+
+**Why wrap encrypted_content directly?** Some clients (like Codex) don't consistently send item IDs in follow-up requests, but they always send the `encrypted_content` itself. By embedding `model_id` into the content, affinity works even when IDs are missing.
+
+**Streaming responses:** The wrapping logic is applied to both:
+- Final response objects (non-streaming)
+- Individual streaming events (`response.output_item.added`, `response.output_item.done`)
+
+This ensures clients receiving streaming responses get wrapped content they can send back.
+
+Before forwarding to the upstream provider, LiteLLM restores the original item IDs and unwraps encrypted_content so the provider never sees the encoded form:
+
+```python
+# In responses/main.py — before calling the handler
+input = ResponsesAPIRequestUtils._restore_encrypted_content_item_ids_in_input(input)
+```
+
+**2. `EncryptedContentAffinityCheck` — routing only** ([`encrypted_content_affinity_check.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py))
+
+No `async_log_success_event` or cache lookups — the `model_id` is decoded directly from the item ID or encrypted_content:
+
+```python
+class EncryptedContentAffinityCheck(CustomLogger):
+    async def async_filter_deployments(self, model, healthy_deployments, ...):
+        """Extract model_id from input items (ID or encrypted_content) and pin to that deployment."""
+        for item in request_kwargs.get("input", []):
+            # Try to extract model_id from two sources:
+            model_id = self._extract_model_id_from_input(item)
+            
+            if model_id:
+                deployment = self._find_deployment_by_model_id(
+                    healthy_deployments, model_id
+                )
+                if deployment:
+                    request_kwargs["_encrypted_content_affinity_pinned"] = True
+                    return [deployment]
+        return healthy_deployments
+    
+    def _extract_model_id_from_input(self, item: dict) -> Optional[str]:
+        """Extract model_id from either encoded ID or wrapped encrypted_content."""
+        # 1. Try decoding from item ID (if present)
+        item_id = item.get("id", "")
+        if item_id:
+            decoded = ResponsesAPIRequestUtils._decode_encrypted_item_id(item_id)
+            if decoded:
+                return decoded["model_id"]
+        
+        # 2. Try unwrapping from encrypted_content (fallback for clients that omit IDs)
+        encrypted_content = item.get("encrypted_content", "")
+        if encrypted_content and encrypted_content.startswith("litellm_enc:"):
+            model_id, _ = ResponsesAPIRequestUtils._unwrap_encrypted_content_with_model_id(
+                encrypted_content
+            )
+            return model_id
+        
+        return None
+```
+
+**3. Rate Limit Bypass** ([`router.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/router.py))
+
+When encrypted content requires a specific deployment, RPM/TPM limits are bypassed (the request would fail on any other deployment anyway):
+
+```python
+# In async_get_available_deployment, after filtering healthy deployments:
+if (
+    request_kwargs.get("_encrypted_content_affinity_pinned")
+    and len(healthy_deployments) == 1
+):
+    return healthy_deployments[0]  # Bypass routing strategy (RPM/TPM checks)
+```
+
+**3. Configuration**
+
+```yaml
+router_settings:
+  routing_strategy: usage-based-routing-v2
+  enable_pre_call_checks: true
+  optional_pre_call_checks:
+    - encrypted_content_affinity
+  deployment_affinity_ttl_seconds: 86400  # 24 hours
+```
+
+### Key Benefits
+
+✅ **No quota reduction**: Only pins requests containing encrypted items  
+✅ **Bypasses rate limits**: When encrypted content requires a specific deployment, RPM/TPM limits don't block it  
+✅ **No `previous_response_id` required**: Works by encoding `model_id` directly into the item ID  
+✅ **No cache required**: `model_id` is decoded on-the-fly from the item ID — no Redis, no TTL  
+✅ **Globally safe**: Can be enabled for all models; non-Responses-API calls are unaffected  
+✅ **Surgical precision**: Normal requests continue to load balance freely
+
+---
+
+## Remediation
+
+| # | Action | Status | Code |
+|---|---|---|---|
+| 1 | Encode `model_id` into encrypted-content item IDs on response | ✅ Done | [`responses/utils.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/responses/utils.py) |
+| 2 | Restore original item IDs before forwarding to upstream provider | ✅ Done | [`responses/main.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/responses/main.py) |
+| 3 | `EncryptedContentAffinityCheck`: decode item IDs to route (no cache) | ✅ Done | [`encrypted_content_affinity_check.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py) |
+| 4 | Add `encrypted_content_affinity` to `OptionalPreCallChecks` type | ✅ Done | [`types/router.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/types/router.py) |
+| 5 | Implement rate limit bypass for affinity-pinned requests | ✅ Done | [`router.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/router.py) |
+| 6 | Unit tests: encoding/decoding utilities, routing, RPM bypass | ✅ Done | [`test_encrypted_content_affinity_check.py`](https://github.com/BerriAI/litellm/blob/main/litellm/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py) |
+| 7 | Documentation: Responses API guide, load balancing guide, config reference | ✅ Done | [Docs](https://docs.litellm.ai/docs/response_api#encrypted-content-affinity-multi-region-load-balancing) |
+| 8 | **[Mar 3]** Fix streaming events to wrap encrypted_content | ✅ Done | [`responses/streaming_iterator.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/responses/streaming_iterator.py) |
+
+---
+
+## Follow-up Fix: Streaming Responses (Mar 3, 2026)
+
+### The Issue
+
+After the initial fix was deployed, users reported that the `invalid_encrypted_content` error **still occurred** when using streaming responses with clients like Codex. Investigation revealed:
+
+- ✅ Non-streaming responses: `encrypted_content` was correctly wrapped with `litellm_enc:` prefix
+- ❌ Streaming responses: Individual `response.output_item.added` and `response.output_item.done` events contained **raw, unwrapped** `encrypted_content`
+
+Since Codex and other clients consume responses as streams, they received unwrapped content in these events and sent it back in follow-up requests, causing the affinity check to fail.
+
+### The Root Cause
+
+The `_update_encrypted_content_item_ids_in_response` function only modified the **final** response object, which is used for non-streaming responses. For streaming responses, individual chunks are processed by `ResponsesAPIStreamingIterator._process_chunk`, which was **not** applying the wrapping logic to streaming events.
+
+### The Fix
+
+Modified `litellm/litellm/responses/streaming_iterator.py` to wrap `encrypted_content` in streaming events:
+
+```python
+# In ResponsesAPIStreamingIterator._process_chunk
+if (
+    self.litellm_metadata
+    and self.litellm_metadata.get("encrypted_content_affinity_enabled")
+):
+    event_type = getattr(openai_responses_api_chunk, "type", None)
+    if event_type in (
+        ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED,
+        ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
+    ):
+        item = getattr(openai_responses_api_chunk, "item", None)
+        if item:
+            encrypted_content = getattr(item, "encrypted_content", None)
+            if encrypted_content and isinstance(encrypted_content, str):
+                model_id = (
+                    self.litellm_metadata.get("model_info", {}).get("id")
+                    if self.litellm_metadata
+                    else None
+                )
+                if model_id:
+                    wrapped_content = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id(
+                        encrypted_content, model_id
+                    )
+                    setattr(item, "encrypted_content", wrapped_content)
+```
+
+This ensures that **all** `encrypted_content` sent to clients (streaming or non-streaming) is wrapped with `model_id` metadata, enabling consistent affinity routing.
+
+---
+
+## Migration Guide
+
+### Before (Using `deployment_affinity`)
+
+```yaml
+router_settings:
+  optional_pre_call_checks:
+    - deployment_affinity  # ❌ Reduces quota by number of users
+```
+
+**Problem:** All requests from a user pin to one deployment, reducing effective quota to 1/N.
+
+### After (Using `encrypted_content_affinity`)
+
+```yaml
+router_settings:
+  optional_pre_call_checks:
+    - encrypted_content_affinity  # ✅ Only pins requests with encrypted content
+```
+
+**Benefit:** Normal requests load balance freely, only encrypted content requests pin when necessary.
+
+---

docs/my-website/blog/server_root_path/index.md

@@ -0,0 +1,154 @@
+---
+slug: server-root-path-incident
+title: "Incident Report: SERVER_ROOT_PATH regression broke UI routing"
+date: 2026-02-21T10:00:00
+authors:
+  - name: Yuneng Jiang
+    title: SWE @ LiteLLM (Full Stack)
+    url: https://www.linkedin.com/in/yunengjiang/
+  - name: Ishaan Jaff
+    title: "CTO, LiteLLM"
+    url: https://www.linkedin.com/in/reffajnaahsi/
+    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+  - name: Krrish Dholakia
+    title: "CEO, LiteLLM"
+    url: https://www.linkedin.com/in/krish-d/
+    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
+tags: [incident-report, ui, stability]
+hide_table_of_contents: false
+---
+
+**Date:** January 22, 2026
+**Duration:** ~4 days (until fix merged January 26, 2026)
+**Severity:** High
+**Status:** Resolved
+
+> **Note:** This fix is available starting from LiteLLM `v1.81.3.rc.6` or higher.
+
+## Summary
+
+A PR ([`#19467`](https://github.com/BerriAI/litellm/pull/19467)) accidentally removed the `root_path=server_root_path` parameter from the FastAPI app initialization in `proxy_server.py`. This caused the proxy to ignore the `SERVER_ROOT_PATH` environment variable when serving the UI. Users who deploy LiteLLM behind a reverse proxy with a path prefix (e.g., `/api/v1` or `/llmproxy`) found that all UI pages returned 404 Not Found.
+
+- **LLM API calls:** No impact. API routing was unaffected.
+- **UI pages:** All UI pages returned 404 for deployments using `SERVER_ROOT_PATH`.
+- **Swagger/OpenAPI docs:** Broken when accessed through the configured root path.
+
+{/* truncate */}
+
+---
+
+## Background
+
+Many LiteLLM deployments run behind a reverse proxy (e.g., Nginx, Traefik, AWS ALB) that routes traffic to LiteLLM under a path prefix. FastAPI's `root_path` parameter tells the application about this prefix so it can correctly serve static files, generate URLs, and handle routing.
+
+```mermaid
+sequenceDiagram
+    participant User as User Browser
+    participant RP as Reverse Proxy
+    participant LP as LiteLLM Proxy
+
+    User->>RP: GET /llmproxy/ui/
+    RP->>LP: GET /ui/ (X-Forwarded-Prefix: /llmproxy)
+
+    Note over LP: Before regression:<br/>FastAPI root_path="/llmproxy"<br/>→ Serves UI correctly
+
+    Note over LP: After regression:<br/>FastAPI root_path=""<br/>→ UI assets resolve to wrong paths<br/>→ 404 Not Found
+```
+
+The `root_path` parameter was present in `proxy_server.py` since early versions of LiteLLM. It was removed as a side effect of PR [#19467](https://github.com/BerriAI/litellm/pull/19467), which was intended to fix a different UI 404 issue.
+
+---
+
+## Root cause
+
+PR [#19467](https://github.com/BerriAI/litellm/pull/19467) (`73d49f8`) removed the `root_path=server_root_path` line from the `FastAPI()` constructor in `proxy_server.py`:
+
+```diff
+ app = FastAPI(
+     docs_url=_get_docs_url(),
+     redoc_url=_get_redoc_url(),
+     title=_title,
+     description=_description,
+     version=version,
+-    root_path=server_root_path,
+     lifespan=proxy_startup_event,
+ )
+```
+
+Without `root_path`, FastAPI treated all requests as if the application was mounted at `/`, causing path mismatches for any deployment using `SERVER_ROOT_PATH`.
+
+The regression went undetected because:
+
+1. **No automated test** verified that `root_path` was set on the FastAPI app.
+2. **No manual test procedure** existed for `SERVER_ROOT_PATH` functionality.
+3. **Default deployments** (without `SERVER_ROOT_PATH`) were unaffected, so most CI tests passed.
+
+---
+
+## Remediation
+
+| #   | Action                                                                                            | Status  | Code                                                                                                                       |
+| --- | ------------------------------------------------------------------------------------------------- | ------- | -------------------------------------------------------------------------------------------------------------------------- |
+| 1   | Restore `root_path=server_root_path` in FastAPI app initialization                                | ✅ Done | [`#19790`](https://github.com/BerriAI/litellm/pull/19790) (`5426b3c`)                                                      |
+| 2   | Add unit tests for `get_server_root_path()` and FastAPI app initialization                        | ✅ Done | [`test_server_root_path.py`](https://github.com/BerriAI/litellm/blob/main/tests/proxy_unit_tests/test_server_root_path.py) |
+| 3   | Add CI workflow that builds Docker image and tests UI routing with `SERVER_ROOT_PATH` on every PR | ✅ Done | [`test_server_root_path.yml`](https://github.com/BerriAI/litellm/blob/main/.github/workflows/test_server_root_path.yml)    |
+| 4   | Document manual test procedure for `SERVER_ROOT_PATH`                                             | ✅ Done | [Discussion #8495](https://github.com/BerriAI/litellm/discussions/8495)                                                    |
+
+---
+
+## CI workflow details
+
+The new [`test_server_root_path.yml`](https://github.com/BerriAI/litellm/blob/main/.github/workflows/test_server_root_path.yml) workflow runs on every PR against `main`. It:
+
+1. Builds the LiteLLM Docker image
+2. Starts a container with `SERVER_ROOT_PATH` set (tests both `/api/v1` and `/llmproxy`)
+3. Verifies the UI returns valid HTML at `{ROOT_PATH}/ui/`
+4. Fails the workflow if the UI is unreachable
+
+```mermaid
+flowchart TD
+    A["PR opened/updated"] --> B["Build Docker image"]
+    B --> C["Start container with SERVER_ROOT_PATH=/api/v1"]
+    B --> D["Start container with SERVER_ROOT_PATH=/llmproxy"]
+    C --> E["curl {ROOT_PATH}/ui/ → expect HTML"]
+    D --> F["curl {ROOT_PATH}/ui/ → expect HTML"]
+    E -->|"HTML found"| G["✅ Pass"]
+    E -->|"404 or no HTML"| H["❌ Fail Workflow"]
+    F -->|"HTML found"| G
+    F -->|"404 or no HTML"| H
+
+    style G fill:#d4edda,stroke:#28a745
+    style H fill:#f8d7da,stroke:#dc3545
+```
+
+This prevents future regressions where changes to `proxy_server.py` accidentally break `SERVER_ROOT_PATH` support.
+
+---
+
+## Timeline
+
+| Time (UTC)         | Event                                                                                                                                                        |
+| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| Jan 22, 2026 04:20 | PR [#19467](https://github.com/BerriAI/litellm/pull/19467) merged, removing `root_path=server_root_path`                                                     |
+| Jan 22–26          | Users on nightly builds report UI 404 errors when using `SERVER_ROOT_PATH`                                                                                   |
+| Jan 26, 2026 17:48 | Fix PR [#19790](https://github.com/BerriAI/litellm/pull/19790) merged, restoring `root_path=server_root_path`                                                |
+| Feb 18, 2026       | CI workflow [`test_server_root_path.yml`](https://github.com/BerriAI/litellm/blob/main/.github/workflows/test_server_root_path.yml) added to run on every PR |
+
+---
+
+## Resolution steps for users
+
+For users still experiencing issues, update to the latest LiteLLM version:
+
+```bash
+pip install --upgrade litellm
+```
+
+Verify your `SERVER_ROOT_PATH` is correctly set:
+
+```bash
+# In your environment or docker-compose.yml
+SERVER_ROOT_PATH="/your-prefix"
+```
+
+Then confirm the UI is accessible at `http://your-host:4000/your-prefix/ui/`.

docs/my-website/blog/vllm_embeddings_incident/index.md

@@ -0,0 +1,117 @@
+---
+slug: vllm-embeddings-incident
+title: "Incident Report: vLLM Embeddings Broken by encoding_format Parameter"
+date: 2026-02-18T10:00:00
+authors:
+  - name: Sameer Kankute
+    title: SWE @ LiteLLM (LLM Translation)
+    url: https://www.linkedin.com/in/sameer-kankute/
+    image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
+  - name: Krrish Dholakia
+    title: "CEO, LiteLLM"
+    url: https://www.linkedin.com/in/krish-d/
+    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
+  - name: Ishaan Jaff
+    title: "CTO, LiteLLM"
+    url: https://www.linkedin.com/in/reffajnaahsi/
+    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+tags: [incident-report, embeddings, vllm]
+hide_table_of_contents: false
+---
+
+**Date:** Feb 16, 2026
+**Duration:** ~3 hours
+**Severity:** High (for vLLM embedding users)
+**Status:** Resolved
+
+## Summary
+
+A commit ([`dbcae4a`](https://github.com/BerriAI/litellm/commit/dbcae4aca5836770d0e9cd43abab0333c3d61ab2)) intended to fix OpenAI SDK behavior broke vLLM embeddings by explicitly passing `encoding_format=None` in API requests. vLLM rejects this with error: `"unknown variant \`\`, expected float or base64"`.
+
+- **vLLM embedding calls:** Complete failure - all requests rejected
+- **Other providers:** No impact - OpenAI and other providers functioned normally
+- **Other vLLM functionality:** No impact - only embeddings were affected
+
+{/* truncate */}
+
+---
+
+## Background
+
+The `encoding_format` parameter for embeddings specifies whether vectors should be returned as `float` arrays or `base64` encoded strings. Different providers have different expectations:
+
+- **OpenAI SDK:** If `encoding_format` is omitted, the SDK adds a default value of `"float"`
+- **vLLM:** Strictly validates `encoding_format` - only accepts `"float"`, `"base64"`, or complete omission. Rejects `None` or empty string values.
+
+```mermaid
+flowchart TD
+    A["1. User calls litellm.embedding()
+    litellm/main.py"] --> B["2. Transform request for provider
+    litellm/llms/openai_like/embedding/handler.py"]
+    B --> C["3. Send request to vLLM endpoint"]
+    C -->|"encoding_format omitted"| D["4a. ✅ vLLM processes request"]
+    C -->|"encoding_format='float' or 'base64'"| D
+    C -->|"encoding_format=None or ''"| E["4b. ❌ vLLM rejects with error:
+    'unknown variant, expected float or base64'"]
+
+    style D fill:#d4edda,stroke:#28a745
+    style E fill:#f8d7da,stroke:#dc3545
+    style B fill:#fff3cd,stroke:#ffc107
+```
+
+---
+
+## Root cause
+
+A well-intentioned fix for OpenAI SDK behavior inadvertently broke vLLM embeddings:
+
+**The Breaking Change ([`dbcae4a`](https://github.com/BerriAI/litellm/commit/dbcae4aca5836770d0e9cd43abab0333c3d61ab2)):**
+
+In `litellm/main.py`, the code was changed to explicitly set `encoding_format=None` instead of omitting it:
+
+```python
+# Added in dbcae4a
+if encoding_format is not None:
+    optional_params["encoding_format"] = encoding_format
+else:
+    # Omitting causes openai sdk to add default value of "float"
+    optional_params["encoding_format"] = None
+```
+
+This fix worked correctly for OpenAI - explicitly passing `None` prevented the SDK from adding its default value. However, vLLM's strict parameter validation rejected `None` values, causing all embedding requests to fail.
+
+---
+
+## The Fix
+
+Fix deployed ([`55348dd`](https://github.com/BerriAI/litellm/commit/55348dd9c51b5b028f676d25ad023b8f052fc071)). The solution filters out `None` and empty string values from `optional_params` before sending requests to OpenAI-like providers (including vLLM).
+
+**In `litellm/llms/openai_like/embedding/handler.py`:**
+
+```python
+# Before (broken)
+data = {"model": model, "input": input, **optional_params}
+
+# After (fixed)
+filtered_optional_params = {k: v for k, v in optional_params.items() if v not in (None, '')}
+data = {"model": model, "input": input, **filtered_optional_params}
+```
+
+This ensures:
+- Valid values (`"float"`, `"base64"`) are preserved and sent
+- `None` and empty string values are filtered out (parameter omitted entirely)
+- OpenAI SDK no longer adds defaults because liteLLM handles the parameter upstream
+
+---
+
+## Remediation
+
+| # | Action | Status | Code |
+|---|---|---|---|
+| 1 | Filter `None` and empty string values in OpenAI-like embedding handler | ✅ Done | [`handler.py#L108`](https://github.com/BerriAI/litellm/blob/main/litellm/llms/openai_like/embedding/handler.py#L108) |
+| 2 | Unit tests for parameter filtering (None, empty string, valid values) | ✅ Done | [`test_openai_like_embedding.py`](https://github.com/BerriAI/litellm/blob/main/tests/test_litellm/llms/openai_like/embedding/test_openai_like_embedding.py) |
+| 3 | Transformation tests for hosted_vllm embedding config | ✅ Done | [`test_hosted_vllm_embedding_transformation.py`](https://github.com/BerriAI/litellm/blob/main/tests/test_litellm/llms/hosted_vllm/embedding/test_hosted_vllm_embedding_transformation.py) |
+| 4 | E2E tests with actual vLLM endpoint | ✅ Done | [`test_hosted_vllm_embedding_e2e.py`](https://github.com/BerriAI/litellm/blob/main/tests/test_litellm/llms/hosted_vllm/embedding/test_hosted_vllm_embedding_e2e.py) |
+| 5 | Validate JSON payload structure matches vLLM expectations | ✅ Done | Tests verify exact JSON sent to endpoint |
+
+---

docs/my-website/docs/adding_provider/generic_guardrail_api.md

@@ -237,12 +237,42 @@ litellm_settings:
         mode: pre_call  # or post_call, during_call
         api_base: https://your-guardrail-api.com
         api_key: os.environ/YOUR_GUARDRAIL_API_KEY  # optional
+        unreachable_fallback: fail_closed  # default: fail_closed. Set to fail_open to proceed if the guardrail endpoint is unreachable (network errors, or HTTP 502/503/504 from an upstream proxy/LB).
         additional_provider_specific_params:
           # your custom parameters
           threshold: 0.8
           language: "en"
 ```
 
+### Static and dynamic headers
+
+You can send two kinds of headers to your guardrail endpoint:
+
+- **Static headers** (`headers`): A key/value map sent with **every** request to your guardrail. Use this for fixed values (e.g. API keys, `X-Service-Name`). Configure in `litellm_params`:
+
+  ```yaml
+  litellm_params:
+    guardrail: generic_guardrail_api
+    api_base: https://your-guardrail-api.com
+    headers:
+      X-Service-Name: "my-app"
+      X-API-Key: "secret"
+  ```
+
+- **Dynamic headers** (`extra_headers`): A list of **header names** that are forwarded from the **client request** to your guardrail. Only headers in this list (plus a small default allowlist such as `x-litellm-*`) have their values sent; others are sent as `[present]`. Use this to pass through client-provided headers (e.g. `x-request-id`, `x-correlation-id`). Configure in `litellm_params`:
+
+  ```yaml
+  litellm_params:
+    guardrail: generic_guardrail_api
+    api_base: https://your-guardrail-api.com
+    extra_headers:
+      - x-request-id
+      - x-correlation-id
+      - x-custom-auth
+  ```
+
+This mirrors the [MCP static and extra headers](/docs/mcp#forwarding-custom-headers-to-mcp-servers) behavior.
+
 ### Example: Pillar Security
 
 [Pillar Security](https://pillar.security) uses the Generic Guardrail API to provide comprehensive AI security scanning including prompt injection protection, PII/PCI detection, secret detection, and content moderation.

docs/my-website/docs/adding_provider/generic_prompt_management_api.md

@@ -0,0 +1,576 @@
+# [BETA] Generic Prompt Management API - Integrate Without a PR
+
+## The Problem
+
+As a prompt management provider, integrating with LiteLLM traditionally requires:
+- Making a PR to the LiteLLM repository
+- Waiting for review and merge
+- Maintaining provider-specific code in LiteLLM's codebase
+- Updating the integration for changes to your API
+
+## The Solution
+
+The **Generic Prompt Management API** lets you integrate with LiteLLM **instantly** by implementing a simple API endpoint. No PR required.
+
+### Key Benefits
+
+1. **No PR Needed** - Deploy and integrate immediately
+3. **Simple Contract** - One GET endpoint, standard JSON response
+4. **Variable Substitution** - Support for prompt variables with `{variable}` syntax
+5. **Custom Parameters** - Pass provider-specific query params via config
+6. **Full Control** - You own and maintain your prompt management API
+7. **Model & Parameters Override** - Optionally override model and parameters from your prompts
+
+## Get Started in 3 Steps
+
+### Step 1: Configure LiteLLM
+
+Add to your `config.yaml`:
+
+```yaml
+prompts:
+  - prompt_id: "simple_prompt"
+    litellm_params:
+      prompt_integration: "generic_prompt_management"
+      api_base: http://localhost:8080
+      api_key: os.environ/YOUR_API_KEY
+```
+
+### Step 2: Implement Your API Endpoint
+
+```python
+from fastapi import FastAPI
+from pydantic import BaseModel
+
+app = FastAPI()
+
+@app.get("/beta/litellm_prompt_management")
+async def get_prompt(prompt_id: str):
+    return {
+        "prompt_id": prompt_id,
+        "prompt_template": [
+            {"role": "system", "content": "You are a helpful assistant."},
+            {"role": "user", "content": "Help me with {task}"}
+        ],
+        "prompt_template_model": "gpt-4",
+        "prompt_template_optional_params": {"temperature": 0.7}
+    }
+```
+
+### Step 3: Use in Your App
+
+```python
+from litellm import completion
+
+response = completion(
+    model="gpt-4",
+    prompt_id="simple_prompt",
+    prompt_variables={"task": "data analysis"},
+    messages=[{"role": "user", "content": "I have sales data"}]
+)
+```
+
+That's it! LiteLLM fetches your prompt, applies variables, and makes the request
+
+## API Contract
+
+### Endpoint
+
+Implement `GET /beta/litellm_prompt_management`
+
+### Request Format
+
+Your endpoint will receive a GET request with query parameters:
+
+```
+GET /beta/litellm_prompt_management?prompt_id={prompt_id}&{custom_params}
+```
+
+**Query Parameters:**
+- `prompt_id` (required): The ID of the prompt to fetch
+- Custom parameters: Any additional parameters you configured in `provider_specific_query_params`
+
+**Example:**
+```
+GET /beta/litellm_prompt_management?prompt_id=hello-world-prompt-2bac&project_name=litellm&slug=hello-world-prompt-2bac
+```
+
+### Response Format
+
+```json
+{
+  "prompt_id": "hello-world-prompt-2bac",
+  "prompt_template": [
+    {
+      "role": "system",
+      "content": "You are a helpful assistant specialized in {domain}."
+    },
+    {
+      "role": "user",
+      "content": "Help me with {task}"
+    }
+  ],
+  "prompt_template_model": "gpt-4",
+  "prompt_template_optional_params": {
+    "temperature": 0.7,
+    "max_tokens": 500,
+    "top_p": 0.9
+  }
+}
+```
+
+**Response Fields:**
+- `prompt_id` (string, required): The ID of the prompt
+- `prompt_template` (array, required): Array of OpenAI-format messages with optional `{variable}` placeholders
+- `prompt_template_model` (string, optional): Model to use for this prompt (overrides client model unless `ignore_prompt_manager_model: true`)
+- `prompt_template_optional_params` (object, optional): Additional parameters like temperature, max_tokens, etc. (merged with client params unless `ignore_prompt_manager_optional_params: true`)
+
+## LiteLLM Configuration
+
+Add to `config.yaml`:
+
+```yaml
+model_list:
+  - model_name: gpt-3.5-turbo
+    litellm_params:
+      model: openai/gpt-3.5-turbo
+      api_key: os.environ/OPENAI_API_KEY
+
+prompts:
+  - prompt_id: "simple_prompt"
+    litellm_params:
+      prompt_integration: "generic_prompt_management"
+      provider_specific_query_params:
+        project_name: litellm
+        slug: hello-world-prompt-2bac
+      api_base: http://localhost:8080
+      api_key: os.environ/YOUR_PROMPT_API_KEY  # optional
+      ignore_prompt_manager_model: true  # optional, keep client's model
+      ignore_prompt_manager_optional_params: true  # optional, don't merge prompt manager's params (e.g. temperature, max_tokens, etc.)
+```
+
+### Configuration Parameters
+
+- `prompt_integration`: Must be `"generic_prompt_management"`
+- `provider_specific_query_params`: Custom query parameters sent to your API (optional)
+- `api_base`: Base URL of your prompt management API
+- `api_key`: Optional API key for authentication (sent as `Bearer` token)
+- `ignore_prompt_manager_model`: If `true`, use the model specified by client instead of prompt's model (default: `false`)
+- `ignore_prompt_manager_optional_params`: If `true`, don't merge prompt's optional params with client params (default: `false`)
+
+## Usage
+
+### Using with LiteLLM SDK
+
+**Basic usage with prompt ID:**
+
+```python
+from litellm import completion
+
+response = completion(
+    model="gpt-4",
+    prompt_id="simple_prompt",
+    messages=[{"role": "user", "content": "Additional message"}]
+)
+```
+
+**With prompt variables:**
+
+```python
+response = completion(
+    model="gpt-4",
+    prompt_id="simple_prompt",
+    prompt_variables={
+        "domain": "data science",
+        "task": "analyzing customer churn"
+    },
+    messages=[{"role": "user", "content": "Please provide a detailed analysis"}]
+)
+```
+
+The prompt template will have `{domain}` replaced with "data science" and `{task}` replaced with "analyzing customer churn".
+
+### Using with LiteLLM Proxy
+
+**1. Start the proxy with your config:**
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+**2. Make requests with prompt_id:**
+
+```bash
+curl http://0.0.0.0:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer sk-1234" \
+  -d '{
+    "model": "gpt-4",
+    "prompt_id": "simple_prompt",
+    "prompt_variables": {
+      "domain": "healthcare",
+      "task": "patient risk assessment"
+    },
+    "messages": [
+      {"role": "user", "content": "Analyze the following data..."}
+    ]
+  }'
+```
+
+**3. Using with OpenAI SDK:**
+
+```python
+from openai import OpenAI
+
+client = OpenAI(
+    base_url="http://0.0.0.0:4000",
+    api_key="sk-1234"
+)
+
+response = client.chat.completions.create(
+    model="gpt-4",
+    messages=[
+        {"role": "user", "content": "Analyze the data"}
+    ],
+    extra_body={
+        "prompt_id": "simple_prompt",
+        "prompt_variables": {
+            "domain": "finance",
+            "task": "fraud detection"
+        }
+    }
+)
+```
+
+## Implementation Example
+
+See [mock_prompt_management_server.py](https://github.com/BerriAI/litellm/blob/main/cookbook/mock_prompt_management_server/mock_prompt_management_server.py) for a complete reference implementation with multiple example prompts, authentication, and convenience endpoints.
+
+**Minimal FastAPI example:**
+
+```python
+from fastapi import FastAPI, HTTPException, Header
+from typing import Optional, Dict, Any, List
+from pydantic import BaseModel
+
+app = FastAPI()
+
+# In-memory prompt storage (replace with your database)
+PROMPTS = {
+    "hello-world-prompt": {
+        "prompt_id": "hello-world-prompt",
+        "prompt_template": [
+            {
+                "role": "system",
+                "content": "You are a helpful assistant specialized in {domain}."
+            },
+            {
+                "role": "user", 
+                "content": "Help me with: {task}"
+            }
+        ],
+        "prompt_template_model": "gpt-4",
+        "prompt_template_optional_params": {
+            "temperature": 0.7,
+            "max_tokens": 500
+        }
+    },
+    "code-review-prompt": {
+        "prompt_id": "code-review-prompt",
+        "prompt_template": [
+            {
+                "role": "system",
+                "content": "You are an expert code reviewer. Review code for {language}."
+            },
+            {
+                "role": "user",
+                "content": "Review the following code:\n\n{code}"
+            }
+        ],
+        "prompt_template_model": "gpt-4-turbo",
+        "prompt_template_optional_params": {
+            "temperature": 0.3,
+            "max_tokens": 1000
+        }
+    }
+}
+
+class PromptResponse(BaseModel):
+    prompt_id: str
+    prompt_template: List[Dict[str, str]]
+    prompt_template_model: Optional[str] = None
+    prompt_template_optional_params: Optional[Dict[str, Any]] = None
+
+@app.get("/beta/litellm_prompt_management", response_model=PromptResponse)
+async def get_prompt(
+    prompt_id: str,
+    authorization: Optional[str] = Header(None),
+    project_name: Optional[str] = None,
+    slug: Optional[str] = None,
+):
+    """
+    Get a prompt by ID with optional filtering by project_name and slug.
+    
+    Args:
+        prompt_id: The ID of the prompt to fetch
+        authorization: Optional Bearer token for authentication
+        project_name: Optional project name filter
+        slug: Optional slug filter
+    """
+    
+    # Optional: Validate authorization
+    if authorization:
+        token = authorization.replace("Bearer ", "")
+        # Validate your token here
+        if not is_valid_token(token):
+            raise HTTPException(status_code=401, detail="Invalid API key")
+    
+    # Optional: Apply additional filtering based on custom params
+    if project_name or slug:
+        # You can use these parameters to filter or validate access
+        # For example, check if the user has access to this project
+        pass
+    
+    # Fetch the prompt from your storage
+    if prompt_id not in PROMPTS:
+        raise HTTPException(
+            status_code=404,
+            detail=f"Prompt '{prompt_id}' not found"
+        )
+    
+    prompt_data = PROMPTS[prompt_id]
+    
+    return PromptResponse(**prompt_data)
+
+def is_valid_token(token: str) -> bool:
+    """Validate API token - implement your logic here"""
+    # Example: Check against your database or secret store
+    valid_tokens = ["your-secret-token", "another-valid-token"]
+    return token in valid_tokens
+
+# Optional: Health check endpoint
+@app.get("/health")
+async def health_check():
+    return {"status": "healthy"}
+
+# Optional: List all prompts endpoint
+@app.get("/prompts")
+async def list_prompts(authorization: Optional[str] = Header(None)):
+    """List all available prompts"""
+    if authorization:
+        token = authorization.replace("Bearer ", "")
+        if not is_valid_token(token):
+            raise HTTPException(status_code=401, detail="Invalid API key")
+    
+    return {
+        "prompts": [
+            {"prompt_id": pid, "model": p.get("prompt_template_model")}
+            for pid, p in PROMPTS.items()
+        ]
+    }
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=8080)
+```
+
+### Running the Example Server
+
+1. Install dependencies:
+```bash
+pip install fastapi uvicorn
+```
+
+2. Save the code above to `prompt_server.py`
+
+3. Run the server:
+```bash
+python prompt_server.py
+```
+
+4. Test the endpoint:
+```bash
+curl "http://localhost:8080/beta/litellm_prompt_management?prompt_id=hello-world-prompt&project_name=litellm&slug=hello-world-prompt-2bac"
+```
+
+Expected response:
+```json
+{
+  "prompt_id": "hello-world-prompt",
+  "prompt_template": [
+    {
+      "role": "system",
+      "content": "You are a helpful assistant specialized in {domain}."
+    },
+    {
+      "role": "user",
+      "content": "Help me with: {task}"
+    }
+  ],
+  "prompt_template_model": "gpt-4",
+  "prompt_template_optional_params": {
+    "temperature": 0.7,
+    "max_tokens": 500
+  }
+}
+```
+
+## Advanced Features
+
+### Variable Substitution
+
+LiteLLM automatically substitutes variables in your prompt templates using the `{variable}` syntax. Both `{variable}` and `{{variable}}` formats are supported.
+
+**Example prompt template:**
+```json
+{
+  "prompt_template": [
+    {
+      "role": "system",
+      "content": "You are an expert in {domain} with {years} years of experience."
+    }
+  ]
+}
+```
+
+**Client request:**
+```python
+completion(
+    model="gpt-4",
+    prompt_id="expert_prompt",
+    prompt_variables={
+        "domain": "machine learning",
+        "years": "10"
+    }
+)
+```
+
+**Result:**
+```
+"You are an expert in machine learning with 10 years of experience."
+```
+
+### Caching
+
+LiteLLM automatically caches fetched prompts in memory. The cache key includes:
+- `prompt_id`
+- `prompt_label` (if provided)
+- `prompt_version` (if provided)
+
+This means your API endpoint is only called once per unique prompt configuration.
+
+### Model Override Behavior
+
+**Default behavior (without `ignore_prompt_manager_model`):**
+```yaml
+prompts:
+  - prompt_id: "my_prompt"
+    litellm_params:
+      prompt_integration: "generic_prompt_management"
+      api_base: http://localhost:8080
+```
+
+If your API returns `"prompt_template_model": "gpt-4"`, LiteLLM will use `gpt-4` regardless of what the client specified.
+
+**With `ignore_prompt_manager_model: true`:**
+```yaml
+prompts:
+  - prompt_id: "my_prompt"
+    litellm_params:
+      prompt_integration: "generic_prompt_management"
+      api_base: http://localhost:8080
+      ignore_prompt_manager_model: true
+```
+
+LiteLLM will use the model specified by the client, ignoring the prompt's model.
+
+### Parameter Merging Behavior
+
+**Default behavior (without `ignore_prompt_manager_optional_params`):**
+
+Client params are merged with prompt params, with prompt params taking precedence:
+```python
+# Prompt returns: {"temperature": 0.7, "max_tokens": 500}
+# Client sends: {"temperature": 0.9, "top_p": 0.95}
+# Final params: {"temperature": 0.7, "max_tokens": 500, "top_p": 0.95}
+```
+
+**With `ignore_prompt_manager_optional_params: true`:**
+
+Only client params are used:
+```python
+# Prompt returns: {"temperature": 0.7, "max_tokens": 500}
+# Client sends: {"temperature": 0.9, "top_p": 0.95}
+# Final params: {"temperature": 0.9, "top_p": 0.95}
+```
+
+## Security Considerations
+
+1. **Authentication**: Use the `api_key` parameter to secure your prompt management API
+2. **Authorization**: Implement team/user-based access control using the custom query parameters
+3. **Rate Limiting**: Add rate limiting to prevent abuse of your API
+4. **Input Validation**: Validate all query parameters before processing
+5. **HTTPS**: Always use HTTPS in production for encrypted communication
+6. **Secrets**: Store API keys in environment variables, not in config files
+
+## Use Cases
+
+✅ **Use Generic Prompt Management API when:**
+- You want instant integration without waiting for PRs
+- You maintain your own prompt management service
+- You need full control over prompt versioning and updates
+- You want to build custom prompt management features
+- You need to integrate with your internal systems
+
+✅ **Common scenarios:**
+- Internal prompt management system for your organization
+- Multi-tenant prompt management with team-based access control
+- A/B testing different prompt versions
+- Prompt experimentation and analytics
+- Integration with existing prompt engineering workflows
+
+## When to Use This
+
+✅ **Use Generic Prompt Management API when:**
+- You want instant integration without waiting for PRs
+- You maintain your own prompt management service
+- You need full control over updates and features
+- You want custom prompt storage and versioning logic
+
+❌ **Make a PR when:**
+- You want deeper integration with LiteLLM internals
+- Your integration requires complex LiteLLM-specific logic
+- You want to be featured as a built-in provider
+- You're building a reusable integration for the community
+
+## Troubleshooting
+
+### Prompt not found
+- Verify the `prompt_id` matches exactly (case-sensitive)
+- Check that your API endpoint is accessible from LiteLLM
+- Verify authentication if using `api_key`
+
+### Variables not substituted
+- Ensure variables use `{variable}` or `{{variable}}` syntax
+- Check that variable names in `prompt_variables` match template exactly
+- Variables are case-sensitive
+
+### Model not being overridden
+- Check if `ignore_prompt_manager_model: true` is set in config
+- Verify your API is returning `prompt_template_model` in the response
+
+### Parameters not being applied
+- Check if `ignore_prompt_manager_optional_params: true` is set
+- Verify your API is returning `prompt_template_optional_params`
+- Ensure parameter names match OpenAI's parameter names
+
+## Questions?
+
+This is a **beta API**. We're actively improving it based on feedback. Open an issue or PR if you need additional capabilities.
+
+## Related Documentation
+
+- [Prompt Management Overview](../proxy/prompt_management.md)
+- [Generic Guardrail API](./generic_guardrail_api.md)
+- [LiteLLM Proxy Setup](../proxy/quick_start.md)
+

docs/my-website/docs/benchmarks.md

@@ -5,6 +5,44 @@ import Image from '@theme/IdealImage';
 
 Benchmarks for LiteLLM Gateway (Proxy Server) tested against a fake OpenAI endpoint.
 
+## Setting Up Benchmarking with Network Mock
+
+The fastest way to benchmark proxy overhead is using `network_mock` mode. This intercepts outbound requests at the httpx transport layer and returns canned responses, no need for setting up a mock provider. 
+
+**1. Create a proxy config:**
+
+```yaml
+model_list:
+  - model_name: db-openai-endpoint
+    litellm_params:
+      model: openai/gpt-4o
+      api_key: "sk-fake-key"
+      api_base: "https://api.openai.com"
+
+litellm_settings:
+  network_mock: true
+  callbacks: []
+  num_retries: 0
+  request_timeout: 30
+
+general_settings:
+  master_key: "sk-1234"
+```
+
+**2. Start the proxy:**
+
+```bash
+litellm --config benchmark_config.yaml --port 4000 --num_workers 8
+```
+
+**3. Run the benchmark script:**
+
+```bash
+python scripts/benchmark_mock.py --requests 2000 --max-concurrent 200 --runs 3
+```
+
+This measures pure proxy overhead on the hot path without any network latency to a real or fake provider.
+
 ## Setting Up a Fake OpenAI Endpoint
 
 For load testing and benchmarking, you can use a fake OpenAI proxy server. LiteLLM provides:

docs/my-website/docs/caching/all_caches.md

@@ -297,6 +297,7 @@ litellm.cache = Cache(
     similarity_threshold=0.7, # similarity threshold for cache hits, 0 == no similarity, 1 = exact matches, 0.5 == 50% similarity
     qdrant_quantization_config ="binary", # can be one of 'binary', 'product' or 'scalar' quantizations that is supported by qdrant
     qdrant_semantic_cache_embedding_model="text-embedding-ada-002", # this model is passed to litellm.embedding(), any litellm.embedding() model is supported here
+    qdrant_semantic_cache_vector_size=1536, # vector size for the embedding model, must match the dimensionality of the embedding model used
 )
 
 response1 = completion(
@@ -635,6 +636,7 @@ def __init__(
     qdrant_quantization_config: Optional[str] = None,
     qdrant_semantic_cache_embedding_model="text-embedding-ada-002",
 
+    qdrant_semantic_cache_vector_size: Optional[int] = None,
     **kwargs
 ):
 ```

docs/my-website/docs/completion/message_sanitization.md

@@ -0,0 +1,465 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Message Sanitization for Tool Calling for anthropic models
+
+**Automatically fix common message formatting issues when using tool calling with `modify_params=True`**
+
+LiteLLM can automatically sanitize messages to handle common issues that occur during tool calling workflows, especially when using OpenAI-compatible clients with providers that have strict message format requirements (like Anthropic Claude).
+
+## Overview
+
+When `litellm.modify_params = True` is enabled, LiteLLM automatically sanitizes messages to fix three common issues:
+
+1. **Orphaned Tool Calls** - Assistant messages with tool_calls but missing tool results
+2. **Orphaned Tool Results** - Tool messages that reference non-existent tool_call_ids
+3. **Empty Message Content** - Messages with empty or whitespace-only text content
+
+This ensures your tool calling workflows work seamlessly across different LLM providers without manual message validation.
+
+## Why Message Sanitization?
+
+Different LLM providers have varying requirements for message formats, especially during tool calling:
+
+- **Anthropic Claude** requires every tool_call to have a corresponding tool result
+- Some providers reject messages with empty content
+- OpenAI-compatible clients may not always maintain perfect message consistency
+
+Without sanitization, these issues cause API errors that interrupt your workflows. With `modify_params=True`, LiteLLM handles these edge cases automatically.
+
+## Quick Start
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+import litellm
+
+# Enable automatic message sanitization
+litellm.modify_params = True
+
+# This will work even if messages have formatting issues
+response = litellm.completion(
+    model="anthropic/claude-3-5-sonnet-20241022",
+    messages=[
+        {"role": "user", "content": "What's the weather in Boston?"},
+        {
+            "role": "assistant",
+            "tool_calls": [
+                {
+                    "id": "call_123",
+                    "type": "function",
+                    "function": {"name": "get_weather", "arguments": '{"city": "Boston"}'}
+                }
+            ]
+            # Missing tool result - LiteLLM will add a dummy result automatically
+        },
+        {"role": "user", "content": "Thanks!"}
+    ],
+    tools=[{
+        "type": "function",
+        "function": {
+            "name": "get_weather",
+            "description": "Get weather for a city",
+            "parameters": {
+                "type": "object",
+                "properties": {"city": {"type": "string"}},
+                "required": ["city"]
+            }
+        }
+    }]
+)
+```
+
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+```yaml
+litellm_settings:
+  modify_params: true  # Enable automatic message sanitization
+
+model_list:
+  - model_name: claude-3-5-sonnet
+    litellm_params:
+      model: anthropic/claude-3-5-sonnet-20241022
+```
+
+</TabItem>
+</Tabs>
+
+## Sanitization Cases
+
+### Case A: Orphaned Tool Calls (Missing Tool Results)
+
+**Problem:** An assistant message contains `tool_calls`, but no corresponding tool result messages follow.
+
+**Solution:** LiteLLM automatically adds dummy tool result messages for any missing tool results.
+
+**Example:**
+
+```python
+import litellm
+litellm.modify_params = True
+
+# Messages with orphaned tool calls
+messages = [
+    {"role": "user", "content": "Search for Python tutorials"},
+    {
+        "role": "assistant",
+        "tool_calls": [
+            {
+                "id": "call_abc123",
+                "type": "function",
+                "function": {"name": "web_search", "arguments": '{"query": "Python tutorials"}'}
+            }
+        ]
+    },
+    # Missing tool result here!
+    {"role": "user", "content": "What about JavaScript?"}
+]
+
+# LiteLLM automatically adds:
+# {
+#     "role": "tool",
+#     "tool_call_id": "call_abc123",
+#     "content": "[System: Tool execution skipped/interrupted by user. No result provided for tool 'web_search'.]"
+# }
+
+response = litellm.completion(
+    model="anthropic/claude-3-5-sonnet-20241022",
+    messages=messages,
+    tools=[...]
+)
+```
+
+**When this happens:**
+- User interrupts tool execution
+- Client loses tool results due to network issues
+- Conversation flow changes before tool completes
+- Multi-turn conversations where tools are optional
+
+### Case B: Orphaned Tool Results (Invalid tool_call_id)
+
+**Problem:** A tool message references a `tool_call_id` that doesn't exist in any previous assistant message.
+
+**Solution:** LiteLLM automatically removes these orphaned tool result messages.
+
+**Example:**
+
+```python
+import litellm
+litellm.modify_params = True
+
+# Messages with orphaned tool result
+messages = [
+    {"role": "user", "content": "Hello"},
+    {"role": "assistant", "content": "Hi! How can I help?"},
+    {
+        "role": "tool",
+        "tool_call_id": "call_nonexistent",  # This tool_call_id doesn't exist!
+        "content": "Some result"
+    }
+]
+
+# LiteLLM automatically removes the orphaned tool message
+
+response = litellm.completion(
+    model="anthropic/claude-3-5-sonnet-20241022",
+    messages=messages
+)
+```
+
+**When this happens:**
+- Message history is manually edited
+- Tool results are duplicated or mismatched
+- Conversation state is restored incorrectly
+- Messages are merged from different conversations
+
+### Case C: Empty Message Content
+
+**Problem:** User or assistant messages have empty or whitespace-only content.
+
+**Solution:** LiteLLM replaces empty content with a system placeholder message.
+
+**Example:**
+
+```python
+import litellm
+litellm.modify_params = True
+
+# Messages with empty content
+messages = [
+    {"role": "user", "content": ""},  # Empty content
+    {"role": "assistant", "content": "   "},  # Whitespace only
+]
+
+# LiteLLM automatically replaces with:
+# {"role": "user", "content": "[System: Empty message content sanitised to satisfy protocol]"}
+# {"role": "assistant", "content": "[System: Empty message content sanitised to satisfy protocol]"}
+
+response = litellm.completion(
+    model="anthropic/claude-3-5-sonnet-20241022",
+    messages=messages
+)
+```
+
+**When this happens:**
+- UI sends empty messages
+- Content is stripped during preprocessing
+- Placeholder messages in conversation history
+- Edge cases in message construction
+
+## Configuration
+
+### Enable Globally
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+import litellm
+
+# Enable for all completion calls
+litellm.modify_params = True
+```
+
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+```yaml
+litellm_settings:
+  modify_params: true
+```
+
+</TabItem>
+<TabItem value="env" label="Environment Variable">
+
+```bash
+export LITELLM_MODIFY_PARAMS=True
+```
+
+</TabItem>
+</Tabs>
+
+### Enable Per-Request
+
+```python
+import litellm
+
+# Enable only for specific requests
+response = litellm.completion(
+    model="anthropic/claude-3-5-sonnet-20241022",
+    messages=messages,
+    modify_params=True  # Override global setting
+)
+```
+
+## Supported Providers
+
+Message sanitization currently works with:
+
+- ✅ Anthropic (Claude)
+
+**Note:** While the sanitization logic is provider-agnostic, it is currently only applied in the Anthropic message transformation pipeline. Support for additional providers may be added in future releases.
+
+## Implementation Details
+
+### How It Works
+
+The message sanitization process runs **before** messages are converted to provider-specific formats:
+
+1. **Input:** OpenAI-format messages with potential issues
+2. **Sanitization:** Three helper functions process the messages:
+   - `_sanitize_empty_text_content()` - Fixes empty content
+   - `_add_missing_tool_results()` - Adds dummy tool results
+   - `_is_orphaned_tool_result()` - Identifies orphaned results
+3. **Output:** Clean, provider-compatible messages
+
+### Code Reference
+
+The sanitization logic is implemented in:
+- `litellm/litellm_core_utils/prompt_templates/factory.py`
+- Function: `sanitize_messages_for_tool_calling()`
+
+### Logging
+
+When sanitization occurs, LiteLLM logs debug messages:
+
+```python
+import litellm
+litellm.set_verbose = True  # Enable debug logging
+
+# You'll see logs like:
+# "_add_missing_tool_results: Found 1 orphaned tool calls. Adding dummy tool results."
+# "_is_orphaned_tool_result: Found orphaned tool result with tool_call_id=call_123"
+# "_sanitize_empty_text_content: Replaced empty text content in user message"
+```
+
+## Best Practices
+
+### 1. Enable for Production Workflows
+
+```python
+# Recommended for production
+litellm.modify_params = True
+
+# Ensures robust handling of edge cases
+response = litellm.completion(
+    model="anthropic/claude-3-5-sonnet-20241022",
+    messages=messages,
+    tools=tools
+)
+```
+
+### 2. Preserve Tool Results When Possible
+
+While sanitization handles missing tool results, it's better to provide actual results:
+
+```python
+# Good: Provide actual tool results
+messages = [
+    {"role": "user", "content": "Search for Python"},
+    {"role": "assistant", "tool_calls": [...]},
+    {"role": "tool", "tool_call_id": "call_123", "content": "Actual search results"}
+]
+
+# Fallback: Sanitization adds dummy result if missing
+messages = [
+    {"role": "user", "content": "Search for Python"},
+    {"role": "assistant", "tool_calls": [...]},
+    # Missing tool result - sanitization adds dummy
+]
+```
+
+### 3. Monitor Sanitization Events
+
+Use logging to track when sanitization occurs:
+
+```python
+import litellm
+import logging
+
+# Enable debug logging
+litellm.set_verbose = True
+logging.basicConfig(level=logging.DEBUG)
+
+# Track sanitization events in your application
+response = litellm.completion(
+    model="anthropic/claude-3-5-sonnet-20241022",
+    messages=messages
+)
+```
+
+### 4. Test Edge Cases
+
+Ensure your application handles sanitized messages correctly:
+
+```python
+import litellm
+litellm.modify_params = True
+
+# Test orphaned tool calls
+test_messages = [
+    {"role": "user", "content": "Test"},
+    {"role": "assistant", "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "test", "arguments": "{}"}}]},
+    {"role": "user", "content": "Continue"}  # No tool result
+]
+
+response = litellm.completion(
+    model="anthropic/claude-3-5-sonnet-20241022",
+    messages=test_messages,
+    tools=[...]
+)
+
+# Verify the response handles the dummy tool result appropriately
+```
+
+## Related Features
+
+- **[Drop Params](./drop_params.md)** - Drop unsupported parameters for specific providers
+- **[Message Trimming](./message_trimming.md)** - Trim messages to fit token limits
+- **[Function Calling](./function_call.md)** - Complete guide to tool/function calling
+- **[Reasoning Content](../reasoning_content.md)** - Extended thinking with tool calling
+
+## Troubleshooting
+
+### Sanitization Not Working
+
+**Issue:** Messages still cause errors despite `modify_params=True`
+
+**Solution:**
+1. Verify `modify_params` is enabled:
+   ```python
+   import litellm
+   print(litellm.modify_params)  # Should be True
+   ```
+
+2. Check if the issue is provider-specific:
+   ```python
+   litellm.set_verbose = True  # Enable debug logging
+   ```
+
+3. Ensure you're using a recent version of LiteLLM:
+   ```bash
+   pip install --upgrade litellm
+   ```
+
+### Unexpected Dummy Tool Results
+
+**Issue:** Dummy tool results appear when you expect actual results
+
+**Cause:** Tool result messages are missing or have incorrect `tool_call_id`
+
+**Solution:**
+1. Verify tool result messages have correct `tool_call_id`:
+   ```python
+   # Correct
+   {"role": "tool", "tool_call_id": "call_123", "content": "result"}
+   
+   # Incorrect - will be treated as orphaned
+   {"role": "tool", "tool_call_id": "wrong_id", "content": "result"}
+   ```
+
+2. Ensure tool results immediately follow assistant messages with tool_calls
+
+### Performance Impact
+
+**Issue:** Concerned about performance overhead
+
+**Details:** Message sanitization has minimal performance impact:
+- Runs in O(n) time where n = number of messages
+- Only processes messages when `modify_params=True`
+- Typically adds < 1ms to request processing time
+
+## FAQ
+
+**Q: Does sanitization modify my original messages?**
+
+A: No, sanitization creates a new list of messages. Your original messages remain unchanged.
+
+**Q: Can I disable specific sanitization cases?**
+
+A: Currently, all three cases are handled together when `modify_params=True`. To disable sanitization entirely, set `modify_params=False`.
+
+**Q: What happens to the dummy tool results?**
+
+A: Dummy tool results are sent to the LLM provider along with other messages. The model sees them as regular tool results with informative error messages.
+
+**Q: Does this work with streaming?**
+
+A: Yes, message sanitization works with both streaming and non-streaming requests.
+
+**Q: Is this related to `drop_params`?**
+
+A: No, they're separate features:
+- `modify_params` - Modifies/fixes message content and structure
+- `drop_params` - Removes unsupported API parameters
+
+Both can be enabled simultaneously.
+
+## See Also
+
+- [Reasoning Content with Tool Calling](../reasoning_content.md)
+- [Function Calling Guide](./function_call.md)
+- [Bedrock Provider Documentation](../providers/bedrock.md)
+- [Anthropic Provider Documentation](../providers/anthropic.md)

docs/my-website/docs/completion/prompt_caching.md

@@ -63,7 +63,6 @@ for _ in range(2):
                     }
                 ],
             },
-            # marked for caching with the cache_control parameter, so that this checkpoint can read from the previous cache.
             {
                 "role": "user",
                 "content": [
@@ -77,7 +76,6 @@ for _ in range(2):
                 "role": "assistant",
                 "content": "Certainly! the key terms and conditions are the following: the contract is 1 year long for $10/mo",
             },
-            # The final turn is marked with cache-control, for continuing in followups.
             {
                 "role": "user",
                 "content": [
@@ -112,16 +110,16 @@ model_list:
         api_key: os.environ/OPENAI_API_KEY
 ```
 
-2. Start proxy 
+2. Start proxy
 
 ```bash
 litellm --config /path/to/config.yaml
 ```
 
-3. Test it! 
+3. Test it!
 
 ```python
-from openai import OpenAI 
+from openai import OpenAI
 import os
 
 client = OpenAI(
@@ -144,7 +142,6 @@ for _ in range(2):
                     }
                 ],
             },
-            # marked for caching with the cache_control parameter, so that this checkpoint can read from the previous cache.
             {
                 "role": "user",
                 "content": [
@@ -158,7 +155,6 @@ for _ in range(2):
                 "role": "assistant",
                 "content": "Certainly! the key terms and conditions are the following: the contract is 1 year long for $10/mo",
             },
-            # The final turn is marked with cache-control, for continuing in followups.
             {
                 "role": "user",
                 "content": [
@@ -183,6 +179,78 @@ assert response.usage.prompt_tokens_details.cached_tokens > 0
 </TabItem>
 </Tabs>
 
+### OpenAI `prompt_cache_key` and `prompt_cache_retention`
+
+OpenAI prompt caching is [**automatic**](https://platform.openai.com/docs/guides/prompt-caching) — no `cache_control` message annotations are needed. Any request with 1024+ prompt tokens is eligible for caching.
+
+OpenAI also supports two optional parameters for more control over caching behavior:
+
+- **`prompt_cache_key`** (string) — A routing hint that improves cache hit rates for requests sharing long common prefixes. Requests with the same cache key are routed to the same backend, increasing the likelihood of a cache hit.
+- **`prompt_cache_retention`** (`"in_memory"` or `"24h"`) — Controls cache TTL. Default is `"in_memory"` (5–10 min). Set to `"24h"` for extended caching that offloads KV tensors to GPU-local storage.
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+from litellm import completion
+import os
+
+os.environ["OPENAI_API_KEY"] = ""
+
+response = completion(
+    model="gpt-4o",
+    messages=[
+        {
+            "role": "system",
+            "content": "You are an AI assistant tasked with analyzing legal documents. "
+            + "Here is the full text of a complex legal agreement " * 400,
+        },
+        {
+            "role": "user",
+            "content": "What are the key terms and conditions?",
+        },
+    ],
+    prompt_cache_key="legal-doc-analysis",
+    prompt_cache_retention="24h",
+)
+print(response.usage)
+```
+
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+```python
+from openai import OpenAI
+
+client = OpenAI(
+    api_key="LITELLM_PROXY_KEY",
+    base_url="LITELLM_PROXY_BASE",
+)
+
+response = client.chat.completions.create(
+    model="gpt-4o",
+    messages=[
+        {
+            "role": "system",
+            "content": "You are an AI assistant tasked with analyzing legal documents. "
+            + "Here is the full text of a complex legal agreement " * 400,
+        },
+        {
+            "role": "user",
+            "content": "What are the key terms and conditions?",
+        },
+    ],
+    extra_body={
+        "prompt_cache_key": "legal-doc-analysis",
+        "prompt_cache_retention": "24h",
+    },
+)
+print(response.usage)
+```
+
+</TabItem>
+</Tabs>
+
 ### Anthropic Example 
 
 Anthropic charges for cache writes. 

docs/my-website/docs/completion/usage.md

@@ -50,3 +50,51 @@ for chunk in completion:
   print(chunk.choices[0].delta)
 
 ```
+
+### Proxy: Always Include Streaming Usage
+
+When using the LiteLLM Proxy, you can configure it to automatically include usage information in all streaming responses, even if the client doesn't send `stream_options={"include_usage": True}`.
+
+#### Configuration
+
+Add the following to your config.yaml:
+
+```yaml
+general_settings:
+  always_include_stream_usage: true
+```
+
+Alternatively, configure it through the UI:
+
+1. Navigate to the LiteLLM Proxy UI
+2. Go to `Settings` >  `Router Settings` > `General`
+3. Find the `always_include_stream_usage` setting
+4. Toggle it to `true`
+5. Click `Update` to save
+
+#### How it works
+
+When `always_include_stream_usage` is enabled:
+- All streaming requests will automatically have `stream_options={"include_usage": True}` added
+- Clients will receive usage information in the final chunk, even if they didn't explicitly request it
+- If a client already provides `stream_options`, `include_usage: True` will be added without overwriting other options
+- Non-streaming requests are not affected
+
+#### Example
+
+With this setting enabled, a simple streaming request like:
+
+```bash
+curl -X POST http://localhost:4000/v1/chat/completions \
+  -H "Authorization: Bearer sk-1234" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "gpt-4o",
+    "messages": [{"role": "user", "content": "Hello!"}],
+    "stream": true
+  }'
+```
+
+Will automatically receive usage information in the response, without needing to explicitly include `stream_options`.
+
+```

docs/my-website/docs/contributing.md

@@ -79,7 +79,27 @@ cp -r out/* ../../litellm/proxy/_experimental/out/
 
 Then restart the proxy and access the UI at `http://localhost:4000/ui`
 
-## 4. Submitting a PR
+## 4. Pre-PR Checklist
+
+Before submitting your pull request, make sure the following pass locally from `ui/litellm-dashboard/`:
+
+**Run tests related to your changes:**
+
+```bash
+npx vitest run src/components/path/to/YourComponent.test.tsx
+```
+
+Tests are co-located with components (e.g., `TeamInfo.tsx` → `TeamInfo.test.tsx`). If you add a new component, add a corresponding `.test.tsx` file next to it.
+
+**Run the build:**
+
+```bash
+npm run build
+```
+
+These map to the `ui_tests` and `ui_build` CI checks.
+
+## 5. Submitting a PR
 
 1. Create a new branch for your changes:
 ```bash

docs/my-website/docs/enterprise.md

@@ -4,7 +4,7 @@ import Image from '@theme/IdealImage';
 
 :::info
 - ✨ SSO is free for up to 5 users. After that, an enterprise license is required. [Get Started with Enterprise here](https://www.litellm.ai/enterprise)
-- Who is Enterprise for? Companies giving access to 100+ users **OR** 10+ AI use-cases. If you're not sure, [get in touch with us](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) to discuss your needs.
+- Who is Enterprise for? Companies giving access to 100+ users **OR** 10+ AI use-cases. If you're not sure, [get in touch with us](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) to discuss your needs.
 :::
 
 For companies that need SSO, user management and professional support for LiteLLM Proxy
@@ -36,7 +36,7 @@ Manage Yourself - you can deploy our Docker Image or build a custom image from o
 
 ### What’s the cost of the Self-Managed Enterprise edition?
 
-Self-Managed Enterprise deployments require our team to understand your exact needs. [Get in touch with us to learn more](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
+Self-Managed Enterprise deployments require our team to understand your exact needs. [Get in touch with us to learn more](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
 
 
 ### How does deployment with Enterprise License work? 
@@ -106,7 +106,7 @@ Professional Support can assist with LLM/Provider integrations, deployment, upgr
 
 Pricing is based on usage. We can figure out a price that works for your team, on the call. 
 
-[**Contact Us to learn more**](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
+[**Contact Us to learn more**](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
 
 
 

docs/my-website/docs/fine_tuning.md

@@ -6,7 +6,7 @@ import TabItem from '@theme/TabItem';
 
 :::info
 
-This is an Enterprise only endpoint [Get Started with Enterprise here](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
+This is an Enterprise only endpoint [Get Started with Enterprise here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
 
 :::
 

docs/my-website/docs/generateContent.md

@@ -15,6 +15,7 @@ Use LiteLLM to call Google AI's generateContent endpoints for text generation, m
 | Streaming | ✅ | |
 | Fallbacks | ✅ | between supported models |
 | Loadbalancing | ✅ | between supported models |
+| Metadata Tracking | ✅ | passes trace ID, metadata to observability callbacks (e.g. S3, Langfuse) |
 
 ## Usage 
 ---

docs/my-website/docs/interactions.md

@@ -130,13 +130,12 @@ Point the Google GenAI SDK to LiteLLM Proxy:
 
 ```python showLineNumbers title="Google GenAI SDK with LiteLLM Proxy"
 from google import genai
-import os
 
 # Point SDK to LiteLLM Proxy
-os.environ["GOOGLE_GENAI_BASE_URL"] = "http://localhost:4000"
-os.environ["GEMINI_API_KEY"] = "sk-1234"  # Your LiteLLM API key
-
-client = genai.Client()
+client = genai.Client(
+    api_key="sk-1234",  # Your LiteLLM API key
+    http_options={"base_url": "http://localhost:4000"},
+)
 
 # Create an interaction
 interaction = client.interactions.create(
@@ -151,12 +150,11 @@ print(interaction.outputs[-1].text)
 
 ```python showLineNumbers title="Google GenAI SDK Streaming"
 from google import genai
-import os
 
-os.environ["GOOGLE_GENAI_BASE_URL"] = "http://localhost:4000"
-os.environ["GEMINI_API_KEY"] = "sk-1234"
-
-client = genai.Client()
+client = genai.Client(
+    api_key="sk-1234",  # Your LiteLLM API key
+    http_options={"base_url": "http://localhost:4000"},
+)
 
 for chunk in client.interactions.create_stream(
     model="gemini/gemini-2.5-flash",

docs/my-website/docs/mcp.md

@@ -641,7 +641,7 @@ import asyncio
 config = {
     "mcpServers": {
         "mcp_group": {
-            "url": "http://localhost:4000/mcp",
+            "url": "http://localhost:4000/mcp/",
             "headers": {
                 "x-mcp-servers": "dev_group", # assume this gives access to github, zapier and deepwiki
                 "x-litellm-api-key": "Bearer sk-1234",
@@ -808,6 +808,68 @@ If your stdio MCP server needs per-request credentials, you can map HTTP headers
 
 In this example, when a client makes a request with the `X-GITHUB_PERSONAL_ACCESS_TOKEN` header, the proxy forwards that value into the stdio process as the `GITHUB_PERSONAL_ACCESS_TOKEN` environment variable.
 
+## Control MCP Access for End Users
+
+Control which MCP servers end users of your AI application can access (e.g. users of an internal chat UI). Pass the customer ID in the `x-litellm-end-user-id` header to:
+- Enforce object permissions (limit which MCP servers they can access)
+- Apply customer-specific budgets
+- Track spend per customer
+
+**FastMCP Client Example:**
+
+```python title="Track customer spend with x-litellm-end-user-id" showLineNumbers
+from fastmcp import Client
+import asyncio
+
+# MCP client configuration with customer tracking
+config = {
+    "mcpServers": {
+        "github": {
+            "url": "http://localhost:4000/github_mcp/mcp",
+            "headers": {
+                "x-litellm-api-key": "Bearer sk-1234",
+                "x-litellm-end-user-id": "customer_123",  # 👈 CUSTOMER ID
+                "Authorization": "Bearer gho_token"
+            }
+        }
+    }
+}
+
+client = Client(config)
+
+async def main():
+    async with client:
+        # All MCP calls will be tracked under customer_123
+        tools = await client.list_tools()
+        result = await client.call_tool(tools[0].name, {})
+        print(f"Tool result: {result}")
+
+asyncio.run(main())
+```
+
+**Cursor IDE Example:**
+
+```json title="Cursor config with customer tracking" showLineNumbers
+{
+  "mcpServers": {
+    "GitHub": {
+      "url": "http://localhost:4000/github_mcp/mcp",
+      "headers": {
+        "x-litellm-api-key": "Bearer $LITELLM_API_KEY",
+        "x-litellm-end-user-id": "customer_123"
+      }
+    }
+  }
+}
+```
+
+**What happens:**
+- Customer-specific object permissions are enforced (only allowed MCP servers are accessible)
+- Customer budgets are applied
+- All tool calls are tracked under `customer_123`
+
+[Learn more about customer management →](./proxy/customers)
+
 ## Using your MCP with client side credentials
 
 Use this if you want to pass a client side authentication token to LiteLLM to then pass to your MCP to auth to your MCP.

docs/my-website/docs/observability/datadog.md

@@ -7,6 +7,7 @@ import TabItem from '@theme/TabItem';
 LiteLLM Supports logging to the following Datdog Integrations:
 - `datadog` [Datadog Logs](https://docs.datadoghq.com/logs/)
 - `datadog_llm_observability` [Datadog LLM Observability](https://www.datadoghq.com/product/llm-observability/)
+- `datadog_metrics` [Datadog Custom Metrics](#datadog-custom-metrics)
 - `datadog_cost_management` [Datadog Cloud Cost Management](#datadog-cloud-cost-management)
 - `ddtrace-run` [Datadog Tracing](#datadog-tracing)
 
@@ -168,6 +169,65 @@ On the Datadog LLM Observability page, you should see that both input messages a
 <Image img={require('../../img/dd_llm_obs.png')} />
 
 
+## Datadog Custom Metrics
+
+| Feature | Details |
+|---------|---------|
+| **What is logged** | Latency metrics, request counts by status code |
+| **Events** | Success + Failure |
+| **Product Link** | [Datadog Metrics](https://docs.datadoghq.com/metrics/) |
+
+Publishes the following metrics to Datadog via the `/api/v2/series` endpoint:
+
+| Metric | Type | Description |
+|--------|------|-------------|
+| `litellm.request.total_latency` | Gauge | End-to-end request latency (seconds) |
+| `litellm.llm_api.latency` | Gauge | Time spent waiting for the LLM provider response (seconds) |
+| `litellm.llm_api.request_count` | Count | Request count, tagged with status code |
+
+Using `total_latency` and `llm_api.latency`, you can derive **internal latency** = `total_latency - llm_api.latency`.
+
+All metrics include the following tags: `env`, `service`, `version`, `HOSTNAME`, `POD_NAME`, `provider`, `model_name`, `model_group`, `team`, `status_code`.
+
+**Step 1**: Create a `config.yaml` file
+
+```yaml
+model_list:
+  - model_name: gpt-3.5-turbo
+    litellm_params:
+      model: gpt-3.5-turbo
+litellm_settings:
+  success_callback: ["datadog_metrics"]
+  failure_callback: ["datadog_metrics"]
+```
+
+**Step 2**: Set required env variables
+
+```shell
+DD_API_KEY="your-api-key"
+DD_SITE="us5.datadoghq.com"  # your datadog site
+```
+
+**Step 3**: Start the proxy and make a test request
+
+```shell
+litellm --config config.yaml
+```
+
+```shell
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+    --header 'Content-Type: application/json' \
+    --header 'Authorization: Bearer sk-1234' \
+    --data '{
+    "model": "gpt-3.5-turbo",
+    "messages": [{"role": "user", "content": "hello"}]
+}'
+```
+
+**Step 4**: View metrics in Datadog Metrics Explorer
+
+Navigate to **Metrics > Explorer** in Datadog and search for `litellm.request.total_latency`, `litellm.llm_api.latency`, or `litellm.llm_api.request_count`.
+
 ## Datadog Cloud Cost Management
 
 | Feature | Details |
@@ -253,3 +313,12 @@ LiteLLM supports customizing the following Datadog environment variables
 \* **Required when using Direct API** (default): `DD_API_KEY` and `DD_SITE` are required  
 \* **Optional when using DataDog Agent**: Set `LITELLM_DD_AGENT_HOST` to use agent mode; `DD_API_KEY` and `DD_SITE` are not required for **Datadog Logs**. (**Note: `DD_API_KEY` IS REQUIRED for Datadog LLM Observability**)
 
+## Automatic Tags
+
+LiteLLM automatically adds the following tags to your Datadog logs and metrics if the information is available in the request:
+
+| Tag | Description | Source |
+|-----|-------------|--------|
+| `team` | The team alias or ID associated with the API Key | `user_api_key_team_alias`, `team_alias`, `user_api_key_team_id`, or `team_id` in metadata |
+| `request_tag` | Custom tags passed in the request | `request_tags` in logging payload |
+

docs/my-website/docs/observability/gcs_bucket_integration.md

@@ -6,7 +6,7 @@ Log LLM Logs to [Google Cloud Storage Buckets](https://cloud.google.com/storage?
 
 :::info
 
-✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
+✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
 
 :::
 

docs/my-website/docs/ocr.md

@@ -61,6 +61,52 @@ async def test_async_ocr():
 asyncio.run(test_async_ocr())
 ```
 
+### Using Local Files
+
+LiteLLM can read local files directly — no manual base64 encoding needed:
+
+```python
+from litellm import ocr
+
+# OCR with a local PDF file path
+response = ocr(
+    model="mistral/mistral-ocr-latest",
+    document={
+        "type": "file",
+        "file": "/path/to/document.pdf"
+    }
+)
+
+# OCR with a file object
+response = ocr(
+    model="mistral/mistral-ocr-latest",
+    document={
+        "type": "file",
+        "file": open("document.pdf", "rb")
+    }
+)
+
+# OCR with raw bytes
+with open("document.pdf", "rb") as f:
+    pdf_bytes = f.read()
+
+response = ocr(
+    model="mistral/mistral-ocr-latest",
+    document={
+        "type": "file",
+        "file": pdf_bytes,
+        "mime_type": "application/pdf"  # recommended for raw bytes (auto-detected from extension for file paths)
+    }
+)
+```
+
+The `file` field accepts:
+- **File path** (`str` or `pathlib.Path`) — LiteLLM reads the file and detects the MIME type from the extension
+- **File object** (binary file-like object) — e.g. `open("doc.pdf", "rb")`
+- **Raw bytes** (`bytes`) — use `mime_type` to specify the content type
+
+LiteLLM automatically converts file inputs to base64 data URIs internally, so all providers work seamlessly.
+
 ### Using Base64 Encoded Documents
 
 ```python
@@ -121,7 +167,7 @@ litellm --config /path/to/config.yaml
 # RUNNING on http://0.0.0.0:4000
 ```
 
-Test request
+**Test request — JSON body**
 
 ```bash
 curl http://0.0.0.0:4000/v1/ocr \
@@ -136,6 +182,27 @@ curl http://0.0.0.0:4000/v1/ocr \
   }'
 ```
 
+**Test request — multipart file upload**
+
+Upload a file directly using multipart form data. No need to base64-encode the file yourself.
+
+```bash
+curl http://0.0.0.0:4000/v1/ocr \
+  -H "Authorization: Bearer sk-1234" \
+  -F "model=mistral-ocr" \
+  -F "file=@/path/to/document.pdf"
+```
+
+You can also pass optional parameters as additional form fields:
+
+```bash
+curl http://0.0.0.0:4000/v1/ocr \
+  -H "Authorization: Bearer sk-1234" \
+  -F "model=mistral-ocr" \
+  -F "file=@screenshot.png" \
+  -F 'pages=[0,1,2]' \
+  -F "include_image_base64=true"
+```
 
 ## **Request/Response Format**
 
@@ -168,10 +235,12 @@ See the [official Mistral OCR documentation](https://docs.mistral.ai/capabilitie
 | Parameter | Type | Required | Description |
 |-----------|------|----------|-------------|
 | `model` | string | Yes | The OCR model to use (e.g., `"mistral/mistral-ocr-latest"`) |
-| `document` | object | Yes | Document to process. Must contain `type` and URL field |
-| `document.type` | string | Yes | Either `"document_url"` for PDFs/docs or `"image_url"` for images |
-| `document.document_url` | string | Conditional | URL to the document (required if `type` is `"document_url"`) |
-| `document.image_url` | string | Conditional | URL to the image (required if `type` is `"image_url"`) |
+| `document` | object | Yes | Document to process. Must contain `type` and the corresponding field |
+| `document.type` | string | Yes | `"document_url"` for PDFs/docs, `"image_url"` for images, or `"file"` for local files |
+| `document.document_url` | string | Conditional | URL or data URI to the document (required if `type` is `"document_url"`) |
+| `document.image_url` | string | Conditional | URL or data URI to the image (required if `type` is `"image_url"`) |
+| `document.file` | string/bytes/file | Conditional | File path, bytes, or file-like object (required if `type` is `"file"`) |
+| `document.mime_type` | string | No | Explicit MIME type for file inputs (auto-detected from extension if not provided) |
 | `pages` | array | No | List of specific page indices to process (0-indexed) |
 | `include_image_base64` | boolean | No | Whether to include extracted images as base64 strings |
 | `image_limit` | integer | No | Maximum number of images to return |
@@ -179,7 +248,7 @@ See the [official Mistral OCR documentation](https://docs.mistral.ai/capabilitie
 
 #### Document Format Examples
 
-**For PDFs and documents:**
+**For PDFs and documents (URL):**
 ```json
 {
   "type": "document_url",
@@ -187,7 +256,7 @@ See the [official Mistral OCR documentation](https://docs.mistral.ai/capabilitie
 }
 ```
 
-**For images:**
+**For images (URL):**
 ```json
 {
   "type": "image_url",
@@ -203,6 +272,21 @@ See the [official Mistral OCR documentation](https://docs.mistral.ai/capabilitie
 }
 ```
 
+**For local files (SDK):**
+```python
+{"type": "file", "file": "/path/to/document.pdf"}
+{"type": "file", "file": open("image.png", "rb")}
+{"type": "file", "file": pdf_bytes, "mime_type": "application/pdf"}
+```
+
+**For file uploads (Proxy — multipart form):**
+```bash
+curl http://0.0.0.0:4000/v1/ocr \
+  -H "Authorization: Bearer sk-1234" \
+  -F "model=mistral-ocr" \
+  -F "file=@document.pdf"
+```
+
 ### Response Format
 
 The response follows Mistral's OCR format with the following structure:

docs/my-website/docs/pass_through/assembly_ai.md

@@ -1,31 +1,36 @@
-# Assembly AI
+# AssemblyAI
 
-Pass-through endpoints for Assembly AI - call Assembly AI endpoints, in native format (no translation).
+Pass-through endpoints for AssemblyAI - call AssemblyAI endpoints, in native format (no translation).
 
-| Feature | Supported | Notes | 
+| Feature | Supported | Notes |
 |-------|-------|-------|
 | Cost Tracking | ✅ | works across all integrations |
 | Logging | ✅ | works across all integrations |
 
 
-Supports **ALL** Assembly AI Endpoints
+Supports **ALL** AssemblyAI Endpoints
 
-[**See All Assembly AI Endpoints**](https://www.assemblyai.com/docs/api-reference)
+[**See All AssemblyAI Endpoints**](https://www.assemblyai.com/docs/api-reference)
 
 
-<iframe width="840" height="500" src="https://www.loom.com/embed/aac3f4d74592448992254bfa79b9f62d?sid=267cd0ab-d92b-42fa-b97a-9f385ef8930c" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>
+## Supported Routes
+
+| AssemblyAI Service | LiteLLM Route | AssemblyAI Base URL |
+|-------------------|---------------|---------------------|
+| Speech-to-Text (US) | `/assemblyai/*` | `api.assemblyai.com` |
+| Speech-to-Text (EU) | `/eu.assemblyai/*` | `eu.api.assemblyai.com` |
 
 ## Quick Start
 
-Let's call the Assembly AI [`/v2/transcripts` endpoint](https://www.assemblyai.com/docs/api-reference/transcripts)
+Let's call the AssemblyAI [`/v2/transcripts` endpoint](https://www.assemblyai.com/docs/api-reference/transcripts)
 
-1. Add Assembly AI API Key to your environment 
+1. Add AssemblyAI API Key to your environment
 
 ```bash
 export ASSEMBLYAI_API_KEY=""
 ```
 
-2. Start LiteLLM Proxy 
+2. Start LiteLLM Proxy
 
 ```bash
 litellm
@@ -33,53 +38,157 @@ litellm
 # RUNNING on http://0.0.0.0:4000
 ```
 
-3. Test it! 
+3. Test it!
 
-Let's call the Assembly AI `/v2/transcripts` endpoint
+Let's call the AssemblyAI [`/v2/transcripts` endpoint](https://www.assemblyai.com/docs/api-reference/transcripts). Includes commented-out [Speech Understanding](https://www.assemblyai.com/docs/speech-understanding) features you can toggle on.
 
 ```python
 import assemblyai as aai
 
-LITELLM_VIRTUAL_KEY = "sk-1234" # <your-virtual-key>
-LITELLM_PROXY_BASE_URL = "http://0.0.0.0:4000/assemblyai" # <your-proxy-base-url>/assemblyai
+aai.settings.base_url = "http://0.0.0.0:4000/assemblyai" # <your-proxy-base-url>/assemblyai
+aai.settings.api_key = "Bearer sk-1234" # Bearer <your-virtual-key>
 
-aai.settings.api_key = f"Bearer {LITELLM_VIRTUAL_KEY}"
-aai.settings.base_url = LITELLM_PROXY_BASE_URL
+# Use a publicly-accessible URL
+audio_file = "https://assembly.ai/wildfires.mp3"
 
-# URL of the file to transcribe
-FILE_URL = "https://assembly.ai/wildfires.mp3"
+# Or use a local file:
+# audio_file = "./example.mp3"
 
-# You can also transcribe a local file by passing in a file path
-# FILE_URL = './path/to/file.mp3'
+config = aai.TranscriptionConfig(
+    speech_models=["universal-3-pro", "universal-2"],
+    language_detection=True,
+    speaker_labels=True,
+    # Speech understanding features
+    # sentiment_analysis=True,
+    # entity_detection=True,
+    # auto_chapters=True,
+    # summarization=True,
+    # summary_type=aai.SummarizationType.bullets,
+    # redact_pii=True,
+    # content_safety=True,
+)
 
-transcriber = aai.Transcriber()
-transcript = transcriber.transcribe(FILE_URL)
-print(transcript)
-print(transcript.id)
+transcript = aai.Transcriber().transcribe(audio_file, config=config)
+
+if transcript.status == aai.TranscriptStatus.error:
+    raise RuntimeError(f"Transcription failed: {transcript.error}")
+
+print(f"\nFull Transcript:\n\n{transcript.text}")
+
+# Optionally print speaker diarization results
+# for utterance in transcript.utterances:
+#     print(f"Speaker {utterance.speaker}: {utterance.text}")
 ```
 
-## Calling Assembly AI EU endpoints
+4. [Prompting with Universal-3 Pro](https://www.assemblyai.com/docs/speech-to-text/prompting) (optional)
 
-If you want to send your request to the Assembly AI EU endpoint, you can do so by setting the `LITELLM_PROXY_BASE_URL` to `<your-proxy-base-url>/eu.assemblyai`
+```python
+import assemblyai as aai
+
+aai.settings.base_url = "http://0.0.0.0:4000/assemblyai" # <your-proxy-base-url>/assemblyai
+aai.settings.api_key = "Bearer sk-1234" # Bearer <your-virtual-key>
+
+audio_file = "https://assemblyaiassets.com/audios/verbatim.mp3"
+
+config = aai.TranscriptionConfig(
+    speech_models=["universal-3-pro", "universal-2"],
+    language_detection=True,
+    prompt="Produce a transcript suitable for conversational analysis. Every disfluency is meaningful data. Include: fillers (um, uh, er, ah, hmm, mhm, like, you know, I mean), repetitions (I I, the the), restarts (I was- I went), stutters (th-that, b-but, no-not), and informal speech (gonna, wanna, gotta)",
+)
+
+transcript = aai.Transcriber().transcribe(audio_file, config)
+
+print(transcript.text)
+```
+
+## Calling AssemblyAI EU endpoints
+
+If you want to send your request to the AssemblyAI EU endpoint, you can do so by setting the `LITELLM_PROXY_BASE_URL` to `<your-proxy-base-url>/eu.assemblyai`
 
 
 ```python
 import assemblyai as aai
 
-LITELLM_VIRTUAL_KEY = "sk-1234" # <your-virtual-key>
-LITELLM_PROXY_BASE_URL = "http://0.0.0.0:4000/eu.assemblyai" # <your-proxy-base-url>/eu.assemblyai
+aai.settings.base_url = "http://0.0.0.0:4000/eu.assemblyai" # <your-proxy-base-url>/eu.assemblyai
+aai.settings.api_key = "Bearer sk-1234" # Bearer <your-virtual-key>
 
-aai.settings.api_key = f"Bearer {LITELLM_VIRTUAL_KEY}"
-aai.settings.base_url = LITELLM_PROXY_BASE_URL
+# Use a publicly-accessible URL
+audio_file = "https://assembly.ai/wildfires.mp3"
 
-# URL of the file to transcribe
-FILE_URL = "https://assembly.ai/wildfires.mp3"
-
-# You can also transcribe a local file by passing in a file path
-# FILE_URL = './path/to/file.mp3'
+# Or use a local file:
+# audio_file = "./path/to/file.mp3"
 
 transcriber = aai.Transcriber()
-transcript = transcriber.transcribe(FILE_URL)
+transcript = transcriber.transcribe(audio_file)
 print(transcript)
 print(transcript.id)
 ```
+
+## LLM Gateway
+
+Use AssemblyAI's [LLM Gateway](https://www.assemblyai.com/docs/llm-gateway) as an OpenAI-compatible provider — a unified API for Claude, GPT, and Gemini models with full LiteLLM logging, guardrails, and cost tracking support.
+
+[**See Available Models**](https://www.assemblyai.com/docs/llm-gateway#available-models)
+
+### Usage
+
+#### LiteLLM Python SDK
+
+```python
+import litellm
+import os
+
+os.environ["ASSEMBLYAI_API_KEY"] = "your-assemblyai-api-key"
+
+response = litellm.completion(
+    model="assemblyai/claude-sonnet-4-5-20250929",
+    messages=[{"role": "user", "content": "What is the capital of France?"}]
+)
+
+print(response.choices[0].message.content)
+```
+
+#### LiteLLM Proxy
+
+1. Config
+
+```yaml
+model_list:
+  - model_name: assemblyai/*
+    litellm_params:
+      model: assemblyai/*
+      api_key: os.environ/ASSEMBLYAI_API_KEY
+```
+
+2. Start proxy
+
+```bash
+litellm --config config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+3. Test it!
+
+```python
+import requests
+
+headers = {
+    "authorization": "Bearer sk-1234"  # Bearer <your-virtual-key>
+}
+
+response = requests.post(
+    "http://0.0.0.0:4000/v1/chat/completions",
+    headers=headers,
+    json={
+        "model": "assemblyai/claude-sonnet-4-5-20250929",
+        "messages": [
+            {"role": "user", "content": "What is the capital of France?"}
+        ],
+        "max_tokens": 1000
+    }
+)
+
+result = response.json()
+print(result["choices"][0]["message"]["content"])
+```

docs/my-website/docs/pass_through/cursor.md

@@ -0,0 +1,157 @@
+import Image from '@theme/IdealImage';
+
+# Cursor Cloud Agents
+
+Pass-through endpoints for the [Cursor Cloud Agents API](https://docs.cursor.com/account/api) — launch and manage cloud agents that work on your repositories, in native format (no translation).
+
+| Feature | Supported | Notes |
+|---------|-----------|-------|
+| Cost Tracking | ✅ | Logged as $0.00 (subscription-based, no per-request pricing) |
+| Logging | ✅ | All requests logged with operation classification |
+| End-user Tracking | ❌ | [Tell us if you need this](https://github.com/BerriAI/litellm/issues/new) |
+| Streaming | ❌ | Cursor API does not use streaming |
+
+Just replace `https://api.cursor.com` with `LITELLM_PROXY_BASE_URL/cursor` 🚀
+
+**Supported endpoints:**
+
+| Endpoint | Method | Description |
+|----------|--------|-------------|
+| `/v0/agents` | GET | List agents |
+| `/v0/agents` | POST | Launch an agent |
+| `/v0/agents/{id}` | GET | Agent status |
+| `/v0/agents/{id}` | DELETE | Delete an agent |
+| `/v0/agents/{id}/conversation` | GET | Agent conversation |
+| `/v0/agents/{id}/followup` | POST | Add follow-up |
+| `/v0/agents/{id}/stop` | POST | Stop an agent |
+| `/v0/me` | GET | API key info |
+| `/v0/models` | GET | List models |
+| `/v0/repositories` | GET | List GitHub repositories |
+
+## Quick Start
+
+### 1. Add Cursor API Key on the UI
+
+Navigate to **Models + Endpoints → LLM Credentials** and click **Add Credential**. Select **Cursor** from the provider dropdown — you'll see the Cursor logo. Enter your API key from [cursor.com/settings](https://cursor.com/settings).
+
+<Image img={require('../../img/cursor_add_credential.png')} alt="Add Cursor credential with logo" style={{maxWidth: '800px'}} />
+
+### 2. Launch a Cursor Agent
+
+```bash
+curl -X POST http://0.0.0.0:4000/cursor/v0/agents \
+  -H "Authorization: Bearer <your-litellm-key>" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "prompt": {
+      "text": "Add a README.md with installation instructions"
+    },
+    "source": {
+      "repository": "https://github.com/your-org/your-repo",
+      "ref": "main"
+    },
+    "target": {
+      "autoCreatePr": true
+    }
+  }'
+```
+
+**Expected Response:**
+
+```json
+{
+  "id": "bc_abc123",
+  "name": "Add README Documentation",
+  "status": "CREATING",
+  "source": {
+    "repository": "https://github.com/your-org/your-repo",
+    "ref": "main"
+  },
+  "target": {
+    "branchName": "cursor/add-readme-1234",
+    "url": "https://cursor.com/agents?id=bc_abc123",
+    "autoCreatePr": true
+  },
+  "createdAt": "2024-01-15T10:30:00Z"
+}
+```
+
+### 3. View Logs
+
+Navigate to **Logs** in the sidebar. Filter by "cursor" to see your agent requests. Each request shows the operation type (e.g., `cursor/cursor:agent:create`), status, duration, and cost.
+
+<Image img={require('../../img/cursor_logs.png')} alt="Cursor requests in Logs page" style={{maxWidth: '800px'}} />
+
+Click on any log entry to see full request details including provider, API base, and metadata.
+
+<Image img={require('../../img/cursor_log_detail.png')} alt="Cursor log entry detail" style={{maxWidth: '800px'}} />
+
+## Examples
+
+Anything after `http://0.0.0.0:4000/cursor` is treated as a provider-specific route, and handled accordingly.
+
+| **Original Endpoint** | **Replace With** |
+|---|---|
+| `https://api.cursor.com` | `http://0.0.0.0:4000/cursor` (LITELLM_PROXY_BASE_URL) |
+| `-u YOUR_API_KEY:` (Basic Auth) | `-H "Authorization: Bearer <your-litellm-key>"` (LiteLLM Virtual Key) |
+
+### List Available Models
+
+```bash
+curl http://0.0.0.0:4000/cursor/v0/models \
+  -H "Authorization: Bearer <your-litellm-key>"
+```
+
+### Check Agent Status
+
+```bash
+curl http://0.0.0.0:4000/cursor/v0/agents/bc_abc123 \
+  -H "Authorization: Bearer <your-litellm-key>"
+```
+
+### List All Agents
+
+```bash
+curl http://0.0.0.0:4000/cursor/v0/agents \
+  -H "Authorization: Bearer <your-litellm-key>"
+```
+
+### Add Follow-up to Agent
+
+```bash
+curl -X POST http://0.0.0.0:4000/cursor/v0/agents/bc_abc123/followup \
+  -H "Authorization: Bearer <your-litellm-key>" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "prompt": {
+      "text": "Also add a section about troubleshooting"
+    }
+  }'
+```
+
+### Stop an Agent
+
+```bash
+curl -X POST http://0.0.0.0:4000/cursor/v0/agents/bc_abc123/stop \
+  -H "Authorization: Bearer <your-litellm-key>"
+```
+
+### Delete an Agent
+
+```bash
+curl -X DELETE http://0.0.0.0:4000/cursor/v0/agents/bc_abc123 \
+  -H "Authorization: Bearer <your-litellm-key>"
+```
+
+### Get API Key Info
+
+```bash
+curl http://0.0.0.0:4000/cursor/v0/me \
+  -H "Authorization: Bearer <your-litellm-key>"
+```
+
+## Related
+
+- [Cursor Cloud Agents API Docs](https://docs.cursor.com/account/api)
+- [Pass-through Endpoints Overview](./intro.md)
+- [Virtual Keys](../proxy/virtual_keys.md)

docs/my-website/docs/pass_through/google_ai_studio.md

@@ -35,26 +35,25 @@ curl 'http://0.0.0.0:4000/gemini/v1beta/models/gemini-1.5-flash:countTokens?key=
 ```
 
 </TabItem>
-<TabItem value="js" label="Google AI Node.js SDK">
+<TabItem value="js" label="Google GenAI JS SDK">
 
 ```javascript
-const { GoogleGenerativeAI } = require("@google/generative-ai");
+const { GoogleGenAI } = require("@google/genai");
 
-const modelParams = {
-    model: 'gemini-pro',
-};
-  
-const requestOptions = {
-    baseUrl: 'http://localhost:4000/gemini', // http://<proxy-base-url>/gemini
-};
-  
-const genAI = new GoogleGenerativeAI("sk-1234"); // litellm proxy API key
-const model = genAI.getGenerativeModel(modelParams, requestOptions);
+const ai = new GoogleGenAI({
+    apiKey: "sk-1234", // litellm proxy API key
+    httpOptions: {
+        baseUrl: "http://localhost:4000/gemini", // http://<proxy-base-url>/gemini
+    },
+});
 
 async function main() {
     try {
-        const result = await model.generateContent("Explain how AI works");
-        console.log(result.response.text());
+        const response = await ai.models.generateContent({
+            model: "gemini-2.5-flash",
+            contents: "Explain how AI works",
+        });
+        console.log(response.text);
     } catch (error) {
         console.error('Error:', error);
     }
@@ -63,12 +62,13 @@ async function main() {
 // For streaming responses
 async function main_streaming() {
     try {
-        const streamingResult = await model.generateContentStream("Explain how AI works");
-        for await (const chunk of streamingResult.stream) {
-            console.log('Stream chunk:', JSON.stringify(chunk));
+        const response = await ai.models.generateContentStream({
+            model: "gemini-2.5-flash",
+            contents: "Explain how AI works",
+        });
+        for await (const chunk of response) {
+            process.stdout.write(chunk.text);
         }
-        const aggregatedResponse = await streamingResult.response;
-        console.log('Aggregated response:', JSON.stringify(aggregatedResponse));
     } catch (error) {
         console.error('Error:', error);
     }
@@ -321,29 +321,28 @@ curl 'http://0.0.0.0:4000/gemini/v1beta/models/gemini-1.5-flash:generateContent?
 ```
 
 </TabItem>
-<TabItem value="js" label="Google AI Node.js SDK">
+<TabItem value="js" label="Google GenAI JS SDK">
 
 ```javascript
-const { GoogleGenerativeAI } = require("@google/generative-ai");
+const { GoogleGenAI } = require("@google/genai");
 
-const modelParams = {
-    model: 'gemini-pro',
-};
-  
-const requestOptions = {
-    baseUrl: 'http://localhost:4000/gemini', // http://<proxy-base-url>/gemini
-    customHeaders: {
-        "tags": "gemini-js-sdk,pass-through-endpoint"
-    }
-};
-  
-const genAI = new GoogleGenerativeAI("sk-1234");
-const model = genAI.getGenerativeModel(modelParams, requestOptions);
+const ai = new GoogleGenAI({
+    apiKey: "sk-1234",
+    httpOptions: {
+        baseUrl: "http://localhost:4000/gemini", // http://<proxy-base-url>/gemini
+        headers: {
+            "tags": "gemini-js-sdk,pass-through-endpoint",
+        },
+    },
+});
 
 async function main() {
     try {
-        const result = await model.generateContent("Explain how AI works");
-        console.log(result.response.text());
+        const response = await ai.models.generateContent({
+            model: "gemini-2.5-flash",
+            contents: "Explain how AI works",
+        });
+        console.log(response.text);
     } catch (error) {
         console.error('Error:', error);
     }

docs/my-website/docs/providers/anthropic.md

@@ -4,6 +4,8 @@ import TabItem from '@theme/TabItem';
 # Anthropic
 LiteLLM supports all anthropic models.
 
+- `claude-opus-4-6` (`claude-opus-4-6-20260205`)
+- `claude-sonnet-4-6`
 - `claude-sonnet-4-5-20250929`
 - `claude-opus-4-5-20251101`
 - `claude-opus-4-1-20250805`
@@ -50,7 +52,7 @@ Check this in code, [here](../completion/input.md#translated-openai-params)
 **Notes:**
 - Anthropic API fails requests when `max_tokens` are not passed. Due to this litellm passes `max_tokens=4096` when no `max_tokens` are passed.
 - `response_format` is fully supported for Claude Sonnet 4.5 and Opus 4.1 models (see [Structured Outputs](#structured-outputs) section)
-- `reasoning_effort` is automatically mapped to `output_config={"effort": ...}` for Claude Opus 4.5 models (see [Effort Parameter](./anthropic_effort.md))
+- `reasoning_effort` is automatically mapped to `output_config={"effort": ...}` for Claude 4.6 and Opus 4.5 models (see [Effort Parameter](./anthropic_effort.md))
 
 :::
 
@@ -415,7 +417,10 @@ print(response)
 
 | Model Name       | Function Call                              |
 |------------------|--------------------------------------------|
+| claude-opus-4-6  | `completion('claude-opus-4-6-20260205', messages)` | `os.environ['ANTHROPIC_API_KEY']`       |
 | claude-sonnet-4-5  | `completion('claude-sonnet-4-5-20250929', messages)` | `os.environ['ANTHROPIC_API_KEY']`       |
+| claude-opus-4-5  | `completion('claude-opus-4-5-20251101', messages)` | `os.environ['ANTHROPIC_API_KEY']`       |
+| claude-opus-4-1  | `completion('claude-opus-4-1-20250805', messages)` | `os.environ['ANTHROPIC_API_KEY']`       |
 | claude-opus-4  | `completion('claude-opus-4-20250514', messages)` | `os.environ['ANTHROPIC_API_KEY']`       |
 | claude-sonnet-4  | `completion('claude-sonnet-4-20250514', messages)` | `os.environ['ANTHROPIC_API_KEY']`       |
 | claude-3.7  | `completion('claude-3-7-sonnet-20250219', messages)` | `os.environ['ANTHROPIC_API_KEY']`       |

docs/my-website/docs/providers/anthropic_effort.md

@@ -9,10 +9,11 @@ Control how many tokens Claude uses when responding with the `effort` parameter,
 
 The `effort` parameter allows you to control how eager Claude is about spending tokens when responding to requests. This gives you the ability to trade off between response thoroughness and token efficiency, all with a single model.
 
-**Note**: The effort parameter is currently in beta and only supported by Claude Opus 4.5. LiteLLM automatically adds the `effort-2025-11-24` beta header when:
-- `reasoning_effort` parameter is provided (for Claude Opus 4.5 only)
+**Supported models:**
+- **Claude 4.6** (Opus 4.6, Sonnet 4.6) — `output_config` is a stable API feature, no beta header needed. Opus 4.6 also supports `effort="max"`.
+- **Claude Opus 4.5** — requires the `effort-2025-11-24` beta header (automatically added by LiteLLM).
 
-For Claude Opus 4.5, `reasoning_effort="medium"`—both are automatically mapped to the correct format.
+LiteLLM automatically maps `reasoning_effort` → `output_config={"effort": ...}` for all supported models.
 
 ## How Effort Works
 
@@ -35,6 +36,7 @@ This gives a much greater degree of control over efficiency.
 
 | Level | Description | Typical use case |
 |-------|-------------|------------------|
+| `max` | Maximum capability beyond high — Claude uses even more tokens for the most thorough outcome. **Only supported by Claude Opus 4.6.** | The hardest reasoning problems, complex multi-step research |
 | `high` | Maximum capability—Claude uses as many tokens as needed for the best possible outcome. Equivalent to not setting the parameter. | Complex reasoning, difficult coding problems, agentic tasks |
 | `medium` | Balanced approach with moderate token savings. | Agentic tasks that require a balance of speed, cost, and performance |
 | `low` | Most efficient—significant token savings with some capability reduction. | Simpler tasks that need the best speed and lowest costs, such as subagents |
@@ -49,16 +51,29 @@ This gives a much greater degree of control over efficiency.
 ```python
 import litellm
 
+# Works with Claude 4.6 models (no beta header needed)
+response = litellm.completion(
+    model="anthropic/claude-sonnet-4-6",
+    messages=[{
+        "role": "user",
+        "content": "Analyze the trade-offs between microservices and monolithic architectures"
+    }],
+    reasoning_effort="medium"  # Automatically mapped to output_config
+)
+
+print(response.choices[0].message.content)
+```
+
+```python
+# Also works with Claude Opus 4.5 (beta header auto-injected)
 response = litellm.completion(
     model="anthropic/claude-opus-4-5-20251101",
     messages=[{
         "role": "user",
         "content": "Analyze the trade-offs between microservices and monolithic architectures"
     }],
-    reasoning_effort="medium"  # Automatically mapped to output_config for Opus 4.5
+    reasoning_effort="medium"
 )
-
-print(response.choices[0].message.content)
 ```
 
 </TabItem>
@@ -71,8 +86,9 @@ const client = new Anthropic({
   apiKey: process.env.ANTHROPIC_API_KEY,
 });
 
+// Claude 4.6 — output_config is a stable API feature (no beta header)
 const response = await client.messages.create({
-  model: "claude-opus-4-5-20251101",
+  model: "claude-sonnet-4-6",
   max_tokens: 4096,
   messages: [{
     role: "user",
@@ -96,7 +112,29 @@ curl http://localhost:4000/v1/chat/completions \
   -H "Content-Type: application/json" \
   -H "Authorization: Bearer $LITELLM_API_KEY" \
   -d '{
-    "model": "anthropic/claude-opus-4-5-20251101",
+    "model": "anthropic/claude-sonnet-4-6",
+    "messages": [{
+      "role": "user",
+      "content": "Analyze the trade-offs between microservices and monolithic architectures"
+    }],
+    "reasoning_effort": "medium"
+  }'
+```
+
+### Direct Anthropic API Call
+
+<Tabs>
+<TabItem value="46" label="Claude 4.6 (stable)">
+
+```bash
+# Claude 4.6 — no beta header needed
+curl https://api.anthropic.com/v1/messages \
+  --header "x-api-key: $ANTHROPIC_API_KEY" \
+  --header "anthropic-version: 2023-06-01" \
+  --header "content-type: application/json" \
+  --data '{
+    "model": "claude-sonnet-4-6",
+    "max_tokens": 4096,
     "messages": [{
       "role": "user",
       "content": "Analyze the trade-offs between microservices and monolithic architectures"
@@ -107,9 +145,11 @@ curl http://localhost:4000/v1/chat/completions \
   }'
 ```
 
-### Direct Anthropic API Call
+</TabItem>
+<TabItem value="45" label="Claude Opus 4.5 (beta)">
 
 ```bash
+# Claude Opus 4.5 — requires beta header
 curl https://api.anthropic.com/v1/messages \
   --header "x-api-key: $ANTHROPIC_API_KEY" \
   --header "anthropic-version: 2023-06-01" \
@@ -128,10 +168,19 @@ curl https://api.anthropic.com/v1/messages \
   }'
 ```
 
+</TabItem>
+</Tabs>
+
 ## Model Compatibility
 
-The effort parameter is currently only supported by:
-- **Claude Opus 4.5** (`claude-opus-4-5-20251101`)
+The effort parameter is supported by:
+- **Claude Opus 4.6** (`claude-opus-4-6`) — supports `high`, `medium`, `low`, and `max`
+- **Claude Sonnet 4.6** (`claude-sonnet-4-6`) — supports `high`, `medium`, `low`
+- **Claude Opus 4.5** (`claude-opus-4-5-20251101`) — supports `high`, `medium`, `low`
+
+:::info
+`effort="max"` is only available on Claude Opus 4.6. Using it with other models will raise a validation error.
+:::
 
 ## When Should I Adjust the Effort Parameter?
 
@@ -154,7 +203,7 @@ Example with tools:
 import litellm
 
 response = litellm.completion(
-    model="anthropic/claude-opus-4-5-20251101",
+    model="anthropic/claude-sonnet-4-6",
     messages=[{
         "role": "user",
         "content": "Check the weather in multiple cities"
@@ -173,9 +222,7 @@ response = litellm.completion(
             }
         }
     }],
-    output_config={
-        "effort": "low"  # Will make fewer tool calls
-    }
+    reasoning_effort="low"  # Mapped to output_config — will make fewer tool calls
 )
 ```
 
@@ -187,18 +234,12 @@ The effort parameter works seamlessly with extended thinking. When both are enab
 import litellm
 
 response = litellm.completion(
-    model="anthropic/claude-opus-4-5-20251101",
+    model="anthropic/claude-sonnet-4-6",
     messages=[{
         "role": "user",
         "content": "Solve this complex problem"
     }],
-    thinking={
-        "type": "enabled",
-        "budget_tokens": 5000
-    },
-    output_config={
-        "effort": "medium"  # Affects both thinking and response tokens
-    }
+    reasoning_effort="medium"  # Mapped to adaptive thinking + output_config for 4.6 models
 )
 ```
 
@@ -218,14 +259,14 @@ response = litellm.completion(
 
 The effort parameter is supported across all Anthropic-compatible providers:
 
-- **Standard Anthropic API**: ✅ Supported (Claude Opus 4.5)
-- **Azure Anthropic / Microsoft Foundry**: ✅ Supported (Claude Opus 4.5)
-- **Amazon Bedrock**: ✅ Supported (Claude Opus 4.5)
-- **Google Cloud Vertex AI**: ✅ Supported (Claude Opus 4.5)
+- **Standard Anthropic API**: ✅ Supported (Claude 4.6, Opus 4.5)
+- **Azure Anthropic / Microsoft Foundry**: ✅ Supported (Claude 4.6, Opus 4.5)
+- **Amazon Bedrock**: ✅ Supported (Claude 4.6, Opus 4.5)
+- **Google Cloud Vertex AI**: ✅ Supported (Claude 4.6, Opus 4.5)
 
 LiteLLM automatically handles:
-- Beta header injection (`effort-2025-11-24`) for all providers
-- Parameter mapping: `reasoning_effort` → `output_config={"effort": ...}` for Claude Opus 4.5
+- Parameter mapping: `reasoning_effort` → `output_config={"effort": ...}` for all supported models
+- Beta header injection (`effort-2025-11-24`) only for Claude Opus 4.5 (not needed for 4.6 models)
 
 ## Usage and Pricing
 
@@ -244,12 +285,13 @@ print(f"Total tokens: {response.usage.total_tokens}")
 
 ## Troubleshooting
 
-### Beta header not being added
+### Beta header not being added (Claude Opus 4.5)
 
-LiteLLM automatically adds the `effort-2025-11-24` beta header when:
-- `reasoning_effort` parameter is provided (for Claude Opus 4.5 only)
+LiteLLM automatically adds the `effort-2025-11-24` beta header for Claude Opus 4.5 when `reasoning_effort` or `output_config` is provided.
 
-If you're not seeing the header:
+**Note:** Claude 4.6 models do NOT need a beta header — `output_config` is a stable API feature for these models.
+
+If you're not seeing the header for Opus 4.5:
 
 1. Ensure you're using `reasoning_effort` parameter
 2. Verify the model is Claude Opus 4.5
@@ -257,7 +299,7 @@ If you're not seeing the header:
 
 ### Invalid effort value error
 
-Only three values are accepted: `"high"`, `"medium"`, `"low"`. Any other value will raise a validation error:
+Accepted values: `"high"`, `"medium"`, `"low"`, and `"max"` (Opus 4.6 only). Any other value will raise a validation error:
 
 ```python
 # ❌ This will raise an error
@@ -265,11 +307,17 @@ output_config={"effort": "very_low"}
 
 # ✅ Use one of the valid values
 output_config={"effort": "low"}
+
+# ❌ This will raise an error (max only works on Opus 4.6)
+litellm.completion(model="anthropic/claude-sonnet-4-6", reasoning_effort="max", ...)
+
+# ✅ max is only for Opus 4.6
+litellm.completion(model="anthropic/claude-opus-4-6", reasoning_effort="max", ...)
 ```
 
 ### Model not supported
 
-Currently, only Claude Opus 4.5 supports the effort parameter. Using it with other models may result in the parameter being ignored or an error.
+The effort parameter is supported by Claude Opus 4.6, Sonnet 4.6, and Opus 4.5. Using it with other models may result in the parameter being ignored or an error.
 
 ## Related Features
 

docs/my-website/docs/providers/bedrock.md

@@ -660,7 +660,7 @@ Same as [Anthropic API response](../providers/anthropic#usage---thinking--reason
 
 LiteLLM supports Anthropic's beta features on AWS Bedrock through the `anthropic-beta` header. This enables access to experimental features like:
 
-- **1M Context Window** - Up to 1 million tokens of context (Claude Sonnet 4)
+- **1M Context Window** - Up to 1 million tokens of context (Claude Opus 4.6, Sonnet 4.5, Sonnet 4)
 - **Computer Use Tools** - AI that can interact with computer interfaces
 - **Token-Efficient Tools** - More efficient tool usage patterns  
 - **Extended Output** - Up to 128K output tokens
@@ -670,7 +670,7 @@ LiteLLM supports Anthropic's beta features on AWS Bedrock through the `anthropic
 
 | Beta Feature | Header Value | Compatible Models | Description |
 |--------------|-------------|------------------|-------------|
-| 1M Context Window | `context-1m-2025-08-07` | Claude Sonnet 4 | Enable 1 million token context window |
+| 1M Context Window | `context-1m-2025-08-07` | Claude Opus 4.6, Sonnet 4.5, Sonnet 4 | Enable 1 million token context window |
 | Computer Use (Latest) | `computer-use-2025-01-24` | Claude 3.7 Sonnet | Latest computer use tools |
 | Computer Use (Legacy) | `computer-use-2024-10-22` | Claude 3.5 Sonnet v2 | Computer use tools for Claude 3.5 |
 | Token-Efficient Tools | `token-efficient-tools-2025-02-19` | Claude 3.7 Sonnet | More efficient tool usage |

docs/my-website/docs/providers/gemini.md

@@ -1196,6 +1196,8 @@ When responding to Computer Use tool calls, include the URL and screenshot:
 
 
 
+
+
 ## Thought Signatures
 
 Thought signatures are encrypted representations of the model's internal reasoning process for a given turn in a conversation. By passing thought signatures back to the model in subsequent requests, you provide it with the context of its previous thoughts, allowing it to build upon its reasoning and maintain a coherent line of inquiry.
@@ -2039,6 +2041,7 @@ response = litellm.completion(
 | gemini-2.0-flash-lite-preview-02-05	     | `completion(model='gemini/gemini-2.0-flash-lite-preview-02-05', messages)`     | `os.environ['GEMINI_API_KEY']` |
 | gemini-2.5-flash-preview-09-2025     | `completion(model='gemini/gemini-2.5-flash-preview-09-2025', messages)`     | `os.environ['GEMINI_API_KEY']` |
 | gemini-2.5-flash-lite-preview-09-2025     | `completion(model='gemini/gemini-2.5-flash-lite-preview-09-2025', messages)`     | `os.environ['GEMINI_API_KEY']` |
+| gemini-3.1-flash-lite-preview     | `completion(model='gemini/gemini-3.1-flash-lite-preview', messages)`     | `os.environ['GEMINI_API_KEY']` |
 | gemini-flash-latest     | `completion(model='gemini/gemini-flash-latest', messages)`     | `os.environ['GEMINI_API_KEY']` |
 | gemini-flash-lite-latest     | `completion(model='gemini/gemini-flash-lite-latest', messages)`     | `os.environ['GEMINI_API_KEY']` |
 

docs/my-website/docs/providers/groq.md

@@ -159,6 +159,7 @@ We support ALL Groq models, just set `groq/` as a prefix when sending completion
 | moonshotai/kimi-k2-instruct-0905 | `completion(model="groq/moonshotai/kimi-k2-instruct-0905", messages)` |
 | openai/gpt-oss-120b | `completion(model="groq/openai/gpt-oss-120b", messages)` |
 | openai/gpt-oss-20b | `completion(model="groq/openai/gpt-oss-20b", messages)` |
+| openai/gpt-oss-safeguard-20b | `completion(model="groq/openai/gpt-oss-safeguard-20b", messages)` |
 
 ## Groq - Tool / Function Calling Example
 

docs/my-website/docs/providers/moonshot.md

@@ -219,6 +219,37 @@ curl http://localhost:4000/v1/chat/completions \
 
 For more detailed information on using the LiteLLM Proxy, see the [LiteLLM Proxy documentation](../providers/litellm_proxy).
 
+## Image / Vision Support
+
+Moonshot vision models (`kimi-k2.5`, `kimi-latest`, `moonshot-v1-*-vision-preview`, etc.) accept the standard OpenAI content array with `image_url` blocks.
+
+LiteLLM automatically detects when your messages contain images and preserves the content array so the image payload reaches the Moonshot API. For text-only requests the content is flattened to a plain string, as required by Moonshot text models.
+
+```python showLineNumbers title="Moonshot Vision Example"
+import os
+import litellm
+
+os.environ["MOONSHOT_API_KEY"] = ""
+
+response = litellm.completion(
+    model="moonshot/kimi-k2.5",
+    messages=[
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "What is in this image?"},
+                {
+                    "type": "image_url",
+                    "image_url": {"url": "https://example.com/image.png"},
+                },
+            ],
+        }
+    ],
+)
+
+print(response.choices[0].message.content)
+```
+
 ## Moonshot AI Limitations & LiteLLM Handling
 
 LiteLLM automatically handles the following [Moonshot AI limitations](https://platform.moonshot.ai/docs/guide/migrating-from-openai-to-kimi#about-api-compatibility) to provide seamless OpenAI compatibility:

docs/my-website/docs/providers/perplexity.md

@@ -120,7 +120,7 @@ All models listed here https://docs.perplexity.ai/docs/model-cards are supported
 
 
 
-## Agentic Research API (Responses API)
+## Agent API (Responses API)
 
 Requires v1.72.6+
 
@@ -196,7 +196,7 @@ import os
 os.environ['PERPLEXITY_API_KEY'] = ""
 
 response = responses(
-    model="perplexity/openai/gpt-4o",
+    model="perplexity/openai/gpt-5.2",
     input="Explain quantum computing in simple terms",
     custom_llm_provider="perplexity",
     max_output_tokens=500,
@@ -215,7 +215,7 @@ import os
 os.environ['PERPLEXITY_API_KEY'] = ""
 
 response = responses(
-    model="perplexity/anthropic/claude-3-5-sonnet-20241022",
+    model="perplexity/anthropic/claude-sonnet-4-5",
     input="Write a short story about a robot learning to paint",
     custom_llm_provider="perplexity",
     max_output_tokens=500,
@@ -234,7 +234,7 @@ import os
 os.environ['PERPLEXITY_API_KEY'] = ""
 
 response = responses(
-    model="perplexity/google/gemini-2.0-flash-exp",
+    model="perplexity/google/gemini-2.5-flash",
     input="Explain the concept of neural networks",
     custom_llm_provider="perplexity",
     max_output_tokens=500,
@@ -253,7 +253,7 @@ import os
 os.environ['PERPLEXITY_API_KEY'] = ""
 
 response = responses(
-    model="perplexity/xai/grok-2-1212",
+    model="perplexity/xai/grok-4-1-fast-non-reasoning",
     input="What makes a good AI assistant?",
     custom_llm_provider="perplexity",
     max_output_tokens=500,
@@ -276,7 +276,7 @@ import os
 os.environ['PERPLEXITY_API_KEY'] = ""
 
 response = responses(
-    model="perplexity/openai/gpt-4o",
+    model="perplexity/openai/gpt-5.2",
     input="What's the weather in San Francisco today?",
     custom_llm_provider="perplexity",
     tools=[{"type": "web_search"}],
@@ -286,6 +286,78 @@ response = responses(
 print(response.output)
 ```
 
+### Function Calling
+
+The Agent API supports custom function tools. Pass function tools through unchanged:
+
+```python
+from litellm import responses
+import os
+
+os.environ['PERPLEXITY_API_KEY'] = ""
+
+response = responses(
+    model="perplexity/openai/gpt-5.2",
+    input="What's the weather in San Francisco?",
+    custom_llm_provider="perplexity",
+    tools=[
+        {"type": "web_search"},
+        {
+            "type": "function",
+            "function": {
+                "name": "get_weather",
+                "description": "Get the current weather for a location",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "location": {"type": "string"},
+                        "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]},
+                    },
+                },
+            },
+        },
+    ],
+    instructions="Use tools when appropriate.",
+)
+
+print(response.output)
+```
+
+### Structured Outputs
+
+Request JSON schema structured outputs via the `text` parameter:
+
+```python
+from litellm import responses
+import os
+
+os.environ['PERPLEXITY_API_KEY'] = ""
+
+response = responses(
+    model="perplexity/preset/pro-search",
+    input="Extract key facts about the Eiffel Tower",
+    custom_llm_provider="perplexity",
+    text={
+        "format": {
+            "type": "json_schema",
+            "name": "facts",
+            "schema": {
+                "type": "object",
+                "properties": {
+                    "name": {"type": "string"},
+                    "height_meters": {"type": "number"},
+                    "year_built": {"type": "integer"},
+                },
+                "required": ["name", "height_meters", "year_built"],
+            },
+            "strict": True,
+        }
+    },
+)
+
+print(response.output)
+```
+
 
 ### Reasoning Effort (Responses API)
 
@@ -319,7 +391,7 @@ import os
 os.environ['PERPLEXITY_API_KEY'] = ""
 
 response = responses(
-    model="perplexity/anthropic/claude-3-5-sonnet-20241022",
+    model="perplexity/anthropic/claude-sonnet-4-5",
     input=[
         {"type": "message", "role": "system", "content": "You are a helpful assistant."},
         {"type": "message", "role": "user", "content": "What are the latest AI developments?"},
@@ -343,7 +415,7 @@ import os
 os.environ['PERPLEXITY_API_KEY'] = ""
 
 response = responses(
-    model="perplexity/openai/gpt-4o",
+    model="perplexity/openai/gpt-5.2",
     input="Tell me a story about space exploration",
     custom_llm_provider="perplexity",
     stream=True,
@@ -360,23 +432,28 @@ for chunk in response:
 
 | Provider | Model Name | Function Call |
 |----------|------------|---------------|
-| OpenAI | gpt-4o | `responses(model="perplexity/openai/gpt-4o", ...)` |
-| OpenAI | gpt-4o-mini | `responses(model="perplexity/openai/gpt-4o-mini", ...)` |
 | OpenAI | gpt-5.2 | `responses(model="perplexity/openai/gpt-5.2", ...)` |
-| Anthropic | claude-3-5-sonnet-20241022 | `responses(model="perplexity/anthropic/claude-3-5-sonnet-20241022", ...)` |
-| Anthropic | claude-3-5-haiku-20241022 | `responses(model="perplexity/anthropic/claude-3-5-haiku-20241022", ...)` |
-| Google | gemini-2.0-flash-exp | `responses(model="perplexity/google/gemini-2.0-flash-exp", ...)` |
-| Google | gemini-2.0-flash-thinking-exp | `responses(model="perplexity/google/gemini-2.0-flash-thinking-exp", ...)` |
-| xAI | grok-2-1212 | `responses(model="perplexity/xai/grok-2-1212", ...)` |
-| xAI | grok-2-vision-1212 | `responses(model="perplexity/xai/grok-2-vision-1212", ...)` |
+| OpenAI | gpt-5.1 | `responses(model="perplexity/openai/gpt-5.1", ...)` |
+| OpenAI | gpt-5-mini | `responses(model="perplexity/openai/gpt-5-mini", ...)` |
+| Anthropic | claude-opus-4-6 | `responses(model="perplexity/anthropic/claude-opus-4-6", ...)` |
+| Anthropic | claude-opus-4-5 | `responses(model="perplexity/anthropic/claude-opus-4-5", ...)` |
+| Anthropic | claude-sonnet-4-5 | `responses(model="perplexity/anthropic/claude-sonnet-4-5", ...)` |
+| Anthropic | claude-haiku-4-5 | `responses(model="perplexity/anthropic/claude-haiku-4-5", ...)` |
+| Google | gemini-3-pro-preview | `responses(model="perplexity/google/gemini-3-pro-preview", ...)` |
+| Google | gemini-3-flash-preview | `responses(model="perplexity/google/gemini-3-flash-preview", ...)` |
+| Google | gemini-2.5-pro | `responses(model="perplexity/google/gemini-2.5-pro", ...)` |
+| Google | gemini-2.5-flash | `responses(model="perplexity/google/gemini-2.5-flash", ...)` |
+| xAI | grok-4-1-fast-non-reasoning | `responses(model="perplexity/xai/grok-4-1-fast-non-reasoning", ...)` |
+| Perplexity | sonar | `responses(model="perplexity/perplexity/sonar", ...)` |
 
 ### Available Presets
 
-| Preset Name    | Function Call                                           |
-|----------------|--------------------------------------------------------|
-| fast-search    | `responses(model="perplexity/preset/fast-search", ...)`|
-| pro-search     | `responses(model="perplexity/preset/pro-search", ...)` |
-| deep-research  | `responses(model="perplexity/preset/deep-research", ...)`|
+| Preset Name | Function Call |
+|-------------|---------------|
+| fast-search | `responses(model="perplexity/preset/fast-search", ...)` |
+| pro-search | `responses(model="perplexity/preset/pro-search", ...)` |
+| deep-research | `responses(model="perplexity/preset/deep-research", ...)` |
+| advanced-deep-research | `responses(model="perplexity/preset/advanced-deep-research", ...)` |
 
 ### Complete Example
 
@@ -388,7 +465,7 @@ os.environ['PERPLEXITY_API_KEY'] = ""
 
 # Comprehensive example with multiple features
 response = responses(
-    model="perplexity/openai/gpt-4o",
+    model="perplexity/openai/gpt-5.2",
     input="Research the latest developments in quantum computing and provide sources",
     custom_llm_provider="perplexity",
     tools=[

docs/my-website/docs/providers/perplexity_embedding.md

@@ -0,0 +1,134 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Perplexity Embeddings
+
+https://docs.perplexity.ai/docs/embeddings/quickstart
+
+LiteLLM supports Perplexity's pplx-embed embedding models for web-scale text retrieval.
+
+## API Key
+
+```python
+# env variable
+os.environ['PERPLEXITYAI_API_KEY']
+```
+
+## Sample Usage - Embedding
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+from litellm import embedding
+import os
+
+os.environ['PERPLEXITYAI_API_KEY'] = ""
+
+response = embedding(
+    model="perplexity/pplx-embed-v1-0.6b",
+    input=["good morning from litellm"],
+)
+print(response)
+```
+
+</TabItem>
+<TabItem value="proxy" label="Proxy">
+
+1. Setup config.yaml
+
+```yaml
+model_list:
+  - model_name: pplx-embed-v1-0.6b
+    litellm_params:
+      model: perplexity/pplx-embed-v1-0.6b
+      api_key: os.environ/PERPLEXITYAI_API_KEY
+  - model_name: pplx-embed-v1-4b
+    litellm_params:
+      model: perplexity/pplx-embed-v1-4b
+      api_key: os.environ/PERPLEXITYAI_API_KEY
+```
+
+2. Start proxy
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+3. Test it!
+
+```bash
+curl http://0.0.0.0:4000/v1/embeddings \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer sk-1234" \
+  -d '{
+    "model": "pplx-embed-v1-0.6b",
+    "input": ["good morning from litellm"]
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+## Supported Parameters
+
+Perplexity embeddings support the following optional parameters:
+
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| `dimensions` | int | Output embedding dimensions. 128–1024 for 0.6b models, 128–2560 for 4b models. Defaults to max. |
+| `encoding_format` | string | `"base64_int8"` (default) or `"base64_binary"` for compressed output. |
+
+### Example with Parameters
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+from litellm import embedding
+import os
+
+os.environ['PERPLEXITYAI_API_KEY'] = ""
+
+response = embedding(
+    model="perplexity/pplx-embed-v1-4b",
+    input=["Your text here"],
+    dimensions=512,
+)
+print(f"Embedding dimensions: {len(response.data[0]['embedding'])}")
+```
+
+</TabItem>
+<TabItem value="proxy" label="Proxy">
+
+```bash
+curl http://0.0.0.0:4000/v1/embeddings \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer sk-1234" \
+  -d '{
+    "model": "pplx-embed-v1-4b",
+    "input": ["Your text here"],
+    "dimensions": 512
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+## Supported Models
+
+All models listed on the [Perplexity Embeddings docs](https://docs.perplexity.ai/docs/embeddings/quickstart) are supported. Use `model=perplexity/<model-name>`.
+
+| Model Name | Dimensions | Max Tokens | Price (per 1M tokens) | Function Call |
+|---|---|---|---|---|
+| pplx-embed-v1-0.6b | 1024 | 32K | $0.004 | `embedding(model="perplexity/pplx-embed-v1-0.6b", input)` |
+| pplx-embed-v1-4b | 2560 | 32K | $0.03 | `embedding(model="perplexity/pplx-embed-v1-4b", input)` |
+
+### Key Specifications
+
+- **Max texts per request:** 512
+- **Max tokens per input:** 32,768
+- **Combined request limit:** 120,000 tokens
+- **Matryoshka dimension reduction** — reduce dimensions to 128+ for faster search and reduced storage
+- **No instruction prefix required** — embed text directly
+- **Unnormalized embeddings** — use cosine similarity for comparison

docs/my-website/docs/providers/vertex.md

@@ -1685,6 +1685,7 @@ litellm.vertex_location = "us-central1 # Your Location
 | gemini-2.5-pro   | `completion('gemini-2.5-pro', messages)`, `completion('vertex_ai/gemini-2.5-pro', messages)` |
 | gemini-2.5-flash-preview-09-2025   | `completion('gemini-2.5-flash-preview-09-2025', messages)`, `completion('vertex_ai/gemini-2.5-flash-preview-09-2025', messages)` |
 | gemini-2.5-flash-lite-preview-09-2025   | `completion('gemini-2.5-flash-lite-preview-09-2025', messages)`, `completion('vertex_ai/gemini-2.5-flash-lite-preview-09-2025', messages)` |
+| gemini-3.1-flash-lite-preview   | `completion('gemini-3.1-flash-lite-preview', messages)`, `completion('vertex_ai/gemini-3.1-flash-lite-preview', messages)` |
 
 ## Private Service Connect (PSC) Endpoints
 

docs/my-website/docs/providers/vertex_realtime.md

@@ -0,0 +1,203 @@
+# Vertex AI Gemini Live - Realtime API
+
+Use Vertex AI's Gemini Live API (BidiGenerateContent) through LiteLLM's unified `/realtime` endpoint, which speaks the OpenAI Realtime protocol.
+
+| Feature | Supported |
+|---------|-----------|
+| Proxy (`/realtime`) | ✅ |
+| Voice in / Voice out | ✅ |
+| Text in / Text out | ✅ |
+| Server VAD | ✅ |
+| Output transcription | ✅ |
+
+## Setup
+
+### 1. Auth
+
+LiteLLM uses your Google Cloud credentials (OAuth2 Bearer token), not an API key.
+
+```bash
+gcloud auth application-default login
+```
+
+Or set a service-account key file:
+
+```bash
+export GOOGLE_APPLICATION_CREDENTIALS=/path/to/sa-key.json
+```
+
+### 2. Proxy config
+
+```yaml
+model_list:
+  - model_name: vertex-gemini-live
+    litellm_params:
+      model: vertex_ai/gemini-2.0-flash-live-001
+      vertex_project: your-gcp-project-id
+      vertex_location: us-east4   # or any supported region, or "global"
+
+general_settings:
+  master_key: sk-your-key
+```
+
+### 3. Start the proxy
+
+```bash
+litellm --config config.yaml --port 4000
+```
+
+## Usage
+
+### Python (websockets)
+
+```python
+import asyncio
+import json
+import websockets
+
+PROXY_URL = "ws://localhost:4000/realtime?model=vertex-gemini-live"
+API_KEY = "sk-your-key"
+
+async def main():
+    async with websockets.connect(
+        PROXY_URL,
+        additional_headers={"api-key": API_KEY},
+    ) as ws:
+        # Wait for session.created
+        event = json.loads(await ws.recv())
+        print(f"session.created: {event['session']['id']}")
+
+        # Send a text message
+        await ws.send(json.dumps({
+            "type": "conversation.item.create",
+            "item": {
+                "type": "message",
+                "role": "user",
+                "content": [{"type": "input_text", "text": "Say hello in one sentence."}],
+            },
+        }))
+
+        # Collect the response
+        async for raw in ws:
+            ev = json.loads(raw)
+            t = ev.get("type", "")
+            if t == "response.text.delta":
+                print(ev.get("delta", ""), end="", flush=True)
+            elif t == "response.done":
+                print("\n[done]")
+                break
+
+asyncio.run(main())
+```
+
+### Node.js
+
+```js
+const WebSocket = require("ws");
+
+const ws = new WebSocket(
+  "ws://localhost:4000/realtime?model=vertex-gemini-live",
+  { headers: { "api-key": "sk-your-key" } }
+);
+
+ws.on("open", () => {
+  ws.send(JSON.stringify({
+    type: "conversation.item.create",
+    item: {
+      type: "message",
+      role: "user",
+      content: [{ type: "input_text", text: "Say hello." }],
+    },
+  }));
+});
+
+ws.on("message", (data) => {
+  const ev = JSON.parse(data);
+  if (ev.type === "response.text.delta") process.stdout.write(ev.delta);
+  if (ev.type === "response.done") ws.close();
+});
+```
+
+### OpenAI SDK (Python)
+
+```python
+import asyncio
+from openai import AsyncOpenAI
+
+client = AsyncOpenAI(
+    base_url="http://localhost:4000",
+    api_key="sk-your-key",
+)
+
+async def main():
+    async with client.beta.realtime.connect(
+        model="vertex-gemini-live"
+    ) as conn:
+        await conn.session.update(session={"modalities": ["text"]})
+
+        await conn.conversation.item.create(
+            item={
+                "type": "message",
+                "role": "user",
+                "content": [{"type": "input_text", "text": "Say hello."}],
+            }
+        )
+
+        async for event in conn:
+            if event.type == "response.text.delta":
+                print(event.delta, end="", flush=True)
+            elif event.type == "response.done":
+                print()
+                break
+
+asyncio.run(main())
+```
+
+## Voice in / Voice out
+
+For a complete voice example see [`voice_realtime_test.py`](https://github.com/BerriAI/litellm/blob/main/voice_realtime_test.py).
+
+Key settings for audio:
+- Microphone input: **16 kHz** PCM16 (`audio/pcm;rate=16000`)
+- Speaker output: **24 kHz** PCM16 (Vertex AI returns audio at 24 kHz)
+- Server VAD is enabled by default with 800 ms silence threshold
+
+```python
+# session.update with server VAD — the proxy ignores this for Vertex AI
+# because VAD is already configured in the initial setup message.
+await ws.send(json.dumps({
+    "type": "session.update",
+    "session": {
+        "modalities": ["audio"],
+        "turn_detection": {"type": "server_vad", "silence_duration_ms": 800},
+    },
+}))
+```
+
+## Supported OpenAI Realtime Events
+
+**Client → Proxy (→ Vertex AI)**
+
+| OpenAI event | Notes |
+|---|---|
+| `input_audio_buffer.append` | Forwarded as `realtime_input.audio` |
+| `conversation.item.create` | Forwarded as `realtime_input.text` |
+| `session.update` | Silently ignored — Vertex AI does not support mid-session reconfiguration |
+| `response.create` | Silently ignored — Vertex AI responds automatically after each turn |
+
+**Vertex AI → Proxy (→ Client)**
+
+| OpenAI event emitted | Vertex AI source |
+|---|---|
+| `session.created` | Synthesized after `setupComplete` |
+| `response.text.delta` | `serverContent.modelTurn.parts[].text` |
+| `response.audio.delta` | `serverContent.modelTurn.parts[].inlineData` |
+| `response.audio_transcript.delta` | `serverContent.outputTranscription.text` |
+| `conversation.item.input_audio_transcription.completed` | `serverContent.inputTranscription.text` |
+| `response.done` | `serverContent.turnComplete` |
+
+## Limitations
+
+- `session.update` is not forwarded (Vertex AI only accepts one setup message per connection).
+- Tool calling / function calling is not yet supported.
+- Audio transcription requires `outputAudioTranscription: {}` to be set in the initial setup (done automatically by LiteLLM).

docs/my-website/docs/providers/watsonx/rerank.md

@@ -0,0 +1,52 @@
+# watsonx.ai Rerank
+
+## Overview
+
+| Property | Details                                                                  |
+|----------|--------------------------------------------------------------------------|
+| Description | watsonx.ai rerank integration                                            |
+| Provider Route on LiteLLM | `watsonx/`                                                               |
+| Supported Operations | `/ml/v1/text/rerank`                                                     |
+| Link to Provider Doc | [IBM WatsonX.ai ↗](https://cloud.ibm.com/apidocs/watsonx-ai#text-rerank) |
+
+## Quick Start
+
+### **LiteLLM SDK**
+
+```python
+import os
+from litellm import rerank
+
+os.environ["WATSONX_APIKEY"] = "YOUR_WATSONX_APIKEY"
+os.environ["WATSONX_API_BASE"] = "YOUR_WATSONX_API_BASE"
+os.environ["WATSONX_PROJECT_ID"] = "YOUR_WATSONX_PROJECT_ID"
+
+query="Best programming language for beginners?"
+documents=[
+    "Python is great for beginners due to simple syntax.",
+    "JavaScript runs in browsers and is versatile.",
+    "Rust has a steep learning curve but is very safe.",
+]
+
+response = rerank(
+    model="watsonx/cross-encoder/ms-marco-minilm-l-12-v2",
+    query=query,
+    documents=documents,
+    top_n=2,
+    return_documents=True,
+)
+
+print(response)
+```
+
+### **LiteLLM Proxy**
+
+```yaml
+model_list:
+  - model_name: cross-encoder/ms-marco-minilm-l-12-v2
+    litellm_params:
+      model: watsonx/cross-encoder/ms-marco-minilm-l-12-v2
+      api_key: os.environ/WATSONX_APIKEY
+      api_base: os.environ/WATSONX_API_BASE
+      project_id: os.environ/WATSONX_PROJECT_ID
+```

docs/my-website/docs/proxy/alerting.md

@@ -438,6 +438,59 @@ curl -X GET --location 'http://0.0.0.0:4000/health/services?service=webhook' \
 
 - `event_message` *str*: A human-readable description of the event.
 
+### Digest Mode (Reducing Alert Noise)
+
+By default, LiteLLM sends a separate Slack message for **every** alert event. For high-frequency alert types like `llm_requests_hanging` or `llm_too_slow`, this can produce hundreds of duplicate messages per day.
+
+**Digest mode** aggregates duplicate alerts within a configurable time window and emits a single summary message with the total count and time range.
+
+#### Configuration
+
+Use `alert_type_config` in `general_settings` to enable digest mode per alert type:
+
+```yaml
+general_settings:
+  alerting: ["slack"]
+  alert_type_config:
+    llm_requests_hanging:
+      digest: true
+      digest_interval: 86400  # 24 hours (default)
+    llm_too_slow:
+      digest: true
+      digest_interval: 3600   # 1 hour
+    llm_exceptions:
+      digest: true
+      # uses default interval (86400 seconds / 24 hours)
+```
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `digest` | bool | `false` | Enable digest mode for this alert type |
+| `digest_interval` | int | `86400` (24h) | Time window in seconds. Alerts are aggregated within this interval. |
+
+#### How It Works
+
+1. When an alert fires for a digest-enabled type, it is **grouped** by `(alert_type, request_model, api_base)` instead of being sent immediately
+2. A counter tracks how many times the alert fires within the interval
+3. When the interval expires, a **single summary message** is sent:
+
+```
+Alert type: `llm_requests_hanging` (Digest)
+Level: `Medium`
+Start: `2026-02-19 03:27:39`
+End: `2026-02-20 03:27:39`
+Count: `847`
+
+Message: `Requests are hanging - 600s+ request time`
+Request Model: `gemini-2.5-flash`
+API Base: `None`
+```
+
+#### Limitations
+
+- **Per-instance**: Digest state is held in memory per proxy instance. If you run multiple instances (e.g., Cloud Run with autoscaling), each instance maintains its own digest and emits its own summary.
+- **Not durable**: If an instance is terminated before the digest interval expires, the aggregated alerts for that instance are lost.
+
 ## Region-outage alerting (✨ Enterprise feature)
 
 :::info

docs/my-website/docs/proxy/auto_routing.md

@@ -219,3 +219,189 @@ curl -X POST http://localhost:4000/v1/chat/completions \
 3. If a route's similarity score exceeds the threshold, the request is routed to that model
 4. If no route matches, the request goes to the default model
 
+---
+
+## Complexity Router
+
+The Complexity Router provides an alternative to semantic routing that uses **rule-based scoring** to classify requests by complexity and route them to appropriate models — with **zero external API calls** and **sub-millisecond latency**.
+
+### When to Use
+
+| Feature | Semantic Auto Router | Complexity Router |
+|---------|---------------------|-------------------|
+| Classification | Embedding-based matching | Rule-based scoring |
+| Latency | ~100-500ms (embedding API) | <1ms |
+| API Calls | Requires embedding model | None |
+| Training | Requires utterance examples | Works out of the box |
+| Best For | Intent-based routing | Cost optimization |
+
+Use **Complexity Router** when you want to:
+- Route simple queries to cheaper/faster models (e.g., gpt-4o-mini)
+- Route complex queries to more capable models (e.g., claude-sonnet-4)
+- Minimize latency overhead from routing decisions
+- Avoid additional API costs for embeddings
+
+### LiteLLM Python SDK
+
+```python
+from litellm import Router
+
+router = Router(
+    model_list=[
+        # Target models for each tier
+        {
+            "model_name": "gpt-4o-mini",
+            "litellm_params": {"model": "gpt-4o-mini"},
+        },
+        {
+            "model_name": "gpt-4o",
+            "litellm_params": {"model": "gpt-4o"},
+        },
+        {
+            "model_name": "claude-sonnet",
+            "litellm_params": {"model": "claude-sonnet-4-20250514"},
+        },
+        {
+            "model_name": "o1-preview",
+            "litellm_params": {"model": "o1-preview"},
+        },
+        # Complexity router configuration
+        {
+            "model_name": "smart-router",
+            "litellm_params": {
+                "model": "auto_router/complexity_router",
+                "complexity_router_config": {
+                    "tiers": {
+                        "SIMPLE": "gpt-4o-mini",
+                        "MEDIUM": "gpt-4o",
+                        "COMPLEX": "claude-sonnet",
+                        "REASONING": "o1-preview",
+                    },
+                },
+                "complexity_router_default_model": "gpt-4o",
+            },
+        },
+    ],
+)
+```
+
+#### Usage
+
+```python
+# Simple query → routes to gpt-4o-mini
+response = await router.acompletion(
+    model="smart-router",
+    messages=[{"role": "user", "content": "What is 2+2?"}],
+)
+
+# Complex technical query → routes to claude-sonnet or higher
+response = await router.acompletion(
+    model="smart-router",
+    messages=[{"role": "user", "content": "Design a distributed microservice architecture with Kubernetes orchestration"}],
+)
+
+# Reasoning request → routes to o1-preview
+response = await router.acompletion(
+    model="smart-router",
+    messages=[{"role": "user", "content": "Think step by step and reason through this problem carefully..."}],
+)
+```
+
+### LiteLLM Proxy Server
+
+Add the complexity router to your `config.yaml`:
+
+```yaml
+model_list:
+  # Target models
+  - model_name: gpt-4o-mini
+    litellm_params:
+      model: gpt-4o-mini
+      
+  - model_name: gpt-4o
+    litellm_params:
+      model: gpt-4o
+      
+  - model_name: claude-sonnet
+    litellm_params:
+      model: claude-sonnet-4-20250514
+      
+  - model_name: o1-preview
+    litellm_params:
+      model: o1-preview
+
+  # Complexity router
+  - model_name: smart-router
+    litellm_params:
+      model: auto_router/complexity_router
+      complexity_router_config:
+        tiers:
+          SIMPLE: gpt-4o-mini
+          MEDIUM: gpt-4o
+          COMPLEX: claude-sonnet
+          REASONING: o1-preview
+      complexity_router_default_model: gpt-4o
+```
+
+### Configuration Options
+
+#### Tier Boundaries
+
+Customize the score thresholds for each tier:
+
+```yaml
+complexity_router_config:
+  tiers:
+    SIMPLE: gpt-4o-mini
+    MEDIUM: gpt-4o
+    COMPLEX: claude-sonnet
+    REASONING: o1-preview
+  tier_boundaries:
+    simple_medium: 0.15    # Below 0.15 → SIMPLE
+    medium_complex: 0.35   # 0.15-0.35 → MEDIUM
+    complex_reasoning: 0.60  # 0.35-0.60 → COMPLEX, above → REASONING
+```
+
+#### Token Thresholds
+
+Adjust when prompts are considered "short" or "long":
+
+```yaml
+complexity_router_config:
+  token_thresholds:
+    simple: 15   # Prompts under 15 tokens are penalized (simple indicator)
+    complex: 400 # Prompts over 400 tokens get complexity boost
+```
+
+#### Dimension Weights
+
+Customize how much each signal contributes to the complexity score:
+
+```yaml
+complexity_router_config:
+  dimension_weights:
+    tokenCount: 0.10        # Prompt length
+    codePresence: 0.30      # Code-related keywords
+    reasoningMarkers: 0.25  # "step by step", "think through", etc.
+    technicalTerms: 0.25    # Domain-specific complexity
+    simpleIndicators: 0.05  # "what is", "define", greetings
+    multiStepPatterns: 0.03 # "first...then", numbered steps
+    questionComplexity: 0.02 # Multiple questions
+```
+
+### How Complexity Routing Works
+
+The router scores each request across 7 dimensions:
+
+| Dimension | What It Detects | Effect |
+|-----------|-----------------|--------|
+| Token Count | Short (<15) or long (>400) prompts | Short = simple, long = complex |
+| Code Presence | "function", "class", "api", "database", etc. | Increases complexity |
+| Reasoning Markers | "step by step", "think through", "analyze" | Triggers REASONING tier |
+| Technical Terms | "architecture", "distributed", "encryption" | Increases complexity |
+| Simple Indicators | "what is", "define", "hello" | Decreases complexity |
+| Multi-Step Patterns | "first...then", "1. 2. 3." | Increases complexity |
+| Question Complexity | Multiple question marks | Increases complexity |
+
+**Special behavior:** If 2+ reasoning markers are detected in the user message, the request automatically routes to the REASONING tier regardless of the weighted score.
+

docs/my-website/docs/proxy/budget_reset_and_tz.md

@@ -1,16 +1,20 @@
-## Budget Reset Times and Timezones
+# Budget Reset Times and Timezones
 
-LiteLLM now supports predictable budget reset times that align with natural calendar boundaries:
+LiteLLM supports predictable budget reset times that align with natural calendar boundaries.
 
-- All budgets reset at midnight (00:00:00) in the configured timezone
-- Special handling for common durations:
-  - Daily (24h/1d): Reset at midnight every day
-  - Weekly (7d): Reset on Monday at midnight
-  - Monthly (30d): Reset on the 1st of each month at midnight
+## How Budget Resets Work
 
-### Configuring the Timezone
+All budgets reset at midnight (00:00:00) in the configured timezone with special handling for common durations:
 
-You can specify the timezone for all budget resets in your configuration file:
+| Duration | Reset Behavior |
+| --- | --- |
+| Daily (24h/1d) | Resets at midnight every day |
+| Weekly (7d) | Resets on Monday at midnight |
+| Monthly (30d) | Resets on the 1st of each month at midnight |
+
+## Configuring the Timezone
+
+Specify the timezone for all budget resets in your configuration file:
 
 ```yaml
 litellm_settings:
@@ -19,16 +23,21 @@ litellm_settings:
   timezone: "US/Eastern" # Any valid timezone string
 ```
 
-This ensures that all budget resets happen at midnight in your specified timezone rather than in UTC.
-If no timezone is specified, UTC will be used by default.
+This ensures that all budget resets happen at midnight in your specified timezone rather than in UTC. If no timezone is specified, UTC will be used by default.
 
-Common timezone values:
+## Supported Timezones
 
-- `UTC` - Coordinated Universal Time
-- `US/Eastern` - Eastern Time
-- `US/Pacific` - Pacific Time
-- `Europe/London` - UK Time
-- `Asia/Kolkata` - Indian Standard Time (IST)
-- `Asia/Bangkok` - Indochina Time (ICT)
-- `Asia/Tokyo` - Japan Standard Time
-- `Australia/Sydney` - Australian Eastern Time
+Any valid [IANA timezone string](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) is supported (powered by Python's `zoneinfo` module). DST transitions are handled automatically.
+
+**Common timezone values:**
+
+| Timezone | Description |
+| --- | --- |
+| `UTC` | Coordinated Universal Time |
+| `US/Eastern` | Eastern Time |
+| `US/Pacific` | Pacific Time |
+| `Europe/London` | UK Time |
+| `Asia/Kolkata` | Indian Standard Time (IST) |
+| `Asia/Bangkok` | Indochina Time (ICT) |
+| `Asia/Tokyo` | Japan Standard Time |
+| `Australia/Sydney` | Australian Eastern Time |

docs/my-website/docs/proxy/caching.md

@@ -340,6 +340,7 @@ litellm_settings:
     qdrant_semantic_cache_embedding_model: openai-embedding # the model should be defined on the model_list
     qdrant_collection_name: test_collection
     qdrant_quantization_config: binary
+    qdrant_semantic_cache_vector_size: 1536 # vector size must match embedding model dimensionality
     similarity_threshold: 0.8 # similarity threshold for semantic cache
 ```
 

docs/my-website/docs/proxy/cli_sso.md

@@ -52,6 +52,10 @@ LITELLM_CLI_JWT_EXPIRATION_HOURS=48 EXPERIMENTAL_UI_LOGIN="True" litellm --confi
 - `LITELLM_CLI_JWT_EXPIRATION_HOURS=168` - Tokens expire after 7 days (168 hours)
 - `LITELLM_CLI_JWT_EXPIRATION_HOURS=720` - Tokens expire after 30 days (720 hours)
 
+:::note[Experimental UI Session]
+When `EXPERIMENTAL_UI_LOGIN` is enabled, the **browser UI login** session uses a fixed 10-minute expiry (not configurable). `LITELLM_UI_SESSION_DURATION` applies only to non-experimental flows.
+:::
+
 :::tip
 You can check your current token's age and expiration status using:
 ```bash

docs/my-website/docs/proxy/config_settings.md

@@ -73,6 +73,7 @@ litellm_settings:
     qdrant_semantic_cache_embedding_model: openai-embedding # the model should be defined on the model_list
     qdrant_collection_name: test_collection
     qdrant_quantization_config: binary
+    qdrant_semantic_cache_vector_size: 1536 # vector size must match embedding model dimensionality
     similarity_threshold: 0.8 # similarity threshold for semantic cache
 
     # Optional - S3 Cache Settings
@@ -195,6 +196,7 @@ router_settings:
 | disable_end_user_cost_tracking_prometheus_only | boolean | If true, turns off end user cost tracking on prometheus metrics only. |
 | key_generation_settings | object | Restricts who can generate keys. [Further docs](./virtual_keys.md#restricting-key-generation) |
 | disable_add_transform_inline_image_block | boolean | For Fireworks AI models - if true, turns off the auto-add of `#transform=inline` to the url of the image_url, if the model is not a vision model. |
+| use_chat_completions_url_for_anthropic_messages | boolean | If true, routes OpenAI `/v1/messages` requests through chat/completions instead of the Responses API. Can also be set via env var `LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES=true`. |
 | disable_hf_tokenizer_download | boolean | If true, it defaults to using the openai tokenizer for all models (including huggingface models). |
 | enable_json_schema_validation | boolean | If true, enables json schema validation for all requests. |
 | disable_copilot_system_to_assistant | boolean | **DEPRECATED** - GitHub Copilot API supports system prompts. |
@@ -358,7 +360,8 @@ router_settings:
 | redis_url | str | URL for Redis server. **Known performance issue with Redis URL.** |
 | cache_responses | boolean | Flag to enable caching LLM Responses, if cache set under `router_settings`. If true, caches responses. Defaults to False. |
 | router_general_settings | RouterGeneralSettings | [SDK-Only] Router general settings - contains optimizations like 'async_only_mode'. [Docs](../routing.md#router-general-settings) |
-| optional_pre_call_checks | List[str] | List of pre-call checks to add to the router. Currently supported: 'router_budget_limiting', 'prompt_caching' |
+| optional_pre_call_checks | List[str] | List of pre-call checks to add to the router. Supported: `router_budget_limiting`, `prompt_caching`, `responses_api_deployment_check`, `encrypted_content_affinity`, `deployment_affinity`, `session_affinity`, `forward_client_headers_by_model_group` |
+| deployment_affinity_ttl_seconds | int | TTL (seconds) for user-key → deployment affinity mapping when `deployment_affinity` is enabled (configured at Router init / proxy startup). Defaults to `3600` (1 hour). |
 | ignore_invalid_deployments | boolean | If true, ignores invalid deployments. Default for proxy is True - to prevent invalid models from blocking other models from being loaded. |
 | search_tools | List[SearchToolTypedDict] | List of search tool configurations for Search API integration. Each tool specifies a search_tool_name and litellm_params with search_provider, api_key, api_base, etc. [Further Docs](../search.md) |
 | guardrail_list | List[GuardrailTypedDict] | List of guardrail configurations for guardrail load balancing. Enables load balancing across multiple guardrail deployments with the same guardrail_name. [Further Docs](./guardrails/guardrail_load_balancing.md) |
@@ -484,6 +487,8 @@ router_settings:
 | CUSTOM_TIKTOKEN_CACHE_DIR | Custom directory for Tiktoken cache
 | CONFIDENT_API_KEY | API key for Confident AI (Deepeval) Logging service
 | COHERE_API_BASE | Base URL for Cohere API. Default is https://api.cohere.com
+| COMPETITOR_LLM_TEMPERATURE | Temperature setting for the LLM used in competitor discovery. Default is 0.3
+| CURSOR_API_BASE | API base URL for Cursor AI provider integration. Default is https://api.cursor.com
 | DATABASE_HOST | Hostname for the database server
 | DATABASE_NAME | Name of the database
 | DATABASE_PASSWORD | Password for the database user
@@ -493,6 +498,7 @@ router_settings:
 | DATABASE_USER | Username for database connection
 | DATABASE_USERNAME | Alias for database user
 | DATABRICKS_API_BASE | Base URL for Databricks API
+| DATABRICKS_API_KEY | API key (Personal Access Token) for Databricks API authentication
 | DATABRICKS_CLIENT_ID | Client ID for Databricks OAuth M2M authentication (Service Principal application ID)
 | DATABRICKS_CLIENT_SECRET | Client secret for Databricks OAuth M2M authentication
 | DATABRICKS_USER_AGENT | Custom user agent string for Databricks API requests. Used for partner telemetry attribution
@@ -540,7 +546,7 @@ router_settings:
 | DEFAULT_IMAGE_WIDTH | Default width for images. Default is 300
 | DEFAULT_IN_MEMORY_TTL | Default time-to-live for in-memory cache in seconds. Default is 5
 | DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL | Default time-to-live in seconds for management objects (User, Team, Key, Organization) in memory cache. Default is 60 seconds.
-| DEFAULT_MAX_LRU_CACHE_SIZE | Default maximum size for LRU cache. Default is 16
+| DEFAULT_MAX_LRU_CACHE_SIZE | Default maximum size for LRU cache. Default is 64
 | DEFAULT_MAX_RECURSE_DEPTH | Default maximum recursion depth. Default is 100
 | DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER | Default maximum recursion depth for sensitive data masker. Default is 10
 | DEFAULT_MAX_RETRIES | Default maximum retry attempts. Default is 2
@@ -551,6 +557,10 @@ router_settings:
 | DEFAULT_MCP_SEMANTIC_FILTER_SIMILARITY_THRESHOLD | Default similarity threshold for MCP semantic tool filtering. Default is 0.3
 | DEFAULT_MCP_SEMANTIC_FILTER_TOP_K | Default number of top results to return for MCP semantic tool filtering. Default is 10
 | MCP_NPM_CACHE_DIR | Directory for npm cache used by STDIO MCP servers. In containers the default (~/.npm) may not exist or be read-only. Default is `/tmp/.npm_mcp_cache`
+| LITELLM_MCP_CLIENT_TIMEOUT | MCP client connection timeout in seconds (stdio and HTTP/SSE transports). Default is 60
+| LITELLM_MCP_TOOL_LISTING_TIMEOUT | Timeout in seconds for listing tools from an MCP server. Default is 30
+| LITELLM_MCP_METADATA_TIMEOUT | HTTP client timeout in seconds for OAuth metadata fetching. Default is 10
+| LITELLM_MCP_HEALTH_CHECK_TIMEOUT | Health check timeout in seconds for MCP servers. Default is 10
 | MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL | Default TTL in seconds for MCP OAuth2 token cache. Default is 3600
 | MCP_OAUTH2_TOKEN_CACHE_MAX_SIZE | Maximum number of entries in MCP OAuth2 token cache. Default is 200
 | MCP_OAUTH2_TOKEN_CACHE_MIN_TTL | Minimum TTL in seconds for MCP OAuth2 token cache. Default is 10
@@ -571,6 +581,8 @@ router_settings:
 | DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_PRO | Default minimal reasoning effort thinking budget for Gemini 2.5 Pro. Default is 512
 | DEFAULT_REDIS_MAJOR_VERSION | Default Redis major version to assume when version cannot be determined. Default is 7
 | DEFAULT_REDIS_SYNC_INTERVAL | Default Redis synchronization interval in seconds. Default is 1
+| DEFAULT_SEMANTIC_GUARD_EMBEDDING_MODEL | Default embedding model for Semantic Guard (route-matching guardrail). Default is "text-embedding-3-small"
+| DEFAULT_SEMANTIC_GUARD_SIMILARITY_THRESHOLD | Default similarity threshold for Semantic Guard route matching. Default is 0.75
 | DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND | Default price per second for Replicate GPU. Default is 0.001400
 | DEFAULT_REPLICATE_POLLING_DELAY_SECONDS | Default delay in seconds for Replicate polling. Default is 1
 | DEFAULT_REPLICATE_POLLING_RETRIES | Default number of retries for Replicate polling. Default is 5
@@ -603,7 +615,6 @@ router_settings:
 | EMAIL_BUDGET_ALERT_TTL | Time-to-live for budget alert deduplication in seconds. Default is 86400 (24 hours)
 | ENKRYPTAI_API_BASE | Base URL for EnkryptAI Guardrails API. **Default is https://api.enkryptai.com**
 | ENKRYPTAI_API_KEY | API key for EnkryptAI Guardrails service
-| EXPERIMENTAL_MULTI_INSTANCE_RATE_LIMITING | Flag to enable new multi-instance rate limiting. **Default is False**
 | FIREWORKS_AI_4_B | Size parameter for Fireworks AI 4B model. Default is 4
 | FIREWORKS_AI_16_B | Size parameter for Fireworks AI 16B model. Default is 16
 | FIREWORKS_AI_56_B_MOE | Size parameter for Fireworks AI 56B MOE model. Default is 56
@@ -751,15 +762,18 @@ router_settings:
 | LITELLM_ANTHROPIC_BETA_HEADERS_URL | Custom URL for fetching Anthropic beta headers configuration. Default is the GitHub main branch URL
 | LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX | Disable automatic URL suffix appending for Anthropic API base URLs. When set to `true`, prevents LiteLLM from automatically adding `/v1/messages` or `/v1/complete` to custom Anthropic API endpoints
 | LITELLM_ASSETS_PATH | Path to directory for UI assets and logos. Used when running with read-only filesystem (e.g., Kubernetes). Default is `/var/lib/litellm/assets` in Docker.
+| LITELLM_BLOG_POSTS_URL | Custom URL for fetching LiteLLM blog posts JSON. Default is the GitHub main branch URL
 | LITELLM_CLI_JWT_EXPIRATION_HOURS | Expiration time in hours for CLI-generated JWT tokens. Default is 24 hours
 | LITELLM_DD_AGENT_HOST | Hostname or IP of DataDog agent for LiteLLM-specific logging. When set, logs are sent to agent instead of direct API
 | LITELLM_DEPLOYMENT_ENVIRONMENT | Environment name for the deployment (e.g., "production", "staging"). Used as a fallback when OTEL_ENVIRONMENT_NAME is not set. Sets the `environment` tag in telemetry data
+| LITELLM_DETAILED_TIMING | When true, adds detailed per-phase timing headers to responses (`x-litellm-timing-{pre-processing,llm-api,post-processing,message-copy}-ms`). Default is false. See [latency overhead docs](../troubleshoot/latency_overhead.md)
 | LITELLM_DD_AGENT_PORT | Port of DataDog agent for LiteLLM-specific log intake. Default is 10518
 | LITELLM_DD_LLM_OBS_PORT | Port for Datadog LLM Observability agent. Default is 8126
 | LITELLM_DONT_SHOW_FEEDBACK_BOX | Flag to hide feedback box in LiteLLM UI
 | LITELLM_DROP_PARAMS | Parameters to drop in LiteLLM requests
 | LITELLM_MODIFY_PARAMS | Parameters to modify in LiteLLM requests
 | LITELLM_EMAIL | Email associated with LiteLLM account
+| LITELLM_FAVICON_URL | Custom URL for the LiteLLM UI favicon. When set, overrides the default favicon
 | LITELLM_GLOBAL_MAX_PARALLEL_REQUEST_RETRIES | Maximum retries for parallel requests in LiteLLM
 | LITELLM_GLOBAL_MAX_PARALLEL_REQUEST_RETRY_TIMEOUT | Timeout for retries of parallel requests in LiteLLM
 | LITELLM_DISABLE_LAZY_LOADING | When set to "1", "true", "yes", or "on", disables lazy loading of attributes (currently only affects encoding/tiktoken). This ensures encoding is initialized before VCR starts recording HTTP requests, fixing VCR cassette creation issues. See [issue #18659](https://github.com/BerriAI/litellm/issues/18659)
@@ -767,12 +781,14 @@ router_settings:
 | LITELLM_HOSTED_UI | URL of the hosted UI for LiteLLM
 | LITELLM_UI_API_DOC_BASE_URL | Optional override for the API Reference base URL (used in sample code/docs) when the admin UI runs on a different host than the proxy. Defaults to `PROXY_BASE_URL` when unset.
 | LITELLM_UI_PATH | Path to directory for Admin UI files. Used when running with read-only filesystem (e.g., Kubernetes). Default is `/var/lib/litellm/ui` in Docker.
+| LITELLM_UI_SESSION_DURATION | Duration for UI login session (username/password, SSO, invitation links). Format: "30s", "30m", "24h", "7d". Does not apply to EXPERIMENTAL_UI_LOGIN flow, which uses a fixed 10-minute expiry for security. Default is "24h"
 | LITELM_ENVIRONMENT | Environment of LiteLLM Instance, used by logging services. Currently only used by DeepEval.
 | LITELLM_KEY_ROTATION_ENABLED | Enable auto-key rotation for LiteLLM (boolean). Default is false.
 | LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS | Interval in seconds for how often to run job that auto-rotates keys. Default is 86400 (24 hours).
 | LITELLM_KEY_ROTATION_GRACE_PERIOD | Duration to keep old key valid after rotation (e.g. "24h", "2d"). Default is empty (immediate revoke). Used for scheduled rotations and as fallback when not specified in regenerate request.
 | LITELLM_LICENSE | License key for LiteLLM usage
 | LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS | Set to `True` to use the local bundled Anthropic beta headers config only, disabling remote fetching. Default is `False`
+| LITELLM_LOCAL_BLOG_POSTS | When set to `True`, uses the local bundled blog posts only, disabling remote fetching from GitHub. Default is `False`
 | LITELLM_LOCAL_MODEL_COST_MAP | Local configuration for model cost mapping in LiteLLM
 | LITELLM_LOCAL_POLICY_TEMPLATES | When set to "true", uses local backup policy templates instead of fetching from GitHub. Policy templates are fetched from https://raw.githubusercontent.com/BerriAI/litellm/main/policy_templates.json by default, with automatic fallback to local backup on failure
 | LITELLM_LOG | Enable detailed logging for LiteLLM
@@ -787,6 +803,8 @@ router_settings:
 | PYROSCOPE_SERVER_ADDRESS | Pyroscope server URL to send profiles to. Required when LITELLM_ENABLE_PYROSCOPE is true. No default.
 | PYROSCOPE_SAMPLE_RATE | Optional. Sample rate for Pyroscope profiling (integer). No default; when unset, the pyroscope-io library default is used.
 | LITELLM_MASTER_KEY | Master key for proxy authentication
+| LITELLM_MAX_ITERATIONS_TTL | TTL in seconds for session iteration counters used by the max-iterations limiter. Default is 3600 (1 hour)
+| LITELLM_MAX_STREAMING_DURATION_SECONDS | Maximum duration in seconds allowed for a streaming response. Streams exceeding this duration are terminated with a Timeout error. Default is None (no limit)
 | LITELLM_MODE | Operating mode for LiteLLM (e.g., production, development)
 | LITELLM_NON_ROOT | Flag to run LiteLLM in non-root mode for enhanced security in Docker containers
 | LITELLM_RATE_LIMIT_WINDOW_SIZE | Rate limit window size for LiteLLM. Default is 60
@@ -795,6 +813,7 @@ router_settings:
 | LITELLM_SSL_CIPHERS | SSL/TLS cipher configuration for faster handshakes. Controls cipher suite preferences for OpenSSL connections.
 | LITELLM_SECRET_AWS_KMS_LITELLM_LICENSE | AWS KMS encrypted license for LiteLLM
 | LITELLM_TOKEN | Access token for LiteLLM integration
+| LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES | When set to "true", routes OpenAI /v1/messages requests through chat/completions instead of the Responses API for Anthropic models. Can also be set via `litellm_settings.use_chat_completions_url_for_anthropic_messages`
 | LITELLM_USER_AGENT | Custom user agent string for LiteLLM API requests. Used for partner telemetry attribution
 | LITELLM_PRINT_STANDARD_LOGGING_PAYLOAD | If true, prints the standard logging payload to the console - useful for debugging
 | LITELM_ENVIRONMENT | Environment for LiteLLM Instance. This is currently only logged to DeepEval to determine the environment for DeepEval integration.
@@ -805,6 +824,8 @@ router_settings:
 | LOGGING_WORKER_MAX_QUEUE_SIZE | Maximum size of the logging worker queue. When the queue is full, the worker aggressively clears tasks to make room instead of dropping logs. Default is 50,000
 | LOGGING_WORKER_MAX_TIME_PER_COROUTINE | Maximum time in seconds allowed for each coroutine in the logging worker before timing out. Default is 20.0
 | LOGGING_WORKER_CLEAR_PERCENTAGE | Percentage of the queue to extract when clearing. Default is 50% 
+| MAX_BASE64_LENGTH_FOR_LOGGING | Maximum number of base64 characters to keep in logging payloads. Data URIs exceeding this are replaced with a size placeholder. Set to 0 to disable truncation. Default is 64
+| MAX_COMPETITOR_NAMES | Maximum number of competitor names allowed in policy template enrichment. Default is 100
 | MAX_EXCEPTION_MESSAGE_LENGTH | Maximum length for exception messages. Default is 2000
 | MAX_ITERATIONS_TO_CLEAR_QUEUE | Maximum number of iterations to attempt when clearing the logging worker queue during shutdown. Default is 200
 | MAX_TIME_TO_CLEAR_QUEUE | Maximum time in seconds to spend clearing the logging worker queue during shutdown. Default is 5.0
@@ -827,6 +848,7 @@ router_settings:
 | MAX_LANGFUSE_INITIALIZED_CLIENTS | Maximum number of Langfuse clients to initialize on proxy. Default is 50. This is set since langfuse initializes 1 thread everytime a client is initialized. We've had an incident in the past where we reached 100% cpu utilization because Langfuse was initialized several times.
 | MAX_MCP_SEMANTIC_FILTER_TOOLS_HEADER_LENGTH | Maximum header length for MCP semantic filter tools. Default is 150
 | MAX_POLICY_ESTIMATE_IMPACT_ROWS | Maximum number of rows returned when estimating the impact of a policy. Default is 1000
+| MAX_PAYLOAD_SIZE_FOR_DEBUG_LOG | Maximum payload size in bytes for full DEBUG serialization. Payloads exceeding this will be truncated in logs. Default is 102400 (100 KB)
 | MIN_NON_ZERO_TEMPERATURE | Minimum non-zero temperature value. Default is 0.0001
 | MINIMUM_PROMPT_CACHE_TOKEN_COUNT | Minimum token count for caching a prompt. Default is 1024
 | MISTRAL_API_BASE | Base URL for Mistral API. Default is https://api.mistral.ai
@@ -890,6 +912,13 @@ router_settings:
 | POSTHOG_API_URL | Base URL for PostHog API (defaults to https://us.i.posthog.com)
 | POSTHOG_MOCK | Enable mock mode for PostHog integration testing. When set to true, intercepts PostHog API calls and returns mock responses without making actual network calls. Default is false
 | POSTHOG_MOCK_LATENCY_MS | Mock latency in milliseconds for PostHog API calls when mock mode is enabled. Simulates network round-trip time. Default is 100ms
+| PRISMA_AUTH_RECONNECT_LOCK_TIMEOUT_SECONDS | Lock timeout in seconds for Prisma auth reconnection. Default is 0.1
+| PRISMA_AUTH_RECONNECT_TIMEOUT_SECONDS | Timeout in seconds for Prisma auth reconnection attempts. Default is 2.0
+| PRISMA_HEALTH_WATCHDOG_ENABLED | Enable the Prisma DB health watchdog that monitors and reconnects on connection loss. Default is true
+| PRISMA_HEALTH_WATCHDOG_INTERVAL_SECONDS | Interval in seconds for Prisma health watchdog probes. Default is 30
+| PRISMA_HEALTH_WATCHDOG_PROBE_TIMEOUT_SECONDS | Timeout in seconds for each Prisma health probe. Default is 5.0
+| PRISMA_RECONNECT_COOLDOWN_SECONDS | Cooldown in seconds between Prisma reconnection attempts. Default is 15
+| PRISMA_WATCHDOG_RECONNECT_TIMEOUT_SECONDS | Timeout in seconds for Prisma watchdog-initiated reconnection. Default is 30.0
 | PREDIBASE_API_BASE | Base URL for Predibase API
 | PRESIDIO_ANALYZER_API_BASE | Base URL for Presidio Analyzer service
 | PRESIDIO_ANONYMIZER_API_BASE | Base URL for Presidio Anonymizer service
@@ -972,6 +1001,7 @@ router_settings:
 | TOGETHER_AI_EMBEDDING_150_M | Size parameter for Together AI 150M embedding model. Default is 150
 | TOGETHER_AI_EMBEDDING_350_M | Size parameter for Together AI 350M embedding model. Default is 350
 | TOOL_CHOICE_OBJECT_TOKEN_COUNT | Token count for tool choice objects. Default is 4
+| TOOL_POLICY_CACHE_TTL_SECONDS | TTL in seconds for caching tool policy guardrail results. Default is 60
 | UI_LOGO_PATH | Path to the logo image used in the UI
 | UI_PASSWORD | Password for accessing the UI
 | UI_USERNAME | Username for accessing the UI

docs/my-website/docs/proxy/cost_tracking.md

@@ -161,7 +161,7 @@ Use this when you want non-proxy admins to access `/spend` endpoints
 
 :::info
 
-Schedule a [meeting with us to get your Enterprise License](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
+Schedule a [meeting with us to get your Enterprise License](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
 
 :::
 
@@ -326,6 +326,10 @@ See our [Swagger API](https://litellm-api.up.railway.app/#/Budget%20%26%20Spend%
 
 ## Custom Tags
 
+:::tip See Full Request Tags Documentation
+For comprehensive documentation on all tag options including `x-litellm-tags` header, request body `tags`, and config-based tags, see the dedicated [Request Tags](./request_tags.md) page.
+:::
+
 Requirements:
 
 - Virtual Keys & a database should be set up, see [virtual keys](https://docs.litellm.ai/docs/proxy/virtual_keys)

docs/my-website/docs/proxy/credential_usage_tracking.md

@@ -0,0 +1,19 @@
+# Credential Usage Tracking
+
+When a model is attached to a [reusable credential](./ui_credentials.md), LiteLLM automatically injects the credential name as a tag on every request that uses that model. This means credential-level spend and usage are tracked with zero extra configuration.
+
+## How It Works
+
+When you attach a model to a reusable credential via `litellm_credential_name`, each request routed through that model is tagged `Credential: <name>` (for example, `Credential: xAI`). This tag flows into `DailyTagSpend` and appears in the **Tag** view on the Usage page, where you can filter spend and usage by credential.
+
+If a model has no credential attached, behavior is unchanged—no credential tag is added.
+
+## Viewing Credential Usage
+
+In the Admin UI, go to **Usage → Tag** and look for tags with the `Credential: ` prefix. These represent aggregated spend and token usage across all requests that used that credential.
+
+## Related Documentation
+
+- [Adding LLM Credentials](./ui_credentials.md) - How to create and attach reusable credentials to models
+- [Tag Budgets](./tag_budgets.md) - Setting spend limits on tags
+- [Tag Routing](./tag_routing.md) - Routing requests based on tags

docs/my-website/docs/proxy/customers.md

@@ -2,29 +2,98 @@ import Image from '@theme/IdealImage';
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
 
-# Customers / End-User Budgets
+# Customers / End-Users
 
-Track spend, set budgets for your customers.
+Track spend, set budgets and permissions for your customers.
 
-## Tracking Customer Spend
+## Tracking Customer Spend + Permissions
 
 ### 1. Make LLM API call w/ Customer ID
 
-Make a /chat/completions call, pass 'user' - First call Works
+LiteLLM checks for a customer/end-user ID in the following order (first match wins):
 
-```bash showLineNumbers title="Make request with customer ID"
+| Priority | Method | Where | Notes |
+|----------|--------|-------|-------|
+| 1 | `x-litellm-customer-id` header | Request headers | Standard header, always checked |
+| 2 | `x-litellm-end-user-id` header | Request headers | Standard header, always checked |
+| 3 | Custom header via `user_header_mappings` | Request headers | Configured in `general_settings` |
+| 4 | Custom header via `user_header_name` | Request headers | Deprecated — use `user_header_mappings` |
+| 5 | `user` field | Request body | Standard OpenAI field |
+| 6 | `litellm_metadata.user` field | Request body | Anthropic-style metadata |
+| 7 | `metadata.user_id` field | Request body | Generic metadata pattern |
+| 8 | `safety_identifier` field | Request body | Responses API |
+
+**Option 1: Standard headers** (recommended — no request body modification needed)
+
+```bash showLineNumbers title="Make request with customer ID in header"
 curl -X POST 'http://0.0.0.0:4000/chat/completions' \
         --header 'Content-Type: application/json' \
-        --header 'Authorization: Bearer sk-1234' \ # 👈 YOUR PROXY KEY
-        --data ' {
+        --header 'Authorization: Bearer sk-1234' \
+        --header 'x-litellm-end-user-id: ishaan3' \
+        --data '{
         "model": "azure-gpt-3.5",
-        "user": "ishaan3", # 👈 CUSTOMER ID
-        "messages": [
-            {
-            "role": "user",
-            "content": "what time is it"
-            }
-        ]
+        "messages": [{"role": "user", "content": "what time is it"}]
+        }'
+```
+
+Both `x-litellm-customer-id` and `x-litellm-end-user-id` are supported and always checked without any configuration.
+
+**Option 2: `user` field in request body** (OpenAI-compatible)
+
+```bash showLineNumbers title="Make request with customer ID in body"
+curl -X POST 'http://0.0.0.0:4000/chat/completions' \
+        --header 'Content-Type: application/json' \
+        --header 'Authorization: Bearer sk-1234' \
+        --data '{
+        "model": "azure-gpt-3.5",
+        "user": "ishaan3",
+        "messages": [{"role": "user", "content": "what time is it"}]
+        }'
+```
+
+**Option 3: Custom header via `user_header_mappings`** (configurable)
+
+```yaml showLineNumbers title="config.yaml"
+general_settings:
+  user_header_mappings:
+    - header_name: "x-my-app-user-id"
+      litellm_user_role: "customer"
+```
+
+```bash showLineNumbers title="Make request with custom header"
+curl -X POST 'http://0.0.0.0:4000/chat/completions' \
+        --header 'Content-Type: application/json' \
+        --header 'Authorization: Bearer sk-1234' \
+        --header 'x-my-app-user-id: ishaan3' \
+        --data '{
+        "model": "azure-gpt-3.5",
+        "messages": [{"role": "user", "content": "what time is it"}]
+        }'
+```
+
+**Option 4: `litellm_metadata.user`** (Anthropic-style)
+
+```bash showLineNumbers title="Make request with litellm_metadata.user"
+curl -X POST 'http://0.0.0.0:4000/chat/completions' \
+        --header 'Content-Type: application/json' \
+        --header 'Authorization: Bearer sk-1234' \
+        --data '{
+        "model": "claude-3-5-sonnet",
+        "messages": [{"role": "user", "content": "what time is it"}],
+        "litellm_metadata": {"user": "ishaan3"}
+        }'
+```
+
+**Option 5: `metadata.user_id`**
+
+```bash showLineNumbers title="Make request with metadata.user_id"
+curl -X POST 'http://0.0.0.0:4000/chat/completions' \
+        --header 'Content-Type: application/json' \
+        --header 'Authorization: Bearer sk-1234' \
+        --data '{
+        "model": "azure-gpt-3.5",
+        "messages": [{"role": "user", "content": "what time is it"}],
+        "metadata": {"user_id": "ishaan3"}
         }'
 ```
 
@@ -123,7 +192,171 @@ Expected Response
 </Tabs>
 
 
-## Setting Customer Budgets 
+## Setting Customer Object Permissions
+
+Control which resources (MCP servers, vector stores, agents) a customer can access.
+
+### What are Object Permissions?
+
+Object permissions allow you to restrict customer access to specific:
+- **MCP Servers**: Limit which MCP servers the customer can call
+- **MCP Access Groups**: Assign customers to predefined groups of MCP servers
+- **MCP Tool Permissions**: Granular control over which tools within an MCP server the customer can use
+- **Vector Stores**: Control which vector stores the customer can query
+- **Agents**: Restrict which agents the customer can interact with
+- **Agent Access Groups**: Assign customers to predefined groups of agents
+
+### Creating a Customer with Object Permissions
+
+```bash showLineNumbers title="Create customer with object permissions"
+curl -L -X POST 'http://localhost:4000/customer/new' \
+-H 'Authorization: Bearer sk-1234' \
+-H 'Content-Type: application/json' \
+-d '{
+    "user_id": "user_1",
+    "object_permission": {
+      "mcp_servers": ["server_1", "server_2"],
+      "mcp_access_groups": ["public_group"],
+      "mcp_tool_permissions": {
+        "server_1": ["tool_a", "tool_b"]
+      },
+      "vector_stores": ["vector_store_1"],
+      "agents": ["agent_1"],
+      "agent_access_groups": ["basic_agents"]
+    }
+  }'
+```
+
+**Parameters:**
+- `mcp_servers` (Optional[List[str]]): List of allowed MCP server IDs
+- `mcp_access_groups` (Optional[List[str]]): List of MCP access group names
+- `mcp_tool_permissions` (Optional[Dict[str, List[str]]]): Map of server ID to allowed tool names
+- `vector_stores` (Optional[List[str]]): List of allowed vector store IDs
+- `agents` (Optional[List[str]]): List of allowed agent IDs
+- `agent_access_groups` (Optional[List[str]]): List of agent access group names
+
+**Note:** If `object_permission` is `null` or `{}`, the customer has no object-level restrictions.
+
+### Updating Customer Object Permissions
+
+You can update object permissions for existing customers:
+
+```bash showLineNumbers title="Update customer object permissions"
+curl -L -X POST 'http://localhost:4000/customer/update' \
+-H 'Authorization: Bearer sk-1234' \
+-H 'Content-Type: application/json' \
+-d '{
+    "user_id": "user_1",
+    "object_permission": {
+      "mcp_servers": ["server_3"],
+      "vector_stores": ["vector_store_2", "vector_store_3"]
+    }
+  }'
+```
+
+### Viewing Customer Object Permissions
+
+When you query customer info, object permissions are included in the response:
+
+```bash showLineNumbers title="Get customer info with object permissions"
+curl -X GET 'http://0.0.0.0:4000/customer/info?end_user_id=user_1' \
+    -H 'Authorization: Bearer sk-1234'
+```
+
+**Response:**
+```json showLineNumbers title="Response with object permissions"
+{
+  "user_id": "user_1",
+  "blocked": false,
+  "alias": "John Doe",
+  "spend": 0.0,
+  "object_permission": {
+    "object_permission_id": "perm_abc123",
+    "mcp_servers": ["server_1", "server_2"],
+    "mcp_access_groups": ["public_group"],
+    "mcp_tool_permissions": {
+      "server_1": ["tool_a", "tool_b"]
+    },
+    "vector_stores": ["vector_store_1"],
+    "agents": ["agent_1"],
+    "agent_access_groups": ["basic_agents"]
+  },
+  "litellm_budget_table": null
+}
+```
+
+### Use Cases
+
+**1. Tiered Access Control**
+Create different permission tiers for your customers:
+
+```bash showLineNumbers title="Free tier customer"
+# Free tier - limited access
+curl -L -X POST 'http://localhost:4000/customer/new' \
+-H 'Authorization: Bearer sk-1234' \
+-H 'Content-Type: application/json' \
+-d '{
+    "user_id": "free_user",
+    "budget_id": "free_tier",
+    "object_permission": {
+      "mcp_access_groups": ["public_group"],
+      "agent_access_groups": ["basic_agents"]
+    }
+  }'
+```
+
+```bash showLineNumbers title="Premium tier customer"
+# Premium tier - full access
+curl -L -X POST 'http://localhost:4000/customer/new' \
+-H 'Authorization: Bearer sk-1234' \
+-H 'Content-Type: application/json' \
+-d '{
+    "user_id": "premium_user",
+    "budget_id": "premium_tier",
+    "object_permission": {
+      "mcp_servers": ["server_1", "server_2", "server_3"],
+      "vector_stores": ["vector_store_1", "vector_store_2"],
+      "agents": ["agent_1", "agent_2", "agent_3"]
+    }
+  }'
+```
+
+**2. Department-Specific Access**
+Restrict customers to resources relevant to their department:
+
+```bash showLineNumbers title="Sales team customer"
+curl -L -X POST 'http://localhost:4000/customer/new' \
+-H 'Authorization: Bearer sk-1234' \
+-H 'Content-Type: application/json' \
+-d '{
+    "user_id": "sales_user",
+    "object_permission": {
+      "mcp_servers": ["crm_server", "email_server"],
+      "agents": ["sales_assistant"],
+      "vector_stores": ["sales_knowledge_base"]
+    }
+  }'
+```
+
+**3. Tool-Level Restrictions**
+Grant access to specific tools within an MCP server:
+
+```bash showLineNumbers title="Limited tool access"
+curl -L -X POST 'http://localhost:4000/customer/new' \
+-H 'Authorization: Bearer sk-1234' \
+-H 'Content-Type: application/json' \
+-d '{
+    "user_id": "restricted_user",
+    "object_permission": {
+      "mcp_servers": ["database_server"],
+      "mcp_tool_permissions": {
+        "database_server": ["read_only_query", "get_table_schema"]
+      }
+    }
+  }'
+```
+
+## Setting Customer Budgets
 
 Set customer budgets (e.g. monthly budgets, tpm/rpm limits) on LiteLLM Proxy 
 

docs/my-website/docs/proxy/email.md

@@ -203,7 +203,7 @@ After regenerating the key, the user will receive an email notification with:
 
 :::info
 
-Customizing Email Branding is an Enterprise Feature [Get in touch with us for a Free Trial](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
+Customizing Email Branding is an Enterprise Feature [Get in touch with us for a Free Trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
 
 :::
 

docs/my-website/docs/proxy/enterprise.md

@@ -5,7 +5,7 @@ import TabItem from '@theme/TabItem';
 # ✨ Enterprise Features
 :::tip
 
-To get a license, get in touch with us [here](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
+To get a license, get in touch with us [here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
 
 :::
 

docs/my-website/docs/proxy/forward_client_headers.md

@@ -37,11 +37,11 @@ The following rules determine which headers are forwarded (see [`_get_forwardabl
 
 | Rule | Example | Forwarded? |
 |---|---|---|
-| Headers starting with `x-` | `x-trace-id`, `x-custom-header`, `x-request-source` | ✅ Yes |
-| `anthropic-beta` header | `anthropic-beta: prompt-caching-2024-07-31` | ✅ Yes |
-| Headers starting with `x-stainless-*` | `x-stainless-lang`, `x-stainless-arch` | ❌ No (causes OpenAI SDK issues) |
-| Standard HTTP headers | `Authorization`, `Content-Type`, `Host` | ❌ No |
-| Other provider headers | `Accept`, `User-Agent` | ❌ No |
+| Headers starting with `x-` | `x-trace-id`, `x-custom-header`, `x-request-source` |  Yes |
+| `anthropic-beta` header | `anthropic-beta: prompt-caching-2024-07-31` |  Yes |
+| Headers starting with `x-stainless-*` | `x-stainless-lang`, `x-stainless-arch` |  No (causes OpenAI SDK issues) |
+| Standard HTTP headers | `Authorization`, `Content-Type`, `Host` |  No |
+| Other provider headers | `Accept`, `User-Agent` |  No |
 
 ### Additional Header Mechanisms
 
@@ -61,6 +61,125 @@ general_settings:
   forward_client_headers_to_llm_api: true
 ```
 
+## Forward LLM Provider Authentication Headers
+
+**New in v1.82+**: By default, LiteLLM strips authentication headers like `x-api-key`, `x-goog-api-key`, and `api-key` from client requests for security (these are typically used to authenticate with the proxy itself). However, you can enable forwarding of these LLM provider authentication headers to allow **Bring Your Own Key (BYOK)** scenarios where clients send their own API keys to the LLM provider.
+
+### Configuration
+
+Add `forward_llm_provider_auth_headers: true` to your `general_settings`:
+
+```yaml
+general_settings:
+  forward_client_headers_to_llm_api: true
+  forward_llm_provider_auth_headers: true  # 👈 Enable BYOK
+```
+
+### Which Headers Are Forwarded
+
+When `forward_llm_provider_auth_headers: true`, the following LLM provider authentication headers are preserved and forwarded:
+
+| Header | Provider | Example |
+|--------|----------|---------|
+| `x-api-key` | Anthropic, Azure AI, Databricks | `x-api-key: sk-ant-api03-...` |
+| `x-goog-api-key` | Google AI Studio | `x-goog-api-key: AIza...` |
+| `api-key` | Azure OpenAI | `api-key: your-azure-key` |
+| `ocp-apim-subscription-key` | Azure APIM | `ocp-apim-subscription-key: your-key` |
+
+:::warning Important Security Note
+The proxy's `Authorization` header (used for proxy authentication) is **never** forwarded to LLM providers, even with this setting enabled. This ensures your proxy authentication remains secure.
+:::
+
+### Use Case: Client-Side API Keys (BYOK)
+
+This feature enables scenarios where:
+1. **Clients bring their own LLM provider API keys** instead of using keys configured in the proxy
+2. **Multi-tenant applications** where each tenant has their own Anthropic/OpenAI account
+3. **Development environments** where developers use their personal API keys through a shared proxy
+
+#### Example: Anthropic BYOK
+
+```yaml
+# proxy_config.yaml
+model_list:
+  - model_name: claude-sonnet-4
+    litellm_params:
+      model: anthropic/claude-sonnet-4-20250514
+      # No api_key configured! Will use client's key
+
+general_settings:
+  forward_client_headers_to_llm_api: true
+  forward_llm_provider_auth_headers: true  # Enable BYOK
+```
+
+Client request:
+```bash
+curl -X POST "http://localhost:4000/v1/messages" \
+  -H "Authorization: Bearer sk-proxy-auth-123" \     # Proxy authentication (stripped)
+  -H "x-api-key: sk-ant-api03-YOUR-KEY..." \        # Client's Anthropic key (forwarded!)
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "claude-sonnet-4",
+    "messages": [{"role": "user", "content": "Hello"}],
+    "max_tokens": 100
+  }'
+```
+
+#### Example: Google AI Studio BYOK
+
+```yaml
+model_list:
+  - model_name: gemini-pro
+    litellm_params:
+      model: gemini/gemini-1.5-pro
+      # No api_key configured
+
+general_settings:
+  forward_client_headers_to_llm_api: true
+  forward_llm_provider_auth_headers: true
+```
+
+Client request:
+```bash
+curl -X POST "http://localhost:4000/v1/chat/completions" \
+  -H "Authorization: Bearer sk-proxy-auth-123" \
+  -H "x-goog-api-key: AIza..." \
+  -d '{
+    "model": "gemini-pro",
+    "messages": [{"role": "user", "content": "Hello"}]
+  }'
+```
+
+### Security Considerations
+
+**When to Use This Feature:**
+-  Internal tools where you trust all clients
+-  Development/testing environments
+-  Multi-tenant apps with proper client authentication
+-  Scenarios where you want clients to use their own API keys
+
+**When NOT to Use:**
+-  Public APIs where you don't trust all clients
+-  When you want centralized billing/cost control
+-  When you need to enforce rate limits at the proxy level
+
+### Backward Compatibility
+
+For backward compatibility, if you have `forward_client_headers_to_llm_api: true` but don't explicitly set `forward_llm_provider_auth_headers`, the behavior is:
+- **Default**: LLM provider auth headers are **NOT** forwarded (safe default)
+- **Explicit `true`**: LLM provider auth headers **ARE** forwarded (BYOK enabled)
+
+```yaml
+# Safe default - auth headers NOT forwarded
+general_settings:
+  forward_client_headers_to_llm_api: true
+
+# BYOK enabled - auth headers ARE forwarded
+general_settings:
+  forward_client_headers_to_llm_api: true
+  forward_llm_provider_auth_headers: true  # 👈 Opt-in required
+```
+
 ## Enable for a Model Group
 
 Add the `forward_client_headers_to_llm_api` setting under `model_group_settings` in your configuration:

docs/my-website/docs/proxy/guardrails/aporia_api.md

@@ -139,7 +139,7 @@ curl -i http://localhost:4000/v1/chat/completions \
 
 :::info
 
-✨ This is an Enterprise only feature [Contact us to get a free trial](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
+✨ This is an Enterprise only feature [Contact us to get a free trial](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
 
 :::