Merge branch 'main' into litellm_fix_nova_pro_max_tokens

.circleci/config.yml

@@ -69,9 +69,11 @@ jobs:
       - run:
           name: Install Python
           command: |
-            choco install python --version=3.11.0 -y
+            choco install python --version=3.11.0 -y --no-progress --force
             refreshenv
             python --version
+          environment:
+            CHOCOLATEY_CONFIRM_ALL: "true"
       - run:
           name: Install Dependencies
           command: |
@@ -1181,7 +1183,7 @@ jobs:
           command: |
             pwd
             ls
-            python -m pytest tests/proxy_unit_tests --ignore=tests/proxy_unit_tests/test_key_generate_prisma.py --ignore=tests/proxy_unit_tests/test_auth_checks.py --ignore=tests/proxy_unit_tests/test_user_api_key_auth.py --cov=litellm --cov-report=xml --junitxml=test-results/junit-part2.xml --durations=10 -n 8 --timeout=300 -vv --log-cli-level=INFO
+            python -m pytest tests/proxy_unit_tests --ignore=tests/proxy_unit_tests/test_key_generate_prisma.py --ignore=tests/proxy_unit_tests/test_auth_checks.py --ignore=tests/proxy_unit_tests/test_user_api_key_auth.py --cov=litellm --cov-report=xml --junitxml=test-results/junit-part2.xml --durations=10 -n 4 --timeout=300 -vv --log-cli-level=INFO
           no_output_timeout: 120m
       - run:
           name: Rename the coverage files
@@ -1456,6 +1458,7 @@ jobs:
             pip install "respx==0.22.0"
             pip install "pydantic==2.10.2"
             pip install "boto3==1.36.0"
+            pip install "semantic_router==0.1.10"
       # Run pytest and generate JUnit XML report
       - run:
           name: Run tests
@@ -1698,7 +1701,7 @@ jobs:
           command: |
             prisma generate
             export PYTHONUNBUFFERED=1
-            python -m pytest tests/test_litellm/proxy --ignore=tests/test_litellm/proxy/guardrails --ignore=tests/test_litellm/proxy/management_endpoints --ignore=tests/test_litellm/proxy/_experimental --ignore=tests/test_litellm/proxy/client --ignore=tests/test_litellm/proxy/auth --cov=litellm --cov-report=xml --junitxml=test-results/junit-proxy-part2.xml --durations=10 -n 8 --maxfail=5 --timeout=60 -vv --log-cli-level=WARNING -r A
+            python -m pytest tests/test_litellm/proxy --ignore=tests/test_litellm/proxy/guardrails --ignore=tests/test_litellm/proxy/management_endpoints --ignore=tests/test_litellm/proxy/_experimental --ignore=tests/test_litellm/proxy/client --ignore=tests/test_litellm/proxy/auth --cov=litellm --cov-report=xml --junitxml=test-results/junit-proxy-part2.xml --durations=10 -n 4 --maxfail=5 --timeout=120 -vv --log-cli-level=WARNING -r A
           no_output_timeout: 60m
       - run:
           name: Rename the coverage files
@@ -3688,6 +3691,114 @@ jobs:
       - store_test_results:
           path: test-results
 
+  proxy_e2e_azure_batches_tests:
+    machine:
+      image: ubuntu-2204:2023.10.1
+    resource_class: xlarge
+    working_directory: ~/project
+    steps:
+      - checkout
+      - setup_google_dns
+      - run:
+          name: Install Docker CLI
+          command: |
+            curl -fsSL https://get.docker.com | sh
+            sudo usermod -aG docker $USER
+            docker version
+      - run:
+          name: Install Python 3.12
+          command: |
+            curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
+            bash miniconda.sh -b -p $HOME/miniconda
+            export PATH="$HOME/miniconda/bin:$PATH"
+            conda init bash
+            source ~/.bashrc
+            conda create -n myenv python=3.12 -y
+            conda activate myenv
+            python --version
+      - run:
+          name: Install Poetry
+          command: |
+            export PATH="$HOME/miniconda/bin:$PATH"
+            source $HOME/miniconda/etc/profile.d/conda.sh
+            conda activate myenv
+            pip install poetry
+      - run:
+          name: Install dockerize
+          command: |
+            wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
+            sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
+            rm dockerize-linux-amd64-v0.6.1.tar.gz
+      - run:
+          name: Start PostgreSQL Database
+          command: |
+            docker run -d \
+              --name postgres-db \
+              -e POSTGRES_USER=llmproxy \
+              -e POSTGRES_PASSWORD=dbpassword9090 \
+              -e POSTGRES_DB=litellm \
+              -p 5432:5432 \
+              postgres:15
+      - run:
+          name: Wait for PostgreSQL to be ready
+          command: dockerize -wait tcp://localhost:5432 -timeout 1m
+      - run:
+          name: Install system dependencies
+          command: |
+            sudo apt-get update -y
+            sudo apt-get install -y libpq-dev
+      - run:
+          name: Install Dependencies
+          command: |
+            export PATH="$HOME/miniconda/bin:$PATH"
+            source $HOME/miniconda/etc/profile.d/conda.sh
+            conda activate myenv
+            poetry config virtualenvs.in-project true
+            poetry install --with dev,proxy-dev --extras "proxy"
+            poetry run pip install psycopg2-binary uvicorn fastapi httpx tenacity
+      - run:
+          name: Setup litellm-enterprise
+          command: |
+            export PATH="$HOME/miniconda/bin:$PATH"
+            source $HOME/miniconda/etc/profile.d/conda.sh
+            conda activate myenv
+            poetry run pip install --force-reinstall --no-deps -e enterprise/
+      - run:
+          name: Generate Prisma client
+          command: |
+            export PATH="$HOME/miniconda/bin:$PATH"
+            source $HOME/miniconda/etc/profile.d/conda.sh
+            conda activate myenv
+            poetry run prisma generate --schema litellm/proxy/schema.prisma
+      - run:
+          name: Run Prisma migrations
+          command: |
+            export PATH="$HOME/miniconda/bin:$PATH"
+            source $HOME/miniconda/etc/profile.d/conda.sh
+            conda activate myenv
+            export DATABASE_URL=postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
+            cd litellm/proxy
+            poetry run prisma migrate deploy --schema schema.prisma
+            cd ../..
+      - run:
+          name: Run Azure Batch E2E Tests
+          command: |
+            export PATH="$HOME/miniconda/bin:$PATH"
+            source $HOME/miniconda/etc/profile.d/conda.sh
+            conda activate myenv
+            export DATABASE_URL=postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
+            export USE_LOCAL_LITELLM=true
+            export USE_MOCK_MODELS=true
+            export USE_STATE_TRACKER=true
+            export LITELLM_LOG=DEBUG
+            poetry run pytest tests/proxy_e2e_azure_batches_tests/test_proxy_e2e_azure_batches.py \
+              -vv -s -k "test_e2e_managed_batch" \
+              --tb=short \
+              --maxfail=3 \
+              --durations=10 \
+              --junitxml=test-results/junit.xml
+          no_output_timeout: 30m
+
   upload-coverage:
     docker:
       - image: cimg/python:3.9
@@ -3885,7 +3996,7 @@ jobs:
           command: |
             cd ~/project
             # Check pyproject.toml
-            CURRENT_VERSION=$(python -c "import toml; print(toml.load('pyproject.toml')['tool']['poetry']['dependencies']['litellm-proxy-extras'].split('\"')[1])")
+            CURRENT_VERSION=$(python -c "import toml; dep = toml.load('pyproject.toml')['tool']['poetry']['dependencies']['litellm-proxy-extras']; print(dep['version'] if isinstance(dep, dict) else dep)")
             if [ "$CURRENT_VERSION" != "$NEW_VERSION" ]; then
               echo "Error: Version in pyproject.toml ($CURRENT_VERSION) doesn't match new version ($NEW_VERSION)"
               exit 1
@@ -4099,6 +4210,63 @@ jobs:
           path: playwright-report
           destination: playwright-report
 
+  prisma_schema_sync:
+    machine:
+      image: ubuntu-2204:2023.10.1
+    resource_class: xlarge
+    working_directory: ~/project
+    steps:
+      - checkout
+      - setup_google_dns
+      - attach_workspace:
+          at: ~/project
+      - run:
+          name: Load Docker Database Image
+          command: |
+            gunzip -c litellm-docker-database.tar.gz | docker load
+            docker images | grep litellm-docker-database
+      - run:
+          name: Install Neon CLI
+          command: |
+            npm i -g neonctl
+      - run:
+          name: Install curl and dockerize
+          command: |
+            sudo apt-get update
+            sudo apt-get install -y curl
+            sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
+            sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
+            sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
+      - run:
+          name: Sync schema on base e2e database
+          command: |
+            BASE_DATABASE_URL=$(neon connection-string \
+              --project-id $NEON_PROJECT_ID \
+              --api-key $NEON_API_KEY \
+              --branch br-fancy-paper-ad1olsb3 \
+              --database-name yuneng-trial-db \
+              --role neondb_owner)
+            docker run -d \
+              -p 4000:4000 \
+              -e DATABASE_URL=$BASE_DATABASE_URL \
+              -e LITELLM_MASTER_KEY="sk-1234" \
+              --name schema-sync \
+              -v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \
+              litellm-docker-database:ci \
+              --config /app/config.yaml \
+              --port 4000 \
+              --use_prisma_db_push
+      - run:
+          name: Start outputting logs
+          command: docker logs -f schema-sync
+          background: true
+      - run:
+          name: Wait for proxy to be ready (schema sync complete)
+          command: dockerize -wait http://localhost:4000 -timeout 5m
+      - run:
+          name: Stop schema sync container
+          command: docker stop schema-sync
+
   test_nonroot_image:
     machine:
       image: ubuntu-2204:2023.10.1
@@ -4297,6 +4465,15 @@ workflows:
               only:
                 - main
                 - /litellm_.*/
+      - prisma_schema_sync:
+          context: e2e_ui_tests
+          requires:
+            - build_docker_database_image
+          filters:
+            branches:
+              only:
+                - main
+                - /litellm_.*/
       - e2e_ui_testing:
           name: e2e_ui_testing_chromium
           browser: chromium
@@ -4304,6 +4481,7 @@ workflows:
           requires:
             - ui_build
             - build_docker_database_image
+            - prisma_schema_sync
           filters:
             branches:
               only:
@@ -4316,6 +4494,7 @@ workflows:
           requires:
             - ui_build
             - build_docker_database_image
+            - prisma_schema_sync
           filters:
             branches:
               only:
@@ -4389,6 +4568,12 @@ workflows:
               only:
                 - main
                 - /litellm_.*/
+      - proxy_e2e_azure_batches_tests:
+          filters:
+            branches:
+              only:
+                - main
+                - /litellm_.*/
       - llm_translation_testing:
           filters:
             branches:

.github/ISSUE_TEMPLATE/config.yml

@@ -1,7 +1,7 @@
 blank_issues_enabled: true
 contact_links:
   - name: Schedule Demo
-    url: https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat
+    url: https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions
     about: Speak directly with Krrish and Ishaan, the founders, to discuss issues, share feedback, or explore improvements for LiteLLM
   - name: Discord
     url: https://discord.com/invite/wuPM9dRgDw

.github/codeql/codeql-config.yml

@@ -0,0 +1,15 @@
+name: "LiteLLM CodeQL config"
+
+# Exclude queries that produce result sets > 2 GiB on this codebase,
+# causing 49+ minute runs that fail and block CI resources.
+query-filters:
+  - exclude:
+      id: py/clear-text-logging-sensitive-data  # CWE-312/CleartextLogging.ql — result set > 2 GiB
+  - exclude:
+      id: py/polynomial-redos                   # CWE-730/PolynomialReDoS.ql — result set > 2 GiB
+
+paths-ignore:
+  - tests
+  - docs
+  - "**/*.md"
+  - litellm/proxy/_experimental/out

.github/observatory/litellm_config.yaml

@@ -0,0 +1,19 @@
+# LiteLLM Observatory Test Configuration
+# This config is used by CI to spin up a temporary LiteLLM instance
+# for running observatory tests against RC/stable releases.
+#
+# Add model definitions for the providers you want to test.
+# Provider API keys are injected via environment variables in CI.
+
+model_list:
+  - model_name: gpt-4o
+    litellm_params:
+      model: azure/gpt-4o
+      api_key: os.environ/AZURE_API_KEY
+      api_base: os.environ/AZURE_API_BASE
+
+  - model_name: gpt-4o-mini
+    litellm_params:
+      model: azure/gpt-4o-mini
+      api_key: os.environ/AZURE_API_KEY
+      api_base: os.environ/AZURE_API_BASE

.github/pull_request_template.md

@@ -6,7 +6,7 @@
 
 **Please complete all items before asking a LiteLLM maintainer to review your PR**
 
-- [ ] I have Added testing in the [`tests/litellm/`](https://github.com/BerriAI/litellm/tree/main/tests/litellm) directory, **Adding at least 1 test is a hard requirement** - [see details](https://docs.litellm.ai/docs/extras/contributing_code)
+- [ ] I have Added testing in the [`tests/test_litellm/`](https://github.com/BerriAI/litellm/tree/main/tests/test_litellm) directory, **Adding at least 1 test is a hard requirement** - [see details](https://docs.litellm.ai/docs/extras/contributing_code)
 - [ ] My PR passes all unit tests on [`make test-unit`](https://docs.litellm.ai/docs/extras/contributing_code)
 - [ ] My PR's scope is as isolated as possible, it only solves 1 specific problem
 - [ ] I have requested a Greptile review by commenting `@greptileai` and received a **Confidence Score of at least 4/5** before requesting a maintainer review

.github/scripts/close_duplicate_issues.py

@@ -0,0 +1,208 @@
+#!/usr/bin/env python3
+"""
+Detect and close duplicate GitHub issues using title similarity.
+
+Modes:
+  --scan           Compare all open issues against each other (batch)
+  --issue-number N Check a single issue against older open issues
+
+Requires the `gh` CLI to be authenticated.
+"""
+
+import argparse
+import difflib
+import json
+import re
+import subprocess
+import sys
+
+
+def normalize_title(title: str) -> str:
+    """Strip common prefixes, lowercase, and collapse whitespace."""
+    title = re.sub(
+        r"^\[?(bug|feature request|enhancement|question|docs)[:\]]?\s*",
+        "",
+        title,
+        flags=re.IGNORECASE,
+    )
+    return " ".join(title.lower().split())
+
+
+def gh(*args: str) -> str:
+    """Run a gh CLI command and return stdout."""
+    result = subprocess.run(
+        ["gh", *args],
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    return result.stdout
+
+
+def fetch_open_issues(repo: str | None) -> list[dict]:
+    """Fetch all open issues (excluding PRs) via gh api --paginate."""
+    if repo:
+        endpoint = f"repos/{repo}/issues?state=open&per_page=100&sort=created&direction=asc"
+    else:
+        endpoint = "repos/{owner}/{repo}/issues?state=open&per_page=100&sort=created&direction=asc"
+    cmd = ["api", "--paginate", endpoint]
+
+    raw = gh(*cmd)
+    # gh --paginate concatenates JSON arrays, so we may get multiple arrays
+    issues = []
+    for line in raw.strip().splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        parsed = json.loads(line)
+        if isinstance(parsed, list):
+            issues.extend(parsed)
+        else:
+            issues.append(parsed)
+
+    # Filter out pull requests (they also appear in the issues endpoint)
+    return [i for i in issues if "pull_request" not in i]
+
+
+def close_as_duplicate(
+    issue_number: int, duplicate_of: int, repo: str | None, dry_run: bool
+) -> None:
+    """Close an issue as duplicate of another, adding a comment and label."""
+    repo_args = ["--repo", repo] if repo else []
+
+    if dry_run:
+        print(f"  [DRY RUN] Would close #{issue_number} as duplicate of #{duplicate_of}")
+        return
+
+    # Add comment
+    comment_body = (
+        f"Closing as duplicate of #{duplicate_of}.\n\n"
+        "If you believe this is not a duplicate, please reopen and add context "
+        "explaining how this differs."
+    )
+    gh("issue", "comment", str(issue_number), "--body", comment_body, *repo_args)
+
+    # Add label
+    gh("issue", "edit", str(issue_number), "--add-label", "duplicate", *repo_args)
+
+    # Close with not_planned reason
+    gh(
+        "api",
+        f"repos/{repo or '{owner}/{repo}'}/issues/{issue_number}",
+        "-X",
+        "PATCH",
+        "-f",
+        "state=closed",
+        "-f",
+        "state_reason=not_planned",
+    )
+
+    print(f"  Closed #{issue_number} as duplicate of #{duplicate_of}")
+
+
+def find_duplicate(
+    issue: dict, candidates: list[dict], threshold: float
+) -> dict | None:
+    """Return the first candidate whose normalized title is above threshold."""
+    norm = normalize_title(issue["title"])
+    for candidate in candidates:
+        if candidate["number"] == issue["number"]:
+            continue
+        cand_norm = normalize_title(candidate["title"])
+        ratio = difflib.SequenceMatcher(None, norm, cand_norm).ratio()
+        if ratio >= threshold:
+            return candidate
+    return None
+
+
+def scan_all(issues: list[dict], threshold: float, repo: str | None, dry_run: bool) -> int:
+    """Compare every issue against all older issues. Returns count of duplicates found."""
+    # Sort oldest first
+    issues.sort(key=lambda i: i["number"])
+    closed_count = 0
+
+    for idx, issue in enumerate(issues):
+        older = issues[:idx]
+        if not older:
+            continue
+        dup = find_duplicate(issue, older, threshold)
+        if dup:
+            ratio = difflib.SequenceMatcher(
+                None,
+                normalize_title(issue["title"]),
+                normalize_title(dup["title"]),
+            ).ratio()
+            print(
+                f"#{issue['number']}: \"{issue['title']}\"\n"
+                f"  -> duplicate of #{dup['number']}: \"{dup['title']}\" "
+                f"({ratio:.0%} similar)"
+            )
+            close_as_duplicate(issue["number"], dup["number"], repo, dry_run)
+            closed_count += 1
+
+    return closed_count
+
+
+def check_single(
+    issue_number: int, issues: list[dict], threshold: float, repo: str | None, dry_run: bool
+) -> bool:
+    """Check a single issue against all older open issues. Returns True if duplicate found."""
+    target = None
+    for i in issues:
+        if i["number"] == issue_number:
+            target = i
+            break
+
+    if target is None:
+        print(f"Issue #{issue_number} not found among open issues.")
+        return False
+
+    older = [i for i in issues if i["number"] < issue_number]
+    dup = find_duplicate(target, older, threshold)
+    if dup:
+        ratio = difflib.SequenceMatcher(
+            None,
+            normalize_title(target["title"]),
+            normalize_title(dup["title"]),
+        ).ratio()
+        print(
+            f"#{target['number']}: \"{target['title']}\"\n"
+            f"  -> duplicate of #{dup['number']}: \"{dup['title']}\" "
+            f"({ratio:.0%} similar)"
+        )
+        close_as_duplicate(issue_number, dup["number"], repo, dry_run)
+        return True
+
+    print(f"#{issue_number}: no duplicate found above threshold {threshold}")
+    return False
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Detect and close duplicate GitHub issues")
+    mode = parser.add_mutually_exclusive_group(required=True)
+    mode.add_argument("--scan", action="store_true", help="Scan all open issues")
+    mode.add_argument("--issue-number", type=int, help="Check a single issue number")
+    parser.add_argument("--threshold", type=float, default=0.85, help="Similarity threshold (0-1)")
+    parser.add_argument("--close", action="store_true", help="Actually close duplicates (default is dry-run)")
+    parser.add_argument("--repo", type=str, help="Repository (owner/repo). Auto-detected if omitted.")
+    args = parser.parse_args()
+
+    dry_run = not args.close
+
+    if dry_run:
+        print("=== DRY RUN MODE (pass --close to actually close issues) ===\n")
+
+    print("Fetching open issues...")
+    issues = fetch_open_issues(args.repo)
+    print(f"Found {len(issues)} open issues.\n")
+
+    if args.scan:
+        count = scan_all(issues, args.threshold, args.repo, dry_run)
+        print(f"\nTotal duplicates {'found' if dry_run else 'closed'}: {count}")
+    else:
+        found = check_single(args.issue_number, issues, args.threshold, args.repo, dry_run)
+        sys.exit(0 if found else 0)  # Always exit 0; finding no dup is not an error
+
+
+if __name__ == "__main__":
+    main()

.github/workflows/auto_update_price_and_context_window.yml

@@ -7,6 +7,7 @@ on:
 
 jobs:
   auto_update_price_and_context_window:
+    if: github.repository == 'BerriAI/litellm'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3

.github/workflows/check_duplicate_issues.yml

@@ -20,10 +20,33 @@ jobs:
           reaction: eyes
           comment: |
             **⚠️ Potential duplicate detected**
-            
+
             This issue appears similar to existing issue(s):
             {{#issues}}
             - [#{{number}}]({{html_url}}) - {{title}} ({{accuracy}}% similar)
             {{/issues}}
-            
+
             Please review the linked issue(s) to see if they address your concern. If this is not a duplicate, please provide additional context to help us understand the difference.
+
+      - name: Checkout close script
+        if: github.event.action == 'opened'
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: .github/scripts
+
+      - name: Set up Python
+        if: github.event.action == 'opened'
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Auto-close if high-confidence duplicate
+        if: github.event.action == 'opened'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          python3 .github/scripts/close_duplicate_issues.py \
+            --issue-number ${{ github.event.issue.number }} \
+            --repo ${{ github.repository }} \
+            --threshold 0.85 \
+            --close

.github/workflows/codeql.yml

@@ -0,0 +1,54 @@
+name: "CodeQL"
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  schedule:
+    # Run weekly on Sundays at 04:00 UTC
+    - cron: "0 4 * * 0"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  analyze:
+    name: Analyze (${{ matrix.language }})
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    permissions:
+      security-events: write
+      packages: read
+      actions: read
+      contents: read
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - language: actions
+            build-mode: none
+          - language: javascript-typescript
+            build-mode: none
+          - language: python
+            build-mode: none
+          - language: ruby
+            build-mode: none
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v3
+        with:
+          languages: ${{ matrix.language }}
+          build-mode: ${{ matrix.build-mode }}
+          config-file: ./.github/codeql/codeql-config.yml
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v3
+        with:
+          category: "/language:${{ matrix.language }}"

.github/workflows/ghcr_deploy.yml

@@ -299,6 +299,15 @@ jobs:
             ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-spend_logs:main-stable', env.REGISTRY) || '' }}
           platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
 
+  run-observatory-tests:
+    if: github.event.inputs.release_type == 'rc' || github.event.inputs.release_type == 'stable'
+    needs: [docker-hub-deploy]
+    uses: ./.github/workflows/run_observatory_tests.yml
+    with:
+      tag: ${{ github.event.inputs.tag }}
+      commit_hash: ${{ github.event.inputs.commit_hash }}
+    secrets: inherit
+
   build-and-push-helm-chart:
     if: github.event.inputs.release_type  != 'dev'
     needs: [docker-hub-deploy, build-and-push-image, build-and-push-image-database]

.github/workflows/interpret_load_test.py

@@ -123,7 +123,7 @@ if __name__ == "__main__":
         + docker_run_command
         + "\n\n"
         + "### Don't want to maintain your internal proxy? get in touch 🎉"
-        + "\nHosted Proxy Alpha: https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat"
+        + "\nHosted Proxy Alpha: https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions"
         + "\n\n"
         + "## Load Test LiteLLM Proxy Results"
         + "\n\n"

.github/workflows/publish_enterprise.yml

@@ -0,0 +1,94 @@
+name: Publish litellm-enterprise to PyPI
+
+on:
+  workflow_dispatch:
+    inputs:
+      bump:
+        description: "Version bump type"
+        required: true
+        default: "patch"
+        type: choice
+        options:
+          - patch
+          - minor
+          - major
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    if: github.repository == 'BerriAI/litellm'
+    permissions:
+      contents: write
+      pull-requests: write
+    defaults:
+      run:
+        working-directory: enterprise
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install Poetry
+        run: pip install poetry
+
+      - name: Bump version
+        id: bump
+        run: |
+          OLD=$(poetry version -s)
+          poetry version ${{ github.event.inputs.bump }}
+          NEW=$(poetry version -s)
+          echo "old=$OLD" >> $GITHUB_OUTPUT
+          echo "new=$NEW" >> $GITHUB_OUTPUT
+
+      - name: Update version refs in root pyproject.toml and requirements.txt
+        run: |
+          OLD=${{ steps.bump.outputs.old }}
+          NEW=${{ steps.bump.outputs.new }}
+          sed -i "s/litellm-enterprise = {version = \"${OLD}\"/litellm-enterprise = {version = \"${NEW}\"/" ../pyproject.toml
+          sed -i "s/litellm-enterprise==${OLD}/litellm-enterprise==${NEW}/" ../requirements.txt
+
+      - name: Update poetry.lock
+        working-directory: .
+        run: poetry lock
+
+      - name: Build
+        run: poetry build
+
+      - name: Commit version bump and create PR
+        id: create-pr
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          cd ..
+          BRANCH="bump/enterprise-${{ steps.bump.outputs.new }}"
+          git checkout -b "$BRANCH"
+          git add enterprise/pyproject.toml pyproject.toml requirements.txt poetry.lock
+          git commit -m "bump: litellm-enterprise ${{ steps.bump.outputs.old }} → ${{ steps.bump.outputs.new }}"
+          git push origin "$BRANCH" --force
+          gh pr create \
+            --title "bump: litellm-enterprise ${{ steps.bump.outputs.old }} → ${{ steps.bump.outputs.new }}" \
+            --body "Version bump for litellm-enterprise. Merge to update main." \
+            --head "$BRANCH" \
+            --base main \
+          || true
+          PR_URL=$(gh pr list --head "$BRANCH" --json url -q '.[0].url')
+          echo "pr_url=$PR_URL" >> $GITHUB_OUTPUT
+        env:
+          GH_TOKEN: ${{ github.token }}
+
+      - name: Enable auto-merge
+        run: |
+          gh pr merge "${{ steps.create-pr.outputs.pr_url }}" --auto --squash
+        env:
+          GH_TOKEN: ${{ github.token }}
+
+      - name: Publish to PyPI
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_ENTERPRISE }}
+        run: |
+          pip install twine
+          twine upload dist/litellm_enterprise-${{ steps.bump.outputs.new }}*

.github/workflows/publish_proxy_extras.yml

@@ -0,0 +1,74 @@
+name: Publish litellm-proxy-extras to PyPI
+
+on:
+  workflow_dispatch:
+    inputs:
+      bump:
+        description: "Version bump type"
+        required: true
+        default: "patch"
+        type: choice
+        options:
+          - patch
+          - minor
+          - major
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    if: github.repository == 'BerriAI/litellm'
+    permissions:
+      contents: write
+    defaults:
+      run:
+        working-directory: litellm-proxy-extras
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install Poetry
+        run: pip install poetry
+
+      - name: Bump version
+        id: bump
+        run: |
+          OLD=$(poetry version -s)
+          poetry version ${{ github.event.inputs.bump }}
+          NEW=$(poetry version -s)
+          echo "old=$OLD" >> $GITHUB_OUTPUT
+          echo "new=$NEW" >> $GITHUB_OUTPUT
+
+      - name: Update version refs in root pyproject.toml and requirements.txt
+        run: |
+          OLD=${{ steps.bump.outputs.old }}
+          NEW=${{ steps.bump.outputs.new }}
+          sed -i "s/litellm-proxy-extras = {version = \"${OLD}\"/litellm-proxy-extras = {version = \"${NEW}\"/" ../pyproject.toml
+          sed -i "s/litellm-proxy-extras==${OLD}/litellm-proxy-extras==${NEW}/" ../requirements.txt
+
+      - name: Update poetry.lock
+        working-directory: .
+        run: poetry lock
+
+      - name: Build
+        run: poetry build
+
+      - name: Commit version bump
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          cd ..
+          git add litellm-proxy-extras/pyproject.toml pyproject.toml requirements.txt poetry.lock
+          git commit -m "bump: litellm-proxy-extras ${{ steps.bump.outputs.old }} → ${{ steps.bump.outputs.new }}"
+          git push
+
+      - name: Publish to PyPI
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_PUBLISH_PASSWORD }}
+        run: |
+          pip install twine
+          twine upload dist/litellm_proxy_extras-${{ steps.bump.outputs.new }}*

.github/workflows/regenerate-poetry-lock.yml

@@ -0,0 +1,80 @@
+name: Regenerate poetry.lock
+
+# Runs whenever pyproject.toml is merged into main (the most common cause of
+# the "pyproject.toml changed significantly since poetry.lock was last generated"
+# CI failure).  Can also be triggered manually.
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - pyproject.toml
+  workflow_dispatch:
+
+permissions:
+  contents: write       # needed to push the auto/regenerate-poetry-lock-* branch
+  pull-requests: write  # needed to open the PR and enable auto-merge
+
+jobs:
+  regenerate-lock:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install Poetry
+        run: pip install poetry
+
+      - name: Regenerate poetry.lock
+        run: poetry lock
+
+      - name: Check whether poetry.lock actually changed
+        id: diff
+        run: |
+          if git diff --quiet poetry.lock; then
+            echo "changed=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Open PR with the refreshed lock file
+        if: steps.diff.outputs.changed == 'true'
+        id: open-pr
+        run: |
+          BRANCH="auto/regenerate-poetry-lock-$(date +'%Y%m%d%H%M%S')"
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git checkout -b "$BRANCH"
+          git add poetry.lock
+          git commit -m "chore: regenerate poetry.lock to match pyproject.toml"
+          git push -f origin "$BRANCH"
+
+          cat > /tmp/pr-body.md << 'BODY'
+          Automated regeneration of `poetry.lock` after `pyproject.toml` was updated on `main`.
+
+          Fixes the recurring CI failure:
+          ```
+          pyproject.toml changed significantly since poetry.lock was last generated.
+          Run `poetry lock` to fix the lock file.
+          ```
+          BODY
+
+          PR_URL=$(gh pr create \
+            --title "chore: regenerate poetry.lock to match pyproject.toml" \
+            --body-file /tmp/pr-body.md \
+            --head "$BRANCH" \
+            --base main)
+          echo "pr_url=$PR_URL" >> "$GITHUB_OUTPUT"
+        env:
+          GH_TOKEN: ${{ github.token }}
+
+      - name: Enable auto-merge
+        if: steps.diff.outputs.changed == 'true'
+        run: |
+          gh pr merge "${{ steps.open-pr.outputs.pr_url }}" --auto --squash
+        env:
+          GH_TOKEN: ${{ github.token }}

.github/workflows/run_observatory_tests.yml

@@ -0,0 +1,225 @@
+name: Run Observatory Tests
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: "Docker image tag to test (e.g. v1.61.0.rc1)"
+        required: true
+        type: string
+      commit_hash:
+        description: "Commit hash (defaults to HEAD of current branch)"
+        required: false
+        type: string
+  workflow_call:
+    inputs:
+      tag:
+        description: "Docker image tag to test"
+        required: true
+        type: string
+      commit_hash:
+        description: "Commit hash of the release"
+        required: true
+        type: string
+
+permissions:
+  contents: read
+
+env:
+  LITELLM_MASTER_KEY: ${{ secrets.LITELLM_MASTER_KEY_STAGING }}
+
+jobs:
+  observatory-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Validate tag input
+        env:
+          TAG: ${{ inputs.tag }}
+        run: |
+          if [[ ! "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+ ]]; then
+            echo "Invalid tag format: $TAG (expected vX.Y.Z...)"
+            exit 1
+          fi
+
+      - name: Start LiteLLM container
+        env:
+          TAG: ${{ inputs.tag }}
+          AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
+          AZURE_API_BASE: ${{ secrets.AZURE_API_BASE }}
+        run: |
+          docker run -d \
+            --name litellm-rc \
+            -p 4000:4000 \
+            -v "${{ github.workspace }}/.github/observatory/litellm_config.yaml:/app/config.yaml" \
+            -e LITELLM_MASTER_KEY="${LITELLM_MASTER_KEY}" \
+            -e AZURE_API_KEY="${AZURE_API_KEY}" \
+            -e AZURE_API_BASE="${AZURE_API_BASE}" \
+            "litellm/litellm:${TAG}" \
+            --config /app/config.yaml --port 4000
+
+      - name: Wait for LiteLLM health check
+        run: |
+          echo "Waiting for LiteLLM to be ready..."
+          for i in $(seq 1 30); do
+            if curl -s -f http://localhost:4000/health/liveliness > /dev/null 2>&1; then
+              echo "LiteLLM is healthy"
+              exit 0
+            fi
+            echo "Attempt $i/30 - not ready yet, waiting 10s..."
+            sleep 10
+          done
+          echo "LiteLLM failed to start within 5 minutes"
+          docker logs litellm-rc
+          exit 1
+
+      - name: Start cloudflared tunnel
+        run: |
+          # Install cloudflared
+          curl -sL https://github.com/cloudflare/cloudflared/releases/download/2025.2.1/cloudflared-linux-amd64 -o /usr/local/bin/cloudflared
+          chmod +x /usr/local/bin/cloudflared
+
+          # Start a quick tunnel (no account needed) and capture the URL
+          cloudflared tunnel --url http://localhost:4000 --no-autoupdate > /tmp/cloudflared.log 2>&1 &
+          CLOUDFLARED_PID=$!
+          echo "CLOUDFLARED_PID=$CLOUDFLARED_PID" >> $GITHUB_ENV
+
+          # Wait for tunnel URL to appear in logs
+          echo "Waiting for tunnel URL..."
+          for i in $(seq 1 30); do
+            TUNNEL_URL=$(grep -oP 'https://[a-z0-9-]+\.trycloudflare\.com' /tmp/cloudflared.log | head -1 || true)
+            if [ -n "$TUNNEL_URL" ]; then
+              echo "Tunnel URL: $TUNNEL_URL"
+              echo "TUNNEL_URL=$TUNNEL_URL" >> $GITHUB_ENV
+              exit 0
+            fi
+            sleep 2
+          done
+          echo "Failed to get tunnel URL"
+          cat /tmp/cloudflared.log
+          exit 1
+
+      - name: Verify tunnel connectivity
+        run: |
+          echo "Testing tunnel at ${{ env.TUNNEL_URL }}..."
+          # Quick tunnels need time for DNS propagation; retry to avoid
+          # transient NXDOMAIN (curl exit code 6) on first attempt.
+          for i in $(seq 1 10); do
+            if curl -sf "${{ env.TUNNEL_URL }}/health/liveliness" > /dev/null 2>&1; then
+              echo "Tunnel is working (attempt $i)"
+              exit 0
+            fi
+            echo "Attempt $i/10 - tunnel not routable yet, waiting 5s..."
+            sleep 5
+          done
+          echo "Tunnel failed to become reachable after 50s"
+          cat /tmp/cloudflared.log
+          exit 1
+
+      - name: Trigger observatory test run
+        id: trigger
+        env:
+          OBSERVATORY_URL: ${{ secrets.OBSERVATORY_URL }}
+          OBSERVATORY_API_KEY: ${{ secrets.OBSERVATORY_API_KEY }}
+        run: |
+          PAYLOAD=$(jq -n \
+            --arg url "${TUNNEL_URL}" \
+            --arg key "${LITELLM_MASTER_KEY}" \
+            '{
+              deployment_url: $url,
+              api_key: $key,
+              test_suite: "TestOAIAzureRelease",
+              models: ["gpt-4o-mini", "gpt-4o"]
+            }')
+          RESPONSE=$(curl -s -w "\n%{http_code}" -X POST "${OBSERVATORY_URL}/run-test" \
+            -H "Content-Type: application/json" \
+            -H "X-LiteLLM-Observatory-API-Key: ${OBSERVATORY_API_KEY}" \
+            -d "$PAYLOAD")
+          HTTP_CODE=$(echo "$RESPONSE" | tail -1)
+          BODY=$(echo "$RESPONSE" | head -n -1)
+          echo "Response ($HTTP_CODE): $BODY"
+          if [ "$HTTP_CODE" -ge 400 ]; then
+            echo "Failed to trigger test run"
+            exit 1
+          fi
+
+          # Extract request_id for polling this specific run
+          REQUEST_ID=$(echo "$BODY" | jq -r '.results.request_id')
+          if [ -z "$REQUEST_ID" ] || [ "$REQUEST_ID" = "null" ]; then
+            echo "Failed to extract request_id from response"
+            exit 1
+          fi
+          echo "Request ID: $REQUEST_ID"
+          echo "request_id=$REQUEST_ID" >> $GITHUB_OUTPUT
+
+      - name: Poll for test completion
+        id: poll
+        env:
+          OBSERVATORY_URL: ${{ secrets.OBSERVATORY_URL }}
+          OBSERVATORY_API_KEY: ${{ secrets.OBSERVATORY_API_KEY }}
+          REQUEST_ID: ${{ steps.trigger.outputs.request_id }}
+        run: |
+          TIMEOUT=900  # 15 minutes
+          INTERVAL=30
+          ELAPSED=0
+          while [ $ELAPSED -lt $TIMEOUT ]; do
+            STATUS=$(curl -s "${OBSERVATORY_URL}/run-status/${REQUEST_ID}" \
+              -H "X-LiteLLM-Observatory-API-Key: ${OBSERVATORY_API_KEY}")
+            RUN_STATUS=$(echo "$STATUS" | jq -r '.status')
+            echo "Run status (${ELAPSED}s elapsed): $RUN_STATUS"
+
+            if [ "$RUN_STATUS" = "completed" ] || [ "$RUN_STATUS" = "failed" ]; then
+              echo "Test finished with status: $RUN_STATUS"
+              echo "$STATUS" > /tmp/observatory_result.json
+              exit 0
+            fi
+
+            sleep $INTERVAL
+            ELAPSED=$((ELAPSED + INTERVAL))
+          done
+          echo "Timed out waiting for test to complete after ${TIMEOUT}s"
+          exit 1
+
+      - name: Verify test results
+        run: |
+          RESULT=$(cat /tmp/observatory_result.json)
+          echo "Full result: $RESULT"
+
+          STATUS=$(echo "$RESULT" | jq -r '.status')
+          TEST_PASSED=$(echo "$RESULT" | jq -r '.result.test_passed // false')
+          FAILURE_RATE=$(echo "$RESULT" | jq -r '.result.failure_rate // "N/A"')
+          ERROR=$(echo "$RESULT" | jq -r '.error // empty')
+
+          echo "Status: $STATUS"
+          echo "Test passed: $TEST_PASSED"
+          echo "Failure rate: $FAILURE_RATE"
+
+          if [ -n "$ERROR" ]; then
+            echo "Error: $ERROR"
+          fi
+
+          if [ "$STATUS" = "failed" ]; then
+            echo "Test run failed"
+            exit 1
+          fi
+
+          if [ "$TEST_PASSED" != "true" ]; then
+            echo "Tests did not pass (failure rate: $FAILURE_RATE)"
+            exit 1
+          fi
+
+          echo "All tests passed!"
+
+      - name: Print LiteLLM logs on failure
+        if: failure()
+        run: |
+          docker logs litellm-rc 2>/dev/null || true
+          cat /tmp/cloudflared.log 2>/dev/null || true
+
+      - name: Cleanup
+        if: always()
+        run: |
+          kill "${{ env.CLOUDFLARED_PID }}" 2>/dev/null || true
+          docker rm -f litellm-rc 2>/dev/null || true

.github/workflows/scan_duplicate_issues.yml

@@ -0,0 +1,47 @@
+name: Scan Duplicate Issues (One-Time)
+
+on:
+  workflow_dispatch:
+    inputs:
+      threshold:
+        description: "Similarity threshold (0-1)"
+        required: false
+        default: "0.85"
+      close:
+        description: "Actually close duplicates (false = dry run)"
+        required: false
+        type: boolean
+        default: false
+
+jobs:
+  scan:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      contents: read
+    steps:
+      - name: Checkout scripts
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: .github/scripts
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Scan for duplicate issues
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          INPUT_THRESHOLD: ${{ inputs.threshold }}
+          INPUT_CLOSE: ${{ inputs.close }}
+        run: |
+          CLOSE_FLAG=""
+          if [ "$INPUT_CLOSE" = "true" ]; then
+            CLOSE_FLAG="--close"
+          fi
+          python3 .github/scripts/close_duplicate_issues.py \
+            --scan \
+            --repo ${{ github.repository }} \
+            --threshold "$INPUT_THRESHOLD" \
+            $CLOSE_FLAG

.github/workflows/test-linting.yml

@@ -32,7 +32,6 @@ jobs:
       run: |
         poetry lock
         poetry install --with dev
-        poetry run pip install openai==1.100.1
 
     - name: Run Black formatting
       run: |
@@ -74,3 +73,35 @@ jobs:
     - name: Check import safety
       run: |
         poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
+
+  secret-scan:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      contents: read
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.12'
+
+    - name: Run secret scan test
+      run: |
+        pip install pytest
+        pytest tests/litellm/test_no_hardcoded_secrets.py -v
+
+    - name: Run ggshield secret scan
+      env:
+        GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }}
+      run: |
+        if [ -n "$GITGUARDIAN_API_KEY" ]; then
+          pip install ggshield
+          ggshield secret scan repo .
+        else
+          echo "GITGUARDIAN_API_KEY not set, skipping ggshield scan"
+        fi

.github/workflows/test-litellm-matrix.yml

@@ -48,8 +48,19 @@ jobs:
             path: "tests/test_litellm/litellm_core_utils"
             workers: 2
             reruns: 1
-          - name: "other"
-            path: "tests/test_litellm/caching tests/test_litellm/responses tests/test_litellm/secret_managers tests/test_litellm/vector_stores tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/enterprise tests/test_litellm/experimental_mcp_client tests/test_litellm/google_genai tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/router_strategy tests/test_litellm/router_utils tests/test_litellm/types"
+          - name: "other-1"
+            # responses (5942) + caching (1723) + types (819) ≈ 8.5k lines
+            path: "tests/test_litellm/responses tests/test_litellm/caching tests/test_litellm/types"
+            workers: 2
+            reruns: 2
+          - name: "other-2"
+            # enterprise (3062) + google_genai (2511) + router_utils (1982) ≈ 7.6k lines
+            path: "tests/test_litellm/enterprise tests/test_litellm/google_genai tests/test_litellm/router_utils"
+            workers: 2
+            reruns: 2
+          - name: "other-3"
+            # remaining dirs ≈ 8.0k lines
+            path: "tests/test_litellm/router_strategy tests/test_litellm/secret_managers tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/experimental_mcp_client tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/vector_stores"
             workers: 2
             reruns: 2
           - name: "root"
@@ -57,12 +68,49 @@ jobs:
             workers: 2
             reruns: 2
           # tests/proxy_unit_tests split alphabetically (~48 files total)
-          - name: "proxy-unit-a"
-            path: "tests/proxy_unit_tests/test_[a-o]*.py"
+          - name: "proxy-unit-a1"
+            # test_[a-j]*.py: jwt (1564) + auth_checks (978) + google_gemini (478) + e2e_pod_lock (437) + rest
+            path: "tests/proxy_unit_tests/test_[a-j]*.py"
             workers: 2
             reruns: 1
-          - name: "proxy-unit-b"
-            path: "tests/proxy_unit_tests/test_[p-z]*.py"
+          - name: "proxy-unit-a2"
+            # test_[k-o]*.py: key_generate_prisma (4346) + key_generate_dynamodb + models_fallback
+            path: "tests/proxy_unit_tests/test_[k-o]*.py"
+            workers: 2
+            reruns: 1
+          - name: "proxy-unit-b1"
+            # lighter config/utility proxy tests (prisma, project, prompt, proxy_[c-r]*)
+            path: "tests/proxy_unit_tests/test_prisma*.py tests/proxy_unit_tests/test_project*.py tests/proxy_unit_tests/test_prompt*.py tests/proxy_unit_tests/test_proxy_[c-r]*.py"
+            workers: 2
+            reruns: 1
+          - name: "proxy-unit-b2"
+            # proxy_server.py alone (2750 lines) - isolated to avoid blocking smaller tests
+            path: "tests/proxy_unit_tests/test_proxy_server.py"
+            workers: 2
+            reruns: 1
+          - name: "proxy-unit-b3"
+            # proxy_server_* (618) + proxy_setting_guardrails (71) - smaller server-related tests
+            path: "tests/proxy_unit_tests/test_proxy_server_*.py tests/proxy_unit_tests/test_proxy_setting_guardrails.py"
+            workers: 2
+            reruns: 1
+          - name: "proxy-unit-b4"
+            # proxy_utils.py alone (2339 lines) - isolated to avoid blocking token counter
+            path: "tests/proxy_unit_tests/test_proxy_utils.py"
+            workers: 2
+            reruns: 1
+          - name: "proxy-unit-b5"
+            # proxy_token_counter (1279) - runs independently from utils
+            path: "tests/proxy_unit_tests/test_proxy_token_counter.py"
+            workers: 2
+            reruns: 1
+          - name: "proxy-unit-b6"
+            # test_[r-t]*.py: response_polling (1399) + search_api_logging (202) + server_root (64) + skills_db (261) + realtime_cache (62)
+            path: "tests/proxy_unit_tests/test_[r-t]*.py"
+            workers: 2
+            reruns: 1
+          - name: "proxy-unit-b7"
+            # test_[u-z]*.py: user_api_key_auth (1136) + zero_cost (590) + update_spend (305) + unit_test_* (206) + ui_path (157)
+            path: "tests/proxy_unit_tests/test_[u-z]*.py"
             workers: 2
             reruns: 1
 

.github/workflows/test-litellm.yml

@@ -38,7 +38,7 @@ jobs:
         poetry run pip install "google-genai==1.22.0"
         poetry run pip install "google-cloud-aiplatform>=1.38"
         poetry run pip install "fastapi-offline==1.7.3"
-        poetry run pip install "python-multipart==0.0.22"
+        poetry run pip install "python-multipart>=0.0.20"
         poetry run pip install "openapi-core"
     - name: Setup litellm-enterprise as local package
       run: |

.github/workflows/test-proxy-e2e-azure-batches.yml

@@ -0,0 +1,90 @@
+name: Proxy E2E Azure Batches Tests
+
+on:
+  pull_request:
+    branches: [main]
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  proxy_e2e_azure_batches_tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+
+    services:
+      postgres:
+        image: postgres:15
+        env:
+          POSTGRES_USER: llmproxy
+          POSTGRES_PASSWORD: dbpassword9090
+          POSTGRES_DB: litellm
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+
+      - name: Cache Poetry dependencies
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/pypoetry
+            ~/.cache/pip
+            .venv
+          key: ${{ runner.os }}-poetry-e2e-batches-${{ hashFiles('poetry.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-poetry-e2e-batches-
+            ${{ runner.os }}-poetry-
+
+      - name: Install dependencies
+        run: |
+          poetry config virtualenvs.in-project true
+          poetry install --with dev,proxy-dev --extras "proxy"
+          poetry run pip install psycopg2-binary uvicorn fastapi httpx tenacity
+
+      - name: Setup litellm-enterprise
+        run: |
+          poetry run pip install --force-reinstall --no-deps -e enterprise/
+
+      - name: Generate Prisma client
+        run: |
+          poetry run prisma generate --schema litellm/proxy/schema.prisma
+
+      - name: Run Prisma migrations
+        env:
+          DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
+        run: |
+          cd litellm/proxy
+          poetry run prisma migrate deploy --schema schema.prisma
+          cd ../..
+
+      - name: Run Azure Batch E2E Tests
+        env:
+          DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
+          USE_LOCAL_LITELLM: "true"
+          USE_MOCK_MODELS: "true"
+          USE_STATE_TRACKER: "true"
+          LITELLM_LOG: DEBUG
+        run: |
+          poetry run pytest tests/proxy_e2e_azure_batches_tests/test_proxy_e2e_azure_batches.py \
+            -vv -s -k "test_e2e_managed_batch" \
+            --tb=short \
+            --maxfail=3 \
+            --durations=10
+

.gitignore

@@ -89,6 +89,7 @@ tests/test_custom_dir/*
 test.py
 
 litellm_config.yaml
+!.github/observatory/litellm_config.yaml
 .cursor
 .vscode/launch.json
 litellm/proxy/to_delete_loadtest_work/*

AGENTS.md

@@ -109,6 +109,8 @@ Key files:
 - `litellm/proxy/auth/` - Authentication logic
 - `litellm/proxy/management_endpoints/` - Admin API endpoints
 
+**Database (proxy)**: Use Prisma model methods (`prisma_client.db.<model>.upsert`, `.find_many`, `.find_unique`, etc.), not raw SQL (`execute_raw`/`query_raw`). See COMMON PITFALLS for details.
+
 ## MCP (MODEL CONTEXT PROTOCOL) SUPPORT
 
 LiteLLM supports MCP for agent workflows:
@@ -174,6 +176,43 @@ When opening issues or pull requests, follow these templates:
 3. **Rate Limits**: Respect provider rate limits in tests
 4. **Memory Usage**: Be mindful of memory usage in streaming scenarios
 5. **Dependencies**: Keep dependencies minimal and well-justified
+6. **UI/Backend Contract Mismatch**: When adding a new entity type to the UI, always check whether the backend endpoint accepts a single value or an array. Match the UI control accordingly (single-select vs. multi-select) to avoid silently dropping user selections
+7. **Missing Tests for New Entity Types**: When adding a new entity type (e.g., in `EntityUsage`, `UsageViewSelect`), always add corresponding tests in the existing test files and update any icon/component mocks
+8. **Raw SQL in proxy DB code**: Do not use `execute_raw` or `query_raw` for proxy database access. Use Prisma model methods (e.g. `prisma_client.db.litellm_tooltable.upsert()`, `.find_many()`, `.find_unique()`) so behavior stays consistent with the schema, the client stays mockable in tests, and you avoid the pitfalls of hand-written SQL (parameter ordering, type casting, schema drift)
+
+8. **Do not hardcode model-specific flags**: Put model-specific capability flags in `model_prices_and_context_window.json` and read them via `get_model_info` (or existing helpers like `supports_reasoning`). This prevents users from needing to upgrade LiteLLM each time a new model supports a feature.
+
+   **Example of BAD** (hardcoded model checks):
+
+   ```python
+   @staticmethod
+   def _is_effort_supported_model(model: str) -> bool:
+       """Check if the model supports the output_config.effort parameter..."""
+       model_lower = model.lower()
+       if AnthropicConfig._is_claude_4_6_model(model):
+           return True
+       return any(
+           v in model_lower for v in ("opus-4-5", "opus_4_5", "opus-4.5", "opus_4.5")
+       )
+   ```
+
+   **Example of GOOD** (config-driven or helper that reads from config):
+
+   ```python
+   if (
+       "claude-3-7-sonnet" in model
+       or AnthropicConfig._is_claude_4_6_model(model)
+       or supports_reasoning(
+           model=model,
+           custom_llm_provider=self.custom_llm_provider,
+       )
+   ):
+       ...
+   ```
+
+   Using helpers like `supports_reasoning` (which read from `model_prices_and_context_window.json` / `get_model_info`) allows future model updates to "just work" without code changes.
+
+9. **Never close HTTP/SDK clients on cache eviction**: Do not add `close()`, `aclose()`, or `create_task(close_fn())` inside `LLMClientCache._remove_key()` or any cache eviction path. Evicted clients may still be held by in-flight requests; closing them causes `RuntimeError: Cannot send a request, as the client has been closed.` in production after the cache TTL (1 hour) expires. Connection cleanup is handled at shutdown by `close_litellm_async_clients()`. See PR #22247 for the full incident history.
 
 ## HELPFUL RESOURCES
 
@@ -187,4 +226,49 @@ When opening issues or pull requests, follow these templates:
 - Check similar provider implementations
 - Ensure comprehensive test coverage
 - Update documentation appropriately
-- Consider backward compatibility impact 
+- Consider backward compatibility impact
+
+## Cursor Cloud specific instructions
+
+### Environment
+
+- Poetry is installed in `~/.local/bin`; the update script ensures it is on `PATH`.
+- Python 3.12, Node 22 are pre-installed.
+- The virtual environment lives under `~/.cache/pypoetry/virtualenvs/`.
+
+### Running the proxy server
+
+Start the proxy with a config file:
+
+```bash
+poetry run litellm --config dev_config.yaml --port 4000
+```
+
+The proxy takes ~15-20 seconds to fully start (it runs Prisma migrations on boot). Wait for `/health` to return before sending requests. Without a PostgreSQL `DATABASE_URL`, the proxy connects to a default Neon dev database embedded in the `litellm-proxy-extras` package.
+
+### Running tests
+
+See `CLAUDE.md` and the `Makefile` for standard commands. Key notes:
+
+- `psycopg-binary` must be installed (`poetry run pip install psycopg-binary`) because the pytest-postgresql plugin requires it and the lock file only includes `psycopg` (no binary).
+- `openapi-core` must be installed (`poetry run pip install openapi-core`) for the OpenAPI compliance tests in `tests/test_litellm/interactions/`.
+- The `--timeout` pytest flag is NOT available; don't pass it.
+- Unit tests: `poetry run pytest tests/test_litellm/ -x -vv -n 4`
+- Black `--check` may report pre-existing formatting issues; this does not block test runs.
+- If `poetry install` fails with "pyproject.toml changed significantly since poetry.lock was last generated", run `poetry lock` first to regenerate the lock file.
+
+### Lint
+
+```bash
+cd litellm && poetry run ruff check .
+```
+
+Ruff is the primary fast linter. For the full lint suite (including mypy, black, circular imports), run `make lint` per `CLAUDE.md`.
+
+### UI Dashboard development
+
+- The UI is at `ui/litellm-dashboard/`. Run `npm run dev` from that directory for the Next.js dev server on port 3000.
+- The proxy at port 4000 serves a **pre-built** static UI from `litellm/proxy/_experimental/out/`. After making UI code changes, you must run `npm run build` in the dashboard directory and copy the output: `cp -r ui/litellm-dashboard/out/* litellm/proxy/_experimental/out/` for the proxy to serve the updated UI.
+- SVGs used as provider logos (loaded via `<img>` tags) must NOT use `fill="currentColor"` — replace with an explicit color like `#000000` or use the `-color` variant from lobehub icons, since CSS color inheritance does not work inside `<img>` elements.
+- Provider logos live in `ui/litellm-dashboard/public/assets/logos/` (source) and `litellm/proxy/_experimental/out/assets/logos/` (pre-built). Both locations must have the file for it to work in dev and proxy-served modes.
+- UI Vitest tests: `cd ui/litellm-dashboard && npx vitest run`

CLAUDE.md

@@ -97,13 +97,34 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components:
 - Integration tests for each provider in `tests/llm_translation/`
 - Proxy tests in `tests/proxy_unit_tests/`
 - Load tests in `tests/load_tests/`
+- **Always add tests when adding new entity types or features** — if the existing test file covers other entity types, add corresponding tests for the new one
+
+### UI / Backend Consistency
+- When wiring a new UI entity type to an existing backend endpoint, verify the backend API contract (single value vs. array, required vs. optional params) and ensure the UI controls match — e.g., use a single-select dropdown when the backend accepts a single value, not a multi-select
 
 ### Database Migrations
 - Prisma handles schema migrations
 - Migration files auto-generated with `prisma migrate dev`
 - Always test migrations against both PostgreSQL and SQLite
 
+### Proxy database access
+- **Do not write raw SQL** for proxy DB operations. Use Prisma model methods instead of `execute_raw` / `query_raw`.
+- Use the generated client: `prisma_client.db.<model>` (e.g. `litellm_tooltable`, `litellm_usertable`) with `.upsert()`, `.find_many()`, `.find_unique()`, `.update()`, `.update_many()` as appropriate. This avoids schema/client drift, keeps code testable with simple mocks, and matches patterns used in spend logs and other proxy code.
+
 ### Enterprise Features
 - Enterprise-specific code in `enterprise/` directory
 - Optional features enabled via environment variables
-- Separate licensing and authentication for enterprise features
+- Separate licensing and authentication for enterprise features
+
+### HTTP Client Cache Safety
+- **Never close HTTP/SDK clients on cache eviction.** `LLMClientCache._remove_key()` must not call `close()`/`aclose()` on evicted clients — they may still be used by in-flight requests. Doing so causes `RuntimeError: Cannot send a request, as the client has been closed.` after the 1-hour TTL expires. Cleanup happens at shutdown via `close_litellm_async_clients()`.
+
+### Troubleshooting: DB schema out of sync after proxy restart
+`litellm-proxy-extras` runs `prisma migrate deploy` on startup using **its own** bundled migration files, which may lag behind schema changes in the current worktree. Symptoms: `Unknown column`, `Invalid prisma invocation`, or missing data on new fields.
+
+**Diagnose:** Run `\d "TableName"` in psql and compare against `schema.prisma` — missing columns confirm the issue.
+
+**Fix options:**
+1. **Create a Prisma migration** (permanent) — run `prisma migrate dev --name <description>` in the worktree. The generated file will be picked up by `prisma migrate deploy` on next startup.
+2. **Apply manually for local dev** — `psql -d litellm -c "ALTER TABLE ... ADD COLUMN IF NOT EXISTS ..."` after each proxy start. Fine for dev, not for production.
+3. **Update litellm-proxy-extras** — if the package is installed from PyPI, its migration directory must include the new file. Either update the package or run the migration manually until the next release ships it.

Dockerfile

@@ -49,7 +49,7 @@ USER root
 
 # Install runtime dependencies (libsndfile needed for audio processing on ARM64)
 RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \
-    npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 && \
+    npm install -g npm@latest tar@7.5.10 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
     # SECURITY FIX: npm bundles tar, glob, and brace-expansion at multiple nested
     # levels inside its dependency tree. `npm install -g <pkg>` only creates a
     # SEPARATE global package, it does NOT replace npm's internal copies.
@@ -64,7 +64,21 @@ RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile
     find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
     done && \
-    npm cache clean --force
+    find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
+    done && \
+    find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
+    done && \
+    # SECURITY FIX: patch npm's own package.json metadata so scanners see the
+    # actual installed versions instead of the stale declared dependencies.
+    find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
+        sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null && \
+    npm cache clean --force && \
+    # Remove the apk-tracked npm so its stale SBOM metadata (tar 7.5.9) is
+    # no longer visible to image scanners.  The globally installed npm@latest
+    # at /usr/local/lib/node_modules/npm/ remains fully functional.
+    { apk del --no-cache npm 2>/dev/null || true; }
 
 WORKDIR /app
 # Copy the current directory contents into the container at /app
@@ -90,14 +104,21 @@ RUN find /usr/lib -type f -path "*/tornado/test/*" -delete && \
 # npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
 # Patch every copy of tar, glob, and brace-expansion inside that tree.
 RUN GLOBAL="$(npm root -g)" && \
-    find /usr/lib -path "*/nodejs_wheel/*/node_modules/tar" -type d | while read d; do \
+    [ -n "$GLOBAL" ] || { echo "ERROR: npm root -g returned empty; aborting"; exit 1; } && \
+    find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
     done && \
-    find /usr/lib -path "*/nodejs_wheel/*/node_modules/glob" -type d | while read d; do \
+    find /usr/lib -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
     done && \
-    find /usr/lib -path "*/nodejs_wheel/*/node_modules/@isaacs/brace-expansion" -type d | while read d; do \
+    find /usr/lib -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
+    done && \
+    find /usr/lib -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
+    done && \
+    find /usr/lib -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
     done
 
 # Install semantic_router and aurelio-sdk using script

README.md

@@ -203,7 +203,7 @@ curl -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
 {
   "mcpServers": {
     "LiteLLM": {
-      "url": "http://localhost:4000/mcp",
+      "url": "http://localhost:4000/mcp/",
       "headers": {
         "x-litellm-api-key": "Bearer sk-1234"
       }
@@ -399,7 +399,7 @@ Support for more providers. Missing a provider or LLM Platform, raise a [feature
 # Enterprise
 For companies that need better security, user management and professional support
 
-[Talk to founders](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
+[Talk to founders](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
 
 This covers:
 - ✅ **Features under the [LiteLLM Commercial License](https://docs.litellm.ai/docs/proxy/enterprise):**

ci_cd/security_scans.sh

@@ -158,6 +158,11 @@ run_grype_scans() {
         "CVE-2025-11468" # No fix available yet
         "CVE-2026-1299" # Python 3.13 email module header injection - not applicable, LiteLLM doesn't use BytesGenerator for email serialization
         "CVE-2026-0775" # npm cli incorrect permission assignment - no fix available yet, npm is only used at build/prisma-generate time
+        "GHSA-3ppc-4f35-3m26" # minimatch ReDoS via repeated wildcards - from nodejs_wheel bundled npm, not used in application runtime code
+        "GHSA-83g3-92jg-28cx" # tar arbitrary file read/write via hardlink - from nodejs_wheel bundled npm, not used in application runtime code
+        "CVE-2026-25639" # axios - full fix requires 1.x major version bump; pinned to >=0.30.2 to clear other axios CVEs, upgrade to 1.x in follow-up
+        "CVE-2026-2297" # Python 3.13 SourcelessFileLoader audit hook bypass - no fix available in base image
+        "GHSA-qffp-2rhf-9h96" # tar hardlink path traversal - from nodejs_wheel bundled npm, not used in application runtime code
     )
 
     # Build JSON array of allowlisted CVE IDs for jq

cookbook/benchmark/readme.md

@@ -178,4 +178,4 @@ Benchmark Results for 'When will BerriAI IPO?':
 ```
 
 ## Support 
-**🤝 Schedule a 1-on-1 Session:** Book a [1-on-1 session](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) with Krrish and Ishaan, the founders, to discuss any issues, provide feedback, or explore how we can improve LiteLLM for you.
+**🤝 Schedule a 1-on-1 Session:** Book a [1-on-1 session](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) with Krrish and Ishaan, the founders, to discuss any issues, provide feedback, or explore how we can improve LiteLLM for you.

cookbook/gollem_go_agent_framework/README.md

@@ -0,0 +1,119 @@
+# Gollem Go Agent Framework with LiteLLM
+
+A working example showing how to use [gollem](https://github.com/fugue-labs/gollem), a production-grade Go agent framework, with LiteLLM as a proxy gateway. This lets Go developers access 100+ LLM providers through a single proxy while keeping compile-time type safety for tools and structured output.
+
+## Quick Start
+
+### 1. Start LiteLLM Proxy
+
+```bash
+# Simple start with a single model
+litellm --model gpt-4o
+
+# Or with the example config for multi-provider access
+litellm --config proxy_config.yaml
+```
+
+### 2. Run the examples
+
+```bash
+# Install Go dependencies
+go mod tidy
+
+# Basic agent
+go run ./basic
+
+# Agent with type-safe tools
+go run ./tools
+
+# Streaming responses
+go run ./streaming
+```
+
+## Configuration
+
+The included `proxy_config.yaml` sets up three providers through LiteLLM:
+
+```yaml
+model_list:
+  - model_name: gpt-4o          # OpenAI
+  - model_name: claude-sonnet    # Anthropic
+  - model_name: gemini-pro       # Google Vertex AI
+```
+
+Switch providers in Go by changing a single string — no code changes needed:
+
+```go
+model := openai.NewLiteLLM("http://localhost:4000",
+    openai.WithModel("gpt-4o"),        // OpenAI
+    // openai.WithModel("claude-sonnet"),  // Anthropic
+    // openai.WithModel("gemini-pro"),     // Google
+)
+```
+
+## Examples
+
+### `basic/` — Basic Agent
+
+Connects gollem to LiteLLM and runs a simple prompt. Demonstrates the `NewLiteLLM` constructor and basic agent creation.
+
+### `tools/` — Type-Safe Tools
+
+Shows gollem's compile-time type-safe tool framework working through LiteLLM's tool-use passthrough. The tool parameters are Go structs with JSON tags — the schema is generated automatically at compile time.
+
+### `streaming/` — Streaming Responses
+
+Real-time token streaming using Go 1.23+ range-over-function iterators, proxied through LiteLLM's SSE passthrough.
+
+## How It Works
+
+Gollem's `openai.NewLiteLLM()` constructor creates an OpenAI-compatible provider pointed at your LiteLLM proxy. Since LiteLLM speaks the OpenAI API protocol, everything works out of the box:
+
+- **Chat completions** — standard request/response
+- **Tool use** — LiteLLM passes tool definitions and calls through transparently
+- **Streaming** — Server-Sent Events proxied through LiteLLM
+- **Structured output** — JSON schema response format works with supporting models
+
+```
+Go App (gollem) → LiteLLM Proxy → OpenAI / Anthropic / Google / ...
+```
+
+## Why Use This?
+
+- **Type-safe Go**: Compile-time type checking for tools, structured output, and agent configuration — no runtime surprises
+- **Single proxy, many models**: Switch between OpenAI, Anthropic, Google, and 100+ other providers by changing a model name string
+- **Zero-dependency core**: gollem's core has no external dependencies — just stdlib
+- **Single binary deployment**: `go build` produces one binary, no pip/venv/Docker needed
+- **Cost tracking & rate limiting**: LiteLLM handles cost tracking, rate limits, and fallbacks at the proxy layer
+
+## Environment Variables
+
+```bash
+# Required for providers you want to use (set in LiteLLM config or env)
+export OPENAI_API_KEY="sk-..."
+export ANTHROPIC_API_KEY="sk-ant-..."
+
+# Optional: point to a non-default LiteLLM proxy
+export LITELLM_PROXY_URL="http://localhost:4000"
+```
+
+## Troubleshooting
+
+**Connection errors?**
+- Make sure LiteLLM is running: `litellm --model gpt-4o`
+- Check the URL is correct (default: `http://localhost:4000`)
+
+**Model not found?**
+- Verify the model name matches what's configured in LiteLLM
+- Run `curl http://localhost:4000/models` to see available models
+
+**Tool calls not working?**
+- Ensure the underlying model supports tool use (GPT-4o, Claude, Gemini)
+- Check LiteLLM logs for any provider-specific errors
+
+## Learn More
+
+- [gollem GitHub](https://github.com/fugue-labs/gollem)
+- [gollem API Reference](https://pkg.go.dev/github.com/fugue-labs/gollem/core)
+- [LiteLLM Proxy Docs](https://docs.litellm.ai/docs/simple_proxy)
+- [LiteLLM Supported Models](https://docs.litellm.ai/docs/providers)

cookbook/gollem_go_agent_framework/basic/main.go

@@ -0,0 +1,41 @@
+// Basic gollem agent connected to a LiteLLM proxy.
+//
+// Usage:
+//
+//	litellm --model gpt-4o   # start proxy in another terminal
+//	go run ./basic
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/fugue-labs/gollem/core"
+	"github.com/fugue-labs/gollem/provider/openai"
+)
+
+func main() {
+	proxyURL := "http://localhost:4000"
+	if u := os.Getenv("LITELLM_PROXY_URL"); u != "" {
+		proxyURL = u
+	}
+
+	// Connect to LiteLLM proxy. NewLiteLLM creates an OpenAI-compatible
+	// provider pointed at the given URL.
+	model := openai.NewLiteLLM(proxyURL,
+		openai.WithModel("gpt-4o"), // any model name configured in LiteLLM
+	)
+
+	// Create and run a simple agent.
+	agent := core.NewAgent[string](model,
+		core.WithSystemPrompt[string]("You are a helpful assistant. Be concise."),
+	)
+
+	result, err := agent.Run(context.Background(), "Explain quantum computing in two sentences.")
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Println(result.Output)
+}

cookbook/gollem_go_agent_framework/go.mod

@@ -0,0 +1,5 @@
+module github.com/BerriAI/litellm/cookbook/gollem_go_agent_framework
+
+go 1.25.1
+
+require github.com/fugue-labs/gollem v0.1.0

cookbook/gollem_go_agent_framework/go.sum

@@ -0,0 +1,2 @@
+github.com/fugue-labs/gollem v0.1.0 h1:QexYnvkb44QZFEljgAePqMIGZjgsbk0Y5GJ2jYYgfa8=
+github.com/fugue-labs/gollem v0.1.0/go.mod h1:htW1YO81uysSKVOkYJtxhGCFrzm+36HBFxEWuECoHKQ=

cookbook/gollem_go_agent_framework/proxy_config.yaml

@@ -0,0 +1,16 @@
+model_list:
+  - model_name: gpt-4o
+    litellm_params:
+      model: openai/gpt-4o
+      api_key: os.environ/OPENAI_API_KEY
+
+  - model_name: claude-sonnet
+    litellm_params:
+      model: anthropic/claude-sonnet-4-20250514
+      api_key: os.environ/ANTHROPIC_API_KEY
+
+  - model_name: gemini-pro
+    litellm_params:
+      model: vertex_ai/gemini-2.0-flash
+      vertex_project: my-project
+      vertex_location: us-central1

cookbook/gollem_go_agent_framework/streaming/main.go

@@ -0,0 +1,56 @@
+// Streaming responses from gollem through LiteLLM.
+//
+// Uses Go 1.23+ range-over-function iterators for real-time token
+// streaming via LiteLLM's SSE passthrough.
+//
+// Usage:
+//
+//	litellm --model gpt-4o
+//	go run ./streaming
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/fugue-labs/gollem/core"
+	"github.com/fugue-labs/gollem/provider/openai"
+)
+
+func main() {
+	proxyURL := "http://localhost:4000"
+	if u := os.Getenv("LITELLM_PROXY_URL"); u != "" {
+		proxyURL = u
+	}
+
+	model := openai.NewLiteLLM(proxyURL,
+		openai.WithModel("gpt-4o"),
+	)
+
+	agent := core.NewAgent[string](model)
+
+	// RunStream returns a streaming result that yields tokens as they arrive.
+	stream, err := agent.RunStream(context.Background(), "Write a haiku about distributed systems")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// StreamText yields text chunks in real-time.
+	// The boolean argument controls whether deltas (true) or accumulated
+	// text (false) is returned.
+	fmt.Print("Response: ")
+	for text, err := range stream.StreamText(true) {
+		if err != nil {
+			log.Fatal(err)
+		}
+		fmt.Print(text)
+	}
+	fmt.Println()
+
+	// After streaming completes, the final response is available.
+	resp := stream.Response()
+	fmt.Printf("\nTokens used: input=%d, output=%d\n",
+		resp.Usage.InputTokens, resp.Usage.OutputTokens)
+}

cookbook/gollem_go_agent_framework/tools/main.go

@@ -0,0 +1,64 @@
+// Gollem agent with type-safe tools through LiteLLM.
+//
+// The tool parameters are Go structs — gollem generates the JSON schema
+// automatically at compile time. LiteLLM passes tool definitions through
+// transparently to the underlying provider.
+//
+// Usage:
+//
+//	litellm --model gpt-4o
+//	go run ./tools
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/fugue-labs/gollem/core"
+	"github.com/fugue-labs/gollem/provider/openai"
+)
+
+// WeatherParams defines the tool's input schema via struct tags.
+// The JSON schema is generated at compile time — no runtime reflection needed.
+type WeatherParams struct {
+	City string `json:"city" description:"City name to get weather for"`
+	Unit string `json:"unit,omitempty" description:"Temperature unit: celsius or fahrenheit"`
+}
+
+func main() {
+	proxyURL := "http://localhost:4000"
+	if u := os.Getenv("LITELLM_PROXY_URL"); u != "" {
+		proxyURL = u
+	}
+
+	model := openai.NewLiteLLM(proxyURL,
+		openai.WithModel("gpt-4o"),
+	)
+
+	// Define a type-safe tool. The function signature enforces correct types.
+	weatherTool := core.FuncTool[WeatherParams](
+		"get_weather",
+		"Get current weather for a city",
+		func(ctx context.Context, p WeatherParams) (string, error) {
+			unit := p.Unit
+			if unit == "" {
+				unit = "fahrenheit"
+			}
+			// In production, call a real weather API here.
+			return fmt.Sprintf("Weather in %s: 72°F (22°C), sunny", p.City), nil
+		},
+	)
+
+	agent := core.NewAgent[string](model,
+		core.WithTools[string](weatherTool),
+		core.WithSystemPrompt[string]("You are a helpful weather assistant. Use the get_weather tool to answer weather questions."),
+	)
+
+	result, err := agent.Run(context.Background(), "What's the weather like in San Francisco and Tokyo?")
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Println(result.Output)
+}

deploy/charts/litellm-helm/README.md

@@ -36,6 +36,10 @@ If `db.useStackgresOperator` is used (not yet implemented):
 | `serviceAccount.create`     | Whether or not to create a Kubernetes Service Account for this deployment. The default is `false` because LiteLLM has no need to access the Kubernetes API.                                                                                   | `false`                   |
 | `service.type`              | Kubernetes Service type (e.g. `LoadBalancer`, `ClusterIP`, etc.)                                                                                                                                                                              | `ClusterIP`               |
 | `service.port`              | TCP port that the Kubernetes Service will listen on. Also the TCP port within the Pod that the proxy will listen on.                                                                                                                          | `4000`                    |
+| `livenessProbe.*`           | Liveness probe settings for the LiteLLM container (`path`, `periodSeconds`, `timeoutSeconds`, thresholds, and initial delay).                                                                                                                  | See `values.yaml`         |
+| `readinessProbe.*`          | Readiness probe settings for the LiteLLM container (`path`, `periodSeconds`, `timeoutSeconds`, thresholds, and initial delay).                                                                                                                 | See `values.yaml`         |
+| `startupProbe.*`            | Startup probe settings for the LiteLLM container (`path`, `periodSeconds`, `timeoutSeconds`, thresholds, and initial delay).                                                                                                                   | See `values.yaml`         |
+| `resources.*`               | CPU/memory requests and limits for the LiteLLM container.                                                                                                                                                                                       | `{}`                      |
 | `service.loadBalancerClass` | Optional LoadBalancer implementation class (only used when `service.type` is `LoadBalancer`)                                                                                                                                                  | `""`                      |
 | `ingress.labels`            | Additional labels for the Ingress resource                                                                                                                                                                                                    | `{}`                      |
 | `ingress.*`                 | See [values.yaml](./values.yaml) for example settings                                                                                                                                                                                         | N/A                       |

deploy/charts/litellm-helm/templates/configmap-litellm.yaml

@@ -6,4 +6,4 @@ metadata:
 data:
   config.yaml: |
 {{ .Values.proxy_config | toYaml | indent 6 }}
-{{- end }}
+{{- end }}

deploy/charts/litellm-helm/templates/deployment.yaml

@@ -158,18 +158,31 @@ spec:
             {{- end }}
           livenessProbe:
             httpGet:
-              path: /health/liveliness
+              path: {{ .Values.livenessProbe.path | quote }}
               port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
+            initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
+            periodSeconds: {{ .Values.livenessProbe.periodSeconds }}
+            timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }}
+            successThreshold: {{ .Values.livenessProbe.successThreshold }}
+            failureThreshold: {{ .Values.livenessProbe.failureThreshold }}
           readinessProbe:
             httpGet:
-              path: /health/readiness
+              path: {{ .Values.readinessProbe.path | quote }}
               port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
+            initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
+            periodSeconds: {{ .Values.readinessProbe.periodSeconds }}
+            timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }}
+            successThreshold: {{ .Values.readinessProbe.successThreshold }}
+            failureThreshold: {{ .Values.readinessProbe.failureThreshold }}
           startupProbe:
             httpGet:
-              path: /health/readiness
+              path: {{ .Values.startupProbe.path | quote }}
               port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
-            failureThreshold: 30
-            periodSeconds: 10
+            initialDelaySeconds: {{ .Values.startupProbe.initialDelaySeconds }}
+            periodSeconds: {{ .Values.startupProbe.periodSeconds }}
+            timeoutSeconds: {{ .Values.startupProbe.timeoutSeconds }}
+            successThreshold: {{ .Values.startupProbe.successThreshold }}
+            failureThreshold: {{ .Values.startupProbe.failureThreshold }}
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
           volumeMounts:
@@ -235,4 +248,4 @@ spec:
       {{- if .Values.topologySpreadConstraints }}
       topologySpreadConstraints:
         {{- toYaml .Values.topologySpreadConstraints | nindent 8 }}
-      {{- end }}
+      {{- end }}

deploy/charts/litellm-helm/tests/deployment_tests.yaml

@@ -159,4 +159,150 @@ tests:
           value: -c
       - equal:
           path: spec.template.spec.containers[0].lifecycle.preStop.exec.command[2]
-          value: echo "Container stopping"
+          value: echo "Container stopping"
+  - it: should render background health check settings from proxy_config.general_settings
+    template: configmap-litellm.yaml
+    set:
+      proxy_config.general_settings.background_health_checks: true
+      proxy_config.general_settings.health_check_interval: 240
+      proxy_config.general_settings.health_check_concurrency: 16
+      proxy_config.general_settings.health_check_details: false
+    asserts:
+      - matchRegex:
+          path: data["config.yaml"]
+          pattern: '(?m)^\s*background_health_checks:\s*true$'
+      - matchRegex:
+          path: data["config.yaml"]
+          pattern: '(?m)^\s*health_check_interval:\s*240$'
+      - matchRegex:
+          path: data["config.yaml"]
+          pattern: '(?m)^\s*health_check_concurrency:\s*16$'
+      - matchRegex:
+          path: data["config.yaml"]
+          pattern: '(?m)^\s*health_check_details:\s*false$'
+  - it: should allow overriding liveness, readiness, and startup probes
+    template: deployment.yaml
+    set:
+      livenessProbe:
+        path: /custom/livez
+        initialDelaySeconds: 5
+        periodSeconds: 15
+        timeoutSeconds: 5
+        successThreshold: 1
+        failureThreshold: 5
+      readinessProbe:
+        path: /custom/readyz
+        initialDelaySeconds: 10
+        periodSeconds: 20
+        timeoutSeconds: 6
+        successThreshold: 1
+        failureThreshold: 6
+      startupProbe:
+        path: /custom/startupz
+        initialDelaySeconds: 15
+        periodSeconds: 25
+        timeoutSeconds: 7
+        successThreshold: 1
+        failureThreshold: 40
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.httpGet.path
+          value: /custom/livez
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.timeoutSeconds
+          value: 5
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.httpGet.path
+          value: /custom/readyz
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.timeoutSeconds
+          value: 6
+      - equal:
+          path: spec.template.spec.containers[0].startupProbe.httpGet.path
+          value: /custom/startupz
+      - equal:
+          path: spec.template.spec.containers[0].startupProbe.failureThreshold
+          value: 40
+  - it: should render container resources from values
+    template: deployment.yaml
+    set:
+      resources:
+        limits:
+          cpu: 500m
+          memory: 2Gi
+        requests:
+          cpu: 250m
+          memory: 1Gi
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].resources.limits.cpu
+          value: 500m
+      - equal:
+          path: spec.template.spec.containers[0].resources.limits.memory
+          value: 2Gi
+      - equal:
+          path: spec.template.spec.containers[0].resources.requests.cpu
+          value: 250m
+      - equal:
+          path: spec.template.spec.containers[0].resources.requests.memory
+          value: 1Gi
+  - it: should keep default probes and empty resources unchanged
+    template: deployment.yaml
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.httpGet.path
+          value: /health/liveliness
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.initialDelaySeconds
+          value: 0
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.periodSeconds
+          value: 10
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.timeoutSeconds
+          value: 1
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.successThreshold
+          value: 1
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.failureThreshold
+          value: 3
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.httpGet.path
+          value: /health/readiness
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.initialDelaySeconds
+          value: 0
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.periodSeconds
+          value: 10
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.timeoutSeconds
+          value: 1
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.successThreshold
+          value: 1
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.failureThreshold
+          value: 3
+      - equal:
+          path: spec.template.spec.containers[0].startupProbe.httpGet.path
+          value: /health/readiness
+      - equal:
+          path: spec.template.spec.containers[0].startupProbe.initialDelaySeconds
+          value: 0
+      - equal:
+          path: spec.template.spec.containers[0].startupProbe.periodSeconds
+          value: 10
+      - equal:
+          path: spec.template.spec.containers[0].startupProbe.timeoutSeconds
+          value: 1
+      - equal:
+          path: spec.template.spec.containers[0].startupProbe.successThreshold
+          value: 1
+      - equal:
+          path: spec.template.spec.containers[0].startupProbe.failureThreshold
+          value: 30
+      - equal:
+          path: spec.template.spec.containers[0].resources
+          value: {}

deploy/charts/litellm-helm/values.yaml

@@ -84,6 +84,31 @@ service:
 separateHealthApp: false
 separateHealthPort: 8081
 
+# Probe tuning for proxy container
+livenessProbe:
+  path: /health/liveliness
+  initialDelaySeconds: 0
+  periodSeconds: 10
+  timeoutSeconds: 1
+  successThreshold: 1
+  failureThreshold: 3
+
+readinessProbe:
+  path: /health/readiness
+  initialDelaySeconds: 0
+  periodSeconds: 10
+  timeoutSeconds: 1
+  successThreshold: 1
+  failureThreshold: 3
+
+startupProbe:
+  path: /health/readiness
+  initialDelaySeconds: 0
+  periodSeconds: 10
+  timeoutSeconds: 1
+  successThreshold: 1
+  failureThreshold: 30
+
 ingress:
   enabled: false
   className: "nginx"

dev_config.yaml

@@ -0,0 +1,13 @@
+model_list:
+  - model_name: fake-openai-endpoint
+    litellm_params:
+      model: openai/fake-model
+      api_key: fake-key
+      api_base: https://exampleopenaiendpoint-production.up.railway.app/
+
+general_settings:
+  master_key: sk-1234
+
+litellm_settings:
+  drop_params: True
+  telemetry: False

docker/Dockerfile.custom_ui

@@ -5,8 +5,21 @@ FROM ghcr.io/berriai/litellm:litellm_fwd_server_root_path-dev
 WORKDIR /app
 
 # Install Node.js and npm (adjust version as needed)
-RUN apt-get update && apt-get install -y nodejs npm && \
-    npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 && \
+RUN apt-get update && apt-get upgrade -y \
+        libxml2 \
+        libexpat1 \
+        openssl \
+        libssl3 \
+        git \
+        libkrb5-3 \
+        libglib2.0-0 \
+        wget \
+        libaom3 \
+        libxslt1.1 \
+        libgnutls30 \
+        libc6 && \
+    apt-get install -y nodejs npm && \
+    npm install -g npm@latest tar@7.5.10 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
     GLOBAL="$(npm root -g)" && \
     find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
@@ -17,7 +30,16 @@ RUN apt-get update && apt-get install -y nodejs npm && \
     find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
     done && \
-    npm cache clean --force
+    find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
+    done && \
+    find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
+    done && \
+    find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
+        sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null && \
+    npm cache clean --force && \
+    apt-get purge -y npm
 
 # Copy the UI source into the container
 COPY ./ui/litellm-dashboard /app/ui/litellm-dashboard

docker/Dockerfile.database

@@ -50,7 +50,7 @@ USER root
 
 # Install runtime dependencies
 RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \
-    npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 && \
+    npm install -g npm@latest tar@7.5.10 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
     GLOBAL="$(npm root -g)" && \
     find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
@@ -61,7 +61,16 @@ RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile
     find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
     done && \
-    npm cache clean --force
+    find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
+    done && \
+    find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
+    done && \
+    find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
+        sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null && \
+    npm cache clean --force && \
+    { apk del --no-cache npm 2>/dev/null || true; }
 
 WORKDIR /app
 # Copy the current directory contents into the container at /app
@@ -79,14 +88,21 @@ RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl
 # npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
 # Patch every copy of tar, glob, and brace-expansion inside that tree.
 RUN GLOBAL="$(npm root -g)" && \
-    find /usr/lib -path "*/nodejs_wheel/*/node_modules/tar" -type d | while read d; do \
+    [ -n "$GLOBAL" ] || { echo "ERROR: npm root -g returned empty; aborting"; exit 1; } && \
+    find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
     done && \
-    find /usr/lib -path "*/nodejs_wheel/*/node_modules/glob" -type d | while read d; do \
+    find /usr/lib -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
     done && \
-    find /usr/lib -path "*/nodejs_wheel/*/node_modules/@isaacs/brace-expansion" -type d | while read d; do \
+    find /usr/lib -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
+    done && \
+    find /usr/lib -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
+    done && \
+    find /usr/lib -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
     done
 
 # Install semantic_router and aurelio-sdk using script

docker/Dockerfile.dev

@@ -56,13 +56,26 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
 USER root
 
 # Install only runtime dependencies
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    libssl3 \
+RUN apt-get update && apt-get upgrade -y \
+        libxml2 \
+        libexpat1 \
+        openssl \
+        libssl3 \
+        git \
+        libkrb5-3 \
+        libglib2.0-0 \
+        wget \
+        libaom3 \
+        libxslt1.1 \
+        libgnutls30 \
+        libc6 \
+    && apt-get install -y --no-install-recommends \
+        libssl3 \
     libatomic1 \
     nodejs \
     npm \
     && rm -rf /var/lib/apt/lists/* \
-    && npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 \
+    && npm install -g npm@latest tar@7.5.10 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \
     && GLOBAL="$(npm root -g)" \
     && find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
           rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
@@ -73,7 +86,16 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     && find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
           rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
        done \
-    && npm cache clean --force
+    && find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
+          rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
+       done \
+    && find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
+          rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
+       done \
+    && find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
+        sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null \
+    && npm cache clean --force \
+    && apt-get purge -y npm
 
 WORKDIR /app
 
@@ -95,14 +117,21 @@ RUN pip install --no-cache-dir *.whl /wheels/* --no-index --find-links=/wheels/
 # npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
 # Patch every copy of tar, glob, and brace-expansion inside that tree.
 RUN GLOBAL="$(npm root -g)" && \
-    find /usr/lib -path "*/nodejs_wheel/*/node_modules/tar" -type d | while read d; do \
+    [ -n "$GLOBAL" ] || { echo "ERROR: npm root -g returned empty; aborting"; exit 1; } && \
+    find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
     done && \
-    find /usr/lib -path "*/nodejs_wheel/*/node_modules/glob" -type d | while read d; do \
+    find /usr/lib -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
     done && \
-    find /usr/lib -path "*/nodejs_wheel/*/node_modules/@isaacs/brace-expansion" -type d | while read d; do \
+    find /usr/lib -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
+    done && \
+    find /usr/lib -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
+    done && \
+    find /usr/lib -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
     done
 
 # Generate prisma client and set permissions

docker/Dockerfile.non_root

@@ -80,7 +80,7 @@ ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
     XDG_CACHE_HOME=/app/.cache \
     PATH="/usr/lib/python3.13/site-packages/nodejs/bin:${PATH}"
 
-RUN pip install --no-cache-dir prisma==0.11.0 nodejs-wheel-binaries==24.12.0 \
+RUN pip install --no-cache-dir prisma==0.11.0 nodejs-wheel-binaries==24.13.1 \
     && mkdir -p /app/.cache/npm
 
 RUN NPM_CONFIG_CACHE=/app/.cache/npm \
@@ -105,7 +105,8 @@ RUN for i in 1 2 3; do \
   && for i in 1 2 3; do \
     apk add --no-cache python3 py3-pip bash openssl tzdata nodejs npm supervisor && break || sleep 5; \
     done \
-  && npm install -g npm@latest tar@7.5.7 glob@11.1.0 @isaacs/brace-expansion@5.0.1 \
+  && apk upgrade --no-cache nodejs \
+  && npm install -g npm@latest tar@7.5.10 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \
   && GLOBAL="$(npm root -g)" \
   && find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
@@ -116,7 +117,16 @@ RUN for i in 1 2 3; do \
   && find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
      done \
-  && npm cache clean --force
+  && find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
+     done \
+  && find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
+     done \
+  && find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
+        sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null \
+  && npm cache clean --force \
+  && { apk del --no-cache npm 2>/dev/null || true; }
 
 # Copy artifacts from builder
 COPY --from=builder /app/requirements.txt /app/requirements.txt
@@ -162,14 +172,21 @@ RUN pip install --no-index --find-links=/wheels/ -r requirements.txt && \
 # npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
 # Patch every copy of tar, glob, and brace-expansion inside that tree.
 RUN GLOBAL="$(npm root -g)" && \
-    find /usr/lib -path "*/nodejs_wheel/*/node_modules/tar" -type d | while read d; do \
+    [ -n "$GLOBAL" ] || { echo "ERROR: npm root -g returned empty; aborting"; exit 1; } && \
+    find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
     done && \
-    find /usr/lib -path "*/nodejs_wheel/*/node_modules/glob" -type d | while read d; do \
+    find /usr/lib -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
     done && \
-    find /usr/lib -path "*/nodejs_wheel/*/node_modules/@isaacs/brace-expansion" -type d | while read d; do \
+    find /usr/lib -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
+    done && \
+    find /usr/lib -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
+    done && \
+    find /usr/lib -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
     done
 
 # Permissions, cleanup, and Prisma prep

docs/my-website/blog/anthropic_wildcard_model_access_incident/index.md

@@ -0,0 +1,147 @@
+---
+slug: anthropic-wildcard-model-access-incident
+title: "Incident Report: Wildcard Blocking New Models After Cost Map Reload"
+date: 2026-02-23T10:00:00
+authors:
+  - name: Sameer Kankute
+    title: SWE @ LiteLLM (LLM Translation)
+    url: https://www.linkedin.com/in/sameer-kankute/
+    image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
+  - name: Krrish Dholakia
+    title: "CEO, LiteLLM"
+    url: https://www.linkedin.com/in/krish-d/
+    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
+  - name: Ishaan Jaff
+    title: "CTO, LiteLLM"
+    url: https://www.linkedin.com/in/reffajnaahsi/
+    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+tags: [incident-report, proxy, auth, model-access]
+hide_table_of_contents: false
+---
+
+**Date:** Feb 23, 2026  
+**Duration:** ~3 hours  
+**Severity:** High (for users with provider wildcard access rules)  
+**Status:** Resolved
+
+## Summary
+
+When a new Anthropic model (e.g. `claude-sonnet-4-6`) was added to the LiteLLM model cost map and a cost map reload was triggered, requests to the new model were rejected with:
+
+```
+key not allowed to access model. This key can only access models=['anthropic/*']. Tried to access claude-sonnet-4-6.
+```
+
+The reload updated `litellm.model_cost` correctly but never re-ran `add_known_models()`, so `litellm.anthropic_models` (the in-memory set used by the wildcard resolver) remained stale. The new model was invisible to the `anthropic/*` wildcard even though the cost map knew about it.
+
+- **LLM calls:** All requests to newly-added Anthropic models were blocked with a 401.
+- **Existing models:** Unaffected — only models missing from the stale provider set were impacted.
+- **Other providers:** Same bug class existed for any provider wildcard (e.g. `openai/*`, `gemini/*`).
+
+{/* truncate */}
+
+---
+
+## Background
+
+LiteLLM supports provider-level wildcard access rules. When an admin configures a key or team with `models=['anthropic/*']`, any model whose provider resolves to `anthropic` should be allowed. The resolution happens in `_model_custom_llm_provider_matches_wildcard_pattern`:
+
+```mermaid
+flowchart TD
+    A["1. Request arrives for claude-sonnet-4-6"] --> B["2. Auth check: can this key call this model?
+    proxy/auth/auth_checks.py"]
+    B --> C["3. Key has models=['anthropic/*']
+    → wildcard match attempted"]
+    C --> D["4. get_llm_provider('claude-sonnet-4-6')
+    checks litellm.anthropic_models set"]
+    D -->|"model IN set"| E["5a. ✅ Provider = 'anthropic'
+    → 'anthropic/claude-sonnet-4-6' matches 'anthropic/*'"]
+    D -->|"model NOT IN set"| F["5b. ❌ Provider unknown
+    → exception raised → wildcard returns False"]
+    E --> G["6. Request allowed"]
+    F --> H["6. 401: key not allowed to access model"]
+
+    style E fill:#d4edda,stroke:#28a745
+    style F fill:#f8d7da,stroke:#dc3545
+    style H fill:#f8d7da,stroke:#dc3545
+    style D fill:#fff3cd,stroke:#ffc107
+```
+
+`litellm.anthropic_models` is a Python `set` populated at import time by `add_known_models()`. It is the source `get_llm_provider()` consults to map a bare model name like `claude-sonnet-4-6` to the provider string `"anthropic"`.
+
+---
+
+## Root Cause
+
+`add_known_models()` is called **once** at module import time. Both reload paths in `proxy_server.py` updated `litellm.model_cost` with the fresh map but never called `add_known_models()` again:
+
+```python
+# Before the fix — both reload paths looked like this:
+new_model_cost_map = get_model_cost_map(url=model_cost_map_url)
+litellm.model_cost = new_model_cost_map          # ✅ cost map updated
+_invalidate_model_cost_lowercase_map()           # ✅ cache cleared
+# ❌ add_known_models() never called
+#    → litellm.anthropic_models still has the old set
+#    → new model not in the set
+#    → get_llm_provider() raises for the new model
+#    → wildcard match returns False
+#    → 401 for every request to the new model
+```
+
+The gap existed in two places:
+1. `_check_and_reload_model_cost_map` — the periodic automatic reload (every 10 s)
+2. The `/reload/model_cost_map` admin endpoint — the manual reload
+
+**Timeline:**
+
+1. New model (`claude-sonnet-4-6`) added to `model_prices_and_context_window.json`
+2. Admin triggers cost map reload via UI → `litellm.model_cost` updated
+3. Users with `anthropic/*` wildcard keys attempt requests to `claude-sonnet-4-6`
+4. `get_llm_provider('claude-sonnet-4-6')` raises → wildcard returns False → 401
+5. Admin reloads cost map again — same result (root cause not addressed)
+6. ~3 hours of investigation → root cause identified → fix deployed
+
+---
+
+## The Fix
+
+After each reload, `add_known_models()` is called with the freshly fetched map passed explicitly. Passing the map directly (rather than relying on the module-level reference) removes any ambiguity about which dict is iterated:
+
+```python
+# After the fix — both reload paths now do:
+new_model_cost_map = get_model_cost_map(url=model_cost_map_url)
+litellm.model_cost = new_model_cost_map
+_invalidate_model_cost_lowercase_map()
+litellm.add_known_models(model_cost_map=new_model_cost_map)  # ✅ sets repopulated
+```
+
+`add_known_models()` was also updated to accept an optional explicit map so callers cannot accidentally iterate a stale module-level reference:
+
+```python
+# Before
+def add_known_models():
+    for key, value in model_cost.items():   # reads module global — ambiguous after reload
+        ...
+
+# After
+def add_known_models(model_cost_map: Optional[Dict] = None):
+    _map = model_cost_map if model_cost_map is not None else model_cost
+    for key, value in _map.items():         # always iterates the map you just fetched
+        ...
+```
+
+After the fix, the provider sets (`anthropic_models`, `open_ai_chat_completion_models`, etc.) are always consistent with `litellm.model_cost` immediately after every reload. New models become accessible via wildcard rules without any proxy restart.
+
+---
+
+## Remediation
+
+| # | Action | Status | Code |
+|---|---|---|---|
+| 1 | Call `add_known_models(model_cost_map=...)` in the periodic reload path | ✅ Done | [`proxy_server.py#L4393`](https://github.com/BerriAI/litellm/blob/main/litellm/proxy/proxy_server.py#L4393) |
+| 2 | Call `add_known_models(model_cost_map=...)` in the `/reload/model_cost_map` endpoint | ✅ Done | [`proxy_server.py#L11904`](https://github.com/BerriAI/litellm/blob/main/litellm/proxy/proxy_server.py#L11904) |
+| 3 | Update `add_known_models()` to accept an explicit map parameter | ✅ Done | [`__init__.py#L617`](https://github.com/BerriAI/litellm/blob/main/litellm/__init__.py#L617) |
+| 4 | Regression test: `add_known_models(model_cost_map=...)` populates provider sets | ✅ Done | [`test_auth_checks.py`](https://github.com/BerriAI/litellm/blob/main/tests/proxy_unit_tests/test_auth_checks.py) |
+| 5 | Regression test: `anthropic/*` wildcard grants/denies access correctly after reload | ✅ Done | [`test_auth_checks.py`](https://github.com/BerriAI/litellm/blob/main/tests/proxy_unit_tests/test_auth_checks.py) |
+
+---

docs/my-website/blog/gemin_3.1/index.md

@@ -37,7 +37,7 @@ LiteLLM now supports `gemini-3.1-pro-preview` and all the new API changes along
 docker run \
 -e STORE_MODEL_IN_DB=True \
 -p 4000:4000 \
-ghcr.io/berriai/litellm:main-v1.80.8-stable.1
+ghcr.io/berriai/litellm:main-v1.81.9-stable.gemini.3.1-pro
 ```
 
 </TabItem>
@@ -45,7 +45,7 @@ ghcr.io/berriai/litellm:main-v1.80.8-stable.1
 <TabItem value="pip" label="Pip">
 
 ``` showLineNumbers title="pip install litellm"
-pip install litellm==1.80.8.post1
+pip install litellm==v1.81.9-stable.gemini.3.1-pro
 ```
 
 </TabItem>

docs/my-website/blog/gemini_3_1_flash_lite/index.md

@@ -0,0 +1,175 @@
+---
+slug: gemini_3_1_flash_lite_preview
+title: "DAY 0 Support: Gemini 3.1 Flash Lite Preview on LiteLLM"
+date: 2026-03-03T08:00:00
+authors:
+  - name: Sameer Kankute
+    title: SWE @ LiteLLM (LLM Translation)
+    url: https://www.linkedin.com/in/sameer-kankute/
+    image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
+  - name: Krrish Dholakia
+    title: "CEO, LiteLLM"
+    url: https://www.linkedin.com/in/krish-d/
+    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
+  - name: Ishaan Jaff
+    title: "CTO, LiteLLM"
+    url: https://www.linkedin.com/in/reffajnaahsi/
+    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+description: "Guide to using Gemini 3.1 Flash Lite Preview on LiteLLM Proxy and SDK with day 0 support."
+tags: [gemini, day 0 support, llms, supernova]
+hide_table_of_contents: false
+---
+
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Gemini 3.1 Flash Lite Preview Day 0 Support 
+
+LiteLLM now supports `gemini-3.1-flash-lite-preview` with full day 0 support!
+
+:::note
+If you only want cost tracking, you need no change in your current Litellm version. But if you want the support for new features introduced along with it like thinking levels, you will need to use v1.80.8-stable.1 or above.
+:::
+
+## Deploy this version
+
+<Tabs>
+<TabItem value="docker" label="Docker">
+
+``` showLineNumbers title="docker run litellm"
+docker run \
+-e STORE_MODEL_IN_DB=True \
+-p 4000:4000 \
+ghcr.io/berriai/litellm:main-v1.80.8-stable.1
+```
+
+</TabItem>
+
+<TabItem value="pip" label="Pip">
+
+``` showLineNumbers title="pip install litellm"
+pip install litellm==v1.80.8-stable.1
+```
+
+</TabItem>
+</Tabs>
+
+## What's New
+
+Supports all four thinking levels:
+- **MINIMAL**: Ultra-fast responses with minimal reasoning
+- **LOW**: Simple instruction following
+- **MEDIUM**: Balanced reasoning for complex tasks
+- **HIGH**: Maximum reasoning depth (dynamic)
+
+---
+
+## Quick Start
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+**Basic Usage**
+
+```python
+from litellm import completion
+
+response = completion(
+    model="gemini/gemini-3.1-flash-lite-preview",
+    messages=[{"role": "user", "content": "Extract key entities from this text: ..."}],
+)
+
+print(response.choices[0].message.content)
+```
+
+**With Thinking Levels**
+
+```python
+from litellm import completion
+
+# Use MEDIUM thinking for complex reasoning tasks
+response = completion(
+    model="gemini/gemini-3.1-flash-lite-preview",
+    messages=[{"role": "user", "content": "Analyze this dataset and identify patterns"}],
+    reasoning_effort="medium",  # low, medium , high
+)
+
+print(response.choices[0].message.content)
+```
+
+</TabItem>
+
+<TabItem value="proxy" label="PROXY">
+
+**1. Setup config.yaml**
+
+```yaml
+model_list:
+  - model_name: gemini-3.1-flash-lite
+    litellm_params:
+      model: gemini/gemini-3.1-flash-lite-preview
+      api_key: os.environ/GEMINI_API_KEY
+  
+  # Or use Vertex AI
+  - model_name: vertex-gemini-3.1-flash-lite
+    litellm_params:
+      model: vertex_ai/gemini-3.1-flash-lite-preview
+      vertex_project: your-project-id
+      vertex_location: us-central1
+```
+
+**2. Start proxy**
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+**3. Make requests**
+
+```bash
+curl -X POST http://localhost:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer <YOUR-LITELLM-KEY>" \
+  -d '{
+    "model": "gemini-3.1-flash-lite",
+    "messages": [{"role": "user", "content": "Extract structured data from this text"}],
+    "reasoning_effort": "low"
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+---
+
+## Supported Endpoints
+
+LiteLLM provides **full end-to-end support** for Gemini 3.1 Flash Lite Preview on:
+
+- ✅ `/v1/chat/completions` - OpenAI-compatible chat completions endpoint
+- ✅ `/v1/responses` - OpenAI Responses API endpoint (streaming and non-streaming)
+- ✅ [`/v1/messages`](../../docs/anthropic_unified) - Anthropic-compatible messages endpoint
+- ✅ `/v1/generateContent` – [Google Gemini API](../../docs/generateContent.md) compatible endpoint 
+
+All endpoints support:
+- Streaming and non-streaming responses
+- Function calling with thought signatures
+- Multi-turn conversations
+- All Gemini 3-specific features (thinking levels, thought signatures)
+- Full multimodal support (text, image, audio, video)
+
+---
+
+## `reasoning_effort` Mapping for Gemini 3.1
+
+LiteLLM automatically maps OpenAI's `reasoning_effort` parameter to Gemini's `thinkingLevel`:
+
+| reasoning_effort | thinking_level | Use Case |
+|------------------|----------------|----------|
+| `minimal` | `minimal` | Ultra-fast responses, simple queries |
+| `low` | `low` | Basic instruction following |
+| `medium` | `medium` | Balanced reasoning for moderate complexity |
+| `high` | `high` | Maximum reasoning depth, complex problems |
+| `disable` | `minimal` | Disable extended reasoning |
+| `none` | `minimal` | No extended reasoning |

docs/my-website/blog/gpt_5_3_codex/index.md

@@ -0,0 +1,145 @@
+---
+slug: gpt_5_3_codex
+title: "Day 0 Support: GPT-5.3-Codex"
+date: 2026-02-24T10:00:00
+authors:
+  - name: Sameer Kankute
+    title: SWE @ LiteLLM (LLM Translation)
+    url: https://www.linkedin.com/in/sameer-kankute/
+    image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
+  - name: Krrish Dholakia
+    title: "CEO, LiteLLM"
+    url: https://www.linkedin.com/in/krish-d/
+    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
+  - name: Ishaan Jaff
+    title: "CTO, LiteLLM"
+    url: https://www.linkedin.com/in/reffajnaahsi/
+    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+description: "Day 0 support for GPT-5.3-Codex on LiteLLM, including phase parameter handling for Responses API."
+tags: [openai, gpt-5.3-codex, codex, day 0 support]
+hide_table_of_contents: false
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+LiteLLM now supports GPT-5.3-Codex on Day 0, including support for the new assistant `phase` metadata on Responses API output items.
+
+## Why `phase` matters for GPT-5.3-Codex
+
+`phase` appears on assistant output items and helps distinguish preamble/commentary turns from final closeout responses.
+
+Reference: [Phase parameter docs](https://developers.openai.com/api/reference/overview)
+
+Supported values:
+- `null`
+- `"commentary"`
+- `"final_answer"`
+
+Important:
+- Persist assistant output items with `phase` exactly as returned.
+- Send those assistant items back on the next turn.
+- Do **not** add `phase` to user messages.
+
+## Docker Image
+
+```bash
+docker pull ghcr.io/berriai/litellm:v1.81.12-stable.gpt-5.3
+```
+
+## Usage 
+
+<Tabs>
+<TabItem value="proxy" label="LiteLLM Proxy">
+
+**1. Setup config.yaml**
+
+```yaml
+model_list:
+  - model_name: gpt-5.3-codex
+    litellm_params:
+      model: openai/gpt-5.3-codex
+```
+
+**2. Start the proxy**
+
+```bash
+docker run -d \
+  -p 4000:4000 \
+  -e ANTHROPIC_API_KEY=$OPENAI_API_KEY \
+  -v $(pwd)/config.yaml:/app/config.yaml \
+  ghcr.io/berriai/litellm:v1.81.12-stable.gpt-5.3 \
+  --config /app/config.yaml
+```
+
+
+**3. Test it**
+
+```bash
+curl -X POST "http://0.0.0.0:4000/v1/responses" \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer $LITELLM_KEY" \
+  -d '{
+    "model": "gpt-5.3-codex",
+    "input": "Write a Python script that checks if a number is prime."
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+## Python Example: Persist `phase` with OpenAI Client + LiteLLM Base URL
+
+```python
+from openai import OpenAI
+
+client = OpenAI(
+    base_url="http://0.0.0.0:4000/v1",  # LiteLLM Proxy
+    api_key="your-litellm-api-key",
+)
+
+items = []  # Persist this per conversation/thread
+
+
+def _item_get(item, key, default=None):
+    if isinstance(item, dict):
+        return item.get(key, default)
+    return getattr(item, key, default)
+
+
+def run_turn(user_text: str):
+    global items
+
+    # User message: no phase field
+    items.append(
+        {
+            "type": "message",
+            "role": "user",
+            "content": [{"type": "input_text", "text": user_text}],
+        }
+    )
+
+    resp = client.responses.create(
+        model="gpt-5.3-codex",
+        input=items,
+    )
+
+    # Persist assistant output items verbatim, including phase
+    for out_item in (resp.output or []):
+        items.append(out_item)
+
+    # Optional: inspect latest phase for UI/telemetry routing
+    latest_phase = None
+    for out_item in reversed(resp.output or []):
+        if _item_get(out_item, "type") == "output_item.done" and _item_get(out_item, "phase") is not None:
+            latest_phase = _item_get(out_item, "phase")
+            break
+
+    return resp, latest_phase
+```
+
+## Notes
+
+- Use `/v1/responses` for GPT Codex models.
+- Preserve full assistant output history for best multi-turn behavior.
+- If `phase` metadata is dropped during history reconstruction, output quality can degrade on long-running tasks.

docs/my-website/blog/gpt_5_4/index.md

@@ -0,0 +1,97 @@
+---
+slug: gpt_5_4
+title: "Day 0 Support: GPT-5.4"
+date: 2026-03-05T10:00:00
+authors:
+  - name: Sameer Kankute
+    title: SWE @ LiteLLM (LLM Translation)
+    url: https://www.linkedin.com/in/sameer-kankute/
+    image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
+  - name: Krrish Dholakia
+    title: "CEO, LiteLLM"
+    url: https://www.linkedin.com/in/krish-d/
+    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
+  - name: Ishaan Jaff
+    title: "CTO, LiteLLM"
+    url: https://www.linkedin.com/in/reffajnaahsi/
+    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+description: "GPT-5.4 model support in LiteLLM"
+tags: [openai, gpt-5.4, completion]
+hide_table_of_contents: false
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+LiteLLM now supports fully GPT-5.4!
+
+## Docker Image
+
+```bash
+docker pull ghcr.io/berriai/litellm:v1.81.14-stable.gpt-5.4_patch
+```
+
+## Usage
+
+<Tabs>
+<TabItem value="proxy" label="LiteLLM Proxy">
+
+**1. Setup config.yaml**
+
+```yaml
+model_list:
+  - model_name: gpt-5.4
+    litellm_params:
+      model: openai/gpt-5.4
+      api_key: os.environ/OPENAI_API_KEY
+```
+
+**2. Start the proxy**
+
+```bash
+docker run -d \
+  -p 4000:4000 \
+  -e OPENAI_API_KEY=$OPENAI_API_KEY \
+  -v $(pwd)/config.yaml:/app/config.yaml \
+  ghcr.io/berriai/litellm:v1.81.14-stable.gpt-5.4_patch \
+  --config /app/config.yaml
+```
+
+**3. Test it**
+
+```bash
+curl -X POST "http://0.0.0.0:4000/chat/completions" \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer $LITELLM_KEY" \
+  -d '{
+    "model": "gpt-5.4",
+    "messages": [
+      {"role": "user", "content": "Write a Python function to check if a number is prime."}
+    ]
+  }'
+```
+
+</TabItem>
+<TabItem value="sdk" label="LiteLLM SDK">
+
+```python
+from litellm import completion
+
+response = completion(
+    model="openai/gpt-5.4",
+    messages=[
+        {"role": "user", "content": "Write a Python function to check if a number is prime."}
+    ],
+)
+
+print(response.choices[0].message.content)
+```
+
+</TabItem>
+</Tabs>
+
+## Notes
+
+- Restart your container to get the cost tracking for this model.
+- Use `/responses` for better model performance.
+- GPT-5.4 supports reasoning, function calling, vision, and tool-use — see the [OpenAI provider docs](../../docs/providers/openai) for advanced usage.

docs/my-website/blog/httpx_cache_eviction_incident/index.md

@@ -0,0 +1,132 @@
+---
+slug: httpx-cache-eviction-incident
+title: "Incident Report: Cache Eviction Closes In-Use httpx Clients"
+date: 2026-02-27T10:00:00
+authors:
+  - name: Ryan Crabbe
+    title: Performance Engineer, LiteLLM
+    url: https://www.linkedin.com/in/ryan-crabbe-0b9687214
+  - name: Ishaan Jaff
+    title: "CTO, LiteLLM"
+    url: https://www.linkedin.com/in/reffajnaahsi/
+    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+  - name: Krrish Dholakia
+    title: "CEO, LiteLLM"
+    url: https://www.linkedin.com/in/krish-d/
+    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
+tags: [incident-report, caching, stability]
+hide_table_of_contents: false
+---
+
+**Date:** February 27, 2026
+**Duration:** ~6 days (Feb 21 merge -> Feb 27 fix)
+**Severity:** High
+**Status:** Resolved
+
+> **Note:** This fix is available starting from LiteLLM `v1.81.14.rc.2` or higher.
+
+## Summary
+
+A change to improve Redis connection pool cleanup introduced a regression that closed **httpx clients** that were still actively being used by the proxy. The `LLMClientCache` (an in-memory TTL cache) stores both Redis clients *and* httpx clients under the same eviction policy. When a cache entry expired or was evicted, the new cleanup code called `aclose()`/`close()` on the evicted value which worked correctly for Redis clients, but destroyed httpx clients that other parts of the system still held references to and were actively using for LLM API calls.
+
+**Impact:** Any proxy instance that hit the cache TTL (default 10 minutes) or capacity limit (200 entries) would have its httpx clients closed out from under it, causing requests to LLM providers to fail with connection errors.
+
+---
+
+## Background
+
+`LLMClientCache` extends `InMemoryCache` and is used to cache SDK clients (OpenAI, Anthropic, etc.) to avoid re-creating them on every request. These clients are keyed by configuration + event loop ID. The cache has:
+
+- **Max size:** 200 entries
+- **Default TTL:** 10 minutes
+
+When the cache is full or entries expire, `InMemoryCache.evict_cache()` calls `_remove_key()` to drop entries.
+
+The cached values are a mix of:
+- **Redis/async Redis clients** — owned exclusively by the cache, safe to close on eviction
+- **httpx-backed SDK clients** (OpenAI, Anthropic, etc.) — shared references, still in use by router/model instances
+
+---
+
+## Root Cause
+
+[PR #21717](https://github.com/BerriAI/litellm/pull/21717) overrode `_remove_key()` in `LLMClientCache` to close async clients on eviction:
+
+<details>
+<summary>Problematic code added in PR #21717</summary>
+
+```python
+class LLMClientCache(InMemoryCache):
+    def _remove_key(self, key: str) -> None:
+        value = self.cache_dict.get(key)
+        super()._remove_key(key)
+        if value is not None:
+            close_fn = getattr(value, "aclose", None) or getattr(value, "close", None)
+            if close_fn and asyncio.iscoroutinefunction(close_fn):
+                try:
+                    asyncio.get_running_loop().create_task(close_fn())
+                except RuntimeError:
+                    pass
+            elif close_fn and callable(close_fn):
+                try:
+                    close_fn()
+                except Exception:
+                    pass
+```
+
+</details>
+
+The intent was correct for Redis clients — prevent connection pool leaks when cached Redis clients expire. But `LLMClientCache` also stores httpx-backed SDK clients (e.g., `AsyncOpenAI`, `AsyncAnthropic`). These clients:
+
+1. Have an `aclose()` method (inherited from httpx)
+2. Are still held by references elsewhere in the codebase (router, model instances)
+3. Were being closed without any check on whether they were still in use
+
+So when the cache evicted an entry, it would call `aclose()` on an httpx client that was still being used for active LLM requests → closed transport → connection errors.
+
+---
+
+## The Fix
+
+[PR #22247](https://github.com/BerriAI/litellm/pull/22247) removed the `_remove_key` override entirely:
+
+<details>
+<summary>The fix (PR #22247)</summary>
+
+```diff
+ class LLMClientCache(InMemoryCache):
+-    def _remove_key(self, key: str) -> None:
+-        """Close async clients before evicting them to prevent connection pool leaks."""
+-        value = self.cache_dict.get(key)
+-        super()._remove_key(key)
+-        if value is not None:
+-            close_fn = getattr(value, "aclose", None) or getattr(
+-                value, "close", None
+-            )
+-            ...
+-
+     def update_cache_key_with_event_loop(self, key):
+```
+
+</details>
+
+The eviction now simply drops the reference and lets Python's GC handle cleanup, which is safe because:
+- httpx clients that are still referenced elsewhere stay alive
+- Unreferenced clients get cleaned up by GC naturally
+
+The other improvements from PR #21717 were kept:
+- **`max_connections` respected for URL-based Redis configs**, previously silently dropped
+- **`disconnect()` now closes both sync and async Redis clients**, sync client was previously leaked
+- **Connection pool passthrough**, when a pool is provided with a URL config, it's used directly instead of creating a duplicate
+
+---
+
+## Remediation
+
+| Action | Status | Code |
+|--------|--------|------|
+| Remove `_remove_key` override that closes shared clients on eviction | ✅ Done | [PR #22247](https://github.com/BerriAI/litellm/pull/22247) |
+| Add e2e test: evicted client still usable (capacity) | ✅ Done | [PR #22313](https://github.com/BerriAI/litellm/pull/22313) |
+| Add e2e test: expired client still usable (TTL) | ✅ Done | [PR #22313](https://github.com/BerriAI/litellm/pull/22313) |
+
+The e2e tests go through `get_async_httpx_client()` the same code path the proxy uses in production and assert the client is still functional after eviction. These run in CI on every PR against `main`. If anyone modifies `LLMClientCache` eviction behavior, overrides `_remove_key`, or adds any form of client cleanup on eviction, these tests will fail regardless of the implementation approach.

docs/my-website/blog/responses_api_encrypted_content_incident/index.md

@@ -0,0 +1,321 @@
+---
+slug: responses-api-encrypted-content-incident
+title: "Incident Report: Encrypted Content Failures in Multi-Region Responses API Load Balancing"
+date: 2026-02-24T10:00:00
+authors:
+  - name: Sameer Kankute
+    title: SWE @ LiteLLM (LLM Translation)
+    url: https://www.linkedin.com/in/sameer-kankute/
+    image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
+  - name: Krrish Dholakia
+    title: "CEO, LiteLLM"
+    url: https://www.linkedin.com/in/krish-d/
+    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
+  - name: Ishaan Jaff
+    title: "CTO, LiteLLM"
+    url: https://www.linkedin.com/in/reffajnaahsi/
+    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+tags: [incident-report, proxy, responses-api, load-balancing]
+hide_table_of_contents: false
+---
+
+**Date:** Feb 24, 2026  
+**Duration:** Ongoing (until fix deployed)  
+**Severity:** High (for users load balancing Responses API across different API keys)  
+**Status:** Resolved
+
+## Summary
+
+When load balancing OpenAI's Responses API across deployments with **different API keys** (e.g., different Azure regions or OpenAI organizations), follow-up requests containing encrypted content items (like `rs_...` reasoning items) would fail with:
+
+```json
+{
+  "error": {
+    "message": "The encrypted content for item rs_0d09d6e56879e76500699d6feee41c8197bd268aae76141f87 could not be verified. Reason: Encrypted content organization_id did not match the target organization.",
+    "type": "invalid_request_error",
+    "code": "invalid_encrypted_content"
+  }
+}
+```
+
+Encrypted content items are cryptographically tied to the API key's organization that created them. When the router load balanced a follow-up request to a deployment with a different API key, decryption failed.
+
+- **Responses API calls with encrypted content:** Complete failure when routed to wrong deployment
+- **Initial requests:** Unaffected — only follow-up requests containing encrypted items failed
+- **Other API endpoints:** No impact — chat completions, embeddings, etc. functioned normally
+
+{/* truncate */}
+
+---
+
+## Background
+
+OpenAI's Responses API can return encrypted "reasoning items" (with IDs like `rs_...`) that contain intermediate reasoning steps. These items are encrypted with the organization's key and can only be decrypted by the same organization's API key.
+
+When load balancing across deployments with different API keys, the existing affinity mechanisms were insufficient:
+
+- **`responses_api_deployment_check`**: Requires `previous_response_id` which some clients (like Codex) don't provide
+- **`deployment_affinity`**: Too broad — pins *all* requests from a user to one deployment, reducing effective quota by the number of users
+- **`session_affinity`**: Requires explicit session IDs and still reduces quota
+
+```mermaid
+flowchart TD
+    A["1. Initial request to Responses API
+    router.aresponses()"] --> B["2. Router load balances to Deployment A
+    (API Key 1, Azure East US)"]
+    B --> C["3. Response contains encrypted item
+    rs_abc123 (encrypted with Org 1 key)"]
+    C --> D["4. Follow-up request includes rs_abc123 in input"]
+    D --> E["5. Router load balances to Deployment B
+    (API Key 2, Azure West Europe)"]
+    E -->|"Different API key"| F["6. ❌ Deployment B cannot decrypt rs_abc123
+    Error: invalid_encrypted_content"]
+    
+    D -.->|"With encrypted_content_affinity"| G["5b. Router detects rs_abc123 was created by Deployment A"]
+    G --> H["6b. ✅ Routes to Deployment A (bypasses rate limits)
+    Request succeeds"]
+
+    style F fill:#f8d7da,stroke:#dc3545
+    style H fill:#d4edda,stroke:#28a745
+    style E fill:#fff3cd,stroke:#ffc107
+    style G fill:#d4edda,stroke:#28a745
+```
+
+---
+
+## Root Cause
+
+LiteLLM's router had no mechanism to track which deployment created specific encrypted content items and route follow-up requests accordingly. The router treated all deployments as interchangeable, leading to decryption failures when encrypted content crossed organizational boundaries.
+
+**The Problem Flow:**
+
+1. User calls `router.aresponses()` with model `gpt-5.1-codex`
+2. Router load balances to Deployment A (Azure East US, API Key 1)
+3. Response contains encrypted reasoning item `rs_abc123` (encrypted with Org 1's key)
+4. User makes follow-up request with `rs_abc123` in the input
+5. Router load balances to Deployment B (Azure West Europe, API Key 2)
+6. Deployment B tries to decrypt `rs_abc123` with Org 2's key → **fails**
+
+**Why Existing Solutions Didn't Work:**
+
+- **`previous_response_id`**: Not provided by all clients (e.g., Codex)
+- **`deployment_affinity`**: Pins *all* user requests to one deployment → reduces quota to 1/N where N = number of deployments
+- **`session_affinity`**: Requires explicit session management and still reduces quota
+
+**Timeline:**
+
+1. Users configured multi-region Responses API load balancing with different API keys
+2. Initial requests succeeded, but follow-up requests with encrypted content failed intermittently
+3. Error rate correlated with number of deployments (more deployments = higher chance of routing to wrong one)
+4. Investigation revealed encrypted content was organization-bound
+5. Existing affinity mechanisms deemed unsuitable (quota reduction, missing `previous_response_id`)
+6. New solution designed and implemented: `encrypted_content_affinity`
+
+---
+
+## The Fix
+
+Implemented a new `encrypted_content_affinity` pre-call check that intelligently tracks encrypted content and routes follow-up requests **only when necessary**.
+
+### Implementation
+
+**1. Encoding `model_id` into output items** ([`responses/utils.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/responses/utils.py))
+
+The same approach used for `previous_response_id` affinity — no cache needed. When a response contains output items with `encrypted_content`, LiteLLM encodes the originating deployment's `model_id` in **two places** for redundancy:
+
+1. **Into the item ID** (if present): `rs_abc123` → `encitem_{base64("litellm:model_id:{model_id};item_id:rs_abc123")}`
+2. **Into the encrypted_content itself**: Wraps the content with `litellm_enc:{base64("model_id:{model_id}")};{original_encrypted_content}`
+
+```python
+# Encoding item IDs (when present)
+def _build_encrypted_item_id(model_id: str, item_id: str) -> str:
+    assembled = f"litellm:model_id:{model_id};item_id:{item_id}"
+    encoded = base64.b64encode(assembled.encode("utf-8")).decode("utf-8")
+    return f"encitem_{encoded}"
+
+# Wrapping encrypted_content (always, for redundancy)
+def _wrap_encrypted_content_with_model_id(encrypted_content: str, model_id: str) -> str:
+    metadata = f"model_id:{model_id}"
+    encoded_metadata = base64.b64encode(metadata.encode("utf-8")).decode("utf-8")
+    return f"litellm_enc:{encoded_metadata};{encrypted_content}"
+```
+
+**Why wrap encrypted_content directly?** Some clients (like Codex) don't consistently send item IDs in follow-up requests, but they always send the `encrypted_content` itself. By embedding `model_id` into the content, affinity works even when IDs are missing.
+
+**Streaming responses:** The wrapping logic is applied to both:
+- Final response objects (non-streaming)
+- Individual streaming events (`response.output_item.added`, `response.output_item.done`)
+
+This ensures clients receiving streaming responses get wrapped content they can send back.
+
+Before forwarding to the upstream provider, LiteLLM restores the original item IDs and unwraps encrypted_content so the provider never sees the encoded form:
+
+```python
+# In responses/main.py — before calling the handler
+input = ResponsesAPIRequestUtils._restore_encrypted_content_item_ids_in_input(input)
+```
+
+**2. `EncryptedContentAffinityCheck` — routing only** ([`encrypted_content_affinity_check.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py))
+
+No `async_log_success_event` or cache lookups — the `model_id` is decoded directly from the item ID or encrypted_content:
+
+```python
+class EncryptedContentAffinityCheck(CustomLogger):
+    async def async_filter_deployments(self, model, healthy_deployments, ...):
+        """Extract model_id from input items (ID or encrypted_content) and pin to that deployment."""
+        for item in request_kwargs.get("input", []):
+            # Try to extract model_id from two sources:
+            model_id = self._extract_model_id_from_input(item)
+            
+            if model_id:
+                deployment = self._find_deployment_by_model_id(
+                    healthy_deployments, model_id
+                )
+                if deployment:
+                    request_kwargs["_encrypted_content_affinity_pinned"] = True
+                    return [deployment]
+        return healthy_deployments
+    
+    def _extract_model_id_from_input(self, item: dict) -> Optional[str]:
+        """Extract model_id from either encoded ID or wrapped encrypted_content."""
+        # 1. Try decoding from item ID (if present)
+        item_id = item.get("id", "")
+        if item_id:
+            decoded = ResponsesAPIRequestUtils._decode_encrypted_item_id(item_id)
+            if decoded:
+                return decoded["model_id"]
+        
+        # 2. Try unwrapping from encrypted_content (fallback for clients that omit IDs)
+        encrypted_content = item.get("encrypted_content", "")
+        if encrypted_content and encrypted_content.startswith("litellm_enc:"):
+            model_id, _ = ResponsesAPIRequestUtils._unwrap_encrypted_content_with_model_id(
+                encrypted_content
+            )
+            return model_id
+        
+        return None
+```
+
+**3. Rate Limit Bypass** ([`router.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/router.py))
+
+When encrypted content requires a specific deployment, RPM/TPM limits are bypassed (the request would fail on any other deployment anyway):
+
+```python
+# In async_get_available_deployment, after filtering healthy deployments:
+if (
+    request_kwargs.get("_encrypted_content_affinity_pinned")
+    and len(healthy_deployments) == 1
+):
+    return healthy_deployments[0]  # Bypass routing strategy (RPM/TPM checks)
+```
+
+**3. Configuration**
+
+```yaml
+router_settings:
+  routing_strategy: usage-based-routing-v2
+  enable_pre_call_checks: true
+  optional_pre_call_checks:
+    - encrypted_content_affinity
+  deployment_affinity_ttl_seconds: 86400  # 24 hours
+```
+
+### Key Benefits
+
+✅ **No quota reduction**: Only pins requests containing encrypted items  
+✅ **Bypasses rate limits**: When encrypted content requires a specific deployment, RPM/TPM limits don't block it  
+✅ **No `previous_response_id` required**: Works by encoding `model_id` directly into the item ID  
+✅ **No cache required**: `model_id` is decoded on-the-fly from the item ID — no Redis, no TTL  
+✅ **Globally safe**: Can be enabled for all models; non-Responses-API calls are unaffected  
+✅ **Surgical precision**: Normal requests continue to load balance freely
+
+---
+
+## Remediation
+
+| # | Action | Status | Code |
+|---|---|---|---|
+| 1 | Encode `model_id` into encrypted-content item IDs on response | ✅ Done | [`responses/utils.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/responses/utils.py) |
+| 2 | Restore original item IDs before forwarding to upstream provider | ✅ Done | [`responses/main.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/responses/main.py) |
+| 3 | `EncryptedContentAffinityCheck`: decode item IDs to route (no cache) | ✅ Done | [`encrypted_content_affinity_check.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py) |
+| 4 | Add `encrypted_content_affinity` to `OptionalPreCallChecks` type | ✅ Done | [`types/router.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/types/router.py) |
+| 5 | Implement rate limit bypass for affinity-pinned requests | ✅ Done | [`router.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/router.py) |
+| 6 | Unit tests: encoding/decoding utilities, routing, RPM bypass | ✅ Done | [`test_encrypted_content_affinity_check.py`](https://github.com/BerriAI/litellm/blob/main/litellm/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py) |
+| 7 | Documentation: Responses API guide, load balancing guide, config reference | ✅ Done | [Docs](https://docs.litellm.ai/docs/response_api#encrypted-content-affinity-multi-region-load-balancing) |
+| 8 | **[Mar 3]** Fix streaming events to wrap encrypted_content | ✅ Done | [`responses/streaming_iterator.py`](https://github.com/BerriAI/litellm/blob/main/litellm/litellm/responses/streaming_iterator.py) |
+
+---
+
+## Follow-up Fix: Streaming Responses (Mar 3, 2026)
+
+### The Issue
+
+After the initial fix was deployed, users reported that the `invalid_encrypted_content` error **still occurred** when using streaming responses with clients like Codex. Investigation revealed:
+
+- ✅ Non-streaming responses: `encrypted_content` was correctly wrapped with `litellm_enc:` prefix
+- ❌ Streaming responses: Individual `response.output_item.added` and `response.output_item.done` events contained **raw, unwrapped** `encrypted_content`
+
+Since Codex and other clients consume responses as streams, they received unwrapped content in these events and sent it back in follow-up requests, causing the affinity check to fail.
+
+### The Root Cause
+
+The `_update_encrypted_content_item_ids_in_response` function only modified the **final** response object, which is used for non-streaming responses. For streaming responses, individual chunks are processed by `ResponsesAPIStreamingIterator._process_chunk`, which was **not** applying the wrapping logic to streaming events.
+
+### The Fix
+
+Modified `litellm/litellm/responses/streaming_iterator.py` to wrap `encrypted_content` in streaming events:
+
+```python
+# In ResponsesAPIStreamingIterator._process_chunk
+if (
+    self.litellm_metadata
+    and self.litellm_metadata.get("encrypted_content_affinity_enabled")
+):
+    event_type = getattr(openai_responses_api_chunk, "type", None)
+    if event_type in (
+        ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED,
+        ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
+    ):
+        item = getattr(openai_responses_api_chunk, "item", None)
+        if item:
+            encrypted_content = getattr(item, "encrypted_content", None)
+            if encrypted_content and isinstance(encrypted_content, str):
+                model_id = (
+                    self.litellm_metadata.get("model_info", {}).get("id")
+                    if self.litellm_metadata
+                    else None
+                )
+                if model_id:
+                    wrapped_content = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id(
+                        encrypted_content, model_id
+                    )
+                    setattr(item, "encrypted_content", wrapped_content)
+```
+
+This ensures that **all** `encrypted_content` sent to clients (streaming or non-streaming) is wrapped with `model_id` metadata, enabling consistent affinity routing.
+
+---
+
+## Migration Guide
+
+### Before (Using `deployment_affinity`)
+
+```yaml
+router_settings:
+  optional_pre_call_checks:
+    - deployment_affinity  # ❌ Reduces quota by number of users
+```
+
+**Problem:** All requests from a user pin to one deployment, reducing effective quota to 1/N.
+
+### After (Using `encrypted_content_affinity`)
+
+```yaml
+router_settings:
+  optional_pre_call_checks:
+    - encrypted_content_affinity  # ✅ Only pins requests with encrypted content
+```
+
+**Benefit:** Normal requests load balance freely, only encrypted content requests pin when necessary.
+
+---

docs/my-website/blog/server_root_path/index.md

@@ -0,0 +1,154 @@
+---
+slug: server-root-path-incident
+title: "Incident Report: SERVER_ROOT_PATH regression broke UI routing"
+date: 2026-02-21T10:00:00
+authors:
+  - name: Yuneng Jiang
+    title: SWE @ LiteLLM (Full Stack)
+    url: https://www.linkedin.com/in/yunengjiang/
+  - name: Ishaan Jaff
+    title: "CTO, LiteLLM"
+    url: https://www.linkedin.com/in/reffajnaahsi/
+    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+  - name: Krrish Dholakia
+    title: "CEO, LiteLLM"
+    url: https://www.linkedin.com/in/krish-d/
+    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
+tags: [incident-report, ui, stability]
+hide_table_of_contents: false
+---
+
+**Date:** January 22, 2026
+**Duration:** ~4 days (until fix merged January 26, 2026)
+**Severity:** High
+**Status:** Resolved
+
+> **Note:** This fix is available starting from LiteLLM `v1.81.3.rc.6` or higher.
+
+## Summary
+
+A PR ([`#19467`](https://github.com/BerriAI/litellm/pull/19467)) accidentally removed the `root_path=server_root_path` parameter from the FastAPI app initialization in `proxy_server.py`. This caused the proxy to ignore the `SERVER_ROOT_PATH` environment variable when serving the UI. Users who deploy LiteLLM behind a reverse proxy with a path prefix (e.g., `/api/v1` or `/llmproxy`) found that all UI pages returned 404 Not Found.
+
+- **LLM API calls:** No impact. API routing was unaffected.
+- **UI pages:** All UI pages returned 404 for deployments using `SERVER_ROOT_PATH`.
+- **Swagger/OpenAPI docs:** Broken when accessed through the configured root path.
+
+{/* truncate */}
+
+---
+
+## Background
+
+Many LiteLLM deployments run behind a reverse proxy (e.g., Nginx, Traefik, AWS ALB) that routes traffic to LiteLLM under a path prefix. FastAPI's `root_path` parameter tells the application about this prefix so it can correctly serve static files, generate URLs, and handle routing.
+
+```mermaid
+sequenceDiagram
+    participant User as User Browser
+    participant RP as Reverse Proxy
+    participant LP as LiteLLM Proxy
+
+    User->>RP: GET /llmproxy/ui/
+    RP->>LP: GET /ui/ (X-Forwarded-Prefix: /llmproxy)
+
+    Note over LP: Before regression:<br/>FastAPI root_path="/llmproxy"<br/>→ Serves UI correctly
+
+    Note over LP: After regression:<br/>FastAPI root_path=""<br/>→ UI assets resolve to wrong paths<br/>→ 404 Not Found
+```
+
+The `root_path` parameter was present in `proxy_server.py` since early versions of LiteLLM. It was removed as a side effect of PR [#19467](https://github.com/BerriAI/litellm/pull/19467), which was intended to fix a different UI 404 issue.
+
+---
+
+## Root cause
+
+PR [#19467](https://github.com/BerriAI/litellm/pull/19467) (`73d49f8`) removed the `root_path=server_root_path` line from the `FastAPI()` constructor in `proxy_server.py`:
+
+```diff
+ app = FastAPI(
+     docs_url=_get_docs_url(),
+     redoc_url=_get_redoc_url(),
+     title=_title,
+     description=_description,
+     version=version,
+-    root_path=server_root_path,
+     lifespan=proxy_startup_event,
+ )
+```
+
+Without `root_path`, FastAPI treated all requests as if the application was mounted at `/`, causing path mismatches for any deployment using `SERVER_ROOT_PATH`.
+
+The regression went undetected because:
+
+1. **No automated test** verified that `root_path` was set on the FastAPI app.
+2. **No manual test procedure** existed for `SERVER_ROOT_PATH` functionality.
+3. **Default deployments** (without `SERVER_ROOT_PATH`) were unaffected, so most CI tests passed.
+
+---
+
+## Remediation
+
+| #   | Action                                                                                            | Status  | Code                                                                                                                       |
+| --- | ------------------------------------------------------------------------------------------------- | ------- | -------------------------------------------------------------------------------------------------------------------------- |
+| 1   | Restore `root_path=server_root_path` in FastAPI app initialization                                | ✅ Done | [`#19790`](https://github.com/BerriAI/litellm/pull/19790) (`5426b3c`)                                                      |
+| 2   | Add unit tests for `get_server_root_path()` and FastAPI app initialization                        | ✅ Done | [`test_server_root_path.py`](https://github.com/BerriAI/litellm/blob/main/tests/proxy_unit_tests/test_server_root_path.py) |
+| 3   | Add CI workflow that builds Docker image and tests UI routing with `SERVER_ROOT_PATH` on every PR | ✅ Done | [`test_server_root_path.yml`](https://github.com/BerriAI/litellm/blob/main/.github/workflows/test_server_root_path.yml)    |
+| 4   | Document manual test procedure for `SERVER_ROOT_PATH`                                             | ✅ Done | [Discussion #8495](https://github.com/BerriAI/litellm/discussions/8495)                                                    |
+
+---
+
+## CI workflow details
+
+The new [`test_server_root_path.yml`](https://github.com/BerriAI/litellm/blob/main/.github/workflows/test_server_root_path.yml) workflow runs on every PR against `main`. It:
+
+1. Builds the LiteLLM Docker image
+2. Starts a container with `SERVER_ROOT_PATH` set (tests both `/api/v1` and `/llmproxy`)
+3. Verifies the UI returns valid HTML at `{ROOT_PATH}/ui/`
+4. Fails the workflow if the UI is unreachable
+
+```mermaid
+flowchart TD
+    A["PR opened/updated"] --> B["Build Docker image"]
+    B --> C["Start container with SERVER_ROOT_PATH=/api/v1"]
+    B --> D["Start container with SERVER_ROOT_PATH=/llmproxy"]
+    C --> E["curl {ROOT_PATH}/ui/ → expect HTML"]
+    D --> F["curl {ROOT_PATH}/ui/ → expect HTML"]
+    E -->|"HTML found"| G["✅ Pass"]
+    E -->|"404 or no HTML"| H["❌ Fail Workflow"]
+    F -->|"HTML found"| G
+    F -->|"404 or no HTML"| H
+
+    style G fill:#d4edda,stroke:#28a745
+    style H fill:#f8d7da,stroke:#dc3545
+```
+
+This prevents future regressions where changes to `proxy_server.py` accidentally break `SERVER_ROOT_PATH` support.
+
+---
+
+## Timeline
+
+| Time (UTC)         | Event                                                                                                                                                        |
+| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| Jan 22, 2026 04:20 | PR [#19467](https://github.com/BerriAI/litellm/pull/19467) merged, removing `root_path=server_root_path`                                                     |
+| Jan 22–26          | Users on nightly builds report UI 404 errors when using `SERVER_ROOT_PATH`                                                                                   |
+| Jan 26, 2026 17:48 | Fix PR [#19790](https://github.com/BerriAI/litellm/pull/19790) merged, restoring `root_path=server_root_path`                                                |
+| Feb 18, 2026       | CI workflow [`test_server_root_path.yml`](https://github.com/BerriAI/litellm/blob/main/.github/workflows/test_server_root_path.yml) added to run on every PR |
+
+---
+
+## Resolution steps for users
+
+For users still experiencing issues, update to the latest LiteLLM version:
+
+```bash
+pip install --upgrade litellm
+```
+
+Verify your `SERVER_ROOT_PATH` is correctly set:
+
+```bash
+# In your environment or docker-compose.yml
+SERVER_ROOT_PATH="/your-prefix"
+```
+
+Then confirm the UI is accessible at `http://your-host:4000/your-prefix/ui/`.

docs/my-website/docs/a2a.md

@@ -20,6 +20,7 @@ Add A2A Agents on LiteLLM AI Gateway, Invoke agents in A2A Protocol, track reque
 | Logging | ✅ |
 | Load Balancing | ✅ |
 | Streaming | ✅ |
+| [Iteration Budgets](a2a_iteration_budgets) | ✅ |
 
 
 :::tip

docs/my-website/docs/a2a_agent_headers.md

@@ -0,0 +1,252 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# A2A Agent Authentication Headers
+
+Forward authentication credentials (Bearer tokens, API keys, etc.) from clients to backend A2A agents.
+
+## Overview
+
+When LiteLLM proxies a request to a backend A2A agent, the agent may require its own authentication headers. There are three ways to supply them:
+
+| Method | Who configures | How it works |
+|---|---|---|
+| **Static headers** | Admin (UI / API) | Always sent, regardless of client request |
+| **Forward client headers** | Admin (UI / API) | Header names to extract from client request and forward |
+| **Convention-based** | Client (no admin config) | Client sends `x-a2a-{agent_name}-{header}` — automatically routed |
+
+All three methods can be combined. **Static headers always win** on key conflicts.
+
+---
+
+## Method 1 — Static Headers
+
+Admin-configured headers that are always sent to the backend agent. Use this for server-to-server tokens or internal credentials that clients should never see or override.
+
+<Tabs>
+<TabItem value="ui" label="UI">
+
+1. Go to **Agents** in the LiteLLM dashboard.
+2. Create or edit an agent.
+3. Open the **Authentication Headers** panel.
+4. Under **Static Headers**, click **Add Static Header** and fill in the header name and value.
+
+</TabItem>
+<TabItem value="api" label="REST API">
+
+```bash
+curl -X POST http://localhost:4000/v1/agents \
+  -H "Authorization: Bearer sk-admin" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "agent_name": "my-agent",
+    "agent_card_params": { ... },
+    "static_headers": {
+      "Authorization": "Bearer internal-server-token",
+      "X-Internal-Service": "litellm-proxy"
+    }
+  }'
+```
+
+To update an existing agent:
+
+```bash
+curl -X PATCH http://localhost:4000/v1/agents/{agent_id} \
+  -H "Authorization: Bearer sk-admin" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "static_headers": {
+      "Authorization": "Bearer new-token"
+    }
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+**Client call — no special headers needed:**
+
+```bash
+curl -X POST http://localhost:4000/a2a/my-agent \
+  -H "Authorization: Bearer sk-client-key" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "jsonrpc": "2.0", "id": "1", "method": "message/send",
+    "params": { "message": { "role": "user", "parts": [{"kind": "text", "text": "Hello"}], "messageId": "msg-1" } }
+  }'
+```
+
+The backend agent receives `Authorization: Bearer internal-server-token` without the client ever knowing the value.
+
+---
+
+## Method 2 — Forward Client Headers
+
+Admin specifies a list of header **names**. When the client sends a request that includes those headers, LiteLLM extracts their values and forwards them to the backend agent. The client controls the values; the admin controls which headers are eligible to be forwarded.
+
+<Tabs>
+<TabItem value="ui" label="UI">
+
+1. Go to **Agents** in the LiteLLM dashboard.
+2. Create or edit an agent.
+3. Open the **Authentication Headers** panel.
+4. Under **Forward Client Headers**, type header names and press **Enter** (e.g. `x-api-key`, `Authorization`).
+
+</TabItem>
+<TabItem value="api" label="REST API">
+
+```bash
+curl -X POST http://localhost:4000/v1/agents \
+  -H "Authorization: Bearer sk-admin" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "agent_name": "my-agent",
+    "agent_card_params": { ... },
+    "extra_headers": ["x-api-key", "x-user-token"]
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+**Client call — include the forwarded headers:**
+
+```bash
+curl -X POST http://localhost:4000/a2a/my-agent \
+  -H "Authorization: Bearer sk-client-key" \
+  -H "x-api-key: user-secret-value" \
+  -H "Content-Type: application/json" \
+  -d '{ ... }'
+```
+
+The backend agent receives `x-api-key: user-secret-value`.
+
+:::note
+Header name matching is **case-insensitive**. If the client sends `X-API-Key` and `extra_headers` lists `x-api-key`, they match.
+:::
+
+---
+
+## Method 3 — Convention-Based Forwarding
+
+Clients can forward headers to a specific agent without any admin pre-configuration by using the naming convention:
+
+```
+x-a2a-{agent_name_or_id}-{header_name}: value
+```
+
+LiteLLM parses these headers automatically and routes them to the matching agent only.
+
+**Examples:**
+
+| Client header sent | Agent name/ID | Forwarded as |
+|---|---|---|
+| `x-a2a-my-agent-authorization: Bearer tok` | `my-agent` | `authorization: Bearer tok` |
+| `x-a2a-my-agent-x-api-key: secret` | `my-agent` | `x-api-key: secret` |
+| `x-a2a-abc123-authorization: Bearer tok` | agent ID `abc123` | `authorization: Bearer tok` |
+
+```bash
+curl -X POST http://localhost:4000/a2a/my-agent \
+  -H "Authorization: Bearer sk-client-key" \
+  -H "x-a2a-my-agent-authorization: Bearer agent-specific-token" \
+  -H "Content-Type: application/json" \
+  -d '{ ... }'
+```
+
+The `x-a2a-other-agent-authorization` header sent in the same request is **not** forwarded to `my-agent` — it is silently ignored.
+
+:::tip Matches both agent name and agent ID
+Both the human-readable name (e.g. `my-agent`) and the UUID (e.g. `abc123-...`) are valid. Use whichever is convenient for the client.
+:::
+
+---
+
+## Merge Precedence
+
+When multiple methods supply the same header name, **static headers win**:
+
+```
+dynamic (forwarded/convention)  →  merged  ←  static (overlays, wins)
+```
+
+Example:
+
+| Source | `Authorization` value |
+|---|---|
+| Client sends (via `extra_headers` or convention) | `Bearer client-token` |
+| Admin-configured `static_headers` | `Bearer server-token` |
+| **What the backend agent receives** | **`Bearer server-token`** |
+
+This ensures admin-controlled credentials cannot be overridden by client requests.
+
+---
+
+## Combining All Three Methods
+
+```bash
+# Register agent with static + forwarded headers
+curl -X POST http://localhost:4000/v1/agents \
+  -H "Authorization: Bearer sk-admin" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "agent_name": "my-agent",
+    "agent_card_params": { ... },
+    "static_headers": {
+      "X-Internal-Token": "secret123"
+    },
+    "extra_headers": ["x-user-id"]
+  }'
+
+# Client call using all three mechanisms
+curl -X POST http://localhost:4000/a2a/my-agent \
+  -H "Authorization: Bearer sk-client-key" \
+  -H "x-user-id: user-42" \
+  -H "x-a2a-my-agent-x-request-id: req-abc" \
+  -H "Content-Type: application/json" \
+  -d '{ ... }'
+```
+
+The backend agent receives:
+
+```
+X-Internal-Token: secret123          ← static header (always)
+x-user-id: user-42                   ← forwarded (in extra_headers)
+x-request-id: req-abc                ← convention-based (x-a2a-my-agent-*)
+X-LiteLLM-Trace-Id: <uuid>           ← LiteLLM internal
+X-LiteLLM-Agent-Id: <agent-id>       ← LiteLLM internal
+```
+
+---
+
+## Header Isolation
+
+Each agent invocation uses an isolated HTTP connection. Headers configured for agent A are **never** sent to agent B, even if both agents are running and receiving requests simultaneously.
+
+---
+
+## API Reference
+
+### `POST /v1/agents` / `PATCH /v1/agents/{agent_id}`
+
+| Field | Type | Description |
+|---|---|---|
+| `static_headers` | `object` | `{"Header-Name": "value"}` — always forwarded |
+| `extra_headers` | `string[]` | Header names to extract from client request and forward |
+
+### Agent Response
+
+Both fields are returned in `GET /v1/agents` and `GET /v1/agents/{agent_id}`:
+
+```json
+{
+  "agent_id": "...",
+  "agent_name": "my-agent",
+  "static_headers": { "X-Internal-Token": "secret123" },
+  "extra_headers": ["x-user-id"],
+  ...
+}
+```
+
+:::caution
+`static_headers` values are stored in the database and returned by the API. Treat them as you would any credential — do not store sensitive long-lived tokens here if your API is publicly accessible. Consider using short-lived tokens or environment-injected secrets instead.
+:::

docs/my-website/docs/a2a_iteration_budgets.md

@@ -0,0 +1,188 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Agent Iteration Budgets
+
+Control runaway costs from agentic loops with per-session iteration and budget caps.
+
+## Overview
+
+When agents run agentic loops, they can make unbounded LLM calls, causing unexpected costs. LiteLLM provides two controls:
+
+| Control | Description |
+|---------|-------------|
+| **Max Iterations** | Hard cap on the number of LLM calls per session |
+| **Max Budget Per Session** | Dollar cap per session (identified by `x-litellm-trace-id`) |
+
+Both controls require a `session_id` (sent via `x-litellm-trace-id` header or `metadata.session_id`) to track calls within a session.
+
+## Trace-ID Enforcement
+
+LiteLLM supports two independent trace-id flags, configured in `litellm_params` on the agent:
+
+| Flag | Description |
+|------|-------------|
+| `require_trace_id_on_calls_to_agent` | Requires callers invoking this agent to include `x-litellm-trace-id`. Use when the agent should only be called as a sub-agent with a trace context. Returns **400** if missing. |
+| `require_trace_id_on_calls_by_agent` | Requires all LLM/MCP calls made **by** this agent (via its virtual key) to include `x-litellm-trace-id`. This is what enables `max_iterations` and `max_budget_per_session` tracking. Returns **400** if missing. |
+
+## Configuring via UI
+
+When creating an agent in the LiteLLM Admin UI:
+
+1. Navigate to the **Agents** tab and click **Add Agent**
+2. In the **Agent Settings** step, expand the **Tracing** section
+3. Toggle **Require x-litellm-trace-id on calls BY this agent** to enable session tracking
+4. Set **Max Iterations** to cap the number of LLM calls per session
+5. Set **Max Budget Per Session ($)** to cap spend per session
+
+The trace-id flags are stored on the agent's `litellm_params`. Budget controls (`max_iterations`, `max_budget_per_session`) are stored in the virtual key's metadata.
+
+## Configuring via API
+
+Set trace-id enforcement on the agent itself:
+
+```bash
+curl -X POST 'http://localhost:4000/v1/agents' \
+  -H 'Authorization: Bearer sk-1234' \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "agent_name": "my-research-agent",
+    "agent_card_params": {
+      "name": "my-research-agent",
+      "description": "A research agent with budget controls",
+      "url": "http://my-agent:8080",
+      "version": "1.0.0"
+    },
+    "litellm_params": {
+      "require_trace_id_on_calls_to_agent": true,
+      "require_trace_id_on_calls_by_agent": true
+    }
+  }'
+```
+
+Budget controls are set on the agent's `litellm_params` (not on individual keys), so they apply across all keys for the agent:
+
+```bash
+curl -X POST 'http://localhost:4000/v1/agents' \
+  -H 'Authorization: Bearer sk-1234' \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "agent_name": "my-research-agent",
+    "agent_card_params": {
+      "name": "my-research-agent",
+      "description": "A research agent with budget controls",
+      "url": "http://my-agent:8080",
+      "version": "1.0.0"
+    },
+    "litellm_params": {
+      "require_trace_id_on_calls_by_agent": true,
+      "max_iterations": 25,
+      "max_budget_per_session": 5.00
+    }
+  }'
+```
+
+## How It Works
+
+### Session Tracking
+
+Callers identify their session by including a `session_id` in one of these ways:
+- **Header**: `x-litellm-trace-id: my-session-123`
+- **Metadata**: `{"metadata": {"session_id": "my-session-123"}}`
+
+### Max Iterations
+
+When `max_iterations` is set in agent `litellm_params`:
+- Each LLM call for a session increments a counter
+- When the counter exceeds `max_iterations`, the request receives a **429 Too Many Requests**
+- Counters expire after 1 hour by default (configurable via `LITELLM_MAX_ITERATIONS_TTL` env var)
+
+### Max Budget Per Session
+
+When `max_budget_per_session` is set in agent `litellm_params`:
+- After each successful LLM call, the response cost is accumulated for the session
+- Before each call, the accumulated spend is checked against the budget
+- When spend exceeds the budget, the request receives a **429 Too Many Requests**
+- Session spend counters expire after 1 hour by default (configurable via `LITELLM_MAX_BUDGET_PER_SESSION_TTL` env var)
+
+## Example
+
+Create an agent with max 25 iterations and a $5 budget cap:
+
+<Tabs>
+<TabItem value="ui" label="Via UI">
+
+1. Go to **Agents** → **Add Agent**
+2. Configure your agent (name, model, etc.)
+3. In **Agent Settings**, expand the **Tracing** section
+4. Toggle on **Require x-litellm-trace-id on calls BY this agent**
+5. Set **Max Iterations** to `25`
+6. Set **Max Budget Per Session** to `5.00`
+7. Proceed to create a new key for the agent
+8. Click **Create Agent**
+
+</TabItem>
+<TabItem value="api" label="Via API">
+
+```bash
+# 1. Create the agent with trace-id enforcement
+curl -X POST 'http://localhost:4000/v1/agents' \
+  -H 'Authorization: Bearer sk-1234' \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "agent_name": "my-research-agent",
+    "agent_card_params": {
+      "name": "my-research-agent",
+      "description": "A research agent with budget controls",
+      "url": "http://my-agent:8080",
+      "version": "1.0.0"
+    },
+    "litellm_params": {
+      "require_trace_id_on_calls_by_agent": true
+    }
+  }'
+
+# 2. Create a key for the agent
+curl -X POST 'http://localhost:4000/key/generate' \
+  -H 'Authorization: Bearer sk-1234' \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "agent_id": "<agent_id_from_step_1>",
+    "key_alias": "my-research-agent-key"
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+### Making Calls with Session Tracking
+
+```bash
+curl -X POST 'http://localhost:4000/chat/completions' \
+  -H 'Authorization: Bearer sk-agent-key-xxx' \
+  -H 'x-litellm-trace-id: session-abc-123' \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "model": "gpt-4o",
+    "messages": [{"role": "user", "content": "Hello"}]
+  }'
+```
+
+After 25 calls or $5 spent within this session, subsequent requests will receive:
+
+```json
+{
+  "error": {
+    "message": "Session budget exceeded for session session-abc-123. Current spend: $5.0032, max_budget_per_session: $5.00.",
+    "type": "budget_exceeded",
+    "code": 429
+  }
+}
+```
+
+## Environment Variables
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `LITELLM_MAX_ITERATIONS_TTL` | `3600` (1 hour) | TTL in seconds for session iteration counters |
+| `LITELLM_MAX_BUDGET_PER_SESSION_TTL` | `3600` (1 hour) | TTL in seconds for session budget counters |

docs/my-website/docs/adding_provider/generic_guardrail_api.md

@@ -244,6 +244,35 @@ litellm_settings:
           language: "en"
 ```
 
+### Static and dynamic headers
+
+You can send two kinds of headers to your guardrail endpoint:
+
+- **Static headers** (`headers`): A key/value map sent with **every** request to your guardrail. Use this for fixed values (e.g. API keys, `X-Service-Name`). Configure in `litellm_params`:
+
+  ```yaml
+  litellm_params:
+    guardrail: generic_guardrail_api
+    api_base: https://your-guardrail-api.com
+    headers:
+      X-Service-Name: "my-app"
+      X-API-Key: "secret"
+  ```
+
+- **Dynamic headers** (`extra_headers`): A list of **header names** that are forwarded from the **client request** to your guardrail. Only headers in this list (plus a small default allowlist such as `x-litellm-*`) have their values sent; others are sent as `[present]`. Use this to pass through client-provided headers (e.g. `x-request-id`, `x-correlation-id`). Configure in `litellm_params`:
+
+  ```yaml
+  litellm_params:
+    guardrail: generic_guardrail_api
+    api_base: https://your-guardrail-api.com
+    extra_headers:
+      - x-request-id
+      - x-correlation-id
+      - x-custom-auth
+  ```
+
+This mirrors the [MCP static and extra headers](/docs/mcp#forwarding-custom-headers-to-mcp-servers) behavior.
+
 ### Example: Pillar Security
 
 [Pillar Security](https://pillar.security) uses the Generic Guardrail API to provide comprehensive AI security scanning including prompt injection protection, PII/PCI detection, secret detection, and content moderation.

docs/my-website/docs/anthropic_unified/messages_to_responses_mapping.md

@@ -0,0 +1,120 @@
+# v1/messages → /responses Parameter Mapping
+
+When you send a request to `/v1/messages` targeting an OpenAI or Azure model, LiteLLM internally routes it through the OpenAI Responses API. This page documents exactly how every parameter gets translated in both directions.
+
+The transformation lives in `litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py`.
+
+
+## Request: Anthropic → Responses API
+
+### Top-level parameters
+
+| Anthropic (`/v1/messages`) | Responses API | Notes |
+|---|---|---|
+| `model` | `model` | Passed through as-is |
+| `messages` | `input` | Structurally transformed — see the messages section below |
+| `system` (string) | `instructions` | Passed as a plain string |
+| `system` (list of content blocks) | `instructions` | Text blocks are joined with `\n`; non-text blocks are ignored |
+| `max_tokens` | `max_output_tokens` | Renamed |
+| `temperature` | `temperature` | Passed through as-is |
+| `top_p` | `top_p` | Passed through as-is |
+| `tools` | `tools` | Format-translated — see the tools section below |
+| `tool_choice` | `tool_choice` | Type-remapped — see the tool_choice section below |
+| `thinking` | `reasoning` | Budget tokens mapped to effort level — see the thinking section below |
+| `output_format` or `output_config.format` | `text` | Wrapped as `{"format": {"type": "json_schema", "name": "structured_output", "schema": ..., "strict": true}}` |
+| `context_management` | `context_management` | Converted from Anthropic dict to OpenAI array format — see the context_management section below |
+| `metadata.user_id` | `user` | Extracted from the metadata object and truncated to 64 characters |
+| `stop_sequences` | ❌ Not mapped | Dropped silently |
+| `top_k` | ❌ Not mapped | Dropped silently |
+| `speed` | ❌ Not mapped | Only used to set Anthropic beta headers on the native path |
+
+
+### How messages get converted
+
+Each Anthropic message is expanded into one or more Responses API input items. The key difference is that `tool_result` and `tool_use` blocks become **top-level items** in the input array rather than being nested inside a message.
+
+| Anthropic message | Responses API input item |
+|---|---|
+| `user` role, string content | `{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "..."}]}` |
+| `user` role, `{"type": "text"}` block | `{"type": "input_text", "text": "..."}` inside a user message |
+| `user` role, `{"type": "image", "source": {"type": "base64"}}` | `{"type": "input_image", "image_url": "data:<media_type>;base64,<data>"}` inside a user message |
+| `user` role, `{"type": "image", "source": {"type": "url"}}` | `{"type": "input_image", "image_url": "<url>"}` inside a user message |
+| `user` role, `{"type": "tool_result"}` block | Top-level `{"type": "function_call_output", "call_id": "...", "output": "..."}` — pulled out of the message entirely |
+| `assistant` role, string content | `{"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "..."}]}` |
+| `assistant` role, `{"type": "text"}` block | `{"type": "output_text", "text": "..."}` inside an assistant message |
+| `assistant` role, `{"type": "tool_use"}` block | Top-level `{"type": "function_call", "call_id": "<id>", "name": "...", "arguments": "<JSON string>"}` — pulled out of the message entirely |
+| `assistant` role, `{"type": "thinking"}` block | `{"type": "output_text", "text": "<thinking text>"}` inside an assistant message |
+
+
+### tools
+
+| Anthropic tool | Responses API tool |
+|---|---|
+| Any tool where `type` starts with `"web_search"` or `name == "web_search"` | `{"type": "web_search_preview"}` |
+| All other tools | `{"type": "function", "name": "...", "description": "...", "parameters": <input_schema>}` |
+
+
+### tool_choice
+
+| Anthropic `tool_choice.type` | Responses API `tool_choice` |
+|---|---|
+| `"auto"` | `{"type": "auto"}` |
+| `"any"` | `{"type": "required"}` |
+| `"tool"` | `{"type": "function", "name": "<tool name>"}` |
+
+
+### thinking → reasoning
+
+The `budget_tokens` value is mapped to a string effort level. `summary` is always set to `"detailed"`.
+
+| `thinking.budget_tokens` | `reasoning.effort` |
+|---|---|
+| >= 10000 | `"high"` |
+| >= 5000 | `"medium"` |
+| >= 2000 | `"low"` |
+| < 2000 | `"minimal"` |
+
+If `thinking.type` is anything other than `"enabled"`, the `reasoning` field is not sent at all.
+
+
+### context_management
+
+Anthropic uses a nested dict with an `edits` array. OpenAI uses a flat array of compaction objects.
+
+```
+Anthropic input:
+{
+  "edits": [
+    {
+      "type": "compact_20260112",
+      "trigger": {"type": "input_tokens", "value": 150000}
+    }
+  ]
+}
+
+Responses API output:
+[
+  {"type": "compaction", "compact_threshold": 150000}
+]
+```
+
+
+## Response: Responses API → Anthropic
+
+When the Responses API reply comes back, LiteLLM converts it into an Anthropic `AnthropicMessagesResponse`.
+
+| Responses API field | Anthropic response field | Notes |
+|---|---|---|
+| `response.id` | `id` | |
+| `response.model` | `model` | Falls back to `"unknown-model"` if missing |
+| `ResponseReasoningItem` — `summary[*].text` | `content` block `{"type": "thinking", "thinking": "..."}` | Each non-empty summary text becomes a thinking block |
+| `ResponseOutputMessage` — `content[*]` where `type == "output_text"` | `content` block `{"type": "text", "text": "..."}` | |
+| `ResponseFunctionToolCall` — `{call_id, name, arguments}` | `content` block `{"type": "tool_use", "id": "...", "name": "...", "input": {...}}` | `arguments` is JSON-parsed back into a dict |
+| Any `function_call` present in output | `stop_reason: "tool_use"` | |
+| `response.status == "incomplete"` | `stop_reason: "max_tokens"` | Takes precedence over the default |
+| Everything else | `stop_reason: "end_turn"` | Default |
+| `response.usage.input_tokens` | `usage.input_tokens` | |
+| `response.usage.output_tokens` | `usage.output_tokens` | |
+| *(hardcoded)* | `type: "message"` | Always set |
+| *(hardcoded)* | `role: "assistant"` | Always set |
+| *(hardcoded)* | `stop_sequence: null` | Always null on this path |

docs/my-website/docs/benchmarks.md

@@ -5,6 +5,44 @@ import Image from '@theme/IdealImage';
 
 Benchmarks for LiteLLM Gateway (Proxy Server) tested against a fake OpenAI endpoint.
 
+## Setting Up Benchmarking with Network Mock
+
+The fastest way to benchmark proxy overhead is using `network_mock` mode. This intercepts outbound requests at the httpx transport layer and returns canned responses, no need for setting up a mock provider. 
+
+**1. Create a proxy config:**
+
+```yaml
+model_list:
+  - model_name: db-openai-endpoint
+    litellm_params:
+      model: openai/gpt-4o
+      api_key: "sk-fake-key"
+      api_base: "https://api.openai.com"
+
+litellm_settings:
+  network_mock: true
+  callbacks: []
+  num_retries: 0
+  request_timeout: 30
+
+general_settings:
+  master_key: "sk-1234"
+```
+
+**2. Start the proxy:**
+
+```bash
+litellm --config benchmark_config.yaml --port 4000 --num_workers 8
+```
+
+**3. Run the benchmark script:**
+
+```bash
+python scripts/benchmark_mock.py --requests 2000 --max-concurrent 200 --runs 3
+```
+
+This measures pure proxy overhead on the hot path without any network latency to a real or fake provider.
+
 ## Setting Up a Fake OpenAI Endpoint
 
 For load testing and benchmarking, you can use a fake OpenAI proxy server. LiteLLM provides:

docs/my-website/docs/caching/all_caches.md

@@ -297,6 +297,7 @@ litellm.cache = Cache(
     similarity_threshold=0.7, # similarity threshold for cache hits, 0 == no similarity, 1 = exact matches, 0.5 == 50% similarity
     qdrant_quantization_config ="binary", # can be one of 'binary', 'product' or 'scalar' quantizations that is supported by qdrant
     qdrant_semantic_cache_embedding_model="text-embedding-ada-002", # this model is passed to litellm.embedding(), any litellm.embedding() model is supported here
+    qdrant_semantic_cache_vector_size=1536, # vector size for the embedding model, must match the dimensionality of the embedding model used
 )
 
 response1 = completion(
@@ -635,6 +636,7 @@ def __init__(
     qdrant_quantization_config: Optional[str] = None,
     qdrant_semantic_cache_embedding_model="text-embedding-ada-002",
 
+    qdrant_semantic_cache_vector_size: Optional[int] = None,
     **kwargs
 ):
 ```

docs/my-website/docs/completion/prompt_caching.md

@@ -63,7 +63,6 @@ for _ in range(2):
                     }
                 ],
             },
-            # marked for caching with the cache_control parameter, so that this checkpoint can read from the previous cache.
             {
                 "role": "user",
                 "content": [
@@ -77,7 +76,6 @@ for _ in range(2):
                 "role": "assistant",
                 "content": "Certainly! the key terms and conditions are the following: the contract is 1 year long for $10/mo",
             },
-            # The final turn is marked with cache-control, for continuing in followups.
             {
                 "role": "user",
                 "content": [
@@ -112,16 +110,16 @@ model_list:
         api_key: os.environ/OPENAI_API_KEY
 ```
 
-2. Start proxy 
+2. Start proxy
 
 ```bash
 litellm --config /path/to/config.yaml
 ```
 
-3. Test it! 
+3. Test it!
 
 ```python
-from openai import OpenAI 
+from openai import OpenAI
 import os
 
 client = OpenAI(
@@ -144,7 +142,6 @@ for _ in range(2):
                     }
                 ],
             },
-            # marked for caching with the cache_control parameter, so that this checkpoint can read from the previous cache.
             {
                 "role": "user",
                 "content": [
@@ -158,7 +155,6 @@ for _ in range(2):
                 "role": "assistant",
                 "content": "Certainly! the key terms and conditions are the following: the contract is 1 year long for $10/mo",
             },
-            # The final turn is marked with cache-control, for continuing in followups.
             {
                 "role": "user",
                 "content": [
@@ -183,6 +179,78 @@ assert response.usage.prompt_tokens_details.cached_tokens > 0
 </TabItem>
 </Tabs>
 
+### OpenAI `prompt_cache_key` and `prompt_cache_retention`
+
+OpenAI prompt caching is [**automatic**](https://platform.openai.com/docs/guides/prompt-caching) — no `cache_control` message annotations are needed. Any request with 1024+ prompt tokens is eligible for caching.
+
+OpenAI also supports two optional parameters for more control over caching behavior:
+
+- **`prompt_cache_key`** (string) — A routing hint that improves cache hit rates for requests sharing long common prefixes. Requests with the same cache key are routed to the same backend, increasing the likelihood of a cache hit.
+- **`prompt_cache_retention`** (`"in_memory"` or `"24h"`) — Controls cache TTL. Default is `"in_memory"` (5–10 min). Set to `"24h"` for extended caching that offloads KV tensors to GPU-local storage.
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+from litellm import completion
+import os
+
+os.environ["OPENAI_API_KEY"] = ""
+
+response = completion(
+    model="gpt-4o",
+    messages=[
+        {
+            "role": "system",
+            "content": "You are an AI assistant tasked with analyzing legal documents. "
+            + "Here is the full text of a complex legal agreement " * 400,
+        },
+        {
+            "role": "user",
+            "content": "What are the key terms and conditions?",
+        },
+    ],
+    prompt_cache_key="legal-doc-analysis",
+    prompt_cache_retention="24h",
+)
+print(response.usage)
+```
+
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+```python
+from openai import OpenAI
+
+client = OpenAI(
+    api_key="LITELLM_PROXY_KEY",
+    base_url="LITELLM_PROXY_BASE",
+)
+
+response = client.chat.completions.create(
+    model="gpt-4o",
+    messages=[
+        {
+            "role": "system",
+            "content": "You are an AI assistant tasked with analyzing legal documents. "
+            + "Here is the full text of a complex legal agreement " * 400,
+        },
+        {
+            "role": "user",
+            "content": "What are the key terms and conditions?",
+        },
+    ],
+    extra_body={
+        "prompt_cache_key": "legal-doc-analysis",
+        "prompt_cache_retention": "24h",
+    },
+)
+print(response.usage)
+```
+
+</TabItem>
+</Tabs>
+
 ### Anthropic Example 
 
 Anthropic charges for cache writes. 

docs/my-website/docs/contributing.md

@@ -79,7 +79,27 @@ cp -r out/* ../../litellm/proxy/_experimental/out/
 
 Then restart the proxy and access the UI at `http://localhost:4000/ui`
 
-## 4. Submitting a PR
+## 4. Pre-PR Checklist
+
+Before submitting your pull request, make sure the following pass locally from `ui/litellm-dashboard/`:
+
+**Run tests related to your changes:**
+
+```bash
+npx vitest run src/components/path/to/YourComponent.test.tsx
+```
+
+Tests are co-located with components (e.g., `TeamInfo.tsx` → `TeamInfo.test.tsx`). If you add a new component, add a corresponding `.test.tsx` file next to it.
+
+**Run the build:**
+
+```bash
+npm run build
+```
+
+These map to the `ui_tests` and `ui_build` CI checks.
+
+## 5. Submitting a PR
 
 1. Create a new branch for your changes:
 ```bash

docs/my-website/docs/enterprise.md

@@ -4,7 +4,7 @@ import Image from '@theme/IdealImage';
 
 :::info
 - ✨ SSO is free for up to 5 users. After that, an enterprise license is required. [Get Started with Enterprise here](https://www.litellm.ai/enterprise)
-- Who is Enterprise for? Companies giving access to 100+ users **OR** 10+ AI use-cases. If you're not sure, [get in touch with us](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) to discuss your needs.
+- Who is Enterprise for? Companies giving access to 100+ users **OR** 10+ AI use-cases. If you're not sure, [get in touch with us](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) to discuss your needs.
 :::
 
 For companies that need SSO, user management and professional support for LiteLLM Proxy
@@ -36,7 +36,7 @@ Manage Yourself - you can deploy our Docker Image or build a custom image from o
 
 ### What’s the cost of the Self-Managed Enterprise edition?
 
-Self-Managed Enterprise deployments require our team to understand your exact needs. [Get in touch with us to learn more](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
+Self-Managed Enterprise deployments require our team to understand your exact needs. [Get in touch with us to learn more](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
 
 
 ### How does deployment with Enterprise License work? 
@@ -106,7 +106,7 @@ Professional Support can assist with LLM/Provider integrations, deployment, upgr
 
 Pricing is based on usage. We can figure out a price that works for your team, on the call. 
 
-[**Contact Us to learn more**](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
+[**Contact Us to learn more**](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
 
 
 

docs/my-website/docs/fine_tuning.md

@@ -6,7 +6,7 @@ import TabItem from '@theme/TabItem';
 
 :::info
 
-This is an Enterprise only endpoint [Get Started with Enterprise here](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
+This is an Enterprise only endpoint [Get Started with Enterprise here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
 
 :::
 

docs/my-website/docs/generateContent.md

@@ -15,6 +15,7 @@ Use LiteLLM to call Google AI's generateContent endpoints for text generation, m
 | Streaming | ✅ | |
 | Fallbacks | ✅ | between supported models |
 | Loadbalancing | ✅ | between supported models |
+| Metadata Tracking | ✅ | passes trace ID, metadata to observability callbacks (e.g. S3, Langfuse) |
 
 ## Usage 
 ---

docs/my-website/docs/image_edits.md

@@ -16,7 +16,7 @@ LiteLLM provides image editing functionality that maps to OpenAI's `/images/edit
 | Supported operations | Create image edits | Single and multiple images supported |
 | Supported LiteLLM SDK Versions | 1.63.8+ | Gemini support requires 1.79.3+ |
 | Supported LiteLLM Proxy Versions | 1.71.1+ | Gemini support requires 1.79.3+ |
-| Supported LLM providers | **OpenAI**, **Gemini (Google AI Studio)**, **Vertex AI**, **Stability AI**, **AWS Bedrock (Stability)** | Gemini supports the new `gemini-2.5-flash-image` family. Vertex AI supports both Gemini and Imagen models. Stability AI and Bedrock Stability support various image editing operations. |
+| Supported LLM providers | **OpenAI**, **Gemini (Google AI Studio)**, **Vertex AI**, **OpenRouter**, **Stability AI**, **AWS Bedrock (Stability)** | Gemini supports the new `gemini-2.5-flash-image` family. Vertex AI supports both Gemini and Imagen models. OpenRouter routes image edits through chat completions. Stability AI and Bedrock Stability support various image editing operations. |
 
  #### ⚡️See all supported models and providers at [models.litellm.ai](https://models.litellm.ai/)
 
@@ -244,6 +244,47 @@ response = litellm.image_edit(
 print(response)
 ```
 
+</TabItem>
+
+<TabItem value="openrouter" label="OpenRouter">
+
+#### Basic Image Edit
+```python showLineNumbers title="OpenRouter Image Edit"
+import os
+from litellm import image_edit
+
+os.environ["OPENROUTER_API_KEY"] = "your-api-key"
+
+response = image_edit(
+    model="openrouter/google/gemini-2.5-flash-image",
+    image=open("original_image.png", "rb"),
+    prompt="Add aurora borealis to the night sky",
+)
+
+print(response)
+```
+
+#### Multiple Images Edit
+```python showLineNumbers title="OpenRouter Multiple Images Edit"
+import os
+from litellm import image_edit
+
+os.environ["OPENROUTER_API_KEY"] = "your-api-key"
+
+response = image_edit(
+    model="openrouter/google/gemini-2.5-flash-image",
+    image=[
+        open("scene.png", "rb"),
+        open("style_reference.png", "rb"),
+    ],
+    prompt="Blend the reference style into the scene",
+    size="1536x1024",   # mapped to aspect_ratio 3:2
+    quality="high",      # mapped to image_size 4K
+)
+
+print(response)
+```
+
 </TabItem>
 </Tabs>
 
@@ -398,6 +439,34 @@ curl -X POST "http://0.0.0.0:4000/v1/images/edits" \
   -F "size=1024x1024"
 ```
 
+</TabItem>
+
+<TabItem value="openrouter" label="OpenRouter">
+
+1. Add the OpenRouter image edit model to your `config.yaml`:
+```yaml showLineNumbers title="OpenRouter Proxy Configuration"
+model_list:
+  - model_name: openrouter-image-edit
+    litellm_params:
+      model: openrouter/google/gemini-2.5-flash-image
+      api_key: os.environ/OPENROUTER_API_KEY
+```
+
+2. Start the LiteLLM proxy server:
+```bash showLineNumbers title="Start LiteLLM Proxy Server"
+litellm --config /path/to/config.yaml
+```
+
+3. Make an image edit request:
+```bash showLineNumbers title="OpenRouter Proxy Image Edit"
+curl -X POST "http://0.0.0.0:4000/v1/images/edits" \
+  -H "Authorization: Bearer <YOUR-LITELLM-KEY>" \
+  -F "model=openrouter-image-edit" \
+  -F "image=@original_image.png" \
+  -F "prompt=Make the sky a vibrant purple sunset" \
+  -F "size=1024x1024"
+```
+
 </TabItem>
 </Tabs>
 

docs/my-website/docs/interactions.md

@@ -130,13 +130,12 @@ Point the Google GenAI SDK to LiteLLM Proxy:
 
 ```python showLineNumbers title="Google GenAI SDK with LiteLLM Proxy"
 from google import genai
-import os
 
 # Point SDK to LiteLLM Proxy
-os.environ["GOOGLE_GENAI_BASE_URL"] = "http://localhost:4000"
-os.environ["GEMINI_API_KEY"] = "sk-1234"  # Your LiteLLM API key
-
-client = genai.Client()
+client = genai.Client(
+    api_key="sk-1234",  # Your LiteLLM API key
+    http_options={"base_url": "http://localhost:4000"},
+)
 
 # Create an interaction
 interaction = client.interactions.create(
@@ -151,12 +150,11 @@ print(interaction.outputs[-1].text)
 
 ```python showLineNumbers title="Google GenAI SDK Streaming"
 from google import genai
-import os
 
-os.environ["GOOGLE_GENAI_BASE_URL"] = "http://localhost:4000"
-os.environ["GEMINI_API_KEY"] = "sk-1234"
-
-client = genai.Client()
+client = genai.Client(
+    api_key="sk-1234",  # Your LiteLLM API key
+    http_options={"base_url": "http://localhost:4000"},
+)
 
 for chunk in client.interactions.create_stream(
     model="gemini/gemini-2.5-flash",

docs/my-website/docs/mcp.md

@@ -336,175 +336,9 @@ litellm_settings:
 
 ## Converting OpenAPI Specs to MCP Servers
 
-LiteLLM can automatically convert OpenAPI specifications into MCP servers, allowing you to expose any REST API as MCP tools. This is useful when you have existing APIs with OpenAPI/Swagger documentation and want to make them available as MCP tools.
+LiteLLM can convert OpenAPI specifications into MCP servers, exposing any REST API as MCP tools without writing custom server code.
 
-**Benefits:**
-
-- **Rapid Integration**: Convert existing APIs to MCP tools without writing custom MCP server code
-- **Automatic Tool Generation**: LiteLLM automatically generates MCP tools from your OpenAPI spec
-- **Unified Interface**: Use the same MCP interface for both native MCP servers and OpenAPI-based APIs
-- **Easy Testing**: Test and iterate on API integrations quickly
-
-**Configuration:**
-
-Add your OpenAPI-based MCP server to your `config.yaml`:
-
-```yaml title="config.yaml - OpenAPI to MCP" showLineNumbers
-model_list:
-  - model_name: gpt-4o
-    litellm_params:
-      model: openai/gpt-4o
-      api_key: sk-xxxxxxx
-
-mcp_servers:
-  # OpenAPI Spec Example - Petstore API
-  petstore_mcp:
-    url: "https://petstore.swagger.io/v2"
-    spec_path: "/path/to/openapi.json"
-    auth_type: "none"
-  
-  # OpenAPI Spec with API Key Authentication
-  my_api_mcp:
-    url: "http://0.0.0.0:8090"
-    spec_path: "/path/to/openapi.json"
-    auth_type: "api_key"
-    auth_value: "your-api-key-here"
-  
-  # OpenAPI Spec with Bearer Token
-  secured_api_mcp:
-    url: "https://api.example.com"
-    spec_path: "/path/to/openapi.json" 
-    auth_type: "bearer_token"
-    auth_value: "your-bearer-token"
-```
-
-**Configuration Parameters:**
-
-| Parameter | Required | Description |
-|-----------|----------|-------------|
-| `url` | Yes | The base URL of your API endpoint |
-| `spec_path` | Yes | Path or URL to your OpenAPI specification file (JSON or YAML) |
-| `auth_type` | No | Authentication type: `none`, `api_key`, `bearer_token`, `basic`, `authorization` |
-| `auth_value` | No | Authentication value (required if `auth_type` is set) |
-| `authorization_url` | No | For `auth_type: oauth2`. Optional override; if omitted LiteLLM auto-discovers it. |
-| `token_url` | No | For `auth_type: oauth2`. Optional override; if omitted LiteLLM auto-discovers it. |
-| `registration_url` | No | For `auth_type: oauth2`. Optional override; if omitted LiteLLM auto-discovers it. |
-| `scopes` | No | For `auth_type: oauth2`. Optional override; if omitted LiteLLM uses the scopes advertised by the server. |
-| `description` | No | Optional description for the MCP server |
-| `allowed_tools` | No | List of specific tools to allow (see [MCP Tool Filtering](#mcp-tool-filtering)) |
-| `disallowed_tools` | No | List of specific tools to block (see [MCP Tool Filtering](#mcp-tool-filtering)) |
-
-### Usage Example
-
-Once configured, you can use the OpenAPI-based MCP server just like any other MCP server:
-
-<Tabs>
-<TabItem value="fastmcp" label="Python FastMCP">
-
-```python title="Using OpenAPI-based MCP Server" showLineNumbers
-from fastmcp import Client
-import asyncio
-
-# Standard MCP configuration
-config = {
-    "mcpServers": {
-        "petstore": {
-            "url": "http://localhost:4000/petstore_mcp/mcp",
-            "headers": {
-                "x-litellm-api-key": "Bearer sk-1234"
-            }
-        }
-    }
-}
-
-# Create a client that connects to the server
-client = Client(config)
-
-async def main():
-    async with client:
-        # List available tools generated from OpenAPI spec
-        tools = await client.list_tools()
-        print(f"Available tools: {[tool.name for tool in tools]}")
-
-        # Example: Get a pet by ID (from Petstore API)
-        response = await client.call_tool(
-            name="getpetbyid", 
-            arguments={"petId": "1"}
-        )
-        print(f"Response:\n{response}\n")
-
-        # Example: Find pets by status
-        response = await client.call_tool(
-            name="findpetsbystatus", 
-            arguments={"status": "available"}
-        )
-        print(f"Response:\n{response}\n")
-
-if __name__ == "__main__":
-    asyncio.run(main())
-```
-
-</TabItem>
-
-<TabItem value="cursor" label="Cursor IDE">
-
-```json title="Cursor MCP Configuration for OpenAPI Server" showLineNumbers
-{
-  "mcpServers": {
-    "Petstore": {
-      "url": "http://localhost:4000/petstore_mcp/mcp",
-      "headers": {
-        "x-litellm-api-key": "Bearer $LITELLM_API_KEY"
-      }
-    }
-  }
-}
-```
-
-</TabItem>
-
-<TabItem value="openai" label="OpenAI Responses API">
-
-```bash title="Using OpenAPI MCP Server with OpenAI" showLineNumbers
-curl --location 'https://api.openai.com/v1/responses' \
---header 'Content-Type: application/json' \
---header "Authorization: Bearer $OPENAI_API_KEY" \
---data '{
-    "model": "gpt-4o",
-    "tools": [
-        {
-            "type": "mcp",
-            "server_label": "petstore",
-            "server_url": "http://localhost:4000/petstore_mcp/mcp",
-            "require_approval": "never",
-            "headers": {
-                "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY"
-            }
-        }
-    ],
-    "input": "Find all available pets in the petstore",
-    "tool_choice": "required"
-}'
-```
-
-</TabItem>
-</Tabs>
-
-**How It Works**
-
-1. **Spec Loading**: LiteLLM loads your OpenAPI specification from the provided `spec_path`
-2. **Tool Generation**: Each API endpoint in the spec becomes an MCP tool
-3. **Parameter Mapping**: OpenAPI parameters are automatically mapped to MCP tool parameters
-4. **Request Handling**: When a tool is called, LiteLLM converts the MCP request to the appropriate HTTP request
-5. **Response Translation**: API responses are converted back to MCP format
-
-**OpenAPI Spec Requirements**
-
-Your OpenAPI specification should follow standard OpenAPI/Swagger conventions:
-- **Supported versions**: OpenAPI 3.0.x, OpenAPI 3.1.x, Swagger 2.0
-- **Required fields**: `paths`, `info` sections should be properly defined
-- **Operation IDs**: Each operation should have a unique `operationId` (this becomes the tool name)
-- **Parameters**: Request parameters should be properly documented with types and descriptions
+See the **[MCP from OpenAPI Specs guide](./mcp_openapi.md)** for full setup, usage examples, and how to override tool names and descriptions.
 
 ## MCP OAuth
 
@@ -641,7 +475,7 @@ import asyncio
 config = {
     "mcpServers": {
         "mcp_group": {
-            "url": "http://localhost:4000/mcp",
+            "url": "http://localhost:4000/mcp/",
             "headers": {
                 "x-mcp-servers": "dev_group", # assume this gives access to github, zapier and deepwiki
                 "x-litellm-api-key": "Bearer sk-1234",
@@ -870,6 +704,63 @@ asyncio.run(main())
 
 [Learn more about customer management →](./proxy/customers)
 
+## Calling the Proxy's /v1/responses Endpoint
+
+When calling your LiteLLM Proxy's `/v1/responses` endpoint to use MCP tools, **always use `server_url: "litellm_proxy"`** in the tools array. This tells the proxy to use its configured MCP servers.
+
+:::important Do not use the full proxy URL
+Using `server_url: "https://your-proxy.com/mcp"` is incorrect when the request is already going to the proxy. The proxy needs the literal value `litellm_proxy` to route to its configured MCP servers.
+:::
+
+```bash title="Correct: Using litellm_proxy" showLineNumbers
+curl --location 'https://your-proxy.com/v1/responses' \
+--header 'Content-Type: application/json' \
+--header "Authorization: Bearer $LITELLM_API_KEY" \
+--data '{
+    "model": "gpt-4",
+    "tools": [
+        {
+            "type": "mcp",
+            "server_label": "litellm",
+            "server_url": "litellm_proxy",
+            "require_approval": "never"
+        }
+    ],
+    "input": "Run available tools",
+    "tool_choice": "required"
+}'
+```
+
+### Sending Custom Headers to MCP Servers
+
+To pass custom headers (e.g., API keys, auth tokens) to specific MCP servers, use either:
+
+**Option 1: Request headers** – Add `x-mcp-{server_alias}-{header_name}` to your request headers. The proxy forwards these to the matching MCP server.
+
+```bash
+# Send Authorization header to the "weather2" MCP server
+--header 'x-mcp-weather2-authorization: Bearer your-token'
+
+# Send custom header to the "github" MCP server  
+--header 'x-mcp-github-x-api-key: your-api-key'
+```
+
+**Option 2: Headers in tool config** – Include a `headers` object in the tool definition. These are merged with request headers.
+
+```json
+{
+    "type": "mcp",
+    "server_label": "litellm",
+    "server_url": "litellm_proxy",
+    "require_approval": "never",
+    "headers": {
+        "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY",
+        "x-mcp-servers": "Zapier_MCP,dev-group",
+        "x-mcp-weather2-authorization": "Bearer your-weather-api-token"
+    }
+}
+```
+
 ## Using your MCP with client side credentials
 
 Use this if you want to pass a client side authentication token to LiteLLM to then pass to your MCP to auth to your MCP.

docs/my-website/docs/mcp_control.md

@@ -323,7 +323,7 @@ curl --location '<your-litellm-proxy-base-url>/v1/responses' \
         {
             "type": "mcp",
             "server_label": "litellm",
-            "server_url": "<your-litellm-proxy-base-url>/dev_group/mcp",
+            "server_url": "litellm_proxy",
             "require_approval": "never",
             "headers": {
                 "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY"
@@ -335,7 +335,7 @@ curl --location '<your-litellm-proxy-base-url>/v1/responses' \
 }'
 ```
 
-This example uses URL namespacing to access all servers in the "dev_group" access group.
+This example uses the `x-mcp-servers` header to access all servers in the "dev_group" access group. Use `server_url: "litellm_proxy"` when calling the proxy's `/v1/responses` endpoint—do not use the full proxy URL.
 
 </TabItem>
 
@@ -423,7 +423,7 @@ curl --location '<your-litellm-proxy-base-url>/v1/responses' \
         {
             "type": "mcp",
             "server_label": "litellm",
-            "server_url": "<your-litellm-proxy-base-url>/mcp/",
+            "server_url": "litellm_proxy",
             "require_approval": "never",
             "headers": {
                 "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY",
@@ -436,7 +436,7 @@ curl --location '<your-litellm-proxy-base-url>/v1/responses' \
 }'
 ```
 
-This configuration restricts the request to only use tools from the specified MCP servers.
+This configuration restricts the request to only use tools from the specified MCP servers. Use `server_url: "litellm_proxy"` when calling the proxy's `/v1/responses` endpoint.
 
 </TabItem>
 

docs/my-website/docs/mcp_openapi.md

@@ -0,0 +1,226 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import Image from '@theme/IdealImage';
+
+# MCP from OpenAPI Specs
+
+LiteLLM can convert any OpenAPI/Swagger spec into an MCP server — no custom MCP server code required.
+
+## Step 1 — Add the MCP Server
+
+Add your OpenAPI-based server in `config.yaml`:
+
+```yaml title="config.yaml" showLineNumbers
+mcp_servers:
+  petstore_mcp:
+    url: "https://petstore.swagger.io/v2"
+    spec_path: "/path/to/openapi.json"
+    auth_type: "none"
+
+  my_api_mcp:
+    url: "http://0.0.0.0:8090"
+    spec_path: "/path/to/openapi.json"
+    auth_type: "api_key"
+    auth_value: "your-api-key-here"
+
+  secured_api_mcp:
+    url: "https://api.example.com"
+    spec_path: "/path/to/openapi.json"
+    auth_type: "bearer_token"
+    auth_value: "your-bearer-token"
+```
+
+Or from the UI: go to **MCP Servers → Add New MCP Server**, fill in the URL and spec path, and LiteLLM will fetch the spec and load all endpoints as tools.
+
+**Configuration parameters:**
+
+| Parameter | Required | Description |
+|-----------|----------|-------------|
+| `url` | Yes | Base URL of your API |
+| `spec_path` | Yes | Path or URL to your OpenAPI spec (JSON or YAML) |
+| `auth_type` | No | `none`, `api_key`, `bearer_token`, `basic`, `authorization`, `oauth2` |
+| `auth_value` | No | Auth value (required if `auth_type` is set) |
+| `description` | No | Optional description |
+| `allowed_tools` | No | Allowlist of specific tools |
+| `disallowed_tools` | No | Blocklist of specific tools |
+
+**Supported spec versions:** OpenAPI 3.0.x, 3.1.x, Swagger 2.0. Each operation's `operationId` becomes the tool name — make sure they're unique.
+
+Once tools are loaded, you'll see them in the Tool Configuration section:
+
+<Image
+  img={require('../img/mcp_openapi_tools_loaded.png')}
+  style={{width: '80%', display: 'block', margin: '0'}}
+/>
+
+<br/>
+
+## Step 2 — Optionally Override Tool Names and Descriptions
+
+By default, tool names and descriptions come from the `operationId` and description fields in your spec. You can rename or rewrite them so MCP clients see something cleaner — without touching the upstream spec.
+
+### From the UI
+
+Each tool card has a pencil icon. Click it to open the inline editor:
+
+<Image
+  img={require('../img/mcp_openapi_tool_edit_panel.png')}
+  style={{width: '80%', display: 'block', margin: '0'}}
+/>
+
+<br/>
+
+- **Display Name** — overrides the name MCP clients see
+- **Description** — overrides the description MCP clients see
+- Leave a field blank to keep the original from the spec
+
+After setting overrides, a purple **Custom name** badge appears on the tool card:
+
+<Image
+  img={require('../img/mcp_openapi_custom_name_badge.png')}
+  style={{width: '80%', display: 'block', margin: '0'}}
+/>
+
+<br/>
+
+### From the API
+
+Pass `tool_name_to_display_name` and `tool_name_to_description` in the create or update request:
+
+```bash title="Create server with tool name overrides" showLineNumbers
+curl -X POST http://localhost:4000/v1/mcp/server \
+  -H "Authorization: Bearer $LITELLM_MASTER_KEY" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "name": "petstore_mcp",
+    "url": "https://petstore.swagger.io/v2",
+    "spec_path": "/path/to/openapi.json",
+    "tool_name_to_display_name": {
+      "getPetById": "Get Pet",
+      "findPetsByStatus": "List Available Pets"
+    },
+    "tool_name_to_description": {
+      "getPetById": "Look up a pet by its ID",
+      "findPetsByStatus": "Returns all pets matching a given status (available, pending, sold)"
+    }
+  }'
+```
+
+```bash title="Update overrides on an existing server" showLineNumbers
+curl -X PUT http://localhost:4000/v1/mcp/server/{server_id} \
+  -H "Authorization: Bearer $LITELLM_MASTER_KEY" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "tool_name_to_display_name": {
+      "getPetById": "Get Pet"
+    },
+    "tool_name_to_description": {
+      "getPetById": "Look up a pet by its ID"
+    }
+  }'
+```
+
+The map key is the **original `operationId`** from the spec — not the prefixed tool name. LiteLLM strips the server prefix before doing the lookup.
+
+For example, if your server is `petstore_mcp`, the tool is exposed as `petstore_mcp-getPetById`. The map key is still `getPetById`.
+
+**Before and after:**
+
+```
+# Without overrides
+Tool: "petstore_mcp-getPetById"
+Description: "Returns a single pet"
+
+Tool: "petstore_mcp-findPetsByStatus"
+Description: "Finds Pets by status"
+
+# After overrides
+Tool: "Get Pet"
+Description: "Look up a pet by its ID"
+
+Tool: "List Available Pets"
+Description: "Returns all pets matching a given status (available, pending, sold)"
+```
+
+## Using the Server
+
+<Tabs>
+<TabItem value="fastmcp" label="Python FastMCP">
+
+```python title="Using OpenAPI-based MCP Server" showLineNumbers
+from fastmcp import Client
+import asyncio
+
+config = {
+    "mcpServers": {
+        "petstore": {
+            "url": "http://localhost:4000/petstore_mcp/mcp",
+            "headers": {
+                "x-litellm-api-key": "Bearer sk-1234"
+            }
+        }
+    }
+}
+
+client = Client(config)
+
+async def main():
+    async with client:
+        tools = await client.list_tools()
+        print(f"Available tools: {[tool.name for tool in tools]}")
+
+        response = await client.call_tool(
+            name="Get Pet",        # overridden name
+            arguments={"petId": "1"}
+        )
+        print(f"Response: {response}")
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+</TabItem>
+
+<TabItem value="cursor" label="Cursor IDE">
+
+```json title="Cursor MCP Configuration" showLineNumbers
+{
+  "mcpServers": {
+    "Petstore": {
+      "url": "http://localhost:4000/petstore_mcp/mcp",
+      "headers": {
+        "x-litellm-api-key": "Bearer $LITELLM_API_KEY"
+      }
+    }
+  }
+}
+```
+
+</TabItem>
+
+<TabItem value="openai" label="OpenAI Responses API">
+
+```bash title="Using OpenAPI MCP Server with OpenAI" showLineNumbers
+curl --location 'https://api.openai.com/v1/responses' \
+--header 'Content-Type: application/json' \
+--header "Authorization: Bearer $OPENAI_API_KEY" \
+--data '{
+    "model": "gpt-4o",
+    "tools": [
+        {
+            "type": "mcp",
+            "server_label": "petstore",
+            "server_url": "http://localhost:4000/petstore_mcp/mcp",
+            "require_approval": "never",
+            "headers": {
+                "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY"
+            }
+        }
+    ],
+    "input": "Find all available pets",
+    "tool_choice": "required"
+}'
+```
+
+</TabItem>
+</Tabs>

docs/my-website/docs/observability/datadog.md

@@ -7,6 +7,7 @@ import TabItem from '@theme/TabItem';
 LiteLLM Supports logging to the following Datdog Integrations:
 - `datadog` [Datadog Logs](https://docs.datadoghq.com/logs/)
 - `datadog_llm_observability` [Datadog LLM Observability](https://www.datadoghq.com/product/llm-observability/)
+- `datadog_metrics` [Datadog Custom Metrics](#datadog-custom-metrics)
 - `datadog_cost_management` [Datadog Cloud Cost Management](#datadog-cloud-cost-management)
 - `ddtrace-run` [Datadog Tracing](#datadog-tracing)
 
@@ -168,6 +169,65 @@ On the Datadog LLM Observability page, you should see that both input messages a
 <Image img={require('../../img/dd_llm_obs.png')} />
 
 
+## Datadog Custom Metrics
+
+| Feature | Details |
+|---------|---------|
+| **What is logged** | Latency metrics, request counts by status code |
+| **Events** | Success + Failure |
+| **Product Link** | [Datadog Metrics](https://docs.datadoghq.com/metrics/) |
+
+Publishes the following metrics to Datadog via the `/api/v2/series` endpoint:
+
+| Metric | Type | Description |
+|--------|------|-------------|
+| `litellm.request.total_latency` | Gauge | End-to-end request latency (seconds) |
+| `litellm.llm_api.latency` | Gauge | Time spent waiting for the LLM provider response (seconds) |
+| `litellm.llm_api.request_count` | Count | Request count, tagged with status code |
+
+Using `total_latency` and `llm_api.latency`, you can derive **internal latency** = `total_latency - llm_api.latency`.
+
+All metrics include the following tags: `env`, `service`, `version`, `HOSTNAME`, `POD_NAME`, `provider`, `model_name`, `model_group`, `team`, `status_code`.
+
+**Step 1**: Create a `config.yaml` file
+
+```yaml
+model_list:
+  - model_name: gpt-3.5-turbo
+    litellm_params:
+      model: gpt-3.5-turbo
+litellm_settings:
+  success_callback: ["datadog_metrics"]
+  failure_callback: ["datadog_metrics"]
+```
+
+**Step 2**: Set required env variables
+
+```shell
+DD_API_KEY="your-api-key"
+DD_SITE="us5.datadoghq.com"  # your datadog site
+```
+
+**Step 3**: Start the proxy and make a test request
+
+```shell
+litellm --config config.yaml
+```
+
+```shell
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+    --header 'Content-Type: application/json' \
+    --header 'Authorization: Bearer sk-1234' \
+    --data '{
+    "model": "gpt-3.5-turbo",
+    "messages": [{"role": "user", "content": "hello"}]
+}'
+```
+
+**Step 4**: View metrics in Datadog Metrics Explorer
+
+Navigate to **Metrics > Explorer** in Datadog and search for `litellm.request.total_latency`, `litellm.llm_api.latency`, or `litellm.llm_api.request_count`.
+
 ## Datadog Cloud Cost Management
 
 | Feature | Details |

docs/my-website/docs/observability/gcs_bucket_integration.md

@@ -6,7 +6,7 @@ Log LLM Logs to [Google Cloud Storage Buckets](https://cloud.google.com/storage?
 
 :::info
 
-✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
+✨ This is an Enterprise only feature [Get Started with Enterprise here](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
 
 :::
 

docs/my-website/docs/ocr.md

@@ -61,6 +61,52 @@ async def test_async_ocr():
 asyncio.run(test_async_ocr())
 ```
 
+### Using Local Files
+
+LiteLLM can read local files directly — no manual base64 encoding needed:
+
+```python
+from litellm import ocr
+
+# OCR with a local PDF file path
+response = ocr(
+    model="mistral/mistral-ocr-latest",
+    document={
+        "type": "file",
+        "file": "/path/to/document.pdf"
+    }
+)
+
+# OCR with a file object
+response = ocr(
+    model="mistral/mistral-ocr-latest",
+    document={
+        "type": "file",
+        "file": open("document.pdf", "rb")
+    }
+)
+
+# OCR with raw bytes
+with open("document.pdf", "rb") as f:
+    pdf_bytes = f.read()
+
+response = ocr(
+    model="mistral/mistral-ocr-latest",
+    document={
+        "type": "file",
+        "file": pdf_bytes,
+        "mime_type": "application/pdf"  # recommended for raw bytes (auto-detected from extension for file paths)
+    }
+)
+```
+
+The `file` field accepts:
+- **File path** (`str` or `pathlib.Path`) — LiteLLM reads the file and detects the MIME type from the extension
+- **File object** (binary file-like object) — e.g. `open("doc.pdf", "rb")`
+- **Raw bytes** (`bytes`) — use `mime_type` to specify the content type
+
+LiteLLM automatically converts file inputs to base64 data URIs internally, so all providers work seamlessly.
+
 ### Using Base64 Encoded Documents
 
 ```python
@@ -121,7 +167,7 @@ litellm --config /path/to/config.yaml
 # RUNNING on http://0.0.0.0:4000
 ```
 
-Test request
+**Test request — JSON body**
 
 ```bash
 curl http://0.0.0.0:4000/v1/ocr \
@@ -136,6 +182,27 @@ curl http://0.0.0.0:4000/v1/ocr \
   }'
 ```
 
+**Test request — multipart file upload**
+
+Upload a file directly using multipart form data. No need to base64-encode the file yourself.
+
+```bash
+curl http://0.0.0.0:4000/v1/ocr \
+  -H "Authorization: Bearer sk-1234" \
+  -F "model=mistral-ocr" \
+  -F "file=@/path/to/document.pdf"
+```
+
+You can also pass optional parameters as additional form fields:
+
+```bash
+curl http://0.0.0.0:4000/v1/ocr \
+  -H "Authorization: Bearer sk-1234" \
+  -F "model=mistral-ocr" \
+  -F "file=@screenshot.png" \
+  -F 'pages=[0,1,2]' \
+  -F "include_image_base64=true"
+```
 
 ## **Request/Response Format**
 
@@ -168,10 +235,12 @@ See the [official Mistral OCR documentation](https://docs.mistral.ai/capabilitie
 | Parameter | Type | Required | Description |
 |-----------|------|----------|-------------|
 | `model` | string | Yes | The OCR model to use (e.g., `"mistral/mistral-ocr-latest"`) |
-| `document` | object | Yes | Document to process. Must contain `type` and URL field |
-| `document.type` | string | Yes | Either `"document_url"` for PDFs/docs or `"image_url"` for images |
-| `document.document_url` | string | Conditional | URL to the document (required if `type` is `"document_url"`) |
-| `document.image_url` | string | Conditional | URL to the image (required if `type` is `"image_url"`) |
+| `document` | object | Yes | Document to process. Must contain `type` and the corresponding field |
+| `document.type` | string | Yes | `"document_url"` for PDFs/docs, `"image_url"` for images, or `"file"` for local files |
+| `document.document_url` | string | Conditional | URL or data URI to the document (required if `type` is `"document_url"`) |
+| `document.image_url` | string | Conditional | URL or data URI to the image (required if `type` is `"image_url"`) |
+| `document.file` | string/bytes/file | Conditional | File path, bytes, or file-like object (required if `type` is `"file"`) |
+| `document.mime_type` | string | No | Explicit MIME type for file inputs (auto-detected from extension if not provided) |
 | `pages` | array | No | List of specific page indices to process (0-indexed) |
 | `include_image_base64` | boolean | No | Whether to include extracted images as base64 strings |
 | `image_limit` | integer | No | Maximum number of images to return |
@@ -179,7 +248,7 @@ See the [official Mistral OCR documentation](https://docs.mistral.ai/capabilitie
 
 #### Document Format Examples
 
-**For PDFs and documents:**
+**For PDFs and documents (URL):**
 ```json
 {
   "type": "document_url",
@@ -187,7 +256,7 @@ See the [official Mistral OCR documentation](https://docs.mistral.ai/capabilitie
 }
 ```
 
-**For images:**
+**For images (URL):**
 ```json
 {
   "type": "image_url",
@@ -203,6 +272,21 @@ See the [official Mistral OCR documentation](https://docs.mistral.ai/capabilitie
 }
 ```
 
+**For local files (SDK):**
+```python
+{"type": "file", "file": "/path/to/document.pdf"}
+{"type": "file", "file": open("image.png", "rb")}
+{"type": "file", "file": pdf_bytes, "mime_type": "application/pdf"}
+```
+
+**For file uploads (Proxy — multipart form):**
+```bash
+curl http://0.0.0.0:4000/v1/ocr \
+  -H "Authorization: Bearer sk-1234" \
+  -F "model=mistral-ocr" \
+  -F "file=@document.pdf"
+```
+
 ### Response Format
 
 The response follows Mistral's OCR format with the following structure:

docs/my-website/docs/pass_through/assembly_ai.md

@@ -1,31 +1,36 @@
-# Assembly AI
+# AssemblyAI
 
-Pass-through endpoints for Assembly AI - call Assembly AI endpoints, in native format (no translation).
+Pass-through endpoints for AssemblyAI - call AssemblyAI endpoints, in native format (no translation).
 
-| Feature | Supported | Notes | 
+| Feature | Supported | Notes |
 |-------|-------|-------|
 | Cost Tracking | ✅ | works across all integrations |
 | Logging | ✅ | works across all integrations |
 
 
-Supports **ALL** Assembly AI Endpoints
+Supports **ALL** AssemblyAI Endpoints
 
-[**See All Assembly AI Endpoints**](https://www.assemblyai.com/docs/api-reference)
+[**See All AssemblyAI Endpoints**](https://www.assemblyai.com/docs/api-reference)
 
 
-<iframe width="840" height="500" src="https://www.loom.com/embed/aac3f4d74592448992254bfa79b9f62d?sid=267cd0ab-d92b-42fa-b97a-9f385ef8930c" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>
+## Supported Routes
+
+| AssemblyAI Service | LiteLLM Route | AssemblyAI Base URL |
+|-------------------|---------------|---------------------|
+| Speech-to-Text (US) | `/assemblyai/*` | `api.assemblyai.com` |
+| Speech-to-Text (EU) | `/eu.assemblyai/*` | `eu.api.assemblyai.com` |
 
 ## Quick Start
 
-Let's call the Assembly AI [`/v2/transcripts` endpoint](https://www.assemblyai.com/docs/api-reference/transcripts)
+Let's call the AssemblyAI [`/v2/transcripts` endpoint](https://www.assemblyai.com/docs/api-reference/transcripts)
 
-1. Add Assembly AI API Key to your environment 
+1. Add AssemblyAI API Key to your environment
 
 ```bash
 export ASSEMBLYAI_API_KEY=""
 ```
 
-2. Start LiteLLM Proxy 
+2. Start LiteLLM Proxy
 
 ```bash
 litellm
@@ -33,53 +38,157 @@ litellm
 # RUNNING on http://0.0.0.0:4000
 ```
 
-3. Test it! 
+3. Test it!
 
-Let's call the Assembly AI `/v2/transcripts` endpoint
+Let's call the AssemblyAI [`/v2/transcripts` endpoint](https://www.assemblyai.com/docs/api-reference/transcripts). Includes commented-out [Speech Understanding](https://www.assemblyai.com/docs/speech-understanding) features you can toggle on.
 
 ```python
 import assemblyai as aai
 
-LITELLM_VIRTUAL_KEY = "sk-1234" # <your-virtual-key>
-LITELLM_PROXY_BASE_URL = "http://0.0.0.0:4000/assemblyai" # <your-proxy-base-url>/assemblyai
+aai.settings.base_url = "http://0.0.0.0:4000/assemblyai" # <your-proxy-base-url>/assemblyai
+aai.settings.api_key = "Bearer sk-1234" # Bearer <your-virtual-key>
 
-aai.settings.api_key = f"Bearer {LITELLM_VIRTUAL_KEY}"
-aai.settings.base_url = LITELLM_PROXY_BASE_URL
+# Use a publicly-accessible URL
+audio_file = "https://assembly.ai/wildfires.mp3"
 
-# URL of the file to transcribe
-FILE_URL = "https://assembly.ai/wildfires.mp3"
+# Or use a local file:
+# audio_file = "./example.mp3"
 
-# You can also transcribe a local file by passing in a file path
-# FILE_URL = './path/to/file.mp3'
+config = aai.TranscriptionConfig(
+    speech_models=["universal-3-pro", "universal-2"],
+    language_detection=True,
+    speaker_labels=True,
+    # Speech understanding features
+    # sentiment_analysis=True,
+    # entity_detection=True,
+    # auto_chapters=True,
+    # summarization=True,
+    # summary_type=aai.SummarizationType.bullets,
+    # redact_pii=True,
+    # content_safety=True,
+)
 
-transcriber = aai.Transcriber()
-transcript = transcriber.transcribe(FILE_URL)
-print(transcript)
-print(transcript.id)
+transcript = aai.Transcriber().transcribe(audio_file, config=config)
+
+if transcript.status == aai.TranscriptStatus.error:
+    raise RuntimeError(f"Transcription failed: {transcript.error}")
+
+print(f"\nFull Transcript:\n\n{transcript.text}")
+
+# Optionally print speaker diarization results
+# for utterance in transcript.utterances:
+#     print(f"Speaker {utterance.speaker}: {utterance.text}")
 ```
 
-## Calling Assembly AI EU endpoints
+4. [Prompting with Universal-3 Pro](https://www.assemblyai.com/docs/speech-to-text/prompting) (optional)
 
-If you want to send your request to the Assembly AI EU endpoint, you can do so by setting the `LITELLM_PROXY_BASE_URL` to `<your-proxy-base-url>/eu.assemblyai`
+```python
+import assemblyai as aai
+
+aai.settings.base_url = "http://0.0.0.0:4000/assemblyai" # <your-proxy-base-url>/assemblyai
+aai.settings.api_key = "Bearer sk-1234" # Bearer <your-virtual-key>
+
+audio_file = "https://assemblyaiassets.com/audios/verbatim.mp3"
+
+config = aai.TranscriptionConfig(
+    speech_models=["universal-3-pro", "universal-2"],
+    language_detection=True,
+    prompt="Produce a transcript suitable for conversational analysis. Every disfluency is meaningful data. Include: fillers (um, uh, er, ah, hmm, mhm, like, you know, I mean), repetitions (I I, the the), restarts (I was- I went), stutters (th-that, b-but, no-not), and informal speech (gonna, wanna, gotta)",
+)
+
+transcript = aai.Transcriber().transcribe(audio_file, config)
+
+print(transcript.text)
+```
+
+## Calling AssemblyAI EU endpoints
+
+If you want to send your request to the AssemblyAI EU endpoint, you can do so by setting the `LITELLM_PROXY_BASE_URL` to `<your-proxy-base-url>/eu.assemblyai`
 
 
 ```python
 import assemblyai as aai
 
-LITELLM_VIRTUAL_KEY = "sk-1234" # <your-virtual-key>
-LITELLM_PROXY_BASE_URL = "http://0.0.0.0:4000/eu.assemblyai" # <your-proxy-base-url>/eu.assemblyai
+aai.settings.base_url = "http://0.0.0.0:4000/eu.assemblyai" # <your-proxy-base-url>/eu.assemblyai
+aai.settings.api_key = "Bearer sk-1234" # Bearer <your-virtual-key>
 
-aai.settings.api_key = f"Bearer {LITELLM_VIRTUAL_KEY}"
-aai.settings.base_url = LITELLM_PROXY_BASE_URL
+# Use a publicly-accessible URL
+audio_file = "https://assembly.ai/wildfires.mp3"
 
-# URL of the file to transcribe
-FILE_URL = "https://assembly.ai/wildfires.mp3"
-
-# You can also transcribe a local file by passing in a file path
-# FILE_URL = './path/to/file.mp3'
+# Or use a local file:
+# audio_file = "./path/to/file.mp3"
 
 transcriber = aai.Transcriber()
-transcript = transcriber.transcribe(FILE_URL)
+transcript = transcriber.transcribe(audio_file)
 print(transcript)
 print(transcript.id)
 ```
+
+## LLM Gateway
+
+Use AssemblyAI's [LLM Gateway](https://www.assemblyai.com/docs/llm-gateway) as an OpenAI-compatible provider — a unified API for Claude, GPT, and Gemini models with full LiteLLM logging, guardrails, and cost tracking support.
+
+[**See Available Models**](https://www.assemblyai.com/docs/llm-gateway#available-models)
+
+### Usage
+
+#### LiteLLM Python SDK
+
+```python
+import litellm
+import os
+
+os.environ["ASSEMBLYAI_API_KEY"] = "your-assemblyai-api-key"
+
+response = litellm.completion(
+    model="assemblyai/claude-sonnet-4-5-20250929",
+    messages=[{"role": "user", "content": "What is the capital of France?"}]
+)
+
+print(response.choices[0].message.content)
+```
+
+#### LiteLLM Proxy
+
+1. Config
+
+```yaml
+model_list:
+  - model_name: assemblyai/*
+    litellm_params:
+      model: assemblyai/*
+      api_key: os.environ/ASSEMBLYAI_API_KEY
+```
+
+2. Start proxy
+
+```bash
+litellm --config config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+3. Test it!
+
+```python
+import requests
+
+headers = {
+    "authorization": "Bearer sk-1234"  # Bearer <your-virtual-key>
+}
+
+response = requests.post(
+    "http://0.0.0.0:4000/v1/chat/completions",
+    headers=headers,
+    json={
+        "model": "assemblyai/claude-sonnet-4-5-20250929",
+        "messages": [
+            {"role": "user", "content": "What is the capital of France?"}
+        ],
+        "max_tokens": 1000
+    }
+)
+
+result = response.json()
+print(result["choices"][0]["message"]["content"])
+```

docs/my-website/docs/pass_through/cursor.md

@@ -0,0 +1,157 @@
+import Image from '@theme/IdealImage';
+
+# Cursor Cloud Agents
+
+Pass-through endpoints for the [Cursor Cloud Agents API](https://docs.cursor.com/account/api) — launch and manage cloud agents that work on your repositories, in native format (no translation).
+
+| Feature | Supported | Notes |
+|---------|-----------|-------|
+| Cost Tracking | ✅ | Logged as $0.00 (subscription-based, no per-request pricing) |
+| Logging | ✅ | All requests logged with operation classification |
+| End-user Tracking | ❌ | [Tell us if you need this](https://github.com/BerriAI/litellm/issues/new) |
+| Streaming | ❌ | Cursor API does not use streaming |
+
+Just replace `https://api.cursor.com` with `LITELLM_PROXY_BASE_URL/cursor` 🚀
+
+**Supported endpoints:**
+
+| Endpoint | Method | Description |
+|----------|--------|-------------|
+| `/v0/agents` | GET | List agents |
+| `/v0/agents` | POST | Launch an agent |
+| `/v0/agents/{id}` | GET | Agent status |
+| `/v0/agents/{id}` | DELETE | Delete an agent |
+| `/v0/agents/{id}/conversation` | GET | Agent conversation |
+| `/v0/agents/{id}/followup` | POST | Add follow-up |
+| `/v0/agents/{id}/stop` | POST | Stop an agent |
+| `/v0/me` | GET | API key info |
+| `/v0/models` | GET | List models |
+| `/v0/repositories` | GET | List GitHub repositories |
+
+## Quick Start
+
+### 1. Add Cursor API Key on the UI
+
+Navigate to **Models + Endpoints → LLM Credentials** and click **Add Credential**. Select **Cursor** from the provider dropdown — you'll see the Cursor logo. Enter your API key from [cursor.com/settings](https://cursor.com/settings).
+
+<Image img={require('../../img/cursor_add_credential.png')} alt="Add Cursor credential with logo" style={{maxWidth: '800px'}} />
+
+### 2. Launch a Cursor Agent
+
+```bash
+curl -X POST http://0.0.0.0:4000/cursor/v0/agents \
+  -H "Authorization: Bearer <your-litellm-key>" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "prompt": {
+      "text": "Add a README.md with installation instructions"
+    },
+    "source": {
+      "repository": "https://github.com/your-org/your-repo",
+      "ref": "main"
+    },
+    "target": {
+      "autoCreatePr": true
+    }
+  }'
+```
+
+**Expected Response:**
+
+```json
+{
+  "id": "bc_abc123",
+  "name": "Add README Documentation",
+  "status": "CREATING",
+  "source": {
+    "repository": "https://github.com/your-org/your-repo",
+    "ref": "main"
+  },
+  "target": {
+    "branchName": "cursor/add-readme-1234",
+    "url": "https://cursor.com/agents?id=bc_abc123",
+    "autoCreatePr": true
+  },
+  "createdAt": "2024-01-15T10:30:00Z"
+}
+```
+
+### 3. View Logs
+
+Navigate to **Logs** in the sidebar. Filter by "cursor" to see your agent requests. Each request shows the operation type (e.g., `cursor/cursor:agent:create`), status, duration, and cost.
+
+<Image img={require('../../img/cursor_logs.png')} alt="Cursor requests in Logs page" style={{maxWidth: '800px'}} />
+
+Click on any log entry to see full request details including provider, API base, and metadata.
+
+<Image img={require('../../img/cursor_log_detail.png')} alt="Cursor log entry detail" style={{maxWidth: '800px'}} />
+
+## Examples
+
+Anything after `http://0.0.0.0:4000/cursor` is treated as a provider-specific route, and handled accordingly.
+
+| **Original Endpoint** | **Replace With** |
+|---|---|
+| `https://api.cursor.com` | `http://0.0.0.0:4000/cursor` (LITELLM_PROXY_BASE_URL) |
+| `-u YOUR_API_KEY:` (Basic Auth) | `-H "Authorization: Bearer <your-litellm-key>"` (LiteLLM Virtual Key) |
+
+### List Available Models
+
+```bash
+curl http://0.0.0.0:4000/cursor/v0/models \
+  -H "Authorization: Bearer <your-litellm-key>"
+```
+
+### Check Agent Status
+
+```bash
+curl http://0.0.0.0:4000/cursor/v0/agents/bc_abc123 \
+  -H "Authorization: Bearer <your-litellm-key>"
+```
+
+### List All Agents
+
+```bash
+curl http://0.0.0.0:4000/cursor/v0/agents \
+  -H "Authorization: Bearer <your-litellm-key>"
+```
+
+### Add Follow-up to Agent
+
+```bash
+curl -X POST http://0.0.0.0:4000/cursor/v0/agents/bc_abc123/followup \
+  -H "Authorization: Bearer <your-litellm-key>" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "prompt": {
+      "text": "Also add a section about troubleshooting"
+    }
+  }'
+```
+
+### Stop an Agent
+
+```bash
+curl -X POST http://0.0.0.0:4000/cursor/v0/agents/bc_abc123/stop \
+  -H "Authorization: Bearer <your-litellm-key>"
+```
+
+### Delete an Agent
+
+```bash
+curl -X DELETE http://0.0.0.0:4000/cursor/v0/agents/bc_abc123 \
+  -H "Authorization: Bearer <your-litellm-key>"
+```
+
+### Get API Key Info
+
+```bash
+curl http://0.0.0.0:4000/cursor/v0/me \
+  -H "Authorization: Bearer <your-litellm-key>"
+```
+
+## Related
+
+- [Cursor Cloud Agents API Docs](https://docs.cursor.com/account/api)
+- [Pass-through Endpoints Overview](./intro.md)
+- [Virtual Keys](../proxy/virtual_keys.md)

docs/my-website/docs/pass_through/google_ai_studio.md

@@ -35,26 +35,25 @@ curl 'http://0.0.0.0:4000/gemini/v1beta/models/gemini-1.5-flash:countTokens?key=
 ```
 
 </TabItem>
-<TabItem value="js" label="Google AI Node.js SDK">
+<TabItem value="js" label="Google GenAI JS SDK">
 
 ```javascript
-const { GoogleGenerativeAI } = require("@google/generative-ai");
+const { GoogleGenAI } = require("@google/genai");
 
-const modelParams = {
-    model: 'gemini-pro',
-};
-  
-const requestOptions = {
-    baseUrl: 'http://localhost:4000/gemini', // http://<proxy-base-url>/gemini
-};
-  
-const genAI = new GoogleGenerativeAI("sk-1234"); // litellm proxy API key
-const model = genAI.getGenerativeModel(modelParams, requestOptions);
+const ai = new GoogleGenAI({
+    apiKey: "sk-1234", // litellm proxy API key
+    httpOptions: {
+        baseUrl: "http://localhost:4000/gemini", // http://<proxy-base-url>/gemini
+    },
+});
 
 async function main() {
     try {
-        const result = await model.generateContent("Explain how AI works");
-        console.log(result.response.text());
+        const response = await ai.models.generateContent({
+            model: "gemini-2.5-flash",
+            contents: "Explain how AI works",
+        });
+        console.log(response.text);
     } catch (error) {
         console.error('Error:', error);
     }
@@ -63,12 +62,13 @@ async function main() {
 // For streaming responses
 async function main_streaming() {
     try {
-        const streamingResult = await model.generateContentStream("Explain how AI works");
-        for await (const chunk of streamingResult.stream) {
-            console.log('Stream chunk:', JSON.stringify(chunk));
+        const response = await ai.models.generateContentStream({
+            model: "gemini-2.5-flash",
+            contents: "Explain how AI works",
+        });
+        for await (const chunk of response) {
+            process.stdout.write(chunk.text);
         }
-        const aggregatedResponse = await streamingResult.response;
-        console.log('Aggregated response:', JSON.stringify(aggregatedResponse));
     } catch (error) {
         console.error('Error:', error);
     }
@@ -321,29 +321,28 @@ curl 'http://0.0.0.0:4000/gemini/v1beta/models/gemini-1.5-flash:generateContent?
 ```
 
 </TabItem>
-<TabItem value="js" label="Google AI Node.js SDK">
+<TabItem value="js" label="Google GenAI JS SDK">
 
 ```javascript
-const { GoogleGenerativeAI } = require("@google/generative-ai");
+const { GoogleGenAI } = require("@google/genai");
 
-const modelParams = {
-    model: 'gemini-pro',
-};
-  
-const requestOptions = {
-    baseUrl: 'http://localhost:4000/gemini', // http://<proxy-base-url>/gemini
-    customHeaders: {
-        "tags": "gemini-js-sdk,pass-through-endpoint"
-    }
-};
-  
-const genAI = new GoogleGenerativeAI("sk-1234");
-const model = genAI.getGenerativeModel(modelParams, requestOptions);
+const ai = new GoogleGenAI({
+    apiKey: "sk-1234",
+    httpOptions: {
+        baseUrl: "http://localhost:4000/gemini", // http://<proxy-base-url>/gemini
+        headers: {
+            "tags": "gemini-js-sdk,pass-through-endpoint",
+        },
+    },
+});
 
 async function main() {
     try {
-        const result = await model.generateContent("Explain how AI works");
-        console.log(result.response.text());
+        const response = await ai.models.generateContent({
+            model: "gemini-2.5-flash",
+            contents: "Explain how AI works",
+        });
+        console.log(response.text);
     } catch (error) {
         console.error('Error:', error);
     }

docs/my-website/docs/providers/anthropic.md

@@ -4,6 +4,8 @@ import TabItem from '@theme/TabItem';
 # Anthropic
 LiteLLM supports all anthropic models.
 
+- `claude-opus-4-6` (`claude-opus-4-6-20260205`)
+- `claude-sonnet-4-6`
 - `claude-sonnet-4-5-20250929`
 - `claude-opus-4-5-20251101`
 - `claude-opus-4-1-20250805`
@@ -50,7 +52,7 @@ Check this in code, [here](../completion/input.md#translated-openai-params)
 **Notes:**
 - Anthropic API fails requests when `max_tokens` are not passed. Due to this litellm passes `max_tokens=4096` when no `max_tokens` are passed.
 - `response_format` is fully supported for Claude Sonnet 4.5 and Opus 4.1 models (see [Structured Outputs](#structured-outputs) section)
-- `reasoning_effort` is automatically mapped to `output_config={"effort": ...}` for Claude Opus 4.5 models (see [Effort Parameter](./anthropic_effort.md))
+- `reasoning_effort` is automatically mapped to `output_config={"effort": ...}` for Claude 4.6 and Opus 4.5 models (see [Effort Parameter](./anthropic_effort.md))
 
 :::
 
@@ -415,7 +417,10 @@ print(response)
 
 | Model Name       | Function Call                              |
 |------------------|--------------------------------------------|
+| claude-opus-4-6  | `completion('claude-opus-4-6-20260205', messages)` | `os.environ['ANTHROPIC_API_KEY']`       |
 | claude-sonnet-4-5  | `completion('claude-sonnet-4-5-20250929', messages)` | `os.environ['ANTHROPIC_API_KEY']`       |
+| claude-opus-4-5  | `completion('claude-opus-4-5-20251101', messages)` | `os.environ['ANTHROPIC_API_KEY']`       |
+| claude-opus-4-1  | `completion('claude-opus-4-1-20250805', messages)` | `os.environ['ANTHROPIC_API_KEY']`       |
 | claude-opus-4  | `completion('claude-opus-4-20250514', messages)` | `os.environ['ANTHROPIC_API_KEY']`       |
 | claude-sonnet-4  | `completion('claude-sonnet-4-20250514', messages)` | `os.environ['ANTHROPIC_API_KEY']`       |
 | claude-3.7  | `completion('claude-3-7-sonnet-20250219', messages)` | `os.environ['ANTHROPIC_API_KEY']`       |

docs/my-website/docs/providers/anthropic_effort.md

@@ -9,10 +9,11 @@ Control how many tokens Claude uses when responding with the `effort` parameter,
 
 The `effort` parameter allows you to control how eager Claude is about spending tokens when responding to requests. This gives you the ability to trade off between response thoroughness and token efficiency, all with a single model.
 
-**Note**: The effort parameter is currently in beta and only supported by Claude Opus 4.5. LiteLLM automatically adds the `effort-2025-11-24` beta header when:
-- `reasoning_effort` parameter is provided (for Claude Opus 4.5 only)
+**Supported models:**
+- **Claude 4.6** (Opus 4.6, Sonnet 4.6) — `output_config` is a stable API feature, no beta header needed. Opus 4.6 also supports `effort="max"`.
+- **Claude Opus 4.5** — requires the `effort-2025-11-24` beta header (automatically added by LiteLLM).
 
-For Claude Opus 4.5, `reasoning_effort="medium"`—both are automatically mapped to the correct format.
+LiteLLM automatically maps `reasoning_effort` → `output_config={"effort": ...}` for all supported models.
 
 ## How Effort Works
 
@@ -35,6 +36,7 @@ This gives a much greater degree of control over efficiency.
 
 | Level | Description | Typical use case |
 |-------|-------------|------------------|
+| `max` | Maximum capability beyond high — Claude uses even more tokens for the most thorough outcome. **Only supported by Claude Opus 4.6.** | The hardest reasoning problems, complex multi-step research |
 | `high` | Maximum capability—Claude uses as many tokens as needed for the best possible outcome. Equivalent to not setting the parameter. | Complex reasoning, difficult coding problems, agentic tasks |
 | `medium` | Balanced approach with moderate token savings. | Agentic tasks that require a balance of speed, cost, and performance |
 | `low` | Most efficient—significant token savings with some capability reduction. | Simpler tasks that need the best speed and lowest costs, such as subagents |
@@ -49,16 +51,29 @@ This gives a much greater degree of control over efficiency.
 ```python
 import litellm
 
+# Works with Claude 4.6 models (no beta header needed)
+response = litellm.completion(
+    model="anthropic/claude-sonnet-4-6",
+    messages=[{
+        "role": "user",
+        "content": "Analyze the trade-offs between microservices and monolithic architectures"
+    }],
+    reasoning_effort="medium"  # Automatically mapped to output_config
+)
+
+print(response.choices[0].message.content)
+```
+
+```python
+# Also works with Claude Opus 4.5 (beta header auto-injected)
 response = litellm.completion(
     model="anthropic/claude-opus-4-5-20251101",
     messages=[{
         "role": "user",
         "content": "Analyze the trade-offs between microservices and monolithic architectures"
     }],
-    reasoning_effort="medium"  # Automatically mapped to output_config for Opus 4.5
+    reasoning_effort="medium"
 )
-
-print(response.choices[0].message.content)
 ```
 
 </TabItem>
@@ -71,8 +86,9 @@ const client = new Anthropic({
   apiKey: process.env.ANTHROPIC_API_KEY,
 });
 
+// Claude 4.6 — output_config is a stable API feature (no beta header)
 const response = await client.messages.create({
-  model: "claude-opus-4-5-20251101",
+  model: "claude-sonnet-4-6",
   max_tokens: 4096,
   messages: [{
     role: "user",
@@ -96,7 +112,29 @@ curl http://localhost:4000/v1/chat/completions \
   -H "Content-Type: application/json" \
   -H "Authorization: Bearer $LITELLM_API_KEY" \
   -d '{
-    "model": "anthropic/claude-opus-4-5-20251101",
+    "model": "anthropic/claude-sonnet-4-6",
+    "messages": [{
+      "role": "user",
+      "content": "Analyze the trade-offs between microservices and monolithic architectures"
+    }],
+    "reasoning_effort": "medium"
+  }'
+```
+
+### Direct Anthropic API Call
+
+<Tabs>
+<TabItem value="46" label="Claude 4.6 (stable)">
+
+```bash
+# Claude 4.6 — no beta header needed
+curl https://api.anthropic.com/v1/messages \
+  --header "x-api-key: $ANTHROPIC_API_KEY" \
+  --header "anthropic-version: 2023-06-01" \
+  --header "content-type: application/json" \
+  --data '{
+    "model": "claude-sonnet-4-6",
+    "max_tokens": 4096,
     "messages": [{
       "role": "user",
       "content": "Analyze the trade-offs between microservices and monolithic architectures"
@@ -107,9 +145,11 @@ curl http://localhost:4000/v1/chat/completions \
   }'
 ```
 
-### Direct Anthropic API Call
+</TabItem>
+<TabItem value="45" label="Claude Opus 4.5 (beta)">
 
 ```bash
+# Claude Opus 4.5 — requires beta header
 curl https://api.anthropic.com/v1/messages \
   --header "x-api-key: $ANTHROPIC_API_KEY" \
   --header "anthropic-version: 2023-06-01" \
@@ -128,10 +168,19 @@ curl https://api.anthropic.com/v1/messages \
   }'
 ```
 
+</TabItem>
+</Tabs>
+
 ## Model Compatibility
 
-The effort parameter is currently only supported by:
-- **Claude Opus 4.5** (`claude-opus-4-5-20251101`)
+The effort parameter is supported by:
+- **Claude Opus 4.6** (`claude-opus-4-6`) — supports `high`, `medium`, `low`, and `max`
+- **Claude Sonnet 4.6** (`claude-sonnet-4-6`) — supports `high`, `medium`, `low`
+- **Claude Opus 4.5** (`claude-opus-4-5-20251101`) — supports `high`, `medium`, `low`
+
+:::info
+`effort="max"` is only available on Claude Opus 4.6. Using it with other models will raise a validation error.
+:::
 
 ## When Should I Adjust the Effort Parameter?
 
@@ -154,7 +203,7 @@ Example with tools:
 import litellm
 
 response = litellm.completion(
-    model="anthropic/claude-opus-4-5-20251101",
+    model="anthropic/claude-sonnet-4-6",
     messages=[{
         "role": "user",
         "content": "Check the weather in multiple cities"
@@ -173,9 +222,7 @@ response = litellm.completion(
             }
         }
     }],
-    output_config={
-        "effort": "low"  # Will make fewer tool calls
-    }
+    reasoning_effort="low"  # Mapped to output_config — will make fewer tool calls
 )
 ```
 
@@ -187,18 +234,12 @@ The effort parameter works seamlessly with extended thinking. When both are enab
 import litellm
 
 response = litellm.completion(
-    model="anthropic/claude-opus-4-5-20251101",
+    model="anthropic/claude-sonnet-4-6",
     messages=[{
         "role": "user",
         "content": "Solve this complex problem"
     }],
-    thinking={
-        "type": "enabled",
-        "budget_tokens": 5000
-    },
-    output_config={
-        "effort": "medium"  # Affects both thinking and response tokens
-    }
+    reasoning_effort="medium"  # Mapped to adaptive thinking + output_config for 4.6 models
 )
 ```
 
@@ -218,14 +259,14 @@ response = litellm.completion(
 
 The effort parameter is supported across all Anthropic-compatible providers:
 
-- **Standard Anthropic API**: ✅ Supported (Claude Opus 4.5)
-- **Azure Anthropic / Microsoft Foundry**: ✅ Supported (Claude Opus 4.5)
-- **Amazon Bedrock**: ✅ Supported (Claude Opus 4.5)
-- **Google Cloud Vertex AI**: ✅ Supported (Claude Opus 4.5)
+- **Standard Anthropic API**: ✅ Supported (Claude 4.6, Opus 4.5)
+- **Azure Anthropic / Microsoft Foundry**: ✅ Supported (Claude 4.6, Opus 4.5)
+- **Amazon Bedrock**: ✅ Supported (Claude 4.6, Opus 4.5)
+- **Google Cloud Vertex AI**: ✅ Supported (Claude 4.6, Opus 4.5)
 
 LiteLLM automatically handles:
-- Beta header injection (`effort-2025-11-24`) for all providers
-- Parameter mapping: `reasoning_effort` → `output_config={"effort": ...}` for Claude Opus 4.5
+- Parameter mapping: `reasoning_effort` → `output_config={"effort": ...}` for all supported models
+- Beta header injection (`effort-2025-11-24`) only for Claude Opus 4.5 (not needed for 4.6 models)
 
 ## Usage and Pricing
 
@@ -244,12 +285,13 @@ print(f"Total tokens: {response.usage.total_tokens}")
 
 ## Troubleshooting
 
-### Beta header not being added
+### Beta header not being added (Claude Opus 4.5)
 
-LiteLLM automatically adds the `effort-2025-11-24` beta header when:
-- `reasoning_effort` parameter is provided (for Claude Opus 4.5 only)
+LiteLLM automatically adds the `effort-2025-11-24` beta header for Claude Opus 4.5 when `reasoning_effort` or `output_config` is provided.
 
-If you're not seeing the header:
+**Note:** Claude 4.6 models do NOT need a beta header — `output_config` is a stable API feature for these models.
+
+If you're not seeing the header for Opus 4.5:
 
 1. Ensure you're using `reasoning_effort` parameter
 2. Verify the model is Claude Opus 4.5
@@ -257,7 +299,7 @@ If you're not seeing the header:
 
 ### Invalid effort value error
 
-Only three values are accepted: `"high"`, `"medium"`, `"low"`. Any other value will raise a validation error:
+Accepted values: `"high"`, `"medium"`, `"low"`, and `"max"` (Opus 4.6 only). Any other value will raise a validation error:
 
 ```python
 # ❌ This will raise an error
@@ -265,11 +307,17 @@ output_config={"effort": "very_low"}
 
 # ✅ Use one of the valid values
 output_config={"effort": "low"}
+
+# ❌ This will raise an error (max only works on Opus 4.6)
+litellm.completion(model="anthropic/claude-sonnet-4-6", reasoning_effort="max", ...)
+
+# ✅ max is only for Opus 4.6
+litellm.completion(model="anthropic/claude-opus-4-6", reasoning_effort="max", ...)
 ```
 
 ### Model not supported
 
-Currently, only Claude Opus 4.5 supports the effort parameter. Using it with other models may result in the parameter being ignored or an error.
+The effort parameter is supported by Claude Opus 4.6, Sonnet 4.6, and Opus 4.5. Using it with other models may result in the parameter being ignored or an error.
 
 ## Related Features
 

docs/my-website/docs/providers/azure_ai/azure_model_router.md

@@ -2,6 +2,32 @@
 
 Azure Model Router is a feature in Azure AI Foundry that automatically routes your requests to the best available model based on your requirements. This allows you to use a single endpoint that intelligently selects the optimal model for each request.
 
+## Quick Start
+
+**Model pattern**: `azure_ai/model_router/<deployment-name>`
+
+```python
+import litellm
+
+response = litellm.completion(
+    model="azure_ai/model_router/model-router",  # Replace with your deployment name
+    messages=[{"role": "user", "content": "Hello!"}],
+    api_base="https://your-endpoint.cognitiveservices.azure.com/openai/v1/",
+    api_key="your-api-key",
+)
+```
+
+**Proxy config** (`config.yaml`):
+
+```yaml
+model_list:
+  - model_name: model-router
+    litellm_params:
+      model: azure_ai/model_router/model-router
+      api_base: https://your-endpoint.cognitiveservices.azure.com/openai/deployments/model-router/chat/completions?api-version=2025-01-01-preview
+      api_key: your-api-key
+```
+
 ## Key Features
 
 - **Automatic Model Selection**: Azure Model Router dynamically selects the best model for your request
@@ -229,19 +255,51 @@ Cost is tracked based on the actual model used (e.g., `gpt-4.1-nano`), plus a fl
 
 ## Cost Tracking
 
-LiteLLM automatically handles cost tracking for Azure Model Router by:
+LiteLLM automatically handles cost tracking for Azure Model Router. Understanding how this works helps you interpret spend and debug billing.
 
-1. **Detecting the actual model**: When Azure Model Router routes your request to a specific model (e.g., `gpt-4.1-nano-2025-04-14`), LiteLLM extracts this from the response
-2. **Calculating accurate costs**: Costs are calculated based on:
-   - The actual model used (e.g., `gpt-4.1-nano` token costs)
-   - Plus a flat infrastructure cost of **$0.14 per million input tokens** for using the Model Router
-3. **Streaming support**: Cost tracking works correctly for both streaming and non-streaming requests
+### How LiteLLM Calculates Cost
+
+When you use Azure Model Router, LiteLLM computes **two cost components**:
+
+| Component | Description | When Applied |
+|-----------|-------------|--------------|
+| **Model Cost** | Token-based cost for the actual model that handled the request (e.g., `gpt-5-nano`, `gpt-4.1-nano`) | Always, when Azure returns the model in the response |
+| **Router Flat Cost** | $0.14 per million input tokens (Azure AI Foundry infrastructure fee) | When the **request** was made via a model router endpoint |
+
+### Cost Calculation Flow
+
+1. **Request model detection**: LiteLLM records the model you requested (e.g., `azure_ai/model_router/model-router`). If it contains `model_router` or `model-router`, the request is treated as a router request.
+
+2. **Response model extraction**: Azure returns the actual model used in the response (e.g., `gpt-5-nano-2025-08-07`). LiteLLM uses this for the model cost lookup.
+
+3. **Model cost**: LiteLLM looks up the response model in its pricing table and computes cost from prompt tokens and completion tokens.
+
+4. **Router flat cost**: Because the original request was to a model router, LiteLLM adds the flat cost ($0.14 per M input tokens) on top of the model cost.
+
+5. **Total cost**: `Total = Model Cost + Router Flat Cost`
+
+### Configuration Requirements
+
+For cost tracking to work correctly:
+
+- **Use the full pattern**: `azure_ai/model_router/<deployment-name>` (e.g., `azure_ai/model_router/model-router`)
+- **Proxy config**: When using the LiteLLM proxy, set `model` in `litellm_params` to the full pattern so the request model is correctly identified as a router
+
+```yaml
+# proxy_server_config.yaml
+model_list:
+  - model_name: model-router
+    litellm_params:
+      model: azure_ai/model_router/model-router  # Required for router cost detection
+      api_base: https://your-endpoint.cognitiveservices.azure.com/openai/deployments/model-router/chat/completions?api-version=2025-01-01-preview
+      api_key: your-api-key
+```
 
 ### Cost Breakdown
 
 When you use Azure Model Router, the total cost includes:
 
-- **Model Cost**: Based on the actual model that handled your request (e.g., `gpt-4.1-nano`)
+- **Model Cost**: Based on the actual model that handled your request (e.g., `gpt-5-nano`, `gpt-4.1-nano`)
 - **Router Flat Cost**: $0.14 per million input tokens (Azure AI Foundry infrastructure fee)
 
 ### Example Response with Cost

docs/my-website/docs/providers/bedrock.md

@@ -660,7 +660,7 @@ Same as [Anthropic API response](../providers/anthropic#usage---thinking--reason
 
 LiteLLM supports Anthropic's beta features on AWS Bedrock through the `anthropic-beta` header. This enables access to experimental features like:
 
-- **1M Context Window** - Up to 1 million tokens of context (Claude Sonnet 4)
+- **1M Context Window** - Up to 1 million tokens of context (Claude Opus 4.6, Sonnet 4.5, Sonnet 4)
 - **Computer Use Tools** - AI that can interact with computer interfaces
 - **Token-Efficient Tools** - More efficient tool usage patterns  
 - **Extended Output** - Up to 128K output tokens
@@ -670,7 +670,7 @@ LiteLLM supports Anthropic's beta features on AWS Bedrock through the `anthropic
 
 | Beta Feature | Header Value | Compatible Models | Description |
 |--------------|-------------|------------------|-------------|
-| 1M Context Window | `context-1m-2025-08-07` | Claude Sonnet 4 | Enable 1 million token context window |
+| 1M Context Window | `context-1m-2025-08-07` | Claude Opus 4.6, Sonnet 4.5, Sonnet 4 | Enable 1 million token context window |
 | Computer Use (Latest) | `computer-use-2025-01-24` | Claude 3.7 Sonnet | Latest computer use tools |
 | Computer Use (Legacy) | `computer-use-2024-10-22` | Claude 3.5 Sonnet v2 | Computer use tools for Claude 3.5 |
 | Token-Efficient Tools | `token-efficient-tools-2025-02-19` | Claude 3.7 Sonnet | More efficient tool usage |

docs/my-website/docs/providers/bedrock_mantle.md

@@ -0,0 +1,157 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Amazon Bedrock Mantle
+
+[Amazon Bedrock Mantle](https://docs.aws.amazon.com/bedrock/latest/userguide/bedrock-mantle.html) is Amazon Bedrock's distributed inference engine (Project Mantle) that exposes an **OpenAI-compatible API** for Bedrock-hosted models.
+
+Use this provider to call Bedrock Mantle models with accurate **AWS Bedrock pricing** instead of OpenAI pricing.
+
+:::tip
+
+**We support ALL Bedrock Mantle models, just set `model=bedrock_mantle/<model-id>` as a prefix when sending litellm requests**
+
+:::
+
+## API Key
+
+```python
+# env variable
+os.environ['BEDROCK_MANTLE_API_KEY'] = "your-aws-bedrock-api-key"
+
+# optional: override region (defaults to us-east-1)
+os.environ['BEDROCK_MANTLE_REGION'] = "us-east-1"  # or use AWS_REGION
+```
+
+## Supported Models
+
+| Model | Context Window | Input (per 1M tokens) | Output (per 1M tokens) |
+|-------|---------------|----------------------|------------------------|
+| `openai.gpt-oss-120b` | 131K | $0.15 | $0.60 |
+| `openai.gpt-oss-20b` | 131K | $0.075 | $0.30 |
+| `openai.gpt-oss-safeguard-120b` | 131K | $0.15 | $0.60 |
+| `openai.gpt-oss-safeguard-20b` | 131K | $0.075 | $0.30 |
+
+## Sample Usage
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+from litellm import completion
+import os
+
+os.environ['BEDROCK_MANTLE_API_KEY'] = "your-bedrock-api-key"
+
+response = completion(
+    model="bedrock_mantle/openai.gpt-oss-120b",
+    messages=[{"role": "user", "content": "hello from litellm"}],
+)
+print(response)
+```
+
+</TabItem>
+<TabItem value="streaming" label="Streaming">
+
+```python
+from litellm import completion
+import os
+
+os.environ['BEDROCK_MANTLE_API_KEY'] = "your-bedrock-api-key"
+
+response = completion(
+    model="bedrock_mantle/openai.gpt-oss-120b",
+    messages=[{"role": "user", "content": "hello from litellm"}],
+    stream=True,
+)
+
+for chunk in response:
+    print(chunk)
+```
+
+</TabItem>
+<TabItem value="async" label="Async">
+
+```python
+import asyncio
+from litellm import acompletion
+import os
+
+os.environ['BEDROCK_MANTLE_API_KEY'] = "your-bedrock-api-key"
+
+async def main():
+    response = await acompletion(
+        model="bedrock_mantle/openai.gpt-oss-120b",
+        messages=[{"role": "user", "content": "hello from litellm"}],
+    )
+    print(response)
+
+asyncio.run(main())
+```
+
+</TabItem>
+</Tabs>
+
+## Region Configuration
+
+The API base URL is `https://bedrock-mantle.{region}.api.aws/v1`. Region is resolved in this order:
+
+1. `BEDROCK_MANTLE_REGION` env var
+2. `AWS_REGION` env var
+3. Default: `us-east-1`
+
+**Supported regions:** `us-east-1`, `us-east-2`, `us-west-2`, `eu-west-1`, `eu-west-2`, `eu-central-1`, `eu-south-1`, `eu-north-1`, `ap-northeast-1`, `ap-south-1`, `ap-southeast-3`, `sa-east-1`
+
+```python
+import os
+os.environ['BEDROCK_MANTLE_REGION'] = "eu-west-1"
+
+# or pass api_base directly
+response = completion(
+    model="bedrock_mantle/openai.gpt-oss-120b",
+    messages=[{"role": "user", "content": "hello"}],
+    api_base="https://bedrock-mantle.eu-west-1.api.aws/v1",
+)
+```
+
+## Usage with LiteLLM Proxy
+
+### 1. Set Bedrock Mantle models on config.yaml
+
+```yaml
+model_list:
+  - model_name: gpt-oss-120b
+    litellm_params:
+      model: bedrock_mantle/openai.gpt-oss-120b
+      api_key: os.environ/BEDROCK_MANTLE_API_KEY
+      # optional region override:
+      api_base: "https://bedrock-mantle.us-east-1.api.aws/v1"
+
+  - model_name: gpt-oss-20b
+    litellm_params:
+      model: bedrock_mantle/openai.gpt-oss-20b
+      api_key: os.environ/BEDROCK_MANTLE_API_KEY
+```
+
+### 2. Start the proxy
+
+```shell
+litellm --config /path/to/config.yaml
+```
+
+### 3. Send a request
+
+```python
+import openai
+
+client = openai.OpenAI(
+    api_key="anything",
+    base_url="http://0.0.0.0:4000",
+)
+
+response = client.chat.completions.create(
+    model="gpt-oss-120b",
+    messages=[{"role": "user", "content": "hello from litellm"}],
+)
+print(response)
+```

docs/my-website/docs/providers/chatgpt.md

@@ -4,12 +4,12 @@ Use ChatGPT Pro/Max subscription models through LiteLLM with OAuth device flow a
 
 | Property | Details |
 |-------|-------|
-| Description | ChatGPT subscription access (Codex + GPT-5.2 family) via ChatGPT backend API |
+| Description | ChatGPT subscription access (Codex + GPT-5.3/5.4 family) via ChatGPT backend API |
 | Provider Route on LiteLLM | `chatgpt/` |
 | Supported Endpoints | `/responses`, `/chat/completions` (bridged to Responses for supported models) |
 | API Reference | https://chatgpt.com |
 
-ChatGPT subscription access is native to the Responses API. Chat Completions requests are bridged to Responses for supported models (for example `chatgpt/gpt-5.2`).
+ChatGPT subscription access is native to the Responses API. Chat Completions requests are bridged to Responses for supported models (for example `chatgpt/gpt-5.4`).
 
 Notes:
 - The ChatGPT subscription backend rejects token limit fields (`max_tokens`, `max_output_tokens`, `max_completion_tokens`) and `metadata`. LiteLLM strips these fields for this provider.
@@ -31,7 +31,7 @@ ChatGPT subscription access uses an OAuth device code flow:
 import litellm
 
 response = litellm.responses(
-    model="chatgpt/gpt-5.2-codex",
+    model="chatgpt/gpt-5.3-codex",
     input="Write a Python hello world"
 )
 
@@ -44,7 +44,7 @@ print(response)
 import litellm
 
 response = litellm.completion(
-    model="chatgpt/gpt-5.2",
+    model="chatgpt/gpt-5.4",
     messages=[{"role": "user", "content": "Write a Python hello world"}]
 )
 
@@ -55,16 +55,36 @@ print(response)
 
 ```yaml showLineNumbers title="config.yaml"
 model_list:
-  - model_name: chatgpt/gpt-5.2
+  - model_name: chatgpt/gpt-5.4
     model_info:
       mode: responses
     litellm_params:
-      model: chatgpt/gpt-5.2
-  - model_name: chatgpt/gpt-5.2-codex
+      model: chatgpt/gpt-5.4
+  - model_name: chatgpt/gpt-5.4-pro
     model_info:
       mode: responses
     litellm_params:
-      model: chatgpt/gpt-5.2-codex
+      model: chatgpt/gpt-5.4-pro
+  - model_name: chatgpt/gpt-5.3-codex
+    model_info:
+      mode: responses
+    litellm_params:
+      model: chatgpt/gpt-5.3-codex
+  - model_name: chatgpt/gpt-5.3-codex-spark
+    model_info:
+      mode: responses
+    litellm_params:
+      model: chatgpt/gpt-5.3-codex-spark
+  - model_name: chatgpt/gpt-5.3-instant
+    model_info:
+      mode: responses
+    litellm_params:
+      model: chatgpt/gpt-5.3-instant
+  - model_name: chatgpt/gpt-5.3-chat-latest
+    model_info:
+      mode: responses
+    litellm_params:
+      model: chatgpt/gpt-5.3-chat-latest
 ```
 
 ```bash showLineNumbers title="Start LiteLLM Proxy"

docs/my-website/docs/providers/gemini.md

@@ -2041,6 +2041,7 @@ response = litellm.completion(
 | gemini-2.0-flash-lite-preview-02-05	     | `completion(model='gemini/gemini-2.0-flash-lite-preview-02-05', messages)`     | `os.environ['GEMINI_API_KEY']` |
 | gemini-2.5-flash-preview-09-2025     | `completion(model='gemini/gemini-2.5-flash-preview-09-2025', messages)`     | `os.environ['GEMINI_API_KEY']` |
 | gemini-2.5-flash-lite-preview-09-2025     | `completion(model='gemini/gemini-2.5-flash-lite-preview-09-2025', messages)`     | `os.environ['GEMINI_API_KEY']` |
+| gemini-3.1-flash-lite-preview     | `completion(model='gemini/gemini-3.1-flash-lite-preview', messages)`     | `os.environ['GEMINI_API_KEY']` |
 | gemini-flash-latest     | `completion(model='gemini/gemini-flash-latest', messages)`     | `os.environ['GEMINI_API_KEY']` |
 | gemini-flash-lite-latest     | `completion(model='gemini/gemini-flash-lite-latest', messages)`     | `os.environ['GEMINI_API_KEY']` |
 

docs/my-website/docs/providers/groq.md

@@ -159,6 +159,7 @@ We support ALL Groq models, just set `groq/` as a prefix when sending completion
 | moonshotai/kimi-k2-instruct-0905 | `completion(model="groq/moonshotai/kimi-k2-instruct-0905", messages)` |
 | openai/gpt-oss-120b | `completion(model="groq/openai/gpt-oss-120b", messages)` |
 | openai/gpt-oss-20b | `completion(model="groq/openai/gpt-oss-20b", messages)` |
+| openai/gpt-oss-safeguard-20b | `completion(model="groq/openai/gpt-oss-safeguard-20b", messages)` |
 
 ## Groq - Tool / Function Calling Example
 

docs/my-website/docs/providers/moonshot.md

@@ -219,6 +219,37 @@ curl http://localhost:4000/v1/chat/completions \
 
 For more detailed information on using the LiteLLM Proxy, see the [LiteLLM Proxy documentation](../providers/litellm_proxy).
 
+## Image / Vision Support
+
+Moonshot vision models (`kimi-k2.5`, `kimi-latest`, `moonshot-v1-*-vision-preview`, etc.) accept the standard OpenAI content array with `image_url` blocks.
+
+LiteLLM automatically detects when your messages contain images and preserves the content array so the image payload reaches the Moonshot API. For text-only requests the content is flattened to a plain string, as required by Moonshot text models.
+
+```python showLineNumbers title="Moonshot Vision Example"
+import os
+import litellm
+
+os.environ["MOONSHOT_API_KEY"] = ""
+
+response = litellm.completion(
+    model="moonshot/kimi-k2.5",
+    messages=[
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "What is in this image?"},
+                {
+                    "type": "image_url",
+                    "image_url": {"url": "https://example.com/image.png"},
+                },
+            ],
+        }
+    ],
+)
+
+print(response.choices[0].message.content)
+```
+
 ## Moonshot AI Limitations & LiteLLM Handling
 
 LiteLLM automatically handles the following [Moonshot AI limitations](https://platform.moonshot.ai/docs/guide/migrating-from-openai-to-kimi#about-api-compatibility) to provide seamless OpenAI compatibility:

docs/my-website/docs/providers/openai.md

@@ -191,8 +191,13 @@ os.environ["OPENAI_BASE_URL"] = "https://your_host/v1"     # OPTIONAL
 | gpt-5.2 | `response = completion(model="gpt-5.2", messages=messages)` |
 | gpt-5.2-2025-12-11 | `response = completion(model="gpt-5.2-2025-12-11", messages=messages)` |
 | gpt-5.2-chat-latest | `response = completion(model="gpt-5.2-chat-latest", messages=messages)` |
+| gpt-5.3-chat-latest | `response = completion(model="gpt-5.3-chat-latest", messages=messages)` |
+| gpt-5.4 | `response = completion(model="gpt-5.4", messages=messages)` |
+| gpt-5.4-2026-03-05 | `response = completion(model="gpt-5.4-2026-03-05", messages=messages)` |
 | gpt-5.2-pro | `response = completion(model="gpt-5.2-pro", messages=messages)` |
 | gpt-5.2-pro-2025-12-11 | `response = completion(model="gpt-5.2-pro-2025-12-11", messages=messages)` |
+| gpt-5.4-pro | `response = completion(model="gpt-5.4-pro", messages=messages)` |
+| gpt-5.4-pro-2026-03-05 | `response = completion(model="gpt-5.4-pro-2026-03-05", messages=messages)` |
 | gpt-5.1 | `response = completion(model="gpt-5.1", messages=messages)` |
 | gpt-5.1-codex | `response = completion(model="gpt-5.1-codex", messages=messages)` |
 | gpt-5.1-codex-mini | `response = completion(model="gpt-5.1-codex-mini", messages=messages)` |

docs/my-website/docs/providers/openrouter.md

@@ -210,3 +210,90 @@ response = image_generation(
 # Cost is available in the response metadata
 print(f"Request cost: ${response._hidden_params['additional_headers']['llm_provider-x-litellm-response-cost']}")
 ```
+
+## Image Edit
+
+OpenRouter supports image editing through select models like Google Gemini image models. LiteLLM routes image edit requests to OpenRouter's chat completions endpoint with the source image sent as a base64 data URL and `modalities: ["image", "text"]`.
+
+### Supported Models
+
+| Model | Description |
+|-------|-------------|
+| `openrouter/google/gemini-2.5-flash-image` | Gemini 2.5 Flash with image editing |
+
+See all available image models on [OpenRouter's model list](https://openrouter.ai/models?modality=image).
+
+### Supported Parameters
+
+| Parameter | OpenRouter Mapping | Notes |
+|-----------|--------------------|-------|
+| `size` | `image_config.aspect_ratio` | `1024x1024` → `1:1`, `1536x1024` → `3:2`, `1024x1536` → `2:3`, `1792x1024` → `16:9`, `1024x1792` → `9:16` |
+| `quality` | `image_config.image_size` | `low`/`standard` → `1K`, `medium` → `2K`, `high`/`hd` → `4K` |
+| `n` | `n` | Number of images |
+
+:::note
+`quality=high` (4K) is only supported by `google/gemini-3-pro-image-preview` and `google/gemini-3.1-flash-image-preview`. The `google/gemini-2.5-flash-image` model supports up to `medium` (2K).
+:::
+
+### Usage
+
+```python
+from litellm import image_edit
+import os
+
+os.environ["OPENROUTER_API_KEY"] = "your-api-key"
+
+# Basic image edit
+response = image_edit(
+    model="openrouter/google/gemini-2.5-flash-image",
+    image=open("original_image.png", "rb"),
+    prompt="Make the sky a vibrant purple sunset",
+)
+
+print(response)
+```
+
+### Advanced Usage with Parameters
+
+```python
+from litellm import image_edit
+import os
+
+os.environ["OPENROUTER_API_KEY"] = "your-api-key"
+
+# Edit with size and quality parameters
+response = image_edit(
+    model="openrouter/google/gemini-2.5-flash-image",
+    image=open("photo.png", "rb"),
+    prompt="Add northern lights to the sky",
+    size="1536x1024",   # Maps to aspect_ratio 3:2
+    quality="high",      # Maps to image_size 4K
+)
+
+# Access the edited image
+image_data = response.data[0]
+if image_data.b64_json:
+    import base64
+    with open("edited.png", "wb") as f:
+        f.write(base64.b64decode(image_data.b64_json))
+```
+
+### Multiple Images Edit
+
+```python
+from litellm import image_edit
+import os
+
+os.environ["OPENROUTER_API_KEY"] = "your-api-key"
+
+response = image_edit(
+    model="openrouter/google/gemini-2.5-flash-image",
+    image=[
+        open("scene.png", "rb"),
+        open("style_reference.png", "rb"),
+    ],
+    prompt="Blend the reference style into the scene",
+)
+
+print(response)
+```

docs/my-website/docs/providers/perplexity.md

@@ -120,7 +120,7 @@ All models listed here https://docs.perplexity.ai/docs/model-cards are supported
 
 
 
-## Agentic Research API (Responses API)
+## Agent API (Responses API)
 
 Requires v1.72.6+
 
@@ -196,7 +196,7 @@ import os
 os.environ['PERPLEXITY_API_KEY'] = ""
 
 response = responses(
-    model="perplexity/openai/gpt-4o",
+    model="perplexity/openai/gpt-5.2",
     input="Explain quantum computing in simple terms",
     custom_llm_provider="perplexity",
     max_output_tokens=500,
@@ -215,7 +215,7 @@ import os
 os.environ['PERPLEXITY_API_KEY'] = ""
 
 response = responses(
-    model="perplexity/anthropic/claude-3-5-sonnet-20241022",
+    model="perplexity/anthropic/claude-sonnet-4-5",
     input="Write a short story about a robot learning to paint",
     custom_llm_provider="perplexity",
     max_output_tokens=500,
@@ -234,7 +234,7 @@ import os
 os.environ['PERPLEXITY_API_KEY'] = ""
 
 response = responses(
-    model="perplexity/google/gemini-2.0-flash-exp",
+    model="perplexity/google/gemini-2.5-flash",
     input="Explain the concept of neural networks",
     custom_llm_provider="perplexity",
     max_output_tokens=500,
@@ -253,7 +253,7 @@ import os
 os.environ['PERPLEXITY_API_KEY'] = ""
 
 response = responses(
-    model="perplexity/xai/grok-2-1212",
+    model="perplexity/xai/grok-4-1-fast-non-reasoning",
     input="What makes a good AI assistant?",
     custom_llm_provider="perplexity",
     max_output_tokens=500,
@@ -276,7 +276,7 @@ import os
 os.environ['PERPLEXITY_API_KEY'] = ""
 
 response = responses(
-    model="perplexity/openai/gpt-4o",
+    model="perplexity/openai/gpt-5.2",
     input="What's the weather in San Francisco today?",
     custom_llm_provider="perplexity",
     tools=[{"type": "web_search"}],
@@ -286,6 +286,78 @@ response = responses(
 print(response.output)
 ```
 
+### Function Calling
+
+The Agent API supports custom function tools. Pass function tools through unchanged:
+
+```python
+from litellm import responses
+import os
+
+os.environ['PERPLEXITY_API_KEY'] = ""
+
+response = responses(
+    model="perplexity/openai/gpt-5.2",
+    input="What's the weather in San Francisco?",
+    custom_llm_provider="perplexity",
+    tools=[
+        {"type": "web_search"},
+        {
+            "type": "function",
+            "function": {
+                "name": "get_weather",
+                "description": "Get the current weather for a location",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "location": {"type": "string"},
+                        "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]},
+                    },
+                },
+            },
+        },
+    ],
+    instructions="Use tools when appropriate.",
+)
+
+print(response.output)
+```
+
+### Structured Outputs
+
+Request JSON schema structured outputs via the `text` parameter:
+
+```python
+from litellm import responses
+import os
+
+os.environ['PERPLEXITY_API_KEY'] = ""
+
+response = responses(
+    model="perplexity/preset/pro-search",
+    input="Extract key facts about the Eiffel Tower",
+    custom_llm_provider="perplexity",
+    text={
+        "format": {
+            "type": "json_schema",
+            "name": "facts",
+            "schema": {
+                "type": "object",
+                "properties": {
+                    "name": {"type": "string"},
+                    "height_meters": {"type": "number"},
+                    "year_built": {"type": "integer"},
+                },
+                "required": ["name", "height_meters", "year_built"],
+            },
+            "strict": True,
+        }
+    },
+)
+
+print(response.output)
+```
+
 
 ### Reasoning Effort (Responses API)
 
@@ -319,7 +391,7 @@ import os
 os.environ['PERPLEXITY_API_KEY'] = ""
 
 response = responses(
-    model="perplexity/anthropic/claude-3-5-sonnet-20241022",
+    model="perplexity/anthropic/claude-sonnet-4-5",
     input=[
         {"type": "message", "role": "system", "content": "You are a helpful assistant."},
         {"type": "message", "role": "user", "content": "What are the latest AI developments?"},
@@ -343,7 +415,7 @@ import os
 os.environ['PERPLEXITY_API_KEY'] = ""
 
 response = responses(
-    model="perplexity/openai/gpt-4o",
+    model="perplexity/openai/gpt-5.2",
     input="Tell me a story about space exploration",
     custom_llm_provider="perplexity",
     stream=True,
@@ -360,23 +432,28 @@ for chunk in response:
 
 | Provider | Model Name | Function Call |
 |----------|------------|---------------|
-| OpenAI | gpt-4o | `responses(model="perplexity/openai/gpt-4o", ...)` |
-| OpenAI | gpt-4o-mini | `responses(model="perplexity/openai/gpt-4o-mini", ...)` |
 | OpenAI | gpt-5.2 | `responses(model="perplexity/openai/gpt-5.2", ...)` |
-| Anthropic | claude-3-5-sonnet-20241022 | `responses(model="perplexity/anthropic/claude-3-5-sonnet-20241022", ...)` |
-| Anthropic | claude-3-5-haiku-20241022 | `responses(model="perplexity/anthropic/claude-3-5-haiku-20241022", ...)` |
-| Google | gemini-2.0-flash-exp | `responses(model="perplexity/google/gemini-2.0-flash-exp", ...)` |
-| Google | gemini-2.0-flash-thinking-exp | `responses(model="perplexity/google/gemini-2.0-flash-thinking-exp", ...)` |
-| xAI | grok-2-1212 | `responses(model="perplexity/xai/grok-2-1212", ...)` |
-| xAI | grok-2-vision-1212 | `responses(model="perplexity/xai/grok-2-vision-1212", ...)` |
+| OpenAI | gpt-5.1 | `responses(model="perplexity/openai/gpt-5.1", ...)` |
+| OpenAI | gpt-5-mini | `responses(model="perplexity/openai/gpt-5-mini", ...)` |
+| Anthropic | claude-opus-4-6 | `responses(model="perplexity/anthropic/claude-opus-4-6", ...)` |
+| Anthropic | claude-opus-4-5 | `responses(model="perplexity/anthropic/claude-opus-4-5", ...)` |
+| Anthropic | claude-sonnet-4-5 | `responses(model="perplexity/anthropic/claude-sonnet-4-5", ...)` |
+| Anthropic | claude-haiku-4-5 | `responses(model="perplexity/anthropic/claude-haiku-4-5", ...)` |
+| Google | gemini-3-pro-preview | `responses(model="perplexity/google/gemini-3-pro-preview", ...)` |
+| Google | gemini-3-flash-preview | `responses(model="perplexity/google/gemini-3-flash-preview", ...)` |
+| Google | gemini-2.5-pro | `responses(model="perplexity/google/gemini-2.5-pro", ...)` |
+| Google | gemini-2.5-flash | `responses(model="perplexity/google/gemini-2.5-flash", ...)` |
+| xAI | grok-4-1-fast-non-reasoning | `responses(model="perplexity/xai/grok-4-1-fast-non-reasoning", ...)` |
+| Perplexity | sonar | `responses(model="perplexity/perplexity/sonar", ...)` |
 
 ### Available Presets
 
-| Preset Name    | Function Call                                           |
-|----------------|--------------------------------------------------------|
-| fast-search    | `responses(model="perplexity/preset/fast-search", ...)`|
-| pro-search     | `responses(model="perplexity/preset/pro-search", ...)` |
-| deep-research  | `responses(model="perplexity/preset/deep-research", ...)`|
+| Preset Name | Function Call |
+|-------------|---------------|
+| fast-search | `responses(model="perplexity/preset/fast-search", ...)` |
+| pro-search | `responses(model="perplexity/preset/pro-search", ...)` |
+| deep-research | `responses(model="perplexity/preset/deep-research", ...)` |
+| advanced-deep-research | `responses(model="perplexity/preset/advanced-deep-research", ...)` |
 
 ### Complete Example
 
@@ -388,7 +465,7 @@ os.environ['PERPLEXITY_API_KEY'] = ""
 
 # Comprehensive example with multiple features
 response = responses(
-    model="perplexity/openai/gpt-4o",
+    model="perplexity/openai/gpt-5.2",
     input="Research the latest developments in quantum computing and provide sources",
     custom_llm_provider="perplexity",
     tools=[

docs/my-website/docs/providers/perplexity_embedding.md

@@ -0,0 +1,134 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Perplexity Embeddings
+
+https://docs.perplexity.ai/docs/embeddings/quickstart
+
+LiteLLM supports Perplexity's pplx-embed embedding models for web-scale text retrieval.
+
+## API Key
+
+```python
+# env variable
+os.environ['PERPLEXITYAI_API_KEY']
+```
+
+## Sample Usage - Embedding
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+from litellm import embedding
+import os
+
+os.environ['PERPLEXITYAI_API_KEY'] = ""
+
+response = embedding(
+    model="perplexity/pplx-embed-v1-0.6b",
+    input=["good morning from litellm"],
+)
+print(response)
+```
+
+</TabItem>
+<TabItem value="proxy" label="Proxy">
+
+1. Setup config.yaml
+
+```yaml
+model_list:
+  - model_name: pplx-embed-v1-0.6b
+    litellm_params:
+      model: perplexity/pplx-embed-v1-0.6b
+      api_key: os.environ/PERPLEXITYAI_API_KEY
+  - model_name: pplx-embed-v1-4b
+    litellm_params:
+      model: perplexity/pplx-embed-v1-4b
+      api_key: os.environ/PERPLEXITYAI_API_KEY
+```
+
+2. Start proxy
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+3. Test it!
+
+```bash
+curl http://0.0.0.0:4000/v1/embeddings \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer sk-1234" \
+  -d '{
+    "model": "pplx-embed-v1-0.6b",
+    "input": ["good morning from litellm"]
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+## Supported Parameters
+
+Perplexity embeddings support the following optional parameters:
+
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| `dimensions` | int | Output embedding dimensions. 128–1024 for 0.6b models, 128–2560 for 4b models. Defaults to max. |
+| `encoding_format` | string | `"base64_int8"` (default) or `"base64_binary"` for compressed output. |
+
+### Example with Parameters
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+from litellm import embedding
+import os
+
+os.environ['PERPLEXITYAI_API_KEY'] = ""
+
+response = embedding(
+    model="perplexity/pplx-embed-v1-4b",
+    input=["Your text here"],
+    dimensions=512,
+)
+print(f"Embedding dimensions: {len(response.data[0]['embedding'])}")
+```
+
+</TabItem>
+<TabItem value="proxy" label="Proxy">
+
+```bash
+curl http://0.0.0.0:4000/v1/embeddings \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer sk-1234" \
+  -d '{
+    "model": "pplx-embed-v1-4b",
+    "input": ["Your text here"],
+    "dimensions": 512
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+## Supported Models
+
+All models listed on the [Perplexity Embeddings docs](https://docs.perplexity.ai/docs/embeddings/quickstart) are supported. Use `model=perplexity/<model-name>`.
+
+| Model Name | Dimensions | Max Tokens | Price (per 1M tokens) | Function Call |
+|---|---|---|---|---|
+| pplx-embed-v1-0.6b | 1024 | 32K | $0.004 | `embedding(model="perplexity/pplx-embed-v1-0.6b", input)` |
+| pplx-embed-v1-4b | 2560 | 32K | $0.03 | `embedding(model="perplexity/pplx-embed-v1-4b", input)` |
+
+### Key Specifications
+
+- **Max texts per request:** 512
+- **Max tokens per input:** 32,768
+- **Combined request limit:** 120,000 tokens
+- **Matryoshka dimension reduction** — reduce dimensions to 128+ for faster search and reduced storage
+- **No instruction prefix required** — embed text directly
+- **Unnormalized embeddings** — use cosine similarity for comparison

docs/my-website/docs/providers/vertex.md

@@ -1472,6 +1472,82 @@ Your WIF credentials JSON file typically looks like this (for AWS federation):
 
 For more details on setting up Workload Identity Federation, see [Google Cloud WIF documentation](https://cloud.google.com/iam/docs/workload-identity-federation).
 
+#### Explicit AWS Credentials for WIF
+
+By default, AWS-based WIF relies on the EC2 instance metadata service to obtain AWS credentials. This works when LiteLLM runs on an EC2 instance or ECS task with an IAM role attached.
+
+If your environment **does not have access to the EC2 metadata service** (e.g., running on-premises, in a container without host networking, or in a different cloud with security restrictions), you can provide explicit AWS credentials directly in the WIF credential JSON file. LiteLLM will use these to authenticate to AWS before performing the GCP token exchange.
+
+Add the `aws_*` keys at the **top level** of your WIF credential JSON (alongside `type`, `audience`, etc.):
+
+```json
+{
+  "type": "external_account",
+  "audience": "//iam.googleapis.com/projects/PROJECT_NUMBER/locations/global/workloadIdentityPools/POOL_ID/providers/PROVIDER_ID",
+  "subject_token_type": "urn:ietf:params:aws:token-type:aws4_request",
+  "service_account_impersonation_url": "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/SERVICE_ACCOUNT_EMAIL:generateAccessToken",
+  "token_url": "https://sts.googleapis.com/v1/token",
+  "credential_source": {
+    "environment_id": "aws1",
+    "region_url": "http://169.254.169.254/latest/meta-data/placement/availability-zone",
+    "url": "http://169.254.169.254/latest/meta-data/iam/security-credentials",
+    "regional_cred_verification_url": "https://sts.{region}.amazonaws.com?Action=GetCallerIdentity&Version=2011-06-15"
+  },
+  "aws_role_name": "arn:aws:iam::123456789012:role/MyWifRole",
+  "aws_region_name": "us-east-1"
+}
+```
+
+**Supported `aws_*` parameters:**
+
+| Parameter | Required | Description |
+|---|---|---|
+| `aws_region_name` | Yes | AWS region for credential verification (e.g. `us-east-1`) |
+| `aws_role_name` | No | IAM role ARN for STS AssumeRole |
+| `aws_access_key_id` | No | Static AWS access key ID |
+| `aws_secret_access_key` | No | Static AWS secret access key |
+| `aws_session_token` | No | Temporary session token |
+| `aws_profile_name` | No | AWS CLI profile name |
+| `aws_session_name` | No | Session name for AssumeRole |
+| `aws_web_identity_token` | No | Web identity token for STS |
+| `aws_sts_endpoint` | No | Custom STS endpoint URL |
+| `aws_external_id` | No | External ID for cross-account AssumeRole |
+
+`aws_region_name` is always required when using explicit AWS credentials. The other parameters follow the same authentication flows as [Bedrock AWS auth](/docs/providers/bedrock#authentication) -- you can use role assumption, static keys, profiles, or web identity tokens.
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+from litellm import completion
+
+response = completion(
+    model="vertex_ai/gemini-1.5-pro",
+    messages=[{"role": "user", "content": "Hello!"}],
+    vertex_credentials="/path/to/wif-credentials-with-aws.json",  # WIF JSON with aws_* keys
+    vertex_project="your-gcp-project-id",
+    vertex_location="us-central1"
+)
+```
+
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+```yaml
+model_list:
+  - model_name: gemini-model
+    litellm_params:
+      model: vertex_ai/gemini-1.5-pro
+      vertex_project: your-gcp-project-id
+      vertex_location: us-central1
+      vertex_credentials: /path/to/wif-credentials-with-aws.json  # WIF JSON with aws_* keys
+```
+
+</TabItem>
+</Tabs>
+
+When `aws_*` keys are present in the JSON, LiteLLM automatically uses explicit AWS authentication instead of the EC2 metadata service. When they are absent, the standard metadata-based flow is used unchanged.
+
 ### **Environment Variables**
 
 You can set:
@@ -1685,6 +1761,21 @@ litellm.vertex_location = "us-central1 # Your Location
 | gemini-2.5-pro   | `completion('gemini-2.5-pro', messages)`, `completion('vertex_ai/gemini-2.5-pro', messages)` |
 | gemini-2.5-flash-preview-09-2025   | `completion('gemini-2.5-flash-preview-09-2025', messages)`, `completion('vertex_ai/gemini-2.5-flash-preview-09-2025', messages)` |
 | gemini-2.5-flash-lite-preview-09-2025   | `completion('gemini-2.5-flash-lite-preview-09-2025', messages)`, `completion('vertex_ai/gemini-2.5-flash-lite-preview-09-2025', messages)` |
+| gemini-3.1-flash-lite-preview   | `completion('gemini-3.1-flash-lite-preview', messages)`, `completion('vertex_ai/gemini-3.1-flash-lite-preview', messages)` |
+
+## PayGo / Priority Cost Tracking
+
+LiteLLM automatically tracks spend for Vertex AI Gemini models using the correct pricing tier based on the response's `usageMetadata.trafficType`:
+
+| Vertex AI `trafficType` | LiteLLM `service_tier` | Pricing applied |
+|-------------------------|-------------------------|-----------------|
+| `ON_DEMAND_PRIORITY` | `priority` | PayGo / priority pricing (`input_cost_per_token_priority`, `output_cost_per_token_priority`) |
+| `ON_DEMAND` | standard | Default on-demand pricing |
+| `FLEX` / `BATCH` | `flex` | Batch/flex pricing |
+
+When you use [Vertex AI PayGo](https://cloud.google.com/vertex-ai/generative-ai/pricing) (on-demand priority) or batch workloads, LiteLLM reads `trafficType` from the response and applies the matching cost per token from the [model cost map](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json). No configuration is required — spend tracking works out of the box for both standard and PayGo requests.
+
+See [Spend Tracking](../proxy/cost_tracking.md) for general cost tracking setup.
 
 ## Private Service Connect (PSC) Endpoints
 

docs/my-website/docs/providers/vertex_realtime.md

@@ -0,0 +1,203 @@
+# Vertex AI Gemini Live - Realtime API
+
+Use Vertex AI's Gemini Live API (BidiGenerateContent) through LiteLLM's unified `/realtime` endpoint, which speaks the OpenAI Realtime protocol.
+
+| Feature | Supported |
+|---------|-----------|
+| Proxy (`/realtime`) | ✅ |
+| Voice in / Voice out | ✅ |
+| Text in / Text out | ✅ |
+| Server VAD | ✅ |
+| Output transcription | ✅ |
+
+## Setup
+
+### 1. Auth
+
+LiteLLM uses your Google Cloud credentials (OAuth2 Bearer token), not an API key.
+
+```bash
+gcloud auth application-default login
+```
+
+Or set a service-account key file:
+
+```bash
+export GOOGLE_APPLICATION_CREDENTIALS=/path/to/sa-key.json
+```
+
+### 2. Proxy config
+
+```yaml
+model_list:
+  - model_name: vertex-gemini-live
+    litellm_params:
+      model: vertex_ai/gemini-2.0-flash-live-001
+      vertex_project: your-gcp-project-id
+      vertex_location: us-east4   # or any supported region, or "global"
+
+general_settings:
+  master_key: sk-your-key
+```
+
+### 3. Start the proxy
+
+```bash
+litellm --config config.yaml --port 4000
+```
+
+## Usage
+
+### Python (websockets)
+
+```python
+import asyncio
+import json
+import websockets
+
+PROXY_URL = "ws://localhost:4000/realtime?model=vertex-gemini-live"
+API_KEY = "sk-your-key"
+
+async def main():
+    async with websockets.connect(
+        PROXY_URL,
+        additional_headers={"api-key": API_KEY},
+    ) as ws:
+        # Wait for session.created
+        event = json.loads(await ws.recv())
+        print(f"session.created: {event['session']['id']}")
+
+        # Send a text message
+        await ws.send(json.dumps({
+            "type": "conversation.item.create",
+            "item": {
+                "type": "message",
+                "role": "user",
+                "content": [{"type": "input_text", "text": "Say hello in one sentence."}],
+            },
+        }))
+
+        # Collect the response
+        async for raw in ws:
+            ev = json.loads(raw)
+            t = ev.get("type", "")
+            if t == "response.text.delta":
+                print(ev.get("delta", ""), end="", flush=True)
+            elif t == "response.done":
+                print("\n[done]")
+                break
+
+asyncio.run(main())
+```
+
+### Node.js
+
+```js
+const WebSocket = require("ws");
+
+const ws = new WebSocket(
+  "ws://localhost:4000/realtime?model=vertex-gemini-live",
+  { headers: { "api-key": "sk-your-key" } }
+);
+
+ws.on("open", () => {
+  ws.send(JSON.stringify({
+    type: "conversation.item.create",
+    item: {
+      type: "message",
+      role: "user",
+      content: [{ type: "input_text", text: "Say hello." }],
+    },
+  }));
+});
+
+ws.on("message", (data) => {
+  const ev = JSON.parse(data);
+  if (ev.type === "response.text.delta") process.stdout.write(ev.delta);
+  if (ev.type === "response.done") ws.close();
+});
+```
+
+### OpenAI SDK (Python)
+
+```python
+import asyncio
+from openai import AsyncOpenAI
+
+client = AsyncOpenAI(
+    base_url="http://localhost:4000",
+    api_key="sk-your-key",
+)
+
+async def main():
+    async with client.beta.realtime.connect(
+        model="vertex-gemini-live"
+    ) as conn:
+        await conn.session.update(session={"modalities": ["text"]})
+
+        await conn.conversation.item.create(
+            item={
+                "type": "message",
+                "role": "user",
+                "content": [{"type": "input_text", "text": "Say hello."}],
+            }
+        )
+
+        async for event in conn:
+            if event.type == "response.text.delta":
+                print(event.delta, end="", flush=True)
+            elif event.type == "response.done":
+                print()
+                break
+
+asyncio.run(main())
+```
+
+## Voice in / Voice out
+
+For a complete voice example see [`voice_realtime_test.py`](https://github.com/BerriAI/litellm/blob/main/voice_realtime_test.py).
+
+Key settings for audio:
+- Microphone input: **16 kHz** PCM16 (`audio/pcm;rate=16000`)
+- Speaker output: **24 kHz** PCM16 (Vertex AI returns audio at 24 kHz)
+- Server VAD is enabled by default with 800 ms silence threshold
+
+```python
+# session.update with server VAD — the proxy ignores this for Vertex AI
+# because VAD is already configured in the initial setup message.
+await ws.send(json.dumps({
+    "type": "session.update",
+    "session": {
+        "modalities": ["audio"],
+        "turn_detection": {"type": "server_vad", "silence_duration_ms": 800},
+    },
+}))
+```
+
+## Supported OpenAI Realtime Events
+
+**Client → Proxy (→ Vertex AI)**
+
+| OpenAI event | Notes |
+|---|---|
+| `input_audio_buffer.append` | Forwarded as `realtime_input.audio` |
+| `conversation.item.create` | Forwarded as `realtime_input.text` |
+| `session.update` | Silently ignored — Vertex AI does not support mid-session reconfiguration |
+| `response.create` | Silently ignored — Vertex AI responds automatically after each turn |
+
+**Vertex AI → Proxy (→ Client)**
+
+| OpenAI event emitted | Vertex AI source |
+|---|---|
+| `session.created` | Synthesized after `setupComplete` |
+| `response.text.delta` | `serverContent.modelTurn.parts[].text` |
+| `response.audio.delta` | `serverContent.modelTurn.parts[].inlineData` |
+| `response.audio_transcript.delta` | `serverContent.outputTranscription.text` |
+| `conversation.item.input_audio_transcription.completed` | `serverContent.inputTranscription.text` |
+| `response.done` | `serverContent.turnComplete` |
+
+## Limitations
+
+- `session.update` is not forwarded (Vertex AI only accepts one setup message per connection).
+- Tool calling / function calling is not yet supported.
+- Audio transcription requires `outputAudioTranscription: {}` to be set in the initial setup (done automatically by LiteLLM).

docs/my-website/docs/proxy/admin_ui_sso.md

@@ -41,12 +41,38 @@ After creating the app, copy your **Client ID** and **Client Secret** from the a
 
 Ensure users are assigned to the app in the **Assignments** tab. If Federation Broker Mode is enabled, you may need to disable it to assign users manually.
 
-#### Step 3: Configure Authorization Server Access Policy
+#### Step 3: Set Environment Variables
 
-:::warning Important
-This step is required. Without an Access Policy for your app, users will get a `no_matching_policy` error when attempting to log in.
+Set the following environment variables. The only difference between the two Okta authorization servers is the endpoint URLs:
+
+**Org Authorization Server** (available on all Okta plans, no additional SKU required):
+```bash
+GENERIC_CLIENT_ID="<your-client-id>"
+GENERIC_CLIENT_SECRET="<your-client-secret>"
+GENERIC_AUTHORIZATION_ENDPOINT="https://<your-okta-domain>/oauth2/v1/authorize"
+GENERIC_TOKEN_ENDPOINT="https://<your-okta-domain>/oauth2/v1/token"
+GENERIC_USERINFO_ENDPOINT="https://<your-okta-domain>/oauth2/v1/userinfo"
+PROXY_BASE_URL="https://<your-proxy-base-url>"
+```
+
+**Custom Authorization Server** (requires the Okta API Access Management SKU):
+```bash
+GENERIC_CLIENT_ID="<your-client-id>"
+GENERIC_CLIENT_SECRET="<your-client-secret>"
+GENERIC_AUTHORIZATION_ENDPOINT="https://<your-okta-domain>/oauth2/default/v1/authorize"
+GENERIC_TOKEN_ENDPOINT="https://<your-okta-domain>/oauth2/default/v1/token"
+GENERIC_USERINFO_ENDPOINT="https://<your-okta-domain>/oauth2/default/v1/userinfo"
+PROXY_BASE_URL="https://<your-proxy-base-url>"
+```
+
+:::tip
+You can find all OAuth endpoints at `https://<your-okta-domain>/.well-known/openid-configuration`
 :::
 
+#### Step 3a: Configure Access Policy (Custom Authorization Server only)
+
+If you are using the Custom Authorization Server, you must configure an Access Policy. Without it, users will get a `no_matching_policy` error. Skip this step if you are using the Org Authorization Server.
+
 1. Go to **Security** → **API**
 
 <Image img={require('../../img/okta_security_api.png')} />
@@ -62,21 +88,21 @@ This step is required. Without an Access Policy for your app, users will get a `
 
 See [Okta's Access Policy documentation](https://help.okta.com/en-us/content/topics/security/api-access-management/access-policies.htm) for more details.
 
-#### Step 4: Configure LiteLLM Environment Variables
+#### Step 4: Configure Okta Security Settings
+
+**GENERIC_CLIENT_STATE** is recommended for Okta to prevent CSRF attacks:
 
 ```bash
-GENERIC_CLIENT_ID="<your-client-id>"
-GENERIC_CLIENT_SECRET="<your-client-secret>"
-GENERIC_AUTHORIZATION_ENDPOINT="https://<your-okta-domain>/oauth2/default/v1/authorize"
-GENERIC_TOKEN_ENDPOINT="https://<your-okta-domain>/oauth2/default/v1/token"
-GENERIC_USERINFO_ENDPOINT="https://<your-okta-domain>/oauth2/default/v1/userinfo"
 GENERIC_CLIENT_STATE="random-string"
-PROXY_BASE_URL="https://<your-proxy-base-url>"
 ```
 
-:::tip
-You can find all OAuth endpoints at `https://<your-okta-domain>/.well-known/openid-configuration`
-:::
+**PKCE (Proof Key for Code Exchange)** — If your Okta application is configured to require PKCE, enable it by setting:
+
+```bash
+GENERIC_CLIENT_USE_PKCE="true"
+```
+
+LiteLLM will automatically handle PKCE parameter generation and verification during the OAuth flow.
 
 #### Step 5: Test the SSO Flow
 
@@ -91,7 +117,7 @@ You can find all OAuth endpoints at `https://<your-okta-domain>/.well-known/open
 |-------|-------|----------|
 | `redirect_uri` error | Redirect URI not configured | Add `<proxy_base_url>/sso/callback` to Sign-in redirect URIs in Okta |
 | `access_denied` | User not assigned to app | Assign the user in the Assignments tab |
-| `no_matching_policy` | Missing Access Policy | Create an Access Policy in the Authorization Server (see Step 3) |
+| `no_matching_policy` | Missing Access Policy (Custom Authorization Server only) | Create an Access Policy in the Authorization Server (see Step 3a) |
 
 </TabItem>
 <TabItem value="google" label="Google SSO">
@@ -456,23 +482,9 @@ PROXY_BASE_URL=http://litellm.platform.com
 PROXY_BASE_URL=litellm.platform.com
 ```
 
-**2. For Okta specifically, ensure GENERIC_CLIENT_STATE is set**
+**2. For Okta specifically, ensure `GENERIC_CLIENT_STATE` is set and PKCE is configured if required**
 
-Okta requires the `GENERIC_CLIENT_STATE` parameter:
-
-```bash
-GENERIC_CLIENT_STATE="random-string" # Required for Okta
-```
-
-### Okta PKCE
-
-If your Okta application is configured to require PKCE (Proof Key for Code Exchange), enable it by setting:
-
-```bash
-GENERIC_CLIENT_USE_PKCE="true"
-```
-
-This is required when your Okta app settings enforce PKCE for enhanced security. LiteLLM will automatically handle PKCE parameter generation and verification during the OAuth flow.
+See [Okta SSO — Step 4: Configure Okta Security Settings](#step-4-configure-okta-security-settings) for details on `GENERIC_CLIENT_STATE` and PKCE configuration.
 
 ### Common Configuration Issues
 

docs/my-website/docs/proxy/alerting.md

@@ -438,6 +438,59 @@ curl -X GET --location 'http://0.0.0.0:4000/health/services?service=webhook' \
 
 - `event_message` *str*: A human-readable description of the event.
 
+### Digest Mode (Reducing Alert Noise)
+
+By default, LiteLLM sends a separate Slack message for **every** alert event. For high-frequency alert types like `llm_requests_hanging` or `llm_too_slow`, this can produce hundreds of duplicate messages per day.
+
+**Digest mode** aggregates duplicate alerts within a configurable time window and emits a single summary message with the total count and time range.
+
+#### Configuration
+
+Use `alert_type_config` in `general_settings` to enable digest mode per alert type:
+
+```yaml
+general_settings:
+  alerting: ["slack"]
+  alert_type_config:
+    llm_requests_hanging:
+      digest: true
+      digest_interval: 86400  # 24 hours (default)
+    llm_too_slow:
+      digest: true
+      digest_interval: 3600   # 1 hour
+    llm_exceptions:
+      digest: true
+      # uses default interval (86400 seconds / 24 hours)
+```
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `digest` | bool | `false` | Enable digest mode for this alert type |
+| `digest_interval` | int | `86400` (24h) | Time window in seconds. Alerts are aggregated within this interval. |
+
+#### How It Works
+
+1. When an alert fires for a digest-enabled type, it is **grouped** by `(alert_type, request_model, api_base)` instead of being sent immediately
+2. A counter tracks how many times the alert fires within the interval
+3. When the interval expires, a **single summary message** is sent:
+
+```
+Alert type: `llm_requests_hanging` (Digest)
+Level: `Medium`
+Start: `2026-02-19 03:27:39`
+End: `2026-02-20 03:27:39`
+Count: `847`
+
+Message: `Requests are hanging - 600s+ request time`
+Request Model: `gemini-2.5-flash`
+API Base: `None`
+```
+
+#### Limitations
+
+- **Per-instance**: Digest state is held in memory per proxy instance. If you run multiple instances (e.g., Cloud Run with autoscaling), each instance maintains its own digest and emits its own summary.
+- **Not durable**: If an instance is terminated before the digest interval expires, the aggregated alerts for that instance are lost.
+
 ## Region-outage alerting (✨ Enterprise feature)
 
 :::info

docs/my-website/docs/proxy/auto_routing.md

@@ -219,3 +219,189 @@ curl -X POST http://localhost:4000/v1/chat/completions \
 3. If a route's similarity score exceeds the threshold, the request is routed to that model
 4. If no route matches, the request goes to the default model
 
+---
+
+## Complexity Router
+
+The Complexity Router provides an alternative to semantic routing that uses **rule-based scoring** to classify requests by complexity and route them to appropriate models — with **zero external API calls** and **sub-millisecond latency**.
+
+### When to Use
+
+| Feature | Semantic Auto Router | Complexity Router |
+|---------|---------------------|-------------------|
+| Classification | Embedding-based matching | Rule-based scoring |
+| Latency | ~100-500ms (embedding API) | <1ms |
+| API Calls | Requires embedding model | None |
+| Training | Requires utterance examples | Works out of the box |
+| Best For | Intent-based routing | Cost optimization |
+
+Use **Complexity Router** when you want to:
+- Route simple queries to cheaper/faster models (e.g., gpt-4o-mini)
+- Route complex queries to more capable models (e.g., claude-sonnet-4)
+- Minimize latency overhead from routing decisions
+- Avoid additional API costs for embeddings
+
+### LiteLLM Python SDK
+
+```python
+from litellm import Router
+
+router = Router(
+    model_list=[
+        # Target models for each tier
+        {
+            "model_name": "gpt-4o-mini",
+            "litellm_params": {"model": "gpt-4o-mini"},
+        },
+        {
+            "model_name": "gpt-4o",
+            "litellm_params": {"model": "gpt-4o"},
+        },
+        {
+            "model_name": "claude-sonnet",
+            "litellm_params": {"model": "claude-sonnet-4-20250514"},
+        },
+        {
+            "model_name": "o1-preview",
+            "litellm_params": {"model": "o1-preview"},
+        },
+        # Complexity router configuration
+        {
+            "model_name": "smart-router",
+            "litellm_params": {
+                "model": "auto_router/complexity_router",
+                "complexity_router_config": {
+                    "tiers": {
+                        "SIMPLE": "gpt-4o-mini",
+                        "MEDIUM": "gpt-4o",
+                        "COMPLEX": "claude-sonnet",
+                        "REASONING": "o1-preview",
+                    },
+                },
+                "complexity_router_default_model": "gpt-4o",
+            },
+        },
+    ],
+)
+```
+
+#### Usage
+
+```python
+# Simple query → routes to gpt-4o-mini
+response = await router.acompletion(
+    model="smart-router",
+    messages=[{"role": "user", "content": "What is 2+2?"}],
+)
+
+# Complex technical query → routes to claude-sonnet or higher
+response = await router.acompletion(
+    model="smart-router",
+    messages=[{"role": "user", "content": "Design a distributed microservice architecture with Kubernetes orchestration"}],
+)
+
+# Reasoning request → routes to o1-preview
+response = await router.acompletion(
+    model="smart-router",
+    messages=[{"role": "user", "content": "Think step by step and reason through this problem carefully..."}],
+)
+```
+
+### LiteLLM Proxy Server
+
+Add the complexity router to your `config.yaml`:
+
+```yaml
+model_list:
+  # Target models
+  - model_name: gpt-4o-mini
+    litellm_params:
+      model: gpt-4o-mini
+      
+  - model_name: gpt-4o
+    litellm_params:
+      model: gpt-4o
+      
+  - model_name: claude-sonnet
+    litellm_params:
+      model: claude-sonnet-4-20250514
+      
+  - model_name: o1-preview
+    litellm_params:
+      model: o1-preview
+
+  # Complexity router
+  - model_name: smart-router
+    litellm_params:
+      model: auto_router/complexity_router
+      complexity_router_config:
+        tiers:
+          SIMPLE: gpt-4o-mini
+          MEDIUM: gpt-4o
+          COMPLEX: claude-sonnet
+          REASONING: o1-preview
+      complexity_router_default_model: gpt-4o
+```
+
+### Configuration Options
+
+#### Tier Boundaries
+
+Customize the score thresholds for each tier:
+
+```yaml
+complexity_router_config:
+  tiers:
+    SIMPLE: gpt-4o-mini
+    MEDIUM: gpt-4o
+    COMPLEX: claude-sonnet
+    REASONING: o1-preview
+  tier_boundaries:
+    simple_medium: 0.15    # Below 0.15 → SIMPLE
+    medium_complex: 0.35   # 0.15-0.35 → MEDIUM
+    complex_reasoning: 0.60  # 0.35-0.60 → COMPLEX, above → REASONING
+```
+
+#### Token Thresholds
+
+Adjust when prompts are considered "short" or "long":
+
+```yaml
+complexity_router_config:
+  token_thresholds:
+    simple: 15   # Prompts under 15 tokens are penalized (simple indicator)
+    complex: 400 # Prompts over 400 tokens get complexity boost
+```
+
+#### Dimension Weights
+
+Customize how much each signal contributes to the complexity score:
+
+```yaml
+complexity_router_config:
+  dimension_weights:
+    tokenCount: 0.10        # Prompt length
+    codePresence: 0.30      # Code-related keywords
+    reasoningMarkers: 0.25  # "step by step", "think through", etc.
+    technicalTerms: 0.25    # Domain-specific complexity
+    simpleIndicators: 0.05  # "what is", "define", greetings
+    multiStepPatterns: 0.03 # "first...then", numbered steps
+    questionComplexity: 0.02 # Multiple questions
+```
+
+### How Complexity Routing Works
+
+The router scores each request across 7 dimensions:
+
+| Dimension | What It Detects | Effect |
+|-----------|-----------------|--------|
+| Token Count | Short (<15) or long (>400) prompts | Short = simple, long = complex |
+| Code Presence | "function", "class", "api", "database", etc. | Increases complexity |
+| Reasoning Markers | "step by step", "think through", "analyze" | Triggers REASONING tier |
+| Technical Terms | "architecture", "distributed", "encryption" | Increases complexity |
+| Simple Indicators | "what is", "define", "hello" | Decreases complexity |
+| Multi-Step Patterns | "first...then", "1. 2. 3." | Increases complexity |
+| Question Complexity | Multiple question marks | Increases complexity |
+
+**Special behavior:** If 2+ reasoning markers are detected in the user message, the request automatically routes to the REASONING tier regardless of the weighted score.
+

docs/my-website/docs/proxy/budget_reset_and_tz.md

@@ -1,16 +1,20 @@
-## Budget Reset Times and Timezones
+# Budget Reset Times and Timezones
 
-LiteLLM now supports predictable budget reset times that align with natural calendar boundaries:
+LiteLLM supports predictable budget reset times that align with natural calendar boundaries.
 
-- All budgets reset at midnight (00:00:00) in the configured timezone
-- Special handling for common durations:
-  - Daily (24h/1d): Reset at midnight every day
-  - Weekly (7d): Reset on Monday at midnight
-  - Monthly (30d): Reset on the 1st of each month at midnight
+## How Budget Resets Work
 
-### Configuring the Timezone
+All budgets reset at midnight (00:00:00) in the configured timezone with special handling for common durations:
 
-You can specify the timezone for all budget resets in your configuration file:
+| Duration | Reset Behavior |
+| --- | --- |
+| Daily (24h/1d) | Resets at midnight every day |
+| Weekly (7d) | Resets on Monday at midnight |
+| Monthly (30d) | Resets on the 1st of each month at midnight |
+
+## Configuring the Timezone
+
+Specify the timezone for all budget resets in your configuration file:
 
 ```yaml
 litellm_settings:
@@ -19,16 +23,21 @@ litellm_settings:
   timezone: "US/Eastern" # Any valid timezone string
 ```
 
-This ensures that all budget resets happen at midnight in your specified timezone rather than in UTC.
-If no timezone is specified, UTC will be used by default.
+This ensures that all budget resets happen at midnight in your specified timezone rather than in UTC. If no timezone is specified, UTC will be used by default.
 
-Common timezone values:
+## Supported Timezones
 
-- `UTC` - Coordinated Universal Time
-- `US/Eastern` - Eastern Time
-- `US/Pacific` - Pacific Time
-- `Europe/London` - UK Time
-- `Asia/Kolkata` - Indian Standard Time (IST)
-- `Asia/Bangkok` - Indochina Time (ICT)
-- `Asia/Tokyo` - Japan Standard Time
-- `Australia/Sydney` - Australian Eastern Time
+Any valid [IANA timezone string](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) is supported (powered by Python's `zoneinfo` module). DST transitions are handled automatically.
+
+**Common timezone values:**
+
+| Timezone | Description |
+| --- | --- |
+| `UTC` | Coordinated Universal Time |
+| `US/Eastern` | Eastern Time |
+| `US/Pacific` | Pacific Time |
+| `Europe/London` | UK Time |
+| `Asia/Kolkata` | Indian Standard Time (IST) |
+| `Asia/Bangkok` | Indochina Time (ICT) |
+| `Asia/Tokyo` | Japan Standard Time |
+| `Australia/Sydney` | Australian Eastern Time |

docs/my-website/docs/proxy/caching.md

@@ -340,6 +340,7 @@ litellm_settings:
     qdrant_semantic_cache_embedding_model: openai-embedding # the model should be defined on the model_list
     qdrant_collection_name: test_collection
     qdrant_quantization_config: binary
+    qdrant_semantic_cache_vector_size: 1536 # vector size must match embedding model dimensionality
     similarity_threshold: 0.8 # similarity threshold for semantic cache
 ```
 

docs/my-website/docs/proxy/cli_sso.md

@@ -52,6 +52,10 @@ LITELLM_CLI_JWT_EXPIRATION_HOURS=48 EXPERIMENTAL_UI_LOGIN="True" litellm --confi
 - `LITELLM_CLI_JWT_EXPIRATION_HOURS=168` - Tokens expire after 7 days (168 hours)
 - `LITELLM_CLI_JWT_EXPIRATION_HOURS=720` - Tokens expire after 30 days (720 hours)
 
+:::note[Experimental UI Session]
+When `EXPERIMENTAL_UI_LOGIN` is enabled, the **browser UI login** session uses a fixed 10-minute expiry (not configurable). `LITELLM_UI_SESSION_DURATION` applies only to non-experimental flows.
+:::
+
 :::tip
 You can check your current token's age and expiration status using:
 ```bash

docs/my-website/docs/proxy/config_settings.md

@@ -73,6 +73,7 @@ litellm_settings:
     qdrant_semantic_cache_embedding_model: openai-embedding # the model should be defined on the model_list
     qdrant_collection_name: test_collection
     qdrant_quantization_config: binary
+    qdrant_semantic_cache_vector_size: 1536 # vector size must match embedding model dimensionality
     similarity_threshold: 0.8 # similarity threshold for semantic cache
 
     # Optional - S3 Cache Settings
@@ -195,8 +196,10 @@ router_settings:
 | disable_end_user_cost_tracking_prometheus_only | boolean | If true, turns off end user cost tracking on prometheus metrics only. |
 | key_generation_settings | object | Restricts who can generate keys. [Further docs](./virtual_keys.md#restricting-key-generation) |
 | disable_add_transform_inline_image_block | boolean | For Fireworks AI models - if true, turns off the auto-add of `#transform=inline` to the url of the image_url, if the model is not a vision model. |
+| use_chat_completions_url_for_anthropic_messages | boolean | If true, routes OpenAI `/v1/messages` requests through chat/completions instead of the Responses API. Can also be set via env var `LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES=true`. |
 | disable_hf_tokenizer_download | boolean | If true, it defaults to using the openai tokenizer for all models (including huggingface models). |
 | enable_json_schema_validation | boolean | If true, enables json schema validation for all requests. |
+| enable_key_alias_format_validation | boolean | If true, validates `key_alias` format on `/key/generate` and `/key/update`. Must be 2-255 chars, start/end with alphanumeric, only allow `a-zA-Z0-9_-/.@`. Default `false`. |
 | disable_copilot_system_to_assistant | boolean | **DEPRECATED** - GitHub Copilot API supports system prompts. |
 
 ### general_settings - Reference
@@ -358,7 +361,7 @@ router_settings:
 | redis_url | str | URL for Redis server. **Known performance issue with Redis URL.** |
 | cache_responses | boolean | Flag to enable caching LLM Responses, if cache set under `router_settings`. If true, caches responses. Defaults to False. |
 | router_general_settings | RouterGeneralSettings | [SDK-Only] Router general settings - contains optimizations like 'async_only_mode'. [Docs](../routing.md#router-general-settings) |
-| optional_pre_call_checks | List[str] | List of pre-call checks to add to the router. Supported: `router_budget_limiting`, `prompt_caching`, `responses_api_deployment_check`, `deployment_affinity`, `forward_client_headers_by_model_group` |
+| optional_pre_call_checks | List[str] | List of pre-call checks to add to the router. Supported: `router_budget_limiting`, `prompt_caching`, `responses_api_deployment_check`, `encrypted_content_affinity`, `deployment_affinity`, `session_affinity`, `forward_client_headers_by_model_group` |
 | deployment_affinity_ttl_seconds | int | TTL (seconds) for user-key → deployment affinity mapping when `deployment_affinity` is enabled (configured at Router init / proxy startup). Defaults to `3600` (1 hour). |
 | ignore_invalid_deployments | boolean | If true, ignores invalid deployments. Default for proxy is True - to prevent invalid models from blocking other models from being loaded. |
 | search_tools | List[SearchToolTypedDict] | List of search tool configurations for Search API integration. Each tool specifies a search_tool_name and litellm_params with search_provider, api_key, api_base, etc. [Further Docs](../search.md) |
@@ -485,6 +488,8 @@ router_settings:
 | CUSTOM_TIKTOKEN_CACHE_DIR | Custom directory for Tiktoken cache
 | CONFIDENT_API_KEY | API key for Confident AI (Deepeval) Logging service
 | COHERE_API_BASE | Base URL for Cohere API. Default is https://api.cohere.com
+| COMPETITOR_LLM_TEMPERATURE | Temperature setting for the LLM used in competitor discovery. Default is 0.3
+| CURSOR_API_BASE | API base URL for Cursor AI provider integration. Default is https://api.cursor.com
 | DATABASE_HOST | Hostname for the database server
 | DATABASE_NAME | Name of the database
 | DATABASE_PASSWORD | Password for the database user
@@ -553,6 +558,10 @@ router_settings:
 | DEFAULT_MCP_SEMANTIC_FILTER_SIMILARITY_THRESHOLD | Default similarity threshold for MCP semantic tool filtering. Default is 0.3
 | DEFAULT_MCP_SEMANTIC_FILTER_TOP_K | Default number of top results to return for MCP semantic tool filtering. Default is 10
 | MCP_NPM_CACHE_DIR | Directory for npm cache used by STDIO MCP servers. In containers the default (~/.npm) may not exist or be read-only. Default is `/tmp/.npm_mcp_cache`
+| LITELLM_MCP_CLIENT_TIMEOUT | MCP client connection timeout in seconds (stdio and HTTP/SSE transports). Default is 60
+| LITELLM_MCP_TOOL_LISTING_TIMEOUT | Timeout in seconds for listing tools from an MCP server. Default is 30
+| LITELLM_MCP_METADATA_TIMEOUT | HTTP client timeout in seconds for OAuth metadata fetching. Default is 10
+| LITELLM_MCP_HEALTH_CHECK_TIMEOUT | Health check timeout in seconds for MCP servers. Default is 10
 | MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL | Default TTL in seconds for MCP OAuth2 token cache. Default is 3600
 | MCP_OAUTH2_TOKEN_CACHE_MAX_SIZE | Maximum number of entries in MCP OAuth2 token cache. Default is 200
 | MCP_OAUTH2_TOKEN_CACHE_MIN_TTL | Minimum TTL in seconds for MCP OAuth2 token cache. Default is 10
@@ -573,6 +582,8 @@ router_settings:
 | DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_PRO | Default minimal reasoning effort thinking budget for Gemini 2.5 Pro. Default is 512
 | DEFAULT_REDIS_MAJOR_VERSION | Default Redis major version to assume when version cannot be determined. Default is 7
 | DEFAULT_REDIS_SYNC_INTERVAL | Default Redis synchronization interval in seconds. Default is 1
+| DEFAULT_SEMANTIC_GUARD_EMBEDDING_MODEL | Default embedding model for Semantic Guard (route-matching guardrail). Default is "text-embedding-3-small"
+| DEFAULT_SEMANTIC_GUARD_SIMILARITY_THRESHOLD | Default similarity threshold for Semantic Guard route matching. Default is 0.75
 | DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND | Default price per second for Replicate GPU. Default is 0.001400
 | DEFAULT_REPLICATE_POLLING_DELAY_SECONDS | Default delay in seconds for Replicate polling. Default is 1
 | DEFAULT_REPLICATE_POLLING_RETRIES | Default number of retries for Replicate polling. Default is 5
@@ -752,15 +763,18 @@ router_settings:
 | LITELLM_ANTHROPIC_BETA_HEADERS_URL | Custom URL for fetching Anthropic beta headers configuration. Default is the GitHub main branch URL
 | LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX | Disable automatic URL suffix appending for Anthropic API base URLs. When set to `true`, prevents LiteLLM from automatically adding `/v1/messages` or `/v1/complete` to custom Anthropic API endpoints
 | LITELLM_ASSETS_PATH | Path to directory for UI assets and logos. Used when running with read-only filesystem (e.g., Kubernetes). Default is `/var/lib/litellm/assets` in Docker.
+| LITELLM_BLOG_POSTS_URL | Custom URL for fetching LiteLLM blog posts JSON. Default is the GitHub main branch URL
 | LITELLM_CLI_JWT_EXPIRATION_HOURS | Expiration time in hours for CLI-generated JWT tokens. Default is 24 hours
 | LITELLM_DD_AGENT_HOST | Hostname or IP of DataDog agent for LiteLLM-specific logging. When set, logs are sent to agent instead of direct API
 | LITELLM_DEPLOYMENT_ENVIRONMENT | Environment name for the deployment (e.g., "production", "staging"). Used as a fallback when OTEL_ENVIRONMENT_NAME is not set. Sets the `environment` tag in telemetry data
+| LITELLM_DETAILED_TIMING | When true, adds detailed per-phase timing headers to responses (`x-litellm-timing-{pre-processing,llm-api,post-processing,message-copy}-ms`). Default is false. See [latency overhead docs](../troubleshoot/latency_overhead.md)
 | LITELLM_DD_AGENT_PORT | Port of DataDog agent for LiteLLM-specific log intake. Default is 10518
 | LITELLM_DD_LLM_OBS_PORT | Port for Datadog LLM Observability agent. Default is 8126
 | LITELLM_DONT_SHOW_FEEDBACK_BOX | Flag to hide feedback box in LiteLLM UI
 | LITELLM_DROP_PARAMS | Parameters to drop in LiteLLM requests
 | LITELLM_MODIFY_PARAMS | Parameters to modify in LiteLLM requests
 | LITELLM_EMAIL | Email associated with LiteLLM account
+| LITELLM_FAVICON_URL | Custom URL for the LiteLLM UI favicon. When set, overrides the default favicon
 | LITELLM_GLOBAL_MAX_PARALLEL_REQUEST_RETRIES | Maximum retries for parallel requests in LiteLLM
 | LITELLM_GLOBAL_MAX_PARALLEL_REQUEST_RETRY_TIMEOUT | Timeout for retries of parallel requests in LiteLLM
 | LITELLM_DISABLE_LAZY_LOADING | When set to "1", "true", "yes", or "on", disables lazy loading of attributes (currently only affects encoding/tiktoken). This ensures encoding is initialized before VCR starts recording HTTP requests, fixing VCR cassette creation issues. See [issue #18659](https://github.com/BerriAI/litellm/issues/18659)
@@ -768,12 +782,14 @@ router_settings:
 | LITELLM_HOSTED_UI | URL of the hosted UI for LiteLLM
 | LITELLM_UI_API_DOC_BASE_URL | Optional override for the API Reference base URL (used in sample code/docs) when the admin UI runs on a different host than the proxy. Defaults to `PROXY_BASE_URL` when unset.
 | LITELLM_UI_PATH | Path to directory for Admin UI files. Used when running with read-only filesystem (e.g., Kubernetes). Default is `/var/lib/litellm/ui` in Docker.
+| LITELLM_UI_SESSION_DURATION | Duration for UI login session (username/password, SSO, invitation links). Format: "30s", "30m", "24h", "7d". Does not apply to EXPERIMENTAL_UI_LOGIN flow, which uses a fixed 10-minute expiry for security. Default is "24h"
 | LITELM_ENVIRONMENT | Environment of LiteLLM Instance, used by logging services. Currently only used by DeepEval.
 | LITELLM_KEY_ROTATION_ENABLED | Enable auto-key rotation for LiteLLM (boolean). Default is false.
 | LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS | Interval in seconds for how often to run job that auto-rotates keys. Default is 86400 (24 hours).
 | LITELLM_KEY_ROTATION_GRACE_PERIOD | Duration to keep old key valid after rotation (e.g. "24h", "2d"). Default is empty (immediate revoke). Used for scheduled rotations and as fallback when not specified in regenerate request.
 | LITELLM_LICENSE | License key for LiteLLM usage
 | LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS | Set to `True` to use the local bundled Anthropic beta headers config only, disabling remote fetching. Default is `False`
+| LITELLM_LOCAL_BLOG_POSTS | When set to `True`, uses the local bundled blog posts only, disabling remote fetching from GitHub. Default is `False`
 | LITELLM_LOCAL_MODEL_COST_MAP | Local configuration for model cost mapping in LiteLLM
 | LITELLM_LOCAL_POLICY_TEMPLATES | When set to "true", uses local backup policy templates instead of fetching from GitHub. Policy templates are fetched from https://raw.githubusercontent.com/BerriAI/litellm/main/policy_templates.json by default, with automatic fallback to local backup on failure
 | LITELLM_LOG | Enable detailed logging for LiteLLM
@@ -788,6 +804,9 @@ router_settings:
 | PYROSCOPE_SERVER_ADDRESS | Pyroscope server URL to send profiles to. Required when LITELLM_ENABLE_PYROSCOPE is true. No default.
 | PYROSCOPE_SAMPLE_RATE | Optional. Sample rate for Pyroscope profiling (integer). No default; when unset, the pyroscope-io library default is used.
 | LITELLM_MASTER_KEY | Master key for proxy authentication
+| LITELLM_MAX_BUDGET_PER_SESSION_TTL | TTL in seconds for session budget counters used by the max-budget-per-session limiter. Default is 3600 (1 hour)
+| LITELLM_MAX_ITERATIONS_TTL | TTL in seconds for session iteration counters used by the max-iterations limiter. Default is 3600 (1 hour)
+| LITELLM_MAX_STREAMING_DURATION_SECONDS | Maximum duration in seconds allowed for a streaming response. Streams exceeding this duration are terminated with a Timeout error. Default is None (no limit)
 | LITELLM_MODE | Operating mode for LiteLLM (e.g., production, development)
 | LITELLM_NON_ROOT | Flag to run LiteLLM in non-root mode for enhanced security in Docker containers
 | LITELLM_RATE_LIMIT_WINDOW_SIZE | Rate limit window size for LiteLLM. Default is 60
@@ -796,7 +815,9 @@ router_settings:
 | LITELLM_SSL_CIPHERS | SSL/TLS cipher configuration for faster handshakes. Controls cipher suite preferences for OpenSSL connections.
 | LITELLM_SECRET_AWS_KMS_LITELLM_LICENSE | AWS KMS encrypted license for LiteLLM
 | LITELLM_TOKEN | Access token for LiteLLM integration
+| LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES | When set to "true", routes OpenAI /v1/messages requests through chat/completions instead of the Responses API for Anthropic models. Can also be set via `litellm_settings.use_chat_completions_url_for_anthropic_messages`
 | LITELLM_USER_AGENT | Custom user agent string for LiteLLM API requests. Used for partner telemetry attribution
+| LITELLM_WORKER_STARTUP_HOOKS | Comma-separated list of `module.path:function_name` callables to run in each worker process during startup. Runs early in the worker lifecycle (before config/DB loading). Useful for re-initializing per-process state like [gflags](https://github.com/google/python-gflags). See [Worker Startup Hooks](/proxy/worker_startup_hooks) for details
 | LITELLM_PRINT_STANDARD_LOGGING_PAYLOAD | If true, prints the standard logging payload to the console - useful for debugging
 | LITELM_ENVIRONMENT | Environment for LiteLLM Instance. This is currently only logged to DeepEval to determine the environment for DeepEval integration.
 | LITELLM_ASYNCIO_QUEUE_MAXSIZE | Maximum size for asyncio queues (e.g. log queues, spend update queues, and cookbook examples such as realtime audio in `nova_sonic_realtime.py`). Bounds in-memory growth to prevent OOM. Default is 1000.
@@ -806,6 +827,8 @@ router_settings:
 | LOGGING_WORKER_MAX_QUEUE_SIZE | Maximum size of the logging worker queue. When the queue is full, the worker aggressively clears tasks to make room instead of dropping logs. Default is 50,000
 | LOGGING_WORKER_MAX_TIME_PER_COROUTINE | Maximum time in seconds allowed for each coroutine in the logging worker before timing out. Default is 20.0
 | LOGGING_WORKER_CLEAR_PERCENTAGE | Percentage of the queue to extract when clearing. Default is 50% 
+| MAX_BASE64_LENGTH_FOR_LOGGING | Maximum number of base64 characters to keep in logging payloads. Data URIs exceeding this are replaced with a size placeholder. Set to 0 to disable truncation. Default is 64
+| MAX_COMPETITOR_NAMES | Maximum number of competitor names allowed in policy template enrichment. Default is 100
 | MAX_EXCEPTION_MESSAGE_LENGTH | Maximum length for exception messages. Default is 2000
 | MAX_ITERATIONS_TO_CLEAR_QUEUE | Maximum number of iterations to attempt when clearing the logging worker queue during shutdown. Default is 200
 | MAX_TIME_TO_CLEAR_QUEUE | Maximum time in seconds to spend clearing the logging worker queue during shutdown. Default is 5.0
@@ -828,6 +851,7 @@ router_settings:
 | MAX_LANGFUSE_INITIALIZED_CLIENTS | Maximum number of Langfuse clients to initialize on proxy. Default is 50. This is set since langfuse initializes 1 thread everytime a client is initialized. We've had an incident in the past where we reached 100% cpu utilization because Langfuse was initialized several times.
 | MAX_MCP_SEMANTIC_FILTER_TOOLS_HEADER_LENGTH | Maximum header length for MCP semantic filter tools. Default is 150
 | MAX_POLICY_ESTIMATE_IMPACT_ROWS | Maximum number of rows returned when estimating the impact of a policy. Default is 1000
+| MAX_PAYLOAD_SIZE_FOR_DEBUG_LOG | Maximum payload size in bytes for full DEBUG serialization. Payloads exceeding this will be truncated in logs. Default is 102400 (100 KB)
 | MIN_NON_ZERO_TEMPERATURE | Minimum non-zero temperature value. Default is 0.0001
 | MINIMUM_PROMPT_CACHE_TOKEN_COUNT | Minimum token count for caching a prompt. Default is 1024
 | MISTRAL_API_BASE | Base URL for Mistral API. Default is https://api.mistral.ai
@@ -891,6 +915,14 @@ router_settings:
 | POSTHOG_API_URL | Base URL for PostHog API (defaults to https://us.i.posthog.com)
 | POSTHOG_MOCK | Enable mock mode for PostHog integration testing. When set to true, intercepts PostHog API calls and returns mock responses without making actual network calls. Default is false
 | POSTHOG_MOCK_LATENCY_MS | Mock latency in milliseconds for PostHog API calls when mock mode is enabled. Simulates network round-trip time. Default is 100ms
+| PRISMA_AUTH_RECONNECT_LOCK_TIMEOUT_SECONDS | Lock timeout in seconds for Prisma auth reconnection. Default is 0.1
+| PRISMA_AUTH_RECONNECT_TIMEOUT_SECONDS | Timeout in seconds for Prisma auth reconnection attempts. Default is 2.0
+| PRISMA_HEALTH_WATCHDOG_ENABLED | Enable the Prisma DB health watchdog that monitors and reconnects on connection loss. Default is true
+| PRISMA_HEALTH_WATCHDOG_INTERVAL_SECONDS | Interval in seconds for Prisma health watchdog probes. Default is 30
+| PRISMA_HEALTH_WATCHDOG_PROBE_TIMEOUT_SECONDS | Timeout in seconds for each Prisma health probe. Default is 5.0
+| PRISMA_RECONNECT_COOLDOWN_SECONDS | Cooldown in seconds between Prisma reconnection attempts. Default is 15
+| PRISMA_RECONNECT_ESCALATION_THRESHOLD | Number of consecutive reconnect failures before escalating the reconnection strategy. Default is 3
+| PRISMA_WATCHDOG_RECONNECT_TIMEOUT_SECONDS | Timeout in seconds for Prisma watchdog-initiated reconnection. Default is 30.0
 | PREDIBASE_API_BASE | Base URL for Predibase API
 | PRESIDIO_ANALYZER_API_BASE | Base URL for Presidio Analyzer service
 | PRESIDIO_ANONYMIZER_API_BASE | Base URL for Presidio Anonymizer service
@@ -912,6 +944,7 @@ router_settings:
 | QDRANT_URL | Connection URL for Qdrant database
 | QDRANT_VECTOR_SIZE | Vector size for Qdrant operations. Default is 1536
 | REDIS_CONNECTION_POOL_TIMEOUT | Timeout in seconds for Redis connection pool. Default is 5
+| REDIS_CLUSTER_NODES | JSON-formatted list of Redis cluster startup nodes for Redis Cluster mode. Example: '[{"host": "node1", "port": 6379}]'
 | REDIS_HOST | Hostname for Redis server
 | REDIS_PASSWORD | Password for Redis service
 | REDIS_PORT | Port number for Redis server
@@ -973,6 +1006,7 @@ router_settings:
 | TOGETHER_AI_EMBEDDING_150_M | Size parameter for Together AI 150M embedding model. Default is 150
 | TOGETHER_AI_EMBEDDING_350_M | Size parameter for Together AI 350M embedding model. Default is 350
 | TOOL_CHOICE_OBJECT_TOKEN_COUNT | Token count for tool choice objects. Default is 4
+| TOOL_POLICY_CACHE_TTL_SECONDS | TTL in seconds for caching tool policy guardrail results. Default is 60
 | UI_LOGO_PATH | Path to the logo image used in the UI
 | UI_PASSWORD | Password for accessing the UI
 | UI_USERNAME | Username for accessing the UI

docs/my-website/docs/proxy/cost_tracking.md

@@ -8,6 +8,8 @@ Track spend for keys, users, and teams across 100+ LLMs.
 
 LiteLLM automatically tracks spend for all known models. See our [model cost map](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json)
 
+Provider-specific cost tracking (e.g., [Vertex AI PayGo / priority pricing](../providers/vertex.md#paygo--priority-cost-tracking), [Bedrock service tiers](../providers/bedrock.md#usage---service-tier), [Azure base model mapping](./custom_pricing.md#set-base_model-for-cost-tracking-eg-azure-deployments)) is applied automatically when the response includes tier metadata.
+
 :::tip Keep Pricing Data Updated
 [Sync model pricing data from GitHub](./sync_models_github.md) to ensure accurate cost tracking.
 :::
@@ -161,7 +163,7 @@ Use this when you want non-proxy admins to access `/spend` endpoints
 
 :::info
 
-Schedule a [meeting with us to get your Enterprise License](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
+Schedule a [meeting with us to get your Enterprise License](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
 
 :::
 
@@ -326,6 +328,10 @@ See our [Swagger API](https://litellm-api.up.railway.app/#/Budget%20%26%20Spend%
 
 ## Custom Tags
 
+:::tip See Full Request Tags Documentation
+For comprehensive documentation on all tag options including `x-litellm-tags` header, request body `tags`, and config-based tags, see the dedicated [Request Tags](./request_tags.md) page.
+:::
+
 Requirements:
 
 - Virtual Keys & a database should be set up, see [virtual keys](https://docs.litellm.ai/docs/proxy/virtual_keys)

docs/my-website/docs/proxy/credential_usage_tracking.md

@@ -0,0 +1,19 @@
+# Credential Usage Tracking
+
+When a model is attached to a [reusable credential](./ui_credentials.md), LiteLLM automatically injects the credential name as a tag on every request that uses that model. This means credential-level spend and usage are tracked with zero extra configuration.
+
+## How It Works
+
+When you attach a model to a reusable credential via `litellm_credential_name`, each request routed through that model is tagged `Credential: <name>` (for example, `Credential: xAI`). This tag flows into `DailyTagSpend` and appears in the **Tag** view on the Usage page, where you can filter spend and usage by credential.
+
+If a model has no credential attached, behavior is unchanged—no credential tag is added.
+
+## Viewing Credential Usage
+
+In the Admin UI, go to **Usage → Tag** and look for tags with the `Credential: ` prefix. These represent aggregated spend and token usage across all requests that used that credential.
+
+## Related Documentation
+
+- [Adding LLM Credentials](./ui_credentials.md) - How to create and attach reusable credentials to models
+- [Tag Budgets](./tag_budgets.md) - Setting spend limits on tags
+- [Tag Routing](./tag_routing.md) - Routing requests based on tags

docs/my-website/docs/proxy/custom_pricing.md

@@ -104,9 +104,18 @@ There are other keys you can use to specify costs for different scenarios and mo
 - `input_cost_per_video_per_second` - Cost per second of video input
 - `input_cost_per_video_per_second_above_128k_tokens` - Video cost for large contexts
 - `input_cost_per_character` - Character-based pricing for some providers
+- `input_cost_per_token_priority` / `output_cost_per_token_priority` - Priority/PayGo pricing (Vertex AI Gemini, Bedrock)
+- `input_cost_per_token_flex` / `output_cost_per_token_flex` - Batch/flex pricing
 
 These keys evolve based on how new models handle multimodality. The latest version can be found at [https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json).
 
+### Service Tier / PayGo Pricing (Vertex AI, Bedrock)
+
+For providers that support multiple pricing tiers (e.g., Vertex AI PayGo, Bedrock service tiers), LiteLLM automatically applies the correct cost based on the response:
+
+- **Vertex AI Gemini**: Uses `usageMetadata.trafficType` (`ON_DEMAND_PRIORITY` → priority, `FLEX`/`BATCH` → flex). See [Vertex AI - PayGo / Priority Cost Tracking](../providers/vertex.md#paygo--priority-cost-tracking).
+- **Bedrock**: Uses `serviceTier` from the response. See [Bedrock - Usage - Service Tier](../providers/bedrock.md#usage---service-tier).
+
 ## Zero-Cost Models (Bypass Budget Checks)
 
 **Use Case**: You have on-premises or free models that should be accessible even when users exceed their budget limits.