Merge branch 'BerriAI:main' into main

.circleci/requirements.txt

@@ -1,5 +1,5 @@
 # used by CI/CD testing
-openai==1.81.0
+openai==1.100.1
 python-dotenv
 tiktoken
 importlib_metadata
@@ -10,6 +10,9 @@ anthropic
 orjson==3.10.12 # fast /embedding responses
 pydantic==2.10.2
 google-cloud-aiplatform==1.43.0
+google-cloud-iam==2.19.1
 fastapi-sso==0.16.0 
 uvloop==0.21.0
-mcp==1.5.0    # for MCP server  
+mcp==1.10.1    # for MCP server  
+semantic_router==0.1.10 # for auto-routing with litellm
+fastuuid==0.12.0

.devcontainer/devcontainer.json

@@ -11,7 +11,12 @@
 	// },
 
 	// Features to add to the dev container. More info: https://containers.dev/features.
-	// "features": {},
+	"features": {
+		"ghcr.io/devcontainers/features/node:1": {
+			"version": "lts"
+		},
+		"ghcr.io/devcontainers/features/docker-in-docker:2": {}
+	},
 
 	// Configure tool-specific properties.
 	"customizations": {
@@ -30,7 +35,7 @@
 	
 	// Use 'forwardPorts' to make a list of ports inside the container available locally.
 	"forwardPorts": [4000],
-
+	
 	"containerEnv": {
 		"LITELLM_LOG": "DEBUG"
 	},
@@ -48,5 +53,5 @@
 	// "remoteUser": "litellm",
 
 	// Use 'postCreateCommand' to run commands after the container is created.
-	"postCreateCommand": "pipx install poetry && poetry install -E extra_proxy -E proxy"
+	"postCreateCommand": "bash ./.devcontainer/post-create.sh"
 }

.devcontainer/post-create.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -e
+
+echo "[post-create] Installing poetry via pip"
+python -m pip install --upgrade pip
+python -m pip install poetry
+
+echo "[post-create] Installing Python dependencies (poetry)"
+poetry install --with dev --extras proxy
+
+echo "[post-create] Generating Prisma client"
+poetry run prisma generate
+
+echo "[post-create] Installing npm dependencies"
+cd ui/litellm-dashboard && npm install --no-audit --no-fund
+
+echo "[post-create] Done"

.github/scripts/scan_keywords.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+import json
+import os
+import sys
+import urllib.request
+import urllib.error
+
+
+def read_event_payload() -> dict:
+    event_path = os.environ.get("GITHUB_EVENT_PATH")
+    if not event_path or not os.path.exists(event_path):
+        return {}
+    with open(event_path, "r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+def get_issue_text(event: dict) -> tuple[str, str, int, str, str]:
+    issue = event.get("issue") or {}
+    title = (issue.get("title") or "").strip()
+    body = (issue.get("body") or "").strip()
+    number = issue.get("number") or 0
+    html_url = issue.get("html_url") or ""
+    author = ((issue.get("user") or {}).get("login") or "").strip()
+    return title, body, number, html_url, author
+
+
+def detect_keywords(text: str, keywords: list[str]) -> list[str]:
+    lowered = text.lower()
+    matches = []
+    for keyword in keywords:
+        k = keyword.strip().lower()
+        if not k:
+            continue
+        if k in lowered:
+            matches.append(keyword.strip())
+    # Deduplicate while preserving order
+    seen = set()
+    unique_matches = []
+    for m in matches:
+        if m not in seen:
+            unique_matches.append(m)
+            seen.add(m)
+    return unique_matches
+
+
+def send_webhook(webhook_url: str, payload: dict) -> None:
+    if not webhook_url:
+        return
+    data = json.dumps(payload).encode("utf-8")
+    req = urllib.request.Request(
+        webhook_url,
+        data=data,
+        headers={"Content-Type": "application/json"},
+        method="POST",
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=10) as resp:
+            resp.read()
+    except urllib.error.HTTPError as e:
+        print(f"Webhook HTTP error: {e.code} {e.reason}", file=sys.stderr)
+    except urllib.error.URLError as e:
+        print(f"Webhook URL error: {e.reason}", file=sys.stderr)
+    except Exception as e:
+        print(f"Webhook unexpected error: {e}", file=sys.stderr)
+
+
+def _excerpt(text: str, max_len: int = 400) -> str:
+    if not text:
+        return ""
+    
+    # Keep original formatting
+    if len(text) <= max_len:
+        return text
+    return text[: max_len - 1] + "…"
+
+
+
+def main() -> int:
+    event = read_event_payload()
+    if not event:
+        print("::warning::No event payload found; exiting without labeling.")
+        return 0
+
+    # Read issue details
+    title, body, number, html_url, author = get_issue_text(event)
+    combined_text = f"{title}\n\n{body}".strip()
+
+    # Keywords from env or defaults
+    keywords_env = os.environ.get("KEYWORDS", "")
+    default_keywords = ["azure", "openai", "bedrock", "vertexai", "vertex ai", "anthropic"]
+    keywords = [k.strip() for k in keywords_env.split(",")] if keywords_env else default_keywords
+
+    matches = detect_keywords(combined_text, keywords)
+    found = bool(matches)
+
+    # Emit outputs
+    github_output = os.environ.get("GITHUB_OUTPUT")
+    if github_output:
+        with open(github_output, "a", encoding="utf-8") as fh:
+            fh.write(f"found={'true' if found else 'false'}\n")
+            fh.write(f"matches={','.join(matches)}\n")
+
+    # Optional webhook notification
+    webhook_url = os.environ.get("PROVIDER_ISSUE_WEBHOOK_URL", "").strip()
+    if found and webhook_url:
+        repo_full = (event.get("repository") or {}).get("full_name", "")
+        title_part = f"*{title}*" if title else "New issue"
+        author_part = f" by @{author}" if author else ""
+        body_preview = _excerpt(body)
+        preview_block = f"\n{body_preview}" if body_preview else ""
+        payload = {
+            "text": (
+                f"New issue 🚨\n"
+                f"{title_part}\n\n{preview_block}\n"
+                f"<{html_url}|View issue>\n"
+                f"Author: {author}"
+            )
+        }
+        send_webhook(webhook_url, payload)
+
+    # Print a short log line for Actions UI
+    if found:
+        print(f"Detected provider keywords: {', '.join(matches)}")
+    else:
+        print("No provider keywords detected.")
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
+
+

.github/workflows/README.md

@@ -0,0 +1,35 @@
+# Simple PyPI Publishing
+
+A GitHub workflow to manually publish LiteLLM packages to PyPI with a specified version.
+
+## How to Use
+
+1. Go to the **Actions** tab in the GitHub repository
+2. Select **Simple PyPI Publish** from the workflow list
+3. Click **Run workflow**
+4. Enter the version to publish (e.g., `1.74.10`)
+
+## What the Workflow Does
+
+1. **Updates** the version in `pyproject.toml`
+2. **Copies** the model prices backup file
+3. **Builds** the Python package
+4. **Publishes** to PyPI
+
+## Prerequisites
+
+Make sure the following secret is configured in the repository:
+- `PYPI_PUBLISH_PASSWORD`: PyPI API token for authentication
+
+## Example Usage
+
+- Version: `1.74.11` → Publishes as v1.74.11
+- Version: `1.74.10-hotfix1` → Publishes as v1.74.10-hotfix1
+
+## Features
+
+- ✅ Manual trigger with version input
+- ✅ Automatic version updates in `pyproject.toml`
+- ✅ Repository safety check (only runs on official repo)
+- ✅ Clean package building and publishing
+- ✅ Success confirmation with PyPI package link 

.github/workflows/auto_update_price_and_context_window_file.py

@@ -43,8 +43,8 @@ def write_to_file(file_path, data):
         # Print an error message if writing to file fails
         print("Error updating JSON file:", e)
 
-# Update the existing models and add the missing models
-def transform_remote_data(data):
+# Update the existing models and add the missing models for OpenRouter
+def transform_openrouter_data(data):
     transformed = {}
     for row in data:
         # Add the fields 'max_tokens' and 'input_cost_per_token'
@@ -81,6 +81,34 @@ def transform_remote_data(data):
 
     return transformed
 
+# Update the existing models and add the missing models for Vercel AI Gateway
+def transform_vercel_ai_gateway_data(data):
+    transformed = {}
+    for row in data:
+        obj = {
+            "max_tokens": row["context_window"],
+            "input_cost_per_token": float(row["pricing"]["input"]),
+            "output_cost_per_token": float(row["pricing"]["output"]),
+            'max_output_tokens': row['max_tokens'],
+            'max_input_tokens': row["context_window"],
+        }
+
+        # Handle cache pricing if available
+        if "pricing" in row:
+            if "input_cache_read" in row["pricing"] and row["pricing"]["input_cache_read"] is not None:
+                obj['cache_read_input_token_cost'] = float(f"{float(row['pricing']['input_cache_read']):e}")
+            
+            if "input_cache_write" in row["pricing"] and row["pricing"]["input_cache_write"] is not None:
+                obj['cache_creation_input_token_cost'] = float(f"{float(row['pricing']['input_cache_write']):e}")
+
+        mode = "embedding" if "embedding" in row["id"].lower() else "chat"
+        
+        obj.update({"litellm_provider": "vercel_ai_gateway", "mode": mode})
+
+        transformed[f'vercel_ai_gateway/{row["id"]}'] = obj
+
+    return transformed
+
 
 # Load local data from a specified file
 def load_local_data(file_path):
@@ -100,22 +128,32 @@ def load_local_data(file_path):
 
 def main():
     local_file_path = "model_prices_and_context_window.json"  # Path to the local data file
-    url = "https://openrouter.ai/api/v1/models"  # URL to fetch remote data
+    openrouter_url = "https://openrouter.ai/api/v1/models"  # URL to fetch OpenRouter data
+    vercel_ai_gateway_url = "https://ai-gateway.vercel.sh/v1/models"  # URL to fetch Vercel AI Gateway data
 
     # Load local data from file
     local_data = load_local_data(local_file_path)
-    # Fetch remote data asynchronously
-    remote_data = asyncio.run(fetch_data(url))
-    # Transform the fetched remote data
-    remote_data = transform_remote_data(remote_data)
+    
+    # Fetch OpenRouter data
+    openrouter_data = asyncio.run(fetch_data(openrouter_url))
+    # Transform the fetched OpenRouter data
+    openrouter_data = transform_openrouter_data(openrouter_data)
+    
+    # Fetch Vercel AI Gateway data
+    vercel_data = asyncio.run(fetch_data(vercel_ai_gateway_url))
+    # Transform the fetched Vercel AI Gateway data
+    vercel_data = transform_vercel_ai_gateway_data(vercel_data)
+    
+    # Combine both datasets
+    all_remote_data = {**openrouter_data, **vercel_data}
 
-    # If both local and remote data are available, synchronize and save
-    if local_data and remote_data:
-        sync_local_data_with_remote(local_data, remote_data)
+    # If both local and openrouter data are available, synchronize and save
+    if local_data and all_remote_data:
+        sync_local_data_with_remote(local_data, all_remote_data)
         write_to_file(local_file_path, local_data)
     else:
         print("Failed to fetch model data from either local file or URL.")
 
 # Entry point of the script
 if __name__ == "__main__":
-    main()
+    main()

.github/workflows/ghcr_deploy.yml

@@ -6,7 +6,7 @@ on:
       tag:
         description: "The tag version you want to build"
       release_type:
-        description: "The release type you want to build. Can be 'latest', 'stable', 'dev'"
+        description: "The release type you want to build. Can be 'latest', 'stable', 'dev', 'rc'"
         type: string
         default: "latest"
       commit_hash:
@@ -73,7 +73,14 @@ jobs:
           push: true
           file: ./litellm-js/spend-logs/Dockerfile
           tags: litellm/litellm-spend_logs:${{ github.event.inputs.tag || 'latest' }}
-      
+      -
+        name: Build and push litellm-non_root image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          file: ./docker/Dockerfile.non_root
+          tags: litellm/litellm-non_root:${{ github.event.inputs.tag || 'latest' }}
   build-and-push-image:
     runs-on: ubuntu-latest
     # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
@@ -114,9 +121,9 @@ jobs:
           tags: |
             ${{ steps.meta.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
             ${{ steps.meta.outputs.tags }}-${{ github.event.inputs.release_type }}
-            ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
+            ${{ (github.event.inputs.release_type == 'stable'  || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
             ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm:main-stable', env.REGISTRY) || '' }},
-            ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm:{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
+            ${{ (github.event.inputs.release_type == 'stable'  || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm:{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
           labels: ${{ steps.meta.outputs.labels }}
           platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
   
@@ -158,7 +165,7 @@ jobs:
           tags: |
             ${{ steps.meta-ee.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
             ${{ steps.meta-ee.outputs.tags }}-${{ github.event.inputs.release_type }}
-            ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-ee:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
+            ${{ (github.event.inputs.release_type == 'stable'  || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm-ee:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
             ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-ee:main-stable', env.REGISTRY) || '' }}
           labels: ${{ steps.meta-ee.outputs.labels }}
           platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
@@ -201,7 +208,7 @@ jobs:
           tags: |
             ${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
             ${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.release_type }}
-            ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-database:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
+            ${{ (github.event.inputs.release_type == 'stable'  || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm-database:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
             ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-database:main-stable', env.REGISTRY) || '' }}
           labels: ${{ steps.meta-database.outputs.labels }}
           platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
@@ -244,7 +251,7 @@ jobs:
           tags: |
             ${{ steps.meta-non_root.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
             ${{ steps.meta-non_root.outputs.tags }}-${{ github.event.inputs.release_type }}
-            ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-non_root:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
+            ${{ (github.event.inputs.release_type == 'stable'  || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm-non_root:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
             ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-non_root:main-stable', env.REGISTRY) || '' }}
           labels: ${{ steps.meta-non_root.outputs.labels }} 
           platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
@@ -287,7 +294,7 @@ jobs:
           tags: |
             ${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
             ${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.release_type }}
-            ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-spend_logs:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
+            ${{ (github.event.inputs.release_type == 'stable'  || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm-spend_logs:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
             ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-spend_logs:main-stable', env.REGISTRY) || '' }}
           platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
 

.github/workflows/issue-keyword-labeler.yml

@@ -0,0 +1,64 @@
+name: Issue Keyword Labeler
+
+on:
+  issues:
+    types: 
+        - opened
+
+jobs:
+  scan-and-label:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      contents: read
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Scan for provider keywords
+        id: scan
+        env:
+          PROVIDER_ISSUE_WEBHOOK_URL: ${{ secrets.PROVIDER_ISSUE_WEBHOOK_URL }}
+          KEYWORDS: azure,openai,bedrock,vertexai,vertex ai,anthropic
+        run: python3 .github/scripts/scan_keywords.py
+
+      - name: Ensure label exists
+        if: steps.scan.outputs.found == 'true'
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const labelName = 'llm translation';
+            try {
+              await github.rest.issues.getLabel({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                name: labelName
+              });
+            } catch (error) {
+              if (error.status === 404) {
+                await github.rest.issues.createLabel({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  name: labelName,
+                  color: 'c1ff72',
+                  description: 'Issues related to LLM provider translation/mapping'
+                });
+              } else {
+                throw error;
+              }
+            }
+
+      - name: Add label to the issue
+        if: steps.scan.outputs.found == 'true'
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            await github.rest.issues.addLabels({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+              labels: ['llm translation']
+            });
+

.github/workflows/llm-translation-testing.yml

@@ -0,0 +1,89 @@
+name: LLM Translation Tests
+
+on:
+  workflow_dispatch:
+    inputs:
+      release_candidate_tag:
+        description: 'Release candidate tag/version'
+        required: true
+        type: string
+  push:
+    tags:
+      - 'v*-rc*'  # Triggers on release candidate tags like v1.0.0-rc1
+  
+jobs:
+  run-llm-translation-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.inputs.release_candidate_tag || github.ref }}
+      
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: latest
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+      
+      - name: Cache Poetry dependencies
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/pypoetry
+            .venv
+          key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-poetry-
+      
+      - name: Install dependencies
+        run: |
+          poetry install --with dev
+          poetry run pip install pytest-xdist pytest-timeout
+      
+      - name: Create test results directory
+        run: mkdir -p test-results
+      
+      - name: Run LLM Translation Tests
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
+          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
+          AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
+          AZURE_API_BASE: ${{ secrets.AZURE_API_BASE }}
+          AZURE_API_VERSION: ${{ secrets.AZURE_API_VERSION }}
+          # Add other API keys as needed
+        run: |
+          python .github/workflows/run_llm_translation_tests.py \
+            --tag "${{ github.event.inputs.release_candidate_tag || github.ref_name }}" \
+            --commit "${{ github.sha }}" \
+            || true  # Continue even if tests fail
+      
+      - name: Display test summary
+        if: always()
+        run: |
+          if [ -f "test-results/llm_translation_report.md" ]; then
+            echo "Test report generated successfully!"
+            echo "Artifact will contain:"
+            echo "- test-results/junit.xml (JUnit XML results)"
+            echo "- test-results/llm_translation_report.md (Beautiful markdown report)"
+          else
+            echo "Warning: Test report was not generated"
+          fi
+      
+      - name: Upload test artifacts
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: LLM-Translation-Artifact-${{ github.event.inputs.release_candidate_tag || github.ref_name }}
+          path: test-results/
+          retention-days: 30

.github/workflows/run_llm_translation_tests.py

@@ -0,0 +1,439 @@
+#!/usr/bin/env python3
+"""
+Run LLM Translation Tests and Generate Beautiful Markdown Report
+
+This script runs the LLM translation tests and generates a comprehensive
+markdown report with provider-specific breakdowns and test statistics.
+"""
+
+import os
+import sys
+import subprocess
+import xml.etree.ElementTree as ET
+from collections import defaultdict
+from datetime import datetime
+from pathlib import Path
+import json
+from typing import Dict, List, Tuple, Optional
+
+# ANSI color codes for terminal output
+class Colors:
+    GREEN = '\033[92m'
+    RED = '\033[91m'
+    YELLOW = '\033[93m'
+    BLUE = '\033[94m'
+    PURPLE = '\033[95m'
+    CYAN = '\033[96m'
+    RESET = '\033[0m'
+    BOLD = '\033[1m'
+
+def print_colored(message: str, color: str = Colors.RESET):
+    """Print colored message to terminal"""
+    print(f"{color}{message}{Colors.RESET}")
+
+def get_provider_from_test_file(test_file: str) -> str:
+    """Map test file names to provider names"""
+    provider_mapping = {
+        'test_anthropic': 'Anthropic',
+        'test_azure': 'Azure',
+        'test_bedrock': 'AWS Bedrock',
+        'test_openai': 'OpenAI',
+        'test_vertex': 'Google Vertex AI',
+        'test_gemini': 'Google Vertex AI',
+        'test_cohere': 'Cohere',
+        'test_databricks': 'Databricks',
+        'test_groq': 'Groq',
+        'test_together': 'Together AI',
+        'test_mistral': 'Mistral',
+        'test_deepseek': 'DeepSeek',
+        'test_replicate': 'Replicate',
+        'test_huggingface': 'HuggingFace',
+        'test_fireworks': 'Fireworks AI',
+        'test_perplexity': 'Perplexity',
+        'test_cloudflare': 'Cloudflare',
+        'test_voyage': 'Voyage AI',
+        'test_xai': 'xAI',
+        'test_nvidia': 'NVIDIA',
+        'test_watsonx': 'IBM watsonx',
+        'test_azure_ai': 'Azure AI',
+        'test_snowflake': 'Snowflake',
+        'test_infinity': 'Infinity',
+        'test_jina': 'Jina AI',
+        'test_deepgram': 'Deepgram',
+        'test_clarifai': 'Clarifai',
+        'test_triton': 'Triton',
+    }
+    
+    for key, provider in provider_mapping.items():
+        if key in test_file:
+            return provider
+    
+    # For cross-provider test files
+    if any(name in test_file for name in ['test_optional_params', 'test_prompt_factory', 
+                                           'test_router', 'test_text_completion']):
+        return f'Cross-Provider Tests ({test_file})'
+    
+    return 'Other Tests'
+
+def format_duration(seconds: float) -> str:
+    """Format duration in human-readable format"""
+    if seconds < 60:
+        return f"{seconds:.2f}s"
+    elif seconds < 3600:
+        minutes = int(seconds // 60)
+        secs = seconds % 60
+        return f"{minutes}m {secs:.0f}s"
+    else:
+        hours = int(seconds // 3600)
+        minutes = int((seconds % 3600) // 60)
+        return f"{hours}h {minutes}m"
+
+
+def generate_markdown_report(junit_xml_path: str, output_path: str, tag: str = None, commit: str = None):
+    """Generate a beautiful markdown report from JUnit XML"""
+    try:
+        tree = ET.parse(junit_xml_path)
+        root = tree.getroot()
+        
+        # Handle both testsuite and testsuites root
+        if root.tag == 'testsuites':
+            suites = root.findall('testsuite')
+        else:
+            suites = [root]
+        
+        # Overall statistics
+        total_tests = 0
+        total_failures = 0
+        total_errors = 0
+        total_skipped = 0
+        total_time = 0.0
+        
+        # Provider breakdown
+        provider_stats = defaultdict(lambda: {'passed': 0, 'failed': 0, 'skipped': 0, 'errors': 0, 'time': 0.0})
+        provider_tests = defaultdict(list)
+        
+        for suite in suites:
+            total_tests += int(suite.get('tests', 0))
+            total_failures += int(suite.get('failures', 0))
+            total_errors += int(suite.get('errors', 0))
+            total_skipped += int(suite.get('skipped', 0))
+            total_time += float(suite.get('time', 0))
+            
+            for testcase in suite.findall('testcase'):
+                classname = testcase.get('classname', '')
+                test_name = testcase.get('name', '')
+                test_time = float(testcase.get('time', 0))
+                
+                # Extract test file name from classname
+                if '.' in classname:
+                    parts = classname.split('.')
+                    test_file = parts[-2] if len(parts) > 1 else 'unknown'
+                else:
+                    test_file = 'unknown'
+                
+                provider = get_provider_from_test_file(test_file)
+                provider_stats[provider]['time'] += test_time
+                
+                # Check test status
+                if testcase.find('failure') is not None:
+                    provider_stats[provider]['failed'] += 1
+                    failure = testcase.find('failure')
+                    failure_msg = failure.get('message', '') if failure is not None else ''
+                    provider_tests[provider].append({
+                        'name': test_name,
+                        'status': 'FAILED',
+                        'time': test_time,
+                        'message': failure_msg
+                    })
+                elif testcase.find('error') is not None:
+                    provider_stats[provider]['errors'] += 1
+                    error = testcase.find('error')
+                    error_msg = error.get('message', '') if error is not None else ''
+                    provider_tests[provider].append({
+                        'name': test_name,
+                        'status': 'ERROR',
+                        'time': test_time,
+                        'message': error_msg
+                    })
+                elif testcase.find('skipped') is not None:
+                    provider_stats[provider]['skipped'] += 1
+                    skip = testcase.find('skipped')
+                    skip_msg = skip.get('message', '') if skip is not None else ''
+                    provider_tests[provider].append({
+                        'name': test_name,
+                        'status': 'SKIPPED',
+                        'time': test_time,
+                        'message': skip_msg
+                    })
+                else:
+                    provider_stats[provider]['passed'] += 1
+                    provider_tests[provider].append({
+                        'name': test_name,
+                        'status': 'PASSED',
+                        'time': test_time,
+                        'message': ''
+                    })
+        
+        passed = total_tests - total_failures - total_errors - total_skipped
+        
+        # Generate the markdown report
+        with open(output_path, 'w') as f:
+            # Header
+            f.write("# LLM Translation Test Results\n\n")
+            
+            # Metadata table
+            f.write("## Test Run Information\n\n")
+            f.write("| Field | Value |\n")
+            f.write("|-------|-------|\n")
+            f.write(f"| **Tag** | `{tag or 'N/A'}` |\n")
+            f.write(f"| **Date** | {datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S UTC')} |\n")
+            f.write(f"| **Commit** | `{commit or 'N/A'}` |\n")
+            f.write(f"| **Duration** | {format_duration(total_time)} |\n")
+            f.write("\n")
+            
+            # Overall statistics with visual elements
+            f.write("## Overall Statistics\n\n")
+            
+            # Summary box
+            f.write("```\n")
+            f.write(f"Total Tests: {total_tests}\n")
+            f.write(f"├── Passed:  {passed:>4} ({(passed/total_tests)*100 if total_tests > 0 else 0:.1f}%)\n")
+            f.write(f"├── Failed:  {total_failures:>4} ({(total_failures/total_tests)*100 if total_tests > 0 else 0:.1f}%)\n")
+            f.write(f"├── Errors:  {total_errors:>4} ({(total_errors/total_tests)*100 if total_tests > 0 else 0:.1f}%)\n")
+            f.write(f"└── Skipped: {total_skipped:>4} ({(total_skipped/total_tests)*100 if total_tests > 0 else 0:.1f}%)\n")
+            f.write("```\n\n")
+            
+            
+            # Provider summary table
+            f.write("## Results by Provider\n\n")
+            f.write("| Provider | Total | Pass | Fail | Error | Skip | Pass Rate | Duration |\n")
+            f.write("|----------|-------|------|------|-------|------|-----------|----------|")
+            
+            # Sort providers: specific providers first, then cross-provider tests
+            sorted_providers = []
+            cross_provider = []
+            for p in sorted(provider_stats.keys()):
+                if 'Cross-Provider' in p or p == 'Other Tests':
+                    cross_provider.append(p)
+                else:
+                    sorted_providers.append(p)
+            
+            all_providers = sorted_providers + cross_provider
+            
+            for provider in all_providers:
+                stats = provider_stats[provider]
+                total = stats['passed'] + stats['failed'] + stats['errors'] + stats['skipped']
+                pass_rate = (stats['passed'] / total * 100) if total > 0 else 0
+                
+                f.write(f"\n| {provider} | {total} | {stats['passed']} | {stats['failed']} | ")
+                f.write(f"{stats['errors']} | {stats['skipped']} | {pass_rate:.1f}% | ")
+                f.write(f"{format_duration(stats['time'])} |")
+            
+            # Detailed test results by provider
+            f.write("\n\n## Detailed Test Results\n\n")
+            
+            for provider in sorted_providers:
+                if provider_tests[provider]:
+                    stats = provider_stats[provider]
+                    total = stats['passed'] + stats['failed'] + stats['errors'] + stats['skipped']
+                    
+                    f.write(f"### {provider}\n\n")
+                    f.write(f"**Summary:** {stats['passed']}/{total} passed ")
+                    f.write(f"({(stats['passed']/total)*100 if total > 0 else 0:.1f}%) ")
+                    f.write(f"in {format_duration(stats['time'])}\n\n")
+                    
+                    # Group tests by status
+                    tests_by_status = defaultdict(list)
+                    for test in provider_tests[provider]:
+                        tests_by_status[test['status']].append(test)
+                    
+                    # Show failed tests first (if any)
+                    if tests_by_status['FAILED']:
+                        f.write("<details>\n<summary>Failed Tests</summary>\n\n")
+                        for test in tests_by_status['FAILED']:
+                            f.write(f"- `{test['name']}` ({test['time']:.2f}s)\n")
+                            if test['message']:
+                                # Truncate long error messages
+                                msg = test['message'][:200] + '...' if len(test['message']) > 200 else test['message']
+                                f.write(f"  > {msg}\n")
+                        f.write("\n</details>\n\n")
+                    
+                    # Show errors (if any)
+                    if tests_by_status['ERROR']:
+                        f.write("<details>\n<summary>Error Tests</summary>\n\n")
+                        for test in tests_by_status['ERROR']:
+                            f.write(f"- `{test['name']}` ({test['time']:.2f}s)\n")
+                        f.write("\n</details>\n\n")
+                    
+                    # Show passed tests in collapsible section
+                    if tests_by_status['PASSED']:
+                        f.write("<details>\n<summary>Passed Tests</summary>\n\n")
+                        for test in tests_by_status['PASSED']:
+                            f.write(f"- `{test['name']}` ({test['time']:.2f}s)\n")
+                        f.write("\n</details>\n\n")
+                    
+                    # Show skipped tests (if any)
+                    if tests_by_status['SKIPPED']:
+                        f.write("<details>\n<summary>Skipped Tests</summary>\n\n")
+                        for test in tests_by_status['SKIPPED']:
+                            f.write(f"- `{test['name']}`\n")
+                        f.write("\n</details>\n\n")
+            
+            # Cross-provider tests in a separate section
+            if cross_provider:
+                f.write("### Cross-Provider Tests\n\n")
+                for provider in cross_provider:
+                    if provider_tests[provider]:
+                        stats = provider_stats[provider]
+                        total = stats['passed'] + stats['failed'] + stats['errors'] + stats['skipped']
+                        
+                        f.write(f"#### {provider}\n\n")
+                        f.write(f"**Summary:** {stats['passed']}/{total} passed ")
+                        f.write(f"({(stats['passed']/total)*100 if total > 0 else 0:.1f}%)\n\n")
+                        
+                        # For cross-provider tests, just show counts
+                        f.write(f"- Passed: {stats['passed']}\n")
+                        if stats['failed'] > 0:
+                            f.write(f"- Failed: {stats['failed']}\n")
+                        if stats['errors'] > 0:
+                            f.write(f"- Errors: {stats['errors']}\n")
+                        if stats['skipped'] > 0:
+                            f.write(f"- Skipped: {stats['skipped']}\n")
+                        f.write("\n")
+            
+        
+        print_colored(f"Report generated: {output_path}", Colors.GREEN)
+        
+    except Exception as e:
+        print_colored(f"Error generating report: {e}", Colors.RED)
+        raise
+
+def run_tests(test_path: str = "tests/llm_translation/", 
+              junit_xml: str = "test-results/junit.xml",
+              report_path: str = "test-results/llm_translation_report.md",
+              tag: str = None,
+              commit: str = None) -> int:
+    """Run the LLM translation tests and generate report"""
+    
+    # Create test results directory
+    os.makedirs(os.path.dirname(junit_xml), exist_ok=True)
+    
+    print_colored("Starting LLM Translation Tests", Colors.BOLD + Colors.BLUE)
+    print_colored(f"Test directory: {test_path}", Colors.CYAN)
+    print_colored(f"Output: {junit_xml}", Colors.CYAN)
+    print()
+    
+    # Run pytest
+    cmd = [
+        "poetry", "run", "pytest", test_path,
+        f"--junitxml={junit_xml}",
+        "-v",
+        "--tb=short",
+        "--maxfail=500",
+        "-n", "auto"
+    ]
+    
+    # Add timeout if pytest-timeout is installed
+    try:
+        subprocess.run(["poetry", "run", "python", "-c", "import pytest_timeout"], 
+                      capture_output=True, check=True)
+        cmd.extend(["--timeout=300"])
+    except:
+        print_colored("Warning: pytest-timeout not installed, skipping timeout option", Colors.YELLOW)
+    
+    print_colored("Running pytest with command:", Colors.YELLOW)
+    print(f"   {' '.join(cmd)}")
+    print()
+    
+    # Run the tests
+    result = subprocess.run(cmd, capture_output=False)
+    
+    # Generate the report regardless of test outcome
+    if os.path.exists(junit_xml):
+        print()
+        print_colored("Generating test report...", Colors.BLUE)
+        generate_markdown_report(junit_xml, report_path, tag, commit)
+        
+        # Print summary to console
+        print()
+        print_colored("Test Summary:", Colors.BOLD + Colors.PURPLE)
+        
+        # Parse XML for quick summary
+        tree = ET.parse(junit_xml)
+        root = tree.getroot()
+        
+        if root.tag == 'testsuites':
+            suites = root.findall('testsuite')
+        else:
+            suites = [root]
+        
+        total = sum(int(s.get('tests', 0)) for s in suites)
+        failures = sum(int(s.get('failures', 0)) for s in suites)
+        errors = sum(int(s.get('errors', 0)) for s in suites)
+        skipped = sum(int(s.get('skipped', 0)) for s in suites)
+        passed = total - failures - errors - skipped
+        
+        print(f"   Total:   {total}")
+        print_colored(f"   Passed:  {passed}", Colors.GREEN)
+        if failures > 0:
+            print_colored(f"   Failed:  {failures}", Colors.RED)
+        if errors > 0:
+            print_colored(f"   Errors:  {errors}", Colors.RED)
+        if skipped > 0:
+            print_colored(f"   Skipped: {skipped}", Colors.YELLOW)
+        
+        if total > 0:
+            pass_rate = (passed / total) * 100
+            color = Colors.GREEN if pass_rate >= 80 else Colors.YELLOW if pass_rate >= 60 else Colors.RED
+            print_colored(f"   Pass Rate: {pass_rate:.1f}%", color)
+    else:
+        print_colored("No test results found!", Colors.RED)
+    
+    print()
+    print_colored("Test run complete!", Colors.BOLD + Colors.GREEN)
+    
+    return result.returncode
+
+if __name__ == "__main__":
+    import argparse
+    
+    parser = argparse.ArgumentParser(description="Run LLM Translation Tests")
+    parser.add_argument("--test-path", default="tests/llm_translation/", 
+                        help="Path to test directory")
+    parser.add_argument("--junit-xml", default="test-results/junit.xml",
+                        help="Path for JUnit XML output")
+    parser.add_argument("--report", default="test-results/llm_translation_report.md",
+                        help="Path for markdown report")
+    parser.add_argument("--tag", help="Git tag or version")
+    parser.add_argument("--commit", help="Git commit SHA")
+    
+    args = parser.parse_args()
+    
+    # Get git info if not provided
+    if not args.commit:
+        try:
+            result = subprocess.run(["git", "rev-parse", "HEAD"], 
+                                    capture_output=True, text=True)
+            if result.returncode == 0:
+                args.commit = result.stdout.strip()
+        except:
+            pass
+    
+    if not args.tag:
+        try:
+            result = subprocess.run(["git", "describe", "--tags", "--abbrev=0"], 
+                                    capture_output=True, text=True)
+            if result.returncode == 0:
+                args.tag = result.stdout.strip()
+        except:
+            pass
+    
+    exit_code = run_tests(
+        test_path=args.test_path,
+        junit_xml=args.junit_xml,
+        report_path=args.report,
+        tag=args.tag,
+        commit=args.commit
+    )
+    
+    sys.exit(exit_code)

.github/workflows/simple_pypi_publish.yml

@@ -0,0 +1,67 @@
+name: Simple PyPI Publish
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Version to publish (e.g., 1.74.10)'
+        required: true
+        type: string
+
+env:
+  TWINE_USERNAME: __token__
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    if: github.repository == 'BerriAI/litellm'
+    
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install toml build wheel twine
+
+      - name: Update version in pyproject.toml
+        run: |
+          python -c "
+          import toml
+          
+          with open('pyproject.toml', 'r') as f:
+              data = toml.load(f)
+          
+          data['tool']['poetry']['version'] = '${{ github.event.inputs.version }}'
+          
+          with open('pyproject.toml', 'w') as f:
+              toml.dump(data, f)
+          
+          print(f'Updated version to ${{ github.event.inputs.version }}')
+          "
+
+      - name: Copy model prices file
+        run: |
+          cp model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json
+
+      - name: Build package
+        run: |
+          rm -rf build dist
+          python -m build
+
+      - name: Publish to PyPI
+        env:
+          TWINE_PASSWORD: ${{ secrets.PYPI_PUBLISH_PASSWORD }}
+        run: |
+          twine upload dist/*
+
+      - name: Output success
+        run: |
+          echo "✅ Successfully published litellm v${{ github.event.inputs.version }} to PyPI"
+          echo "📦 Package: https://pypi.org/project/litellm/${{ github.event.inputs.version }}/" 

.github/workflows/test-linting.yml

@@ -11,6 +11,9 @@ jobs:
 
     steps:
     - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+        clean: true
 
     - name: Set up Python
       uses: actions/setup-python@v4
@@ -20,13 +23,15 @@ jobs:
     - name: Install Poetry
       uses: snok/install-poetry@v1
 
+    - name: Clean Python cache
+      run: |
+        find . -type d -name "__pycache__" -exec rm -rf {} + || true
+        find . -name "*.pyc" -delete || true
+
     - name: Install dependencies
       run: |
-        pip install openai==1.81.0
         poetry install --with dev
-        pip install openai==1.81.0
-
-
+        poetry run pip install openai==1.100.1
 
     - name: Run Black formatting
       run: |
@@ -34,16 +39,29 @@ jobs:
         poetry run black .
         cd ..
 
+    - name: Debug - Check file state
+      run: |
+        echo "Current branch:"
+        git branch --show-current
+        echo "Last 3 commits:"
+        git log --oneline -3
+        echo "File content around line 43:"
+        head -50 litellm/litellm_core_utils/custom_logger_registry.py | tail -10
+        
     - name: Run Ruff linting
       run: |
         cd litellm
         poetry run ruff check .
         cd ..
 
+    - name: Print OpenAI version
+      run: |
+        poetry run python -c "import openai; print(f'OpenAI version: {openai.__version__}')"
+
     - name: Run MyPy type checking
       run: |
         cd litellm
-        poetry run mypy . --ignore-missing-imports
+        poetry run mypy . 
         cd ..
 
     - name: Check for circular imports

.github/workflows/test-litellm.yml

@@ -7,7 +7,7 @@ on:
 jobs:
   test:
     runs-on: ubuntu-latest
-    timeout-minutes: 15
+    timeout-minutes: 25
 
     steps:
     - uses: actions/checkout@v4
@@ -27,9 +27,12 @@ jobs:
 
     - name: Install dependencies
       run: |
-        poetry install --with dev,proxy-dev --extras proxy
+        poetry install --with dev,proxy-dev --extras "proxy semantic-router"
         poetry run pip install "pytest-retry==1.6.3"
         poetry run pip install pytest-xdist
+        poetry run pip install "google-genai==1.22.0"
+        poetry run pip install "google-cloud-aiplatform>=1.38"
+        poetry run pip install "fastapi-offline==1.7.3"
     - name: Setup litellm-enterprise as local package
       run: |
         cd enterprise
@@ -37,4 +40,4 @@ jobs:
         cd ..
     - name: Run tests
       run: |
-        poetry run pytest tests/test_litellm -x -vv -n 4 
+        poetry run pytest tests/test_litellm --tb=short -vv --maxfail=10 -n 4 

.github/workflows/test-mcp.yml

@@ -0,0 +1,48 @@
+name: LiteLLM MCP Tests (folder - tests/mcp_tests)
+
+on:
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Thank You Message
+      run: |
+        echo "### 🙏 Thank you for contributing to LiteLLM!" >> $GITHUB_STEP_SUMMARY
+        echo "Your PR is being tested now. We appreciate your help in making LiteLLM better!" >> $GITHUB_STEP_SUMMARY
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.12'
+
+    - name: Install Poetry
+      uses: snok/install-poetry@v1
+
+    - name: Install dependencies
+      run: |
+        poetry install --with dev,proxy-dev --extras "proxy semantic-router"
+        poetry run pip install "pytest==7.3.1"
+        poetry run pip install "pytest-retry==1.6.3"
+        poetry run pip install "pytest-cov==5.0.0"
+        poetry run pip install "pytest-asyncio==0.21.1"
+        poetry run pip install "respx==0.22.0"
+        poetry run pip install "pydantic==2.10.2"
+        poetry run pip install "mcp==1.10.1"
+        poetry run pip install pytest-xdist
+
+    - name: Setup litellm-enterprise as local package
+      run: |
+        cd enterprise
+        python -m pip install -e .
+        cd ..
+
+    - name: Run MCP tests
+      run: |
+        poetry run pytest tests/mcp_tests -x -vv -n 4 --cov=litellm --cov-report=xml --durations=5

.gitignore

@@ -86,7 +86,14 @@ litellm/proxy/db/migrations/0_init/migration.sql
 litellm/proxy/db/migrations/*
 litellm/proxy/migrations/*config.yaml
 litellm/proxy/migrations/*
+litellm/proxy/to_delete_loadtest_work/*
 config.yaml
 tests/litellm/litellm_core_utils/llm_cost_calc/log.txt
 tests/test_custom_dir/*
 test.py
+
+litellm_config.yaml
+.cursor
+.vscode/launch.json
+litellm/proxy/to_delete_loadtest_work/*
+update_model_cost_map.py

.pre-commit-config.yaml

@@ -14,17 +14,17 @@ repos:
         types: [python]
         files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py
         exclude: ^litellm/__init__.py$
-    -   id: black
-        name: black
-        entry: poetry run black
-        language: system
-        types: [python]
-        files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py
+    # -   id: black
+    #     name: black
+    #     entry: poetry run black
+    #     language: system
+    #     types: [python]
+    #     files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py
 -   repo: https://github.com/pycqa/flake8
     rev: 7.0.0  # The version of flake8 to use
     hooks:
     -  id: flake8
-       exclude: ^litellm/tests/|^litellm/proxy/tests/|^litellm/tests/test_litellm/|^tests/test_litellm/
+       exclude: ^litellm/tests/|^litellm/proxy/tests/|^litellm/tests/test_litellm/|^tests/test_litellm/|^tests/enterprise/
        additional_dependencies: [flake8-print]
        files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py
 -   repo: https://github.com/python-poetry/poetry

AGENTS.md

@@ -0,0 +1,144 @@
+# INSTRUCTIONS FOR LITELLM
+
+This document provides comprehensive instructions for AI agents working in the LiteLLM repository.
+
+## OVERVIEW
+
+LiteLLM is a unified interface for 100+ LLMs that:
+- Translates inputs to provider-specific completion, embedding, and image generation endpoints
+- Provides consistent OpenAI-format output across all providers
+- Includes retry/fallback logic across multiple deployments (Router)
+- Offers a proxy server (LLM Gateway) with budgets, rate limits, and authentication
+- Supports advanced features like function calling, streaming, caching, and observability
+
+## REPOSITORY STRUCTURE
+
+### Core Components
+- `litellm/` - Main library code
+  - `llms/` - Provider-specific implementations (OpenAI, Anthropic, Azure, etc.)
+  - `proxy/` - Proxy server implementation (LLM Gateway)
+  - `router_utils/` - Load balancing and fallback logic
+  - `types/` - Type definitions and schemas
+  - `integrations/` - Third-party integrations (observability, caching, etc.)
+
+### Key Directories
+- `tests/` - Comprehensive test suites
+- `docs/my-website/` - Documentation website
+- `ui/litellm-dashboard/` - Admin dashboard UI
+- `enterprise/` - Enterprise-specific features
+
+## DEVELOPMENT GUIDELINES
+
+### MAKING CODE CHANGES
+
+1. **Provider Implementations**: When adding/modifying LLM providers:
+   - Follow existing patterns in `litellm/llms/{provider}/`
+   - Implement proper transformation classes that inherit from `BaseConfig`
+   - Support both sync and async operations
+   - Handle streaming responses appropriately
+   - Include proper error handling with provider-specific exceptions
+
+2. **Type Safety**: 
+   - Use proper type hints throughout
+   - Update type definitions in `litellm/types/`
+   - Ensure compatibility with both Pydantic v1 and v2
+
+3. **Testing**:
+   - Add tests in appropriate `tests/` subdirectories
+   - Include both unit tests and integration tests
+   - Test provider-specific functionality thoroughly
+   - Consider adding load tests for performance-critical changes
+
+### IMPORTANT PATTERNS
+
+1. **Function/Tool Calling**:
+   - LiteLLM standardizes tool calling across providers
+   - OpenAI format is the standard, with transformations for other providers
+   - See `litellm/llms/anthropic/chat/transformation.py` for complex tool handling
+
+2. **Streaming**:
+   - All providers should support streaming where possible
+   - Use consistent chunk formatting across providers
+   - Handle both sync and async streaming
+
+3. **Error Handling**:
+   - Use provider-specific exception classes
+   - Maintain consistent error formats across providers
+   - Include proper retry logic and fallback mechanisms
+
+4. **Configuration**:
+   - Support both environment variables and programmatic configuration
+   - Use `BaseConfig` classes for provider configurations
+   - Allow dynamic parameter passing
+
+## PROXY SERVER (LLM GATEWAY)
+
+The proxy server is a critical component that provides:
+- Authentication and authorization
+- Rate limiting and budget management
+- Load balancing across multiple models/deployments
+- Observability and logging
+- Admin dashboard UI
+- Enterprise features
+
+Key files:
+- `litellm/proxy/proxy_server.py` - Main server implementation
+- `litellm/proxy/auth/` - Authentication logic
+- `litellm/proxy/management_endpoints/` - Admin API endpoints
+
+## MCP (MODEL CONTEXT PROTOCOL) SUPPORT
+
+LiteLLM supports MCP for agent workflows:
+- MCP server integration for tool calling
+- Transformation between OpenAI and MCP tool formats
+- Support for external MCP servers (Zapier, Jira, Linear, etc.)
+- See `litellm/experimental_mcp_client/` and `litellm/proxy/_experimental/mcp_server/`
+
+## TESTING CONSIDERATIONS
+
+1. **Provider Tests**: Test against real provider APIs when possible
+2. **Proxy Tests**: Include authentication, rate limiting, and routing tests
+3. **Performance Tests**: Load testing for high-throughput scenarios
+4. **Integration Tests**: End-to-end workflows including tool calling
+
+## DOCUMENTATION
+
+- Keep documentation in sync with code changes
+- Update provider documentation when adding new providers
+- Include code examples for new features
+- Update changelog and release notes
+
+## SECURITY CONSIDERATIONS
+
+- Handle API keys securely
+- Validate all inputs, especially for proxy endpoints
+- Consider rate limiting and abuse prevention
+- Follow security best practices for authentication
+
+## ENTERPRISE FEATURES
+
+- Some features are enterprise-only
+- Check `enterprise/` directory for enterprise-specific code
+- Maintain compatibility between open-source and enterprise versions
+
+## COMMON PITFALLS TO AVOID
+
+1. **Breaking Changes**: LiteLLM has many users - avoid breaking existing APIs
+2. **Provider Specifics**: Each provider has unique quirks - handle them properly
+3. **Rate Limits**: Respect provider rate limits in tests
+4. **Memory Usage**: Be mindful of memory usage in streaming scenarios
+5. **Dependencies**: Keep dependencies minimal and well-justified
+
+## HELPFUL RESOURCES
+
+- Main documentation: https://docs.litellm.ai/
+- Provider-specific docs in `docs/my-website/docs/providers/`
+- Admin UI for testing proxy features
+
+## WHEN IN DOUBT
+
+- Follow existing patterns in the codebase
+- Check similar provider implementations
+- Ensure comprehensive test coverage
+- Update documentation appropriately
+- Consider backward compatibility impact 

CLAUDE.md

@@ -0,0 +1,89 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Development Commands
+
+### Installation
+- `make install-dev` - Install core development dependencies
+- `make install-proxy-dev` - Install proxy development dependencies with full feature set
+- `make install-test-deps` - Install all test dependencies
+
+### Testing
+- `make test` - Run all tests
+- `make test-unit` - Run unit tests (tests/test_litellm) with 4 parallel workers
+- `make test-integration` - Run integration tests (excludes unit tests)
+- `pytest tests/` - Direct pytest execution
+
+### Code Quality
+- `make lint` - Run all linting (Ruff, MyPy, Black, circular imports, import safety)
+- `make format` - Apply Black code formatting
+- `make lint-ruff` - Run Ruff linting only
+- `make lint-mypy` - Run MyPy type checking only
+
+### Single Test Files
+- `poetry run pytest tests/path/to/test_file.py -v` - Run specific test file
+- `poetry run pytest tests/path/to/test_file.py::test_function -v` - Run specific test
+
+## Architecture Overview
+
+LiteLLM is a unified interface for 100+ LLM providers with two main components:
+
+### Core Library (`litellm/`)
+- **Main entry point**: `litellm/main.py` - Contains core completion() function
+- **Provider implementations**: `litellm/llms/` - Each provider has its own subdirectory
+- **Router system**: `litellm/router.py` + `litellm/router_utils/` - Load balancing and fallback logic
+- **Type definitions**: `litellm/types/` - Pydantic models and type hints
+- **Integrations**: `litellm/integrations/` - Third-party observability, caching, logging
+- **Caching**: `litellm/caching/` - Multiple cache backends (Redis, in-memory, S3, etc.)
+
+### Proxy Server (`litellm/proxy/`)
+- **Main server**: `proxy_server.py` - FastAPI application
+- **Authentication**: `auth/` - API key management, JWT, OAuth2
+- **Database**: `db/` - Prisma ORM with PostgreSQL/SQLite support
+- **Management endpoints**: `management_endpoints/` - Admin APIs for keys, teams, models
+- **Pass-through endpoints**: `pass_through_endpoints/` - Provider-specific API forwarding
+- **Guardrails**: `guardrails/` - Safety and content filtering hooks
+- **UI Dashboard**: Served from `_experimental/out/` (Next.js build)
+
+## Key Patterns
+
+### Provider Implementation
+- Providers inherit from base classes in `litellm/llms/base.py`
+- Each provider has transformation functions for input/output formatting
+- Support both sync and async operations
+- Handle streaming responses and function calling
+
+### Error Handling
+- Provider-specific exceptions mapped to OpenAI-compatible errors
+- Fallback logic handled by Router system
+- Comprehensive logging through `litellm/_logging.py`
+
+### Configuration
+- YAML config files for proxy server (see `proxy/example_config_yaml/`)
+- Environment variables for API keys and settings
+- Database schema managed via Prisma (`proxy/schema.prisma`)
+
+## Development Notes
+
+### Code Style
+- Uses Black formatter, Ruff linter, MyPy type checker
+- Pydantic v2 for data validation
+- Async/await patterns throughout
+- Type hints required for all public APIs
+
+### Testing Strategy
+- Unit tests in `tests/test_litellm/`
+- Integration tests for each provider in `tests/llm_translation/`
+- Proxy tests in `tests/proxy_unit_tests/`
+- Load tests in `tests/load_tests/`
+
+### Database Migrations
+- Prisma handles schema migrations
+- Migration files auto-generated with `prisma migrate dev`
+- Always test migrations against both PostgreSQL and SQLite
+
+### Enterprise Features
+- Enterprise-specific code in `enterprise/` directory
+- Optional features enabled via environment variables
+- Separate licensing and authentication for enterprise features

CONTRIBUTING.md

@@ -0,0 +1,275 @@
+# Contributing to LiteLLM
+
+Thank you for your interest in contributing to LiteLLM! We welcome contributions of all kinds - from bug fixes and documentation improvements to new features and integrations.
+
+## **Checklist before submitting a PR**
+
+Here are the core requirements for any PR submitted to LiteLLM:
+
+- [ ] **Sign the Contributor License Agreement (CLA)** - [see details](#contributor-license-agreement-cla)
+- [ ] **Add testing** - Adding at least 1 test is a hard requirement - [see details](#adding-testing)
+- [ ] **Ensure your PR passes all checks**:
+  - [ ] [Unit Tests](#running-unit-tests) - `make test-unit`
+  - [ ] [Linting / Formatting](#running-linting-and-formatting-checks) - `make lint`
+- [ ] **Keep scope isolated** - Your changes should address 1 specific problem at a time
+
+## **Contributor License Agreement (CLA)**
+
+Before contributing code to LiteLLM, you must sign our [Contributor License Agreement (CLA)](https://cla-assistant.io/BerriAI/litellm). This is a legal requirement for all contributions to be merged into the main repository.
+
+**Important:** We strongly recommend reviewing and signing the CLA before starting work on your contribution to avoid any delays in the PR process.
+
+## Quick Start
+
+### 1. Setup Your Local Development Environment
+
+```bash
+# Clone the repository
+git clone https://github.com/BerriAI/litellm.git
+cd litellm
+
+# Create a new branch for your feature
+git checkout -b your-feature-branch
+
+# Install development dependencies
+make install-dev
+
+# Verify your setup works
+make help
+```
+
+That's it! Your local development environment is ready.
+
+### 2. Development Workflow
+
+Here's the recommended workflow for making changes:
+
+```bash
+# Make your changes to the code
+# ...
+
+# Format your code (auto-fixes formatting issues)
+make format
+
+# Run all linting checks (matches CI exactly)
+make lint
+
+# Run unit tests to ensure nothing is broken
+make test-unit
+
+# Commit your changes
+git add .
+git commit -m "Your descriptive commit message"
+
+# Push and create a PR
+git push origin your-feature-branch
+```
+
+## Adding Testing
+
+**Adding at least 1 test is a hard requirement for all PRs.**
+
+### Where to Add Tests
+
+Add your tests to the [`tests/test_litellm/` directory](https://github.com/BerriAI/litellm/tree/main/tests/test_litellm).
+
+- This directory mirrors the structure of the `litellm/` directory
+- **Only add mocked tests** - no real LLM API calls in this directory
+- For integration tests with real APIs, use the appropriate test directories
+
+### File Naming Convention
+
+The `tests/test_litellm/` directory follows the same structure as `litellm/`:
+
+- `litellm/proxy/caching_routes.py` → `tests/test_litellm/proxy/test_caching_routes.py`
+- `litellm/utils.py` → `tests/test_litellm/test_utils.py`
+
+### Example Test
+
+```python
+import pytest
+from litellm import completion
+
+def test_your_feature():
+    """Test your feature with a descriptive docstring."""
+    # Arrange
+    messages = [{"role": "user", "content": "Hello"}]
+    
+    # Act
+    # Use mocked responses, not real API calls
+    
+    # Assert
+    assert expected_result == actual_result
+```
+
+## Running Tests and Checks
+
+### Running Unit Tests
+
+Run all unit tests (uses parallel execution for speed):
+
+```bash
+make test-unit
+```
+
+Run specific test files:
+```bash
+poetry run pytest tests/test_litellm/test_your_file.py -v
+```
+
+### Running Linting and Formatting Checks
+
+Run all linting checks (matches CI exactly):
+
+```bash
+make lint
+```
+
+Individual linting commands:
+```bash
+make format-check       # Check Black formatting
+make lint-ruff          # Run Ruff linting
+make lint-mypy          # Run MyPy type checking
+make check-circular-imports    # Check for circular imports
+make check-import-safety       # Check import safety
+```
+
+Apply formatting (auto-fixes issues):
+```bash
+make format
+```
+
+### CI Compatibility
+
+To ensure your changes will pass CI, run the exact same checks locally:
+
+```bash
+# This runs the same checks as the GitHub workflows
+make lint
+make test-unit
+```
+
+For exact CI compatibility (pins OpenAI version like CI):
+```bash
+make install-dev-ci     # Installs exact CI dependencies
+```
+
+## Available Make Commands
+
+Run `make help` to see all available commands:
+
+```bash
+make help                       # Show all available commands
+make install-dev               # Install development dependencies
+make install-proxy-dev         # Install proxy development dependencies
+make install-test-deps         # Install test dependencies (for running tests)
+make format                    # Apply Black code formatting
+make format-check              # Check Black formatting (matches CI)
+make lint                      # Run all linting checks
+make test-unit                 # Run unit tests
+make test-integration          # Run integration tests
+make test-unit-helm            # Run Helm unit tests
+```
+
+## Code Quality Standards
+
+LiteLLM follows the [Google Python Style Guide](https://google.github.io/styleguide/pyguide.html).
+
+Our automated quality checks include:
+- **Black** for consistent code formatting
+- **Ruff** for linting and code quality
+- **MyPy** for static type checking
+- **Circular import detection**
+- **Import safety validation**
+
+All checks must pass before your PR can be merged.
+
+## Common Issues and Solutions
+
+### 1. Linting Failures
+
+If `make lint` fails:
+
+1. **Formatting issues**: Run `make format` to auto-fix
+2. **Ruff issues**: Check the output and fix manually
+3. **MyPy issues**: Add proper type hints
+4. **Circular imports**: Refactor import dependencies
+5. **Import safety**: Fix any unprotected imports
+
+### 2. Test Failures
+
+If `make test-unit` fails:
+
+1. Check if you broke existing functionality
+2. Add tests for your new code
+3. Ensure tests use mocks, not real API calls
+4. Check test file naming conventions
+
+### 3. Common Development Tips
+
+- **Use type hints**: MyPy requires proper type annotations
+- **Write descriptive commit messages**: Help reviewers understand your changes
+- **Keep PRs focused**: One feature/fix per PR
+- **Test edge cases**: Don't just test the happy path
+- **Update documentation**: If you change APIs, update docs
+
+## Building and Running Locally
+
+### LiteLLM Proxy Server
+
+To run the proxy server locally:
+
+```bash
+# Install proxy dependencies
+make install-proxy-dev
+
+# Start the proxy server
+poetry run litellm --config your_config.yaml
+```
+
+### Docker Development
+
+If you want to build the Docker image yourself:
+
+```bash
+# Build using the non-root Dockerfile
+docker build -f docker/Dockerfile.non_root -t litellm_dev .
+
+# Run with your config
+docker run \
+    -v $(pwd)/proxy_config.yaml:/app/config.yaml \
+    -e LITELLM_MASTER_KEY="sk-1234" \
+    -p 4000:4000 \
+    litellm_dev \
+    --config /app/config.yaml --detailed_debug
+```
+
+## Submitting Your PR
+
+1. **Push your branch**: `git push origin your-feature-branch`
+2. **Create a PR**: Go to GitHub and create a pull request
+3. **Fill out the PR template**: Provide clear description of changes
+4. **Wait for review**: Maintainers will review and provide feedback
+5. **Address feedback**: Make requested changes and push updates
+6. **Merge**: Once approved, your PR will be merged!
+
+## Getting Help
+
+If you need help:
+
+- 💬 [Join our Discord](https://discord.gg/wuPM9dRgDw)
+- 💬 [Join our Slack](https://join.slack.com/share/enQtOTE0ODczMzk2Nzk4NC01YjUxNjY2YjBlYTFmNDRiZTM3NDFiYTM3MzVkODFiMDVjOGRjMmNmZTZkZTMzOWQzZGQyZWIwYjQ0MWExYmE3)
+- 📧 Email us: ishaan@berri.ai / krrish@berri.ai
+- 🐛 [Create an issue](https://github.com/BerriAI/litellm/issues/new)
+
+## What to Contribute
+
+Looking for ideas? Check out:
+
+- 🐛 [Good first issues](https://github.com/BerriAI/litellm/labels/good%20first%20issue)
+- 🚀 [Feature requests](https://github.com/BerriAI/litellm/labels/enhancement)
+- 📚 Documentation improvements
+- 🧪 Test coverage improvements
+- 🔌 New LLM provider integrations
+
+Thank you for contributing to LiteLLM! 🚀 

Dockerfile

@@ -15,7 +15,7 @@ USER root
 RUN apk add --no-cache gcc python3-dev openssl openssl-dev
 
 
-RUN pip install --upgrade pip && \
+RUN pip install --upgrade pip>=24.3.1 && \
     pip install build
 
 # Copy the current directory contents into the container at /app
@@ -41,9 +41,6 @@ RUN pip uninstall jwt -y
 RUN pip uninstall PyJWT -y
 RUN pip install PyJWT==2.9.0 --no-cache-dir
 
-# Build Admin UI
-RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
-
 # Runtime stage
 FROM $LITELLM_RUNTIME_IMAGE AS runtime
 
@@ -53,6 +50,9 @@ USER root
 # Install runtime dependencies
 RUN apk add --no-cache openssl tzdata
 
+# Upgrade pip to fix CVE-2025-8869
+RUN pip install --upgrade pip>=24.3.1
+
 WORKDIR /app
 # Copy the current directory contents into the container at /app
 COPY . .
@@ -65,6 +65,9 @@ COPY --from=builder /wheels/ /wheels/
 # Install the built wheel using pip; again using a wildcard if it's the only file
 RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
 
+# Install semantic_router and aurelio-sdk using script
+RUN chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
+
 # Generate prisma client
 RUN prisma generate
 RUN chmod +x docker/entrypoint.sh
@@ -72,6 +75,9 @@ RUN chmod +x docker/prod_entrypoint.sh
 
 EXPOSE 4000/tcp
 
+RUN apk add --no-cache supervisor
+COPY docker/supervisord.conf /etc/supervisord.conf
+
 ENTRYPOINT ["docker/prod_entrypoint.sh"]
 
 # Append "--detailed_debug" to the end of CMD to view detailed debug logs

GEMINI.md

@@ -0,0 +1,89 @@
+# GEMINI.md
+
+This file provides guidance to Gemini when working with code in this repository.
+
+## Development Commands
+
+### Installation
+- `make install-dev` - Install core development dependencies
+- `make install-proxy-dev` - Install proxy development dependencies with full feature set
+- `make install-test-deps` - Install all test dependencies
+
+### Testing
+- `make test` - Run all tests
+- `make test-unit` - Run unit tests (tests/test_litellm) with 4 parallel workers
+- `make test-integration` - Run integration tests (excludes unit tests)
+- `pytest tests/` - Direct pytest execution
+
+### Code Quality
+- `make lint` - Run all linting (Ruff, MyPy, Black, circular imports, import safety)
+- `make format` - Apply Black code formatting
+- `make lint-ruff` - Run Ruff linting only
+- `make lint-mypy` - Run MyPy type checking only
+
+### Single Test Files
+- `poetry run pytest tests/path/to/test_file.py -v` - Run specific test file
+- `poetry run pytest tests/path/to/test_file.py::test_function -v` - Run specific test
+
+## Architecture Overview
+
+LiteLLM is a unified interface for 100+ LLM providers with two main components:
+
+### Core Library (`litellm/`)
+- **Main entry point**: `litellm/main.py` - Contains core completion() function
+- **Provider implementations**: `litellm/llms/` - Each provider has its own subdirectory
+- **Router system**: `litellm/router.py` + `litellm/router_utils/` - Load balancing and fallback logic
+- **Type definitions**: `litellm/types/` - Pydantic models and type hints
+- **Integrations**: `litellm/integrations/` - Third-party observability, caching, logging
+- **Caching**: `litellm/caching/` - Multiple cache backends (Redis, in-memory, S3, etc.)
+
+### Proxy Server (`litellm/proxy/`)
+- **Main server**: `proxy_server.py` - FastAPI application
+- **Authentication**: `auth/` - API key management, JWT, OAuth2
+- **Database**: `db/` - Prisma ORM with PostgreSQL/SQLite support
+- **Management endpoints**: `management_endpoints/` - Admin APIs for keys, teams, models
+- **Pass-through endpoints**: `pass_through_endpoints/` - Provider-specific API forwarding
+- **Guardrails**: `guardrails/` - Safety and content filtering hooks
+- **UI Dashboard**: Served from `_experimental/out/` (Next.js build)
+
+## Key Patterns
+
+### Provider Implementation
+- Providers inherit from base classes in `litellm/llms/base.py`
+- Each provider has transformation functions for input/output formatting
+- Support both sync and async operations
+- Handle streaming responses and function calling
+
+### Error Handling
+- Provider-specific exceptions mapped to OpenAI-compatible errors
+- Fallback logic handled by Router system
+- Comprehensive logging through `litellm/_logging.py`
+
+### Configuration
+- YAML config files for proxy server (see `proxy/example_config_yaml/`)
+- Environment variables for API keys and settings
+- Database schema managed via Prisma (`proxy/schema.prisma`)
+
+## Development Notes
+
+### Code Style
+- Uses Black formatter, Ruff linter, MyPy type checker
+- Pydantic v2 for data validation
+- Async/await patterns throughout
+- Type hints required for all public APIs
+
+### Testing Strategy
+- Unit tests in `tests/test_litellm/`
+- Integration tests for each provider in `tests/llm_translation/`
+- Proxy tests in `tests/proxy_unit_tests/`
+- Load tests in `tests/load_tests/`
+
+### Database Migrations
+- Prisma handles schema migrations
+- Migration files auto-generated with `prisma migrate dev`
+- Always test migrations against both PostgreSQL and SQLite
+
+### Enterprise Features
+- Enterprise-specific code in `enterprise/` directory
+- Optional features enabled via environment variables
+- Separate licensing and authentication for enterprise features

Makefile

@@ -1,35 +1,103 @@
 # LiteLLM Makefile
 # Simple Makefile for running tests and basic development tasks
 
-.PHONY: help test test-unit test-integration lint format
+.PHONY: help test test-unit test-integration test-unit-helm lint format install-dev install-proxy-dev install-test-deps install-helm-unittest check-circular-imports check-import-safety
 
 # Default target
 help:
 	@echo "Available commands:"
+	@echo "  make install-dev        - Install development dependencies"
+	@echo "  make install-proxy-dev  - Install proxy development dependencies"
+	@echo "  make install-dev-ci     - Install dev dependencies (CI-compatible, pins OpenAI)"
+	@echo "  make install-proxy-dev-ci - Install proxy dev dependencies (CI-compatible)"
+	@echo "  make install-test-deps  - Install test dependencies"
+	@echo "  make install-helm-unittest - Install helm unittest plugin"
+	@echo "  make format             - Apply Black code formatting"
+	@echo "  make format-check       - Check Black code formatting (matches CI)"
+	@echo "  make lint               - Run all linting (Ruff, MyPy, Black check, circular imports, import safety)"
+	@echo "  make lint-ruff          - Run Ruff linting only"
+	@echo "  make lint-mypy          - Run MyPy type checking only"
+	@echo "  make lint-black         - Check Black formatting (matches CI)"
+	@echo "  make check-circular-imports - Check for circular imports"
+	@echo "  make check-import-safety - Check import safety"
 	@echo "  make test               - Run all tests"
-	@echo "  make test-unit          - Run unit tests"
+	@echo "  make test-unit          - Run unit tests (tests/test_litellm)"
 	@echo "  make test-integration   - Run integration tests"
 	@echo "  make test-unit-helm     - Run helm unit tests"
 
+# Installation targets
 install-dev:
 	poetry install --with dev
 
 install-proxy-dev:
-	poetry install --with dev,proxy-dev
+	poetry install --with dev,proxy-dev --extras proxy
 
-lint: install-dev
+# CI-compatible installations (matches GitHub workflows exactly)
+install-dev-ci:
+	pip install openai==1.99.5
+	poetry install --with dev
+	pip install openai==1.99.5
+
+install-proxy-dev-ci:
+	poetry install --with dev,proxy-dev --extras proxy
+	pip install openai==1.99.5
+
+install-test-deps: install-proxy-dev
+	poetry run pip install "pytest-retry==1.6.3"
+	poetry run pip install pytest-xdist
+	cd enterprise && python -m pip install -e . && cd ..
+
+install-helm-unittest:
+	helm plugin install https://github.com/helm-unittest/helm-unittest --version v0.4.4 || echo "ignore error if plugin exists"
+
+# Formatting
+format: install-dev
+	cd litellm && poetry run black . && cd ..
+
+format-check: install-dev
+	cd litellm && poetry run black --check . && cd ..
+
+# Linting targets
+lint-ruff: install-dev
+	cd litellm && poetry run ruff check . && cd ..
+
+lint-mypy: install-dev
 	poetry run pip install types-requests types-setuptools types-redis types-PyYAML
-	cd litellm && poetry run mypy . --ignore-missing-imports
+	cd litellm && poetry run mypy . --ignore-missing-imports && cd ..
 
-# Testing
+lint-black: format-check
+
+check-circular-imports: install-dev
+	cd litellm && poetry run python ../tests/documentation_tests/test_circular_imports.py && cd ..
+
+check-import-safety: install-dev
+	poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
+
+# Combined linting (matches test-linting.yml workflow)
+lint: format-check lint-ruff lint-mypy check-circular-imports check-import-safety
+
+# Testing targets
 test:
 	poetry run pytest tests/
 
-test-unit:
-	poetry run pytest tests/test_litellm/
+test-unit: install-test-deps
+	poetry run pytest tests/test_litellm -x -vv -n 4
 
 test-integration:
 	poetry run pytest tests/ -k "not test_litellm"
 
-test-unit-helm:
-	helm unittest -f 'tests/*.yaml' deploy/charts/litellm-helm
+test-unit-helm: install-helm-unittest
+	helm unittest -f 'tests/*.yaml' deploy/charts/litellm-helm
+
+# LLM Translation testing targets
+test-llm-translation: install-test-deps
+	@echo "Running LLM translation tests..."
+	@python .github/workflows/run_llm_translation_tests.py
+
+test-llm-translation-single: install-test-deps
+	@echo "Running single LLM translation test file..."
+	@if [ -z "$(FILE)" ]; then echo "Usage: make test-llm-translation-single FILE=test_filename.py"; exit 1; fi
+	@mkdir -p test-results
+	poetry run pytest tests/llm_translation/$(FILE) \
+		--junitxml=test-results/junit.xml \
+		-v --tb=short --maxfail=100 --timeout=300

README.md

@@ -25,6 +25,9 @@
     <a href="https://discord.gg/wuPM9dRgDw">
         <img src="https://img.shields.io/static/v1?label=Chat%20on&message=Discord&color=blue&logo=Discord&style=flat-square" alt="Discord">
     </a>
+    <a href="https://www.litellm.ai/support">
+        <img src="https://img.shields.io/static/v1?label=Chat%20on&message=Slack&color=black&logo=Slack&style=flat-square" alt="Slack">
+    </a>
 </h4>
 
 LiteLLM manages:
@@ -34,7 +37,7 @@ LiteLLM manages:
 - Retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - [Router](https://docs.litellm.ai/docs/routing)
 - Set Budgets & Rate limits per project, api key, model [LiteLLM Proxy Server (LLM Gateway)](https://docs.litellm.ai/docs/simple_proxy)
 
-[**Jump to LiteLLM Proxy (LLM Gateway) Docs**](https://github.com/BerriAI/litellm?tab=readme-ov-file#openai-proxy---docs) <br>
+[**Jump to LiteLLM Proxy (LLM Gateway) Docs**](https://github.com/BerriAI/litellm?tab=readme-ov-file#litellm-proxy-server-llm-gateway---docs) <br>
 [**Jump to Supported LLM Providers**](https://github.com/BerriAI/litellm?tab=readme-ov-file#supported-providers-docs)
 
 🚨 **Stable Release:** Use docker images with the `-stable` tag. These have undergone 12 hour load tests, before being published. [More information about the release cycle here](https://docs.litellm.ai/docs/proxy/release_cycle)
@@ -44,7 +47,7 @@ Support for more providers. Missing a provider or LLM Platform, raise a [feature
 # Usage ([**Docs**](https://docs.litellm.ai/docs/))
 
 > [!IMPORTANT]
-> LiteLLM v1.0.0 now requires `openai>=1.0.0`. Migration guide [here](https://docs.litellm.ai/docs/migration)  
+> LiteLLM v1.0.0 now requires `openai>=1.0.0`. Migration guide [here](https://docs.litellm.ai/docs/migration)
 > LiteLLM v1.40.14+ now requires `pydantic>=2.0.0`. No changes required.
 
 <a target="_blank" href="https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/liteLLM_Getting_Started.ipynb">
@@ -69,7 +72,7 @@ messages = [{ "content": "Hello, how are you?","role": "user"}]
 response = completion(model="openai/gpt-4o", messages=messages)
 
 # anthropic call
-response = completion(model="anthropic/claude-3-sonnet-20240229", messages=messages)
+response = completion(model="anthropic/claude-sonnet-4-20250514", messages=messages)
 print(response)
 ```
 
@@ -77,9 +80,9 @@ print(response)
 
 ```json
 {
-    "id": "chatcmpl-565d891b-a42e-4c39-8d14-82a1f5208885",
-    "created": 1734366691,
-    "model": "claude-3-sonnet-20240229",
+    "id": "chatcmpl-1214900a-6cdd-4148-b663-b5e2f642b4de",
+    "created": 1751494488,
+    "model": "claude-sonnet-4-20250514",
     "object": "chat.completion",
     "system_fingerprint": null,
     "choices": [
@@ -87,7 +90,7 @@ print(response)
             "finish_reason": "stop",
             "index": 0,
             "message": {
-                "content": "Hello! As an AI language model, I don't have feelings, but I'm operating properly and ready to assist you with any questions or tasks you may have. How can I help you today?",
+                "content": "Hello! I'm doing well, thank you for asking. I'm here and ready to help with whatever you'd like to discuss or work on. How are you doing today?",
                 "role": "assistant",
                 "tool_calls": null,
                 "function_call": null
@@ -95,9 +98,9 @@ print(response)
         }
     ],
     "usage": {
-        "completion_tokens": 43,
+        "completion_tokens": 39,
         "prompt_tokens": 13,
-        "total_tokens": 56,
+        "total_tokens": 52,
         "completion_tokens_details": null,
         "prompt_tokens_details": {
             "audio_tokens": null,
@@ -129,7 +132,7 @@ print(response)
 
 ## Streaming ([Docs](https://docs.litellm.ai/docs/completion/stream))
 
-liteLLM supports streaming the model response back, pass `stream=True` to get a streaming iterator in response.  
+liteLLM supports streaming the model response back, pass `stream=True` to get a streaming iterator in response.
 Streaming is supported for all models (Bedrock, Huggingface, TogetherAI, Azure, OpenAI, etc.)
 
 ```python
@@ -138,8 +141,8 @@ response = completion(model="openai/gpt-4o", messages=messages, stream=True)
 for part in response:
     print(part.choices[0].delta.content or "")
 
-# claude 2
-response = completion('anthropic/claude-3-sonnet-20240229', messages, stream=True)
+# claude sonnet 4
+response = completion('anthropic/claude-sonnet-4-20250514', messages, stream=True)
 for part in response:
     print(part)
 ```
@@ -148,9 +151,9 @@ for part in response:
 
 ```json
 {
-    "id": "chatcmpl-2be06597-eb60-4c70-9ec5-8cd2ab1b4697",
-    "created": 1734366925,
-    "model": "claude-3-sonnet-20240229",
+    "id": "chatcmpl-fe575c37-5004-4926-ae5e-bfbc31f356ca",
+    "created": 1751494808,
+    "model": "claude-sonnet-4-20250514",
     "object": "chat.completion.chunk",
     "system_fingerprint": null,
     "choices": [
@@ -158,6 +161,7 @@ for part in response:
             "finish_reason": null,
             "index": 0,
             "delta": {
+                "provider_specific_fields": null,
                 "content": "Hello",
                 "role": "assistant",
                 "function_call": null,
@@ -166,7 +170,10 @@ for part in response:
             },
             "logprobs": null
         }
-    ]
+    ],
+    "provider_specific_fields": null,
+    "stream_options": null,
+    "citations": null
 }
 ```
 
@@ -227,7 +234,7 @@ $ litellm --model huggingface/bigcode/starcoder
 
 
 > [!IMPORTANT]
-> 💡 [Use LiteLLM Proxy with Langchain (Python, JS), OpenAI SDK (Python, JS) Anthropic SDK, Mistral SDK, LlamaIndex, Instructor, Curl](https://docs.litellm.ai/docs/proxy/user_keys)  
+> 💡 [Use LiteLLM Proxy with Langchain (Python, JS), OpenAI SDK (Python, JS) Anthropic SDK, Mistral SDK, LlamaIndex, Instructor, Curl](https://docs.litellm.ai/docs/proxy/user_keys)
 
 ```python
 import openai # openai v1.0.0+
@@ -259,14 +266,14 @@ echo 'LITELLM_MASTER_KEY="sk-1234"' > .env
 
 # Add the litellm salt key - you cannot change this after adding a model
 # It is used to encrypt / decrypt your LLM API Key credentials
-# We recommend - https://1password.com/password-generator/ 
+# We recommend - https://1password.com/password-generator/
 # password generator to get a random hash for litellm salt key
-echo 'LITELLM_SALT_KEY="sk-1234"' > .env
+echo 'LITELLM_SALT_KEY="sk-1234"' >> .env
 
 source .env
 
 # Start
-docker-compose up
+docker compose up
 ```
 
 
@@ -309,6 +316,7 @@ curl 'http://0.0.0.0:4000/key/generate' \
 | [google AI Studio - gemini](https://docs.litellm.ai/docs/providers/gemini)          | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                               |                                                                         |
 | [mistral ai api](https://docs.litellm.ai/docs/providers/mistral)                    | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 | ✅                                                                             |                                                                         |
 | [cloudflare AI Workers](https://docs.litellm.ai/docs/providers/cloudflare_workers)  | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                               |                                                                         |
+| [CompactifAI](https://docs.litellm.ai/docs/providers/compactifai)                   | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                               |                                                                         |
 | [cohere](https://docs.litellm.ai/docs/providers/cohere)                             | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 | ✅                                                                             |                                                                         |
 | [anthropic](https://docs.litellm.ai/docs/providers/anthropic)                       | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                               |                                                                         |
 | [empower](https://docs.litellm.ai/docs/providers/empower)                    | ✅                                                      | ✅                                                                              | ✅                                                                                  | ✅                                                                                |
@@ -333,22 +341,37 @@ curl 'http://0.0.0.0:4000/key/generate' \
 | [xinference [Xorbits Inference]](https://docs.litellm.ai/docs/providers/xinference) |                                                         |                                                                                 |                                                                                     |                                                                                   | ✅                                                                             |                                                                         |
 | [FriendliAI](https://docs.litellm.ai/docs/providers/friendliai)                              | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                               |                                                                         |
 | [Galadriel](https://docs.litellm.ai/docs/providers/galadriel)                              | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                               |                                                                         |
+| [GradientAI](https://docs.litellm.ai/docs/providers/gradient_ai)                              | ✅                                                       | ✅                                                                               |                                                                                   |                                                                                  |                                                                               |                                                                         |
 | [Novita AI](https://novita.ai/models/llm?utm_source=github_litellm&utm_medium=github_readme&utm_campaign=github_link)                     | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                               |                                                                         |
 | [Featherless AI](https://docs.litellm.ai/docs/providers/featherless_ai)                              | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                               |                                                                         |
 | [Nebius AI Studio](https://docs.litellm.ai/docs/providers/nebius)                             | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 | ✅                                                                             |                                                                         |
+| [Heroku](https://docs.litellm.ai/docs/providers/heroku)                             | ✅                                                       | ✅                                                                               |                                                                                    |                                                                                  |                                                                              |                                                                         |
+| [OVHCloud AI Endpoints](https://docs.litellm.ai/docs/providers/ovhcloud)                             | ✅                                                       | ✅                                                                               |                                                                                    |                                                                                  |                                                                              |                                                                         |
 
 [**Read the Docs**](https://docs.litellm.ai/docs/)
 
-## Contributing
+## Run in Developer mode
+### Services
+1. Setup .env file in root
+2. Run dependant services `docker-compose up db prometheus`
 
-Interested in contributing? Contributions to LiteLLM Python SDK, Proxy Server, and contributing LLM integrations are both accepted and highly encouraged! [See our Contribution Guide for more details](https://docs.litellm.ai/docs/extras/contributing_code)
+### Backend
+1. (In root) create virtual environment `python -m venv .venv`
+2. Activate virtual environment `source .venv/bin/activate`
+3. Install dependencies `pip install -e ".[all]"`
+4. Start proxy backend `python litellm/proxy_cli.py`
+
+### Frontend
+1. Navigate to `ui/litellm-dashboard`
+2. Install dependencies `npm install`
+3. Run `npm run dev` to start the dashboard
 
 # Enterprise
 For companies that need better security, user management and professional support
 
 [Talk to founders](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
 
-This covers: 
+This covers:
 - ✅ **Features under the [LiteLLM Commercial License](https://docs.litellm.ai/docs/proxy/enterprise):**
 - ✅ **Feature Prioritization**
 - ✅ **Custom Integrations**
@@ -356,24 +379,46 @@ This covers:
 - ✅ **Custom SLAs**
 - ✅ **Secure access with Single Sign-On**
 
-# Code Quality / Linting
+# Contributing
+
+We welcome contributions to LiteLLM! Whether you're fixing bugs, adding features, or improving documentation, we appreciate your help.
+
+## Quick Start for Contributors
+
+This requires poetry to be installed.
+
+```bash
+git clone https://github.com/BerriAI/litellm.git
+cd litellm
+make install-dev    # Install development dependencies
+make format         # Format your code
+make lint           # Run all linting checks
+make test-unit      # Run unit tests
+make format-check   # Check formatting only
+```
+
+For detailed contributing guidelines, see [CONTRIBUTING.md](CONTRIBUTING.md).
+
+## Code Quality / Linting
 
 LiteLLM follows the [Google Python Style Guide](https://google.github.io/styleguide/pyguide.html).
 
-We run: 
-- Ruff for [formatting and linting checks](https://github.com/BerriAI/litellm/blob/e19bb55e3b4c6a858b6e364302ebbf6633a51de5/.circleci/config.yml#L320)
-- Mypy + Pyright for typing [1](https://github.com/BerriAI/litellm/blob/e19bb55e3b4c6a858b6e364302ebbf6633a51de5/.circleci/config.yml#L90), [2](https://github.com/BerriAI/litellm/blob/e19bb55e3b4c6a858b6e364302ebbf6633a51de5/.pre-commit-config.yaml#L4)
-- Black for [formatting](https://github.com/BerriAI/litellm/blob/e19bb55e3b4c6a858b6e364302ebbf6633a51de5/.circleci/config.yml#L79)
-- isort for [import sorting](https://github.com/BerriAI/litellm/blob/e19bb55e3b4c6a858b6e364302ebbf6633a51de5/.pre-commit-config.yaml#L10)
+Our automated checks include:
+- **Black** for code formatting
+- **Ruff** for linting and code quality
+- **MyPy** for type checking
+- **Circular import detection**
+- **Import safety checks**
 
 
-If you have suggestions on how to improve the code quality feel free to open an issue or a PR.
+All these checks must pass before your PR can be merged.
 
 
 # Support / talk with founders
 
 - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version)
 - [Community Discord 💭](https://discord.gg/wuPM9dRgDw)
+- [Community Slack 💭](https://www.litellm.ai/support)
 - Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬
 - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai
 
@@ -397,18 +442,3 @@ If you have suggestions on how to improve the code quality feel free to open an
 </a>
 
 
-## Run in Developer mode
-### Services
-1. Setup .env file in root
-2. Run dependant services `docker-compose up db prometheus`
-
-### Backend
-1. (In root) create virtual environment `python -m venv .venv`
-2. Activate virtual environment `source .venv/bin/activate`
-3. Install dependencies `pip install -e ".[all]"`
-4. Start proxy backend `uvicorn litellm.proxy.proxy_server:app --host localhost --port 4000 --reload`
-
-### Frontend
-1. Navigate to `ui/litellm-dashboard`
-2. Install dependencies `npm install`
-3. Run `npm run dev` to start the dashboard

ci_cd/security_scans.sh

@@ -0,0 +1,121 @@
+#!/bin/bash
+
+# Security Scans Script for LiteLLM
+# This script runs comprehensive security scans including Trivy and Grype
+
+set -e
+
+echo "Starting security scans for LiteLLM..."
+
+# Function to install Trivy and required tools
+install_trivy() {
+    echo "Installing Trivy and required tools..."
+    sudo apt-get update
+    sudo apt-get install -y wget apt-transport-https gnupg lsb-release jq curl
+    wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | sudo apt-key add -
+    echo "deb https://aquasecurity.github.io/trivy-repo/deb $(lsb_release -sc) main" | sudo tee -a /etc/apt/sources.list.d/trivy.list
+    sudo apt-get update
+    sudo apt-get install trivy
+    echo "Trivy and required tools installed successfully"
+}
+
+# Function to install Grype
+install_grype() {
+    echo "Installing Grype..."
+    curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sudo sh -s -- -b /usr/local/bin
+    echo "Grype installed successfully"
+}
+
+# Function to run Trivy scans
+run_trivy_scans() {
+    echo "Running Trivy scans..."
+    
+    echo "Scanning LiteLLM Docs..."
+    trivy fs --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./docs/
+    
+    echo "Scanning LiteLLM UI..."
+    trivy fs --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./ui/
+    
+    echo "Trivy scans completed successfully"
+}
+
+# Function to build and scan Docker images with Grype
+run_grype_scans() {
+    echo "Running Grype scans..."
+    
+    # Temporarily add wheel files to .dockerignore for security scans
+    echo "Temporarily modifying .dockerignore to exclude problematic wheel files..."
+    cp .dockerignore .dockerignore.backup 2>/dev/null || touch .dockerignore.backup
+    echo "/*.whl" >> .dockerignore
+    
+    # Build and scan Dockerfile.database
+    echo "Building and scanning Dockerfile.database..."
+    docker build --no-cache -t litellm-database:latest -f ./docker/Dockerfile.database .
+    grype litellm-database:latest --fail-on critical
+    
+    # Build and scan main Dockerfile
+    echo "Building and scanning main Dockerfile..."
+    docker build --no-cache -t litellm:latest .
+    grype litellm:latest --fail-on critical
+    
+    # Restore original .dockerignore
+    echo "Restoring original .dockerignore..."
+    mv .dockerignore.backup .dockerignore
+    
+    # Scan the locally built LiteLLM image for vulnerabilities with CVSS >= 4.0
+    echo "Scanning locally built LiteLLM image for high-severity vulnerabilities..."
+    echo "Using locally built image: litellm:latest"
+    
+    # Allowlist of CVEs to be ignored in failure threshold/reporting
+    # - CVE-2025-8869: Not applicable on Python >=3.13 (PEP 706 implemented); pip fallback unused; no OS-level fix
+    ALLOWED_CVES=(
+        "CVE-2025-8869"
+    )
+
+    # Build JSON array of allowlisted CVE IDs for jq
+    ALLOWED_IDS_JSON=$(printf '%s\n' "${ALLOWED_CVES[@]}" | jq -R . | jq -s .)
+
+    echo "Checking for vulnerabilities with CVSS score >= 4.0..."
+    echo "Allowlisted CVEs (ignored in threshold): ${ALLOWED_CVES[*]}"
+
+    HIGH_SEVERITY_COUNT=$(grype litellm:latest -o json | jq --argjson allow "$ALLOWED_IDS_JSON" -r '
+        .matches[]
+        | select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0)
+        | select((.vulnerability.id as $id | $allow | index($id) | not))
+        | .vulnerability.id' | wc -l)
+    
+    if [ "$HIGH_SEVERITY_COUNT" -gt 0 ]; then
+        echo "ERROR: Found $HIGH_SEVERITY_COUNT vulnerabilities with CVSS score >= 4.0 in litellm:latest"
+        echo "Detailed vulnerability report:"
+        grype litellm:latest -o json | jq --argjson allow "$ALLOWED_IDS_JSON" -r '
+        ["Package", "Version", "Vulnerability ID", "CVSS Score", "Severity", "Fix Version", "Description"],
+        (.matches[]
+          | select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0)
+          | select((.vulnerability.id as $id | $allow | index($id) | not))
+          | [.artifact.name, .artifact.version, .vulnerability.id, .vulnerability.cvss[0].metrics.baseScore, .vulnerability.severity, (.vulnerability.fix.versions[0] // "No fix available"), .vulnerability.description])
+        | @tsv' | column -t -s $'\t'
+        exit 1
+    else
+        echo "No high-severity vulnerabilities (CVSS >= 4.0) found in litellm:latest"
+    fi
+    
+    echo "Grype scans completed successfully"
+}
+
+# Main execution
+main() {
+    echo "Installing security scanning tools..."
+    install_trivy
+    install_grype
+    
+    echo "Running filesystem vulnerability scans..."
+    run_trivy_scans
+    
+    echo "Running Docker image vulnerability scans..."
+    run_grype_scans
+    
+    echo "All security scans completed successfully!"
+}
+
+# Execute main function
+main "$@"

ci_cd/security_scans_readme.md

@@ -0,0 +1,9 @@
+# Security Scans
+
+## Scans that run:
+
+- Trivy scan on `./docs/` (HIGH/CRITICAL/MEDIUM)
+- Trivy scan on `./ui/` (HIGH/CRITICAL/MEDIUM) 
+- Grype scan on `Dockerfile.database` (fails on CRITICAL)
+- Grype scan on main `Dockerfile` (fails on CRITICAL)
+- Grype CVSS ≥ 4.0 scan on main `Dockerfile` (fails any vulnerabilities with CVSS ≥ 4.0)

cookbook/liteLLM_Baseten.ipynb

@@ -6,19 +6,21 @@
     "id": "gZx-wHJapG5w"
    },
    "source": [
-    "# Use liteLLM to call Falcon, Wizard, MPT 7B using OpenAI chatGPT Input/output\n",
+    "# LiteLLM with Baseten Model APIs\n",
     "\n",
-    "* Falcon 7B: https://app.baseten.co/explore/falcon_7b\n",
-    "* Wizard LM: https://app.baseten.co/explore/wizardlm\n",
-    "* MPT 7B Base: https://app.baseten.co/explore/mpt_7b_instruct\n",
+    "This notebook demonstrates how to use LiteLLM with Baseten's Model APIs instead of dedicated deployments.\n",
     "\n",
-    "\n",
-    "## Call all baseten llm models using OpenAI chatGPT Input/Output using liteLLM\n",
-    "Example call\n",
+    "## Example Usage\n",
     "```python\n",
-    "model = \"q841o8w\" # baseten model version ID\n",
-    "response = completion(model=model, messages=messages, custom_llm_provider=\"baseten\")\n",
-    "```"
+    "response = completion(\n",
+    "    model=\"baseten/openai/gpt-oss-120b\",\n",
+    "    messages=[{\"role\": \"user\", \"content\": \"Hello!\"}],\n",
+    "    max_tokens=1000,\n",
+    "    temperature=0.7\n",
+    ")\n",
+    "```\n",
+    "\n",
+    "## Setup"
    ]
   },
   {
@@ -29,20 +31,25 @@
    },
    "outputs": [],
    "source": [
-    "!pip install litellm==0.1.399\n",
-    "!pip install baseten urllib3"
+    "%pip install litellm"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "metadata": {
     "id": "VEukLhDzo4vw"
    },
    "outputs": [],
    "source": [
     "import os\n",
-    "from litellm import completion"
+    "from litellm import completion\n",
+    "\n",
+    "# Set your Baseten API key\n",
+    "os.environ['BASETEN_API_KEY'] = \"\" #@param {type:\"string\"}\n",
+    "\n",
+    "# Test message\n",
+    "messages = [{\"role\": \"user\", \"content\": \"What is AGI?\"}]"
    ]
   },
   {
@@ -51,19 +58,31 @@
     "id": "4STYM2OHFNlc"
    },
    "source": [
-    "## Setup"
+    "## Example 1: Basic Completion\n",
+    "\n",
+    "Simple completion with the GPT-OSS 120B model"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 21,
+   "execution_count": null,
    "metadata": {
     "id": "DorpLxw1FHbC"
    },
    "outputs": [],
    "source": [
-    "os.environ['BASETEN_API_KEY'] = \"\" #@param\n",
-    "messages = [{ \"content\": \"what does Baseten do? \",\"role\": \"user\"}]"
+    "print(\"=== Basic Completion ===\")\n",
+    "response = completion(\n",
+    "    model=\"baseten/openai/gpt-oss-120b\",\n",
+    "    messages=messages,\n",
+    "    max_tokens=1000,\n",
+    "    temperature=0.7,\n",
+    "    top_p=0.9,\n",
+    "    presence_penalty=0.1,\n",
+    "    frequency_penalty=0.1,\n",
+    ")\n",
+    "print(f\"Response: {response.choices[0].message.content}\")\n",
+    "print(f\"Usage: {response.usage}\")"
    ]
   },
   {
@@ -72,13 +91,14 @@
     "id": "syF3dTdKFSQQ"
    },
    "source": [
-    "## Calling Falcon 7B: https://app.baseten.co/explore/falcon_7b\n",
-    "### Pass Your Baseten model `Version ID` as `model`"
+    "## Example 2: Streaming Completion\n",
+    "\n",
+    "Streaming completion with usage statistics"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 18,
+   "execution_count": null,
    "metadata": {
     "colab": {
      "base_uri": "https://localhost:8080/"
@@ -86,137 +106,26 @@
     "id": "rPgSoMlsojz0",
     "outputId": "81d6dc7b-1681-4ae4-e4c8-5684eb1bd050"
    },
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "\u001b[32mINFO\u001b[0m API key set.\n",
-      "INFO:baseten:API key set.\n"
-     ]
-    },
-    {
-     "data": {
-      "text/plain": [
-       "{'choices': [{'finish_reason': 'stop',\n",
-       "   'index': 0,\n",
-       "   'message': {'role': 'assistant',\n",
-       "    'content': \"what does Baseten do? \\nI'm sorry, I cannot provide a specific answer as\"}}],\n",
-       " 'created': 1692135883.699066,\n",
-       " 'model': 'qvv0xeq'}"
-      ]
-     },
-     "execution_count": 18,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
-    "model = \"qvv0xeq\"\n",
-    "response = completion(model=model, messages=messages, custom_llm_provider=\"baseten\")\n",
-    "response"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "id": "7n21UroEGCGa"
-   },
-   "source": [
-    "## Calling Wizard LM https://app.baseten.co/explore/wizardlm\n",
-    "### Pass Your Baseten model `Version ID` as `model`"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 19,
-   "metadata": {
-    "colab": {
-     "base_uri": "https://localhost:8080/"
-    },
-    "id": "uLVWFH899lAF",
-    "outputId": "61c2bc74-673b-413e-bb40-179cf408523d"
-   },
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "\u001b[32mINFO\u001b[0m API key set.\n",
-      "INFO:baseten:API key set.\n"
-     ]
-    },
-    {
-     "data": {
-      "text/plain": [
-       "{'choices': [{'finish_reason': 'stop',\n",
-       "   'index': 0,\n",
-       "   'message': {'role': 'assistant',\n",
-       "    'content': 'As an AI language model, I do not have personal beliefs or practices, but based on the information available online, Baseten is a popular name for a traditional Ethiopian dish made with injera, a spongy flatbread, and wat, a spicy stew made with meat or vegetables. It is typically served for breakfast or dinner and is a staple in Ethiopian cuisine. The name Baseten is also used to refer to a traditional Ethiopian coffee ceremony, where coffee is brewed and served in a special ceremony with music and food.'}}],\n",
-       " 'created': 1692135900.2806294,\n",
-       " 'model': 'q841o8w'}"
-      ]
-     },
-     "execution_count": 19,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "model = \"q841o8w\"\n",
-    "response = completion(model=model, messages=messages, custom_llm_provider=\"baseten\")\n",
-    "response"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "id": "6-TFwmPAGPXq"
-   },
-   "source": [
-    "## Calling mosaicml/mpt-7b https://app.baseten.co/explore/mpt_7b_instruct\n",
-    "### Pass Your Baseten model `Version ID` as `model`"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 20,
-   "metadata": {
-    "colab": {
-     "base_uri": "https://localhost:8080/"
-    },
-    "id": "gbeYZOrUE_Bp",
-    "outputId": "838d86ea-2143-4cb3-bc80-2acc2346c37a"
-   },
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "\u001b[32mINFO\u001b[0m API key set.\n",
-      "INFO:baseten:API key set.\n"
-     ]
-    },
-    {
-     "data": {
-      "text/plain": [
-       "{'choices': [{'finish_reason': 'stop',\n",
-       "   'index': 0,\n",
-       "   'message': {'role': 'assistant',\n",
-       "    'content': \"\\n===================\\n\\nIt's a tool to build a local version of a game on your own machine to host\\non your website.\\n\\nIt's used to make game demos and show them on Twitter, Tumblr, and Facebook.\\n\\n\\n\\n## What's built\\n\\n- A directory of all your game directories, named with a version name and build number, with images linked to.\\n- Includes HTML to include in another site.\\n- Includes images for your icons and\"}}],\n",
-       " 'created': 1692135914.7472186,\n",
-       " 'model': '31dxrj3'}"
-      ]
-     },
-     "execution_count": 20,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "model = \"31dxrj3\"\n",
-    "response = completion(model=model, messages=messages, custom_llm_provider=\"baseten\")\n",
-    "response"
+    "print(\"=== Streaming Completion ===\")\n",
+    "response = completion(\n",
+    "    model=\"baseten/openai/gpt-oss-120b\",\n",
+    "    messages=[{\"role\": \"user\", \"content\": \"Write a short poem about AI\"}],\n",
+    "    stream=True,\n",
+    "    max_tokens=500,\n",
+    "    temperature=0.8,\n",
+    "    stream_options={\n",
+    "        \"include_usage\": True,\n",
+    "        \"continuous_usage_stats\": True\n",
+    "    },\n",
+    ")\n",
+    "\n",
+    "print(\"Streaming response:\")\n",
+    "for chunk in response:\n",
+    "    if chunk.choices and chunk.choices[0].delta.content:\n",
+    "        print(chunk.choices[0].delta.content, end=\"\", flush=True)\n",
+    "print(\"\\n\")"
    ]
   }
  ],
@@ -234,4 +143,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 0
-}
+}

cookbook/litellm_proxy_server/batch_api/bedrock/bedrock.py

@@ -0,0 +1,25 @@
+from openai import OpenAI
+
+client = OpenAI(
+    base_url="http://0.0.0.0:4000",
+    api_key="sk-1234",
+)
+
+BEDROCK_BATCH_MODEL = "bedrock/batch-anthropic.claude-3-5-sonnet-20240620-v1:0"
+
+# Upload file
+batch_input_file = client.files.create(
+    file=open("./bedrock_batch_completions.jsonl", "rb"),
+    purpose="batch",
+    extra_body={"target_model_names": BEDROCK_BATCH_MODEL}
+)
+print(batch_input_file)
+
+# Create batch
+batch = client.batches.create( 
+    input_file_id=batch_input_file.id,
+    endpoint="/v1/chat/completions",
+    completion_window="24h",
+    metadata={"description": "Test batch job"},
+)
+print(batch)

cookbook/litellm_proxy_server/batch_api/bedrock/bedrock_batch_completions.jsonl

@@ -0,0 +1,128 @@
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}

cookbook/litellm_proxy_server/cli_token_usage.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+"""
+Example: Using CLI token with LiteLLM SDK
+
+This example shows how to use the CLI authentication token
+in your Python scripts after running `litellm-proxy login`.
+"""
+
+from textwrap import indent
+import litellm
+LITELLM_BASE_URL = "http://localhost:4000/"
+
+
+def main():
+    """Using CLI token with LiteLLM SDK"""
+    print("🚀 Using CLI Token with LiteLLM SDK")
+    print("=" * 40)
+    #litellm._turn_on_debug()
+    
+    # Get the CLI token
+    api_key = litellm.get_litellm_gateway_api_key()
+    
+    if not api_key:
+        print("❌ No CLI token found. Please run 'litellm-proxy login' first.")
+        return
+    
+    print("✅ Found CLI token.")
+
+    available_models = litellm.get_valid_models(
+        check_provider_endpoint=True,
+        custom_llm_provider="litellm_proxy",
+        api_key=api_key,
+        api_base=LITELLM_BASE_URL
+    )
+    
+    print("✅ Available models:")
+    if available_models:
+        for i, model in enumerate(available_models, 1):
+            print(f"   {i:2d}. {model}")
+    else:
+        print("   No models available")
+    
+    # Use with LiteLLM
+    try:
+        response = litellm.completion(
+            model="litellm_proxy/gemini/gemini-2.5-flash",
+            messages=[{"role": "user", "content": "Hello from CLI token!"}],
+            api_key=api_key,
+            base_url=LITELLM_BASE_URL
+        )
+        print(f"✅ LLM Response: {response.model_dump_json(indent=4)}")
+    except Exception as e:
+        print(f"❌ Error: {e}")
+
+
+if __name__ == "__main__":
+    main()
+    
+    print("\n💡 Tips:")
+    print("1. Run 'litellm-proxy login' to authenticate first")
+    print("2. Replace 'https://your-proxy.com' with your actual proxy URL")
+    print("3. The token is stored locally at ~/.litellm/token.json")

cookbook/litellm_proxy_server/mcp/mcp_with_litellm_proxy.py

@@ -0,0 +1,36 @@
+"""
+Use LiteLLM Proxy MCP Gateway to call MCP tools.
+
+When using LiteLLM Proxy, you can use the same MCP tools across all your LLM providers.
+"""
+import openai
+
+client = openai.OpenAI(
+    api_key="sk-1234", # paste your litellm proxy api key here
+    base_url="http://localhost:4000" # paste your litellm proxy base url here
+)
+print("Making API request to Responses API with MCP tools")
+
+response = client.responses.create(
+    model="gpt-5",
+    input=[
+        {
+            "role": "user",
+            "content": "give me TLDR of what BerriAI/litellm repo is about",
+            "type": "message"
+        }
+    ],
+    tools=[
+        {
+            "type": "mcp",
+            "server_label": "litellm",
+            "server_url": "litellm_proxy",
+            "require_approval": "never"
+        }
+    ],
+    stream=True,
+    tool_choice="required"
+)
+
+for chunk in response:
+    print("response chunk: ", chunk)

cookbook/litellm_router_load_test/memory_usage/router_endpoint.py

@@ -5,7 +5,7 @@ import os
 import litellm
 from litellm import Router
 from dotenv import load_dotenv
-import uuid
+from litellm._uuid import uuid
 
 load_dotenv()
 

cookbook/litellm_router_load_test/memory_usage/router_memory_usage copy.py

@@ -12,7 +12,7 @@ sys.path.insert(
 import litellm
 from litellm import Router
 from dotenv import load_dotenv
-import uuid
+from litellm._uuid import uuid
 
 load_dotenv()
 

cookbook/litellm_router_load_test/memory_usage/router_memory_usage.py

@@ -12,7 +12,7 @@ sys.path.insert(
 import litellm
 from litellm import Router
 from dotenv import load_dotenv
-import uuid
+from litellm._uuid import uuid
 
 load_dotenv()
 

cookbook/misc/RELEASE_NOTES_GENERATION_INSTRUCTIONS.md

@@ -0,0 +1,400 @@
+# LiteLLM Release Notes Generation Instructions
+
+This document provides comprehensive instructions for AI agents to generate release notes for LiteLLM following the established format and style.
+
+## Required Inputs
+
+1. **Release Version** (e.g., `v1.77.3-stable`)
+2. **PR Diff/Changelog** - List of PRs with titles and contributors
+3. **Previous Version Commit Hash** - To compare model pricing changes
+4. **Reference Release Notes** - Use recent stable releases (v1.76.3-stable, v1.77.2-stable) as templates for consistent formatting
+
+## Step-by-Step Process
+
+### 1. Initial Setup and Analysis
+
+```bash
+# Check git diff for model pricing changes
+git diff <previous_commit_hash> HEAD -- model_prices_and_context_window.json
+```
+
+**Key Analysis Points:**
+- New models added (look for new entries)
+- Deprecated models removed (look for deleted entries)
+- Pricing updates (look for cost changes)
+- Feature support changes (tool calling, reasoning, etc.)
+
+### 2. Release Notes Structure
+
+Follow this exact structure based on recent stable releases (v1.76.3-stable, v1.77.2-stable, v1.77.5-stable):
+
+```markdown
+---
+title: "v1.77.X-stable - [Key Theme]"
+slug: "v1-77-X"
+date: YYYY-MM-DDTHH:mm:ss
+authors: [standard author block]
+hide_table_of_contents: false
+---
+
+## Deploy this version
+[Docker and pip installation tabs]
+
+## Key Highlights
+[3-5 bullet points of major features - prioritize MCP OAuth 2.0, scheduled key rotations, and major model updates]
+
+## New Models / Updated Models
+#### New Model Support
+[Model pricing table]
+
+#### Features
+[Provider-specific features organized by provider]
+
+### Bug Fixes
+[Provider-specific bug fixes organized by provider]
+
+#### New Provider Support
+[New provider integrations]
+
+## LLM API Endpoints
+#### Features
+[API-specific features organized by API type]
+
+#### Bugs
+[General bug fixes]
+
+## Management Endpoints / UI
+#### Features
+[UI and management features - group by functionality like Proxy CLI Auth, Virtual Keys, Models + Endpoints]
+
+#### Bugs
+[Management-related bug fixes]
+
+## Logging / Guardrail / Prompt Management Integrations
+#### Features
+[Organized by integration provider with proper doc links]
+
+#### Guardrails
+[Guardrail-specific features and fixes]
+
+#### Prompt Management
+[Prompt management integrations like BitBucket]
+
+## Spend Tracking, Budgets and Rate Limiting
+[Cost tracking, service tier pricing, rate limiting improvements]
+
+## MCP Gateway
+[MCP-specific features, OAuth 2.0, configuration improvements]
+
+## Performance / Loadbalancing / Reliability improvements
+[Infrastructure improvements, memory fixes, performance optimizations]
+
+## Documentation Updates
+[Documentation improvements, guides, corrections - separate section for visibility]
+
+## New Contributors
+[List of first-time contributors]
+
+## Full Changelog
+[Link to GitHub comparison]
+```
+
+### 3. Categorization Rules
+
+**Performance Improvements:**
+- RPS improvements
+- Memory optimizations
+- CPU usage optimizations
+- Timeout controls
+- Worker configuration
+- Memory leak fixes
+- Cache performance improvements
+- Database connection management
+- Dependency management (fastuuid, etc.)
+- Configuration management
+
+**New Models/Updated Models:**
+- Extract from model_prices_and_context_window.json diff
+- Create tables with: Provider, Model, Context Window, Input Cost, Output Cost, Features
+- **Structure:**
+  - `#### New Model Support` - pricing table
+  - `#### Features` - organized by provider with documentation links
+  - `### Bug Fixes` - provider-specific bug fixes
+  - `#### New Provider Support` - major new provider integrations
+- Group by provider with proper doc links: `**[Provider Name](../../docs/providers/[provider])**`
+- Use bullet points under each provider for multiple features
+- Separate features from bug fixes clearly
+
+**LLM API Endpoints:**
+- **Structure:**
+  - `#### Features` - organized by API type (Responses API, Batch API, etc.)
+  - `#### Bugs` - general bug fixes under **General** category
+- **API Categories:**
+  - Responses API
+  - Batch API  
+  - CountTokens API
+  - Images API
+  - Video Generation (if applicable)
+  - General (miscellaneous improvements)
+- Use proper documentation links for each API type
+
+**UI/Management:**
+- Authentication changes
+- Dashboard improvements
+- Team management
+- Key management
+- Proxy CLI authentication and improvements
+- Virtual key management and scheduled rotations
+- SSO configuration fixes
+- Admin settings updates
+- Management routes and endpoints
+
+**Logging / Guardrail / Prompt Management Integrations:**
+- **Structure:**
+  - `#### Features` - organized by integration provider with proper doc links
+  - `#### Guardrails` - guardrail-specific features and fixes
+  - `#### Prompt Management` - prompt management integrations
+  - `#### New Integration` - major new integrations
+- **Integration Categories:**
+  - **[DataDog](../../docs/proxy/logging#datadog)** - group all DataDog-related changes
+  - **[Langfuse](../../docs/proxy/logging#langfuse)** - Langfuse-specific features
+  - **[Prometheus](../../docs/proxy/logging#prometheus)** - monitoring improvements
+  - **[PostHog](../../docs/observability/posthog)** - observability integration
+  - **[SQS](../../docs/proxy/logging#sqs)** - SQS logging features
+  - **[Opik](../../docs/proxy/logging#opik)** - Opik integration improvements
+  - Other logging providers with proper doc links
+- **Guardrail Categories:**
+  - LakeraAI, Presidio, Noma, and other guardrail providers
+- **Prompt Management:**
+  - BitBucket, GitHub, and other prompt management integrations
+- Use bullet points under each provider for multiple features
+- Separate logging features from guardrails and prompt management clearly
+
+### 4. Documentation Linking Strategy
+
+**Link to docs when:**
+- New provider support added
+- Significant feature additions
+- API endpoint changes
+- Integration additions
+
+**Link format:** `../../docs/[category]/[specific_doc]`
+
+**Common doc paths:**
+- `../../docs/providers/[provider]` - Provider-specific docs
+- `../../docs/image_generation` - Image generation
+- `../../docs/video_generation` - Video generation (if exists)
+- `../../docs/response_api` - Responses API
+- `../../docs/proxy/logging` - Logging integrations
+- `../../docs/proxy/guardrails` - Guardrails
+- `../../docs/pass_through/[provider]` - Passthrough endpoints
+
+### 5. Model Table Generation
+
+From git diff analysis, create tables like:
+
+```markdown
+| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Features |
+| -------- | ----- | -------------- | ------------------- | -------------------- | -------- |
+| OpenRouter | `openrouter/openai/gpt-4.1` | 1M | $2.00 | $8.00 | Chat completions with vision |
+```
+
+**Extract from JSON:**
+- `max_input_tokens` → Context Window
+- `input_cost_per_token` × 1,000,000 → Input cost
+- `output_cost_per_token` × 1,000,000 → Output cost
+- `supports_*` fields → Features
+- Special pricing fields (per image, per second) for generation models
+
+### 6. PR Categorization Logic
+
+**By Keywords in PR Title:**
+- `[Perf]`, `Performance`, `RPS` → Performance Improvements
+- `[Bug]`, `[Bug Fix]`, `Fix` → Bug Fixes section
+- `[Feat]`, `[Feature]`, `Add support` → Features section
+- `[Docs]` → Documentation Updates section
+- Provider names (Gemini, OpenAI, etc.) → Group under provider
+- `MCP`, `oauth`, `Model Context Protocol` → MCP Gateway
+- `service_tier`, `priority`, `cost tracking` → Spend Tracking, Budgets and Rate Limiting
+
+**By PR Content Analysis:**
+- New model additions → New Models section
+- UI changes → Management Endpoints/UI
+- Logging/observability → Logging/Guardrail/Prompt Management Integrations
+- Rate limiting/budgets → Spend Tracking, Budgets and Rate Limiting
+- Authentication → Management Endpoints/UI
+- MCP-related changes → MCP Gateway
+- Documentation updates → Documentation Updates
+- Performance/memory fixes → Performance/Loadbalancing/Reliability improvements
+
+**Special Categorization Rules:**
+- **Service tier pricing** (OpenAI priority/flex) → Spend Tracking section (NOT provider features)
+- **Cost breakdown in logging** → Spend Tracking section
+- **MCP configuration/OAuth** → MCP Gateway (NOT General Proxy Improvements)
+- **All documentation PRs** → Documentation Updates section for visibility
+
+### 7. Writing Style Guidelines
+
+**Tone:**
+- Professional but accessible
+- Focus on user impact
+- Highlight breaking changes clearly
+- Use active voice
+
+**Formatting:**
+- Use consistent markdown formatting
+- Include PR links: `[PR #XXXXX](https://github.com/BerriAI/litellm/pull/XXXXX)`
+- Use code blocks for configuration examples
+- Bold important terms and section headers
+
+**Warnings/Notes:**
+- Add warning boxes for breaking changes
+- Include migration instructions when needed
+- Provide override options for default changes
+
+### 8. Quality Checks
+
+**Before finalizing:**
+- Verify all PR links work
+- Check documentation links are valid
+- Ensure model pricing is accurate
+- Confirm provider names are consistent
+- Review for typos and formatting issues
+- **Count PRs by section** - Provide final count like:
+  ```
+  ## MM/DD/YYYY
+  * New Models / Updated Models: XX
+  * LLM API Endpoints: XX
+  * Management Endpoints / UI: XX
+  * Logging / Guardrail / Prompt Management Integrations: XX
+  * Spend Tracking, Budgets and Rate Limiting: XX
+  * MCP Gateway: XX
+  * Performance / Loadbalancing / Reliability improvements: XX
+  * Documentation Updates: XX
+  ```
+
+### 9. Common Patterns to Follow
+
+**Performance Changes:**
+```markdown
+- **+400 RPS Performance Boost** - Description - [PR #XXXXX](link)
+```
+
+**New Models:**
+Always include pricing table and feature highlights
+
+**Breaking Changes:**
+```markdown
+:::warning
+This release has a known issue...
+:::
+```
+
+**Provider Features (New Models / Updated Models section):**
+```markdown
+#### Features
+
+- **[Provider Name](../../docs/providers/provider)**
+    - Feature description - [PR #XXXXX](link)
+    - Another feature description - [PR #YYYYY](link)
+```
+
+**API Features (LLM API Endpoints section):**
+```markdown
+#### Features
+
+- **[API Name](../../docs/api_path)**
+    - Feature description - [PR #XXXXX](link)
+    - Another feature - [PR #YYYYY](link)
+- **General**
+    - Miscellaneous improvements - [PR #ZZZZZ](link)
+```
+
+**Integration Features (Logging / Guardrail Integrations section):**
+```markdown
+#### Features
+
+- **[Integration Name](../../docs/proxy/logging#integration)**
+    - Feature description - [PR #XXXXX](link)
+    - Bug fix description - [PR #YYYYY](link)
+```
+
+**Bug Fixes Pattern:**
+```markdown
+### Bug Fixes
+
+- **[Provider/Component Name](../../docs/providers/provider)**
+    - Bug fix description - [PR #XXXXX](link)
+```
+
+### 10. Missing Documentation Check
+
+**Review for missing docs:**
+- New providers without documentation
+- New API endpoints without examples
+- Complex features without guides
+- Integration setup instructions
+
+**Flag for documentation needs:**
+- New provider integrations
+- Significant API changes
+- Complex configuration options
+- Migration requirements
+
+### 11. New Sections and Categories (Added in v1.77.5)
+
+**MCP Gateway Section:**
+- All MCP-related changes go here (not in General Proxy Improvements)
+- OAuth 2.0 flow improvements
+- MCP configuration and tools
+- Server management features
+
+**Spend Tracking, Budgets and Rate Limiting Section:**
+- Service tier pricing (OpenAI priority/flex pricing)
+- Cost tracking and breakdown features
+- Rate limiting improvements (Parallel Request Limiter v3)
+- Priority reservation fixes
+- Metadata handling for rate limiting
+
+**Documentation Updates Section:**
+- Create separate section for all documentation improvements
+- Include provider documentation fixes
+- Model reference updates
+- New guides and tutorials
+- Documentation corrections and clarifications
+- This gives documentation changes proper visibility
+
+**Management Endpoints / UI Grouping:**
+- Group related features under sub-categories:
+  - **Proxy CLI Auth** - CLI authentication improvements
+  - **Virtual Keys** - Key rotation and management
+  - **Models + Endpoints** - Provider and endpoint management
+
+**Logging Section Expansion:**
+- Rename to "Logging / Guardrail / Prompt Management Integrations"
+- Add **Prompt Management** subsection for BitBucket, GitHub integrations
+- Keep guardrails separate from logging features
+
+## Example Command Workflow
+
+```bash
+# 1. Get model changes
+git diff <commit> HEAD -- model_prices_and_context_window.json
+
+# 2. Analyze PR list for categorization
+# 3. Create release notes following template
+# 4. Link to appropriate documentation
+# 5. Review for missing documentation needs
+```
+
+## Output Requirements
+
+- Follow exact markdown structure from reference
+- Include all PR links and contributors
+- Provide accurate model pricing tables
+- Link to relevant documentation
+- Highlight breaking changes with warnings
+- Include deployment instructions
+- End with full changelog link
+
+This process ensures consistent, comprehensive release notes that help users understand changes and upgrade smoothly.

cookbook/misc/test_responses_api.py

@@ -0,0 +1,53 @@
+import base64
+from openai import OpenAI
+import time
+client = OpenAI(
+    base_url="http://0.0.0.0:4001",
+    api_key="sk-1234"
+)
+
+# Function to encode the image
+def encode_image(image_path):
+    with open(image_path, "rb") as image_file:
+        return base64.b64encode(image_file.read()).decode("utf-8")
+
+
+# Path to your image
+image_path = "litellm/proxy/logo.jpg"
+
+# Getting the Base64 string
+base64_image = encode_image(image_path)
+
+
+response = client.responses.create(
+    model="bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0",
+    input=[
+        {
+            "role": "user",
+            "content": [
+                { "type": "input_text", "text": "what color is the image"},
+                {
+                    "type": "input_image",
+                    "image_url": f"data:image/jpeg;base64,{base64_image}",
+                },
+            ],
+        }
+    ],
+)
+
+
+
+print(response.output_text)
+print("response1 id===", response.id)
+print("sleeping for 20 seconds...")
+time.sleep(20)
+print("making follow up request for existing id")
+response2 = client.responses.create(
+    model="bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0",
+    previous_response_id=response.id,
+    input="ok, and what objects are in the image?"
+)
+
+print(response2.output_text)
+
+

cookbook/veo_video_generation.py

@@ -0,0 +1,311 @@
+#!/usr/bin/env python3
+"""
+Complete example for Veo video generation through LiteLLM proxy.
+
+This script demonstrates how to:
+1. Generate videos using Google's Veo model
+2. Poll for completion status
+3. Download the generated video file
+
+Requirements:
+- LiteLLM proxy running with Google AI Studio pass-through configured
+- Google AI Studio API key with Veo access
+"""
+
+import json
+import os
+import time
+import requests
+from typing import Optional
+
+
+class VeoVideoGenerator:
+    """Complete Veo video generation client using LiteLLM proxy."""
+    
+    def __init__(self, base_url: str = "http://localhost:4000/gemini/v1beta", 
+                 api_key: str = "sk-1234"):
+        """
+        Initialize the Veo video generator.
+        
+        Args:
+            base_url: Base URL for the LiteLLM proxy with Gemini pass-through
+            api_key: API key for LiteLLM proxy authentication
+        """
+        self.base_url = base_url
+        self.api_key = api_key
+        self.headers = {
+            "x-goog-api-key": api_key,
+            "Content-Type": "application/json"
+        }
+    
+    def generate_video(self, prompt: str) -> Optional[str]:
+        """
+        Initiate video generation with Veo.
+        
+        Args:
+            prompt: Text description of the video to generate
+            
+        Returns:
+            Operation name if successful, None otherwise
+        """
+        print(f"🎬 Generating video with prompt: '{prompt}'")
+        
+        url = f"{self.base_url}/models/veo-3.0-generate-preview:predictLongRunning"
+        payload = {
+            "instances": [{
+                "prompt": prompt
+            }]
+        }
+        
+        try:
+            response = requests.post(url, headers=self.headers, json=payload)
+            response.raise_for_status()
+            
+            data = response.json()
+            operation_name = data.get("name")
+            
+            if operation_name:
+                print(f"✅ Video generation started: {operation_name}")
+                return operation_name
+            else:
+                print("❌ No operation name returned")
+                print(f"Response: {json.dumps(data, indent=2)}")
+                return None
+                
+        except requests.RequestException as e:
+            print(f"❌ Failed to start video generation: {e}")
+            if hasattr(e, 'response') and e.response is not None:
+                try:
+                    error_data = e.response.json()
+                    print(f"Error details: {json.dumps(error_data, indent=2)}")
+                except:
+                    print(f"Error response: {e.response.text}")
+            return None
+    
+    def wait_for_completion(self, operation_name: str, max_wait_time: int = 600) -> Optional[str]:
+        """
+        Poll operation status until video generation is complete.
+        
+        Args:
+            operation_name: Name of the operation to monitor
+            max_wait_time: Maximum time to wait in seconds (default: 10 minutes)
+            
+        Returns:
+            Video URI if successful, None otherwise
+        """
+        print("⏳ Waiting for video generation to complete...")
+        
+        operation_url = f"{self.base_url}/{operation_name}"
+        start_time = time.time()
+        poll_interval = 10  # Start with 10 seconds
+        
+        while time.time() - start_time < max_wait_time:
+            try:
+                print(f"🔍 Polling status... ({int(time.time() - start_time)}s elapsed)")
+                
+                response = requests.get(operation_url, headers=self.headers)
+                response.raise_for_status()
+                
+                data = response.json()
+                
+                # Check for errors
+                if "error" in data:
+                    print("❌ Error in video generation:")
+                    print(json.dumps(data["error"], indent=2))
+                    return None
+                
+                # Check if operation is complete
+                is_done = data.get("done", False)
+                
+                if is_done:
+                    print("🎉 Video generation complete!")
+                    
+                    try:
+                        # Extract video URI from nested response
+                        video_uri = data["response"]["generateVideoResponse"]["generatedSamples"][0]["video"]["uri"]
+                        print(f"📹 Video URI: {video_uri}")
+                        return video_uri
+                    except KeyError as e:
+                        print(f"❌ Could not extract video URI: {e}")
+                        print("Full response:")
+                        print(json.dumps(data, indent=2))
+                        return None
+                
+                # Wait before next poll, with exponential backoff
+                time.sleep(poll_interval)
+                poll_interval = min(poll_interval * 1.2, 30)  # Cap at 30 seconds
+                
+            except requests.RequestException as e:
+                print(f"❌ Error polling operation status: {e}")
+                time.sleep(poll_interval)
+        
+        print(f"⏰ Timeout after {max_wait_time} seconds")
+        return None
+    
+    def download_video(self, video_uri: str, output_filename: str = "generated_video.mp4") -> bool:
+        """
+        Download the generated video file.
+        
+        Args:
+            video_uri: URI of the video to download (from Google's response)
+            output_filename: Local filename to save the video
+            
+        Returns:
+            True if download successful, False otherwise
+        """
+        print(f"⬇️  Downloading video...")
+        print(f"Original URI: {video_uri}")
+        
+        # Convert Google URI to LiteLLM proxy URI
+        # Example: files/abc123 -> /gemini/v1beta/files/abc123:download?alt=media
+        if video_uri.startswith("files/"):
+            download_path = f"{video_uri}:download?alt=media"
+        else:
+            download_path = video_uri
+            
+        litellm_download_url = f"{self.base_url}/{download_path}"
+        print(f"Download URL: {litellm_download_url}")
+        
+        try:
+            # Download with streaming and redirect handling
+            response = requests.get(
+                litellm_download_url, 
+                headers=self.headers, 
+                stream=True,
+                allow_redirects=True  # Handle redirects automatically
+            )
+            response.raise_for_status()
+            
+            # Save video file
+            with open(output_filename, 'wb') as f:
+                downloaded_size = 0
+                for chunk in response.iter_content(chunk_size=8192):
+                    if chunk:
+                        f.write(chunk)
+                        downloaded_size += len(chunk)
+                        
+                        # Progress indicator for large files
+                        if downloaded_size % (1024 * 1024) == 0:  # Every MB
+                            print(f"📦 Downloaded {downloaded_size / (1024*1024):.1f} MB...")
+            
+            # Verify file was created and has content
+            if os.path.exists(output_filename):
+                file_size = os.path.getsize(output_filename)
+                if file_size > 0:
+                    print(f"✅ Video downloaded successfully!")
+                    print(f"📁 Saved as: {output_filename}")
+                    print(f"📏 File size: {file_size / (1024*1024):.2f} MB")
+                    return True
+                else:
+                    print("❌ Downloaded file is empty")
+                    os.remove(output_filename)
+                    return False
+            else:
+                print("❌ File was not created")
+                return False
+                
+        except requests.RequestException as e:
+            print(f"❌ Download failed: {e}")
+            if hasattr(e, 'response') and e.response is not None:
+                print(f"Status code: {e.response.status_code}")
+                print(f"Response headers: {dict(e.response.headers)}")
+            return False
+    
+    def generate_and_download(self, prompt: str, output_filename: str = None) -> bool:
+        """
+        Complete workflow: generate video and download it.
+        
+        Args:
+            prompt: Text description for video generation
+            output_filename: Output filename (auto-generated if None)
+            
+        Returns:
+            True if successful, False otherwise
+        """
+        # Auto-generate filename if not provided
+        if output_filename is None:
+            timestamp = int(time.time())
+            safe_prompt = "".join(c for c in prompt[:30] if c.isalnum() or c in (' ', '-', '_')).rstrip()
+            output_filename = f"veo_video_{safe_prompt.replace(' ', '_')}_{timestamp}.mp4"
+        
+        print("=" * 60)
+        print("🎬 VEO VIDEO GENERATION WORKFLOW")
+        print("=" * 60)
+        
+        # Step 1: Generate video
+        operation_name = self.generate_video(prompt)
+        if not operation_name:
+            return False
+        
+        # Step 2: Wait for completion
+        video_uri = self.wait_for_completion(operation_name)
+        if not video_uri:
+            return False
+        
+        # Step 3: Download video
+        success = self.download_video(video_uri, output_filename)
+        
+        if success:
+            print("=" * 60)
+            print("🎉 SUCCESS! Video generation complete!")
+            print(f"📁 Video saved as: {output_filename}")
+            print("=" * 60)
+        else:
+            print("=" * 60)
+            print("❌ FAILED! Video generation or download failed")
+            print("=" * 60)
+        
+        return success
+
+
+def main():
+    """
+    Example usage of the VeoVideoGenerator.
+    
+    Configure these environment variables:
+    - LITELLM_BASE_URL: Your LiteLLM proxy URL (default: http://localhost:4000/gemini/v1beta)
+    - LITELLM_API_KEY: Your LiteLLM API key (default: sk-1234)
+    """
+    
+    # Configuration from environment or defaults
+    base_url = os.getenv("LITELLM_BASE_URL", "http://localhost:4000/gemini/v1beta")
+    api_key = os.getenv("LITELLM_API_KEY", "sk-1234")
+    
+    print("🚀 Starting Veo Video Generation Example")
+    print(f"📡 Using LiteLLM proxy at: {base_url}")
+    
+    # Initialize generator
+    generator = VeoVideoGenerator(base_url=base_url, api_key=api_key)
+    
+    # Example prompts - try different ones!
+    example_prompts = [
+        "A cat playing with a ball of yarn in a sunny garden",
+        "Ocean waves crashing against rocky cliffs at sunset",
+        "A bustling city street with people walking and cars passing by",
+        "A peaceful forest with sunlight filtering through the trees"
+    ]
+    
+    # Use first example or get from user
+    prompt = example_prompts[0]
+    print(f"🎬 Using prompt: '{prompt}'")
+    
+    # Generate and download video
+    success = generator.generate_and_download(prompt)
+    
+    if success:
+        print("\n✅ Example completed successfully!")
+        print("💡 Try modifying the prompt in the script for different videos!")
+    else:
+        print("\n❌ Example failed!")
+        print("🔧 Check your LiteLLM proxy configuration and Google AI Studio API key")
+        
+        # Troubleshooting tips
+        print("\n🔍 Troubleshooting:")
+        print("1. Ensure LiteLLM proxy is running with Google AI Studio pass-through")
+        print("2. Verify your Google AI Studio API key has Veo access")
+        print("3. Check that your prompt meets Veo's content guidelines")
+        print("4. Review the LiteLLM proxy logs for detailed error information")
+
+
+if __name__ == "__main__":
+    main()

db_scripts/migrate_keys.py

@@ -0,0 +1,187 @@
+from prisma import Prisma
+import csv
+import json
+import asyncio
+from datetime import datetime
+from typing import Optional, List, Dict, Any
+
+import os
+
+## VARIABLES
+DATABASE_URL = "postgresql://postgres:postgres@localhost:5432/litellm"
+CSV_FILE_PATH = "./path_to_csv.csv"
+
+os.environ["DATABASE_URL"] = DATABASE_URL
+
+
+async def parse_csv_value(value: str, field_type: str) -> Any:
+    """Parse CSV values according to their expected types"""
+    if value == "NULL" or value == "" or value is None:
+        return None
+
+    if field_type == "boolean":
+        return value.lower() == "true"
+    elif field_type == "float":
+        return float(value)
+    elif field_type == "int":
+        return int(value) if value.isdigit() else None
+    elif field_type == "bigint":
+        return int(value) if value.isdigit() else None
+    elif field_type == "datetime":
+        try:
+            return datetime.fromisoformat(value.replace("Z", "+00:00"))
+        except:
+            return None
+    elif field_type == "json":
+        try:
+            return value if value else json.dumps({})
+        except:
+            return json.dumps({})
+    elif field_type == "string_array":
+        # Handle string arrays like {default-models}
+        if value.startswith("{") and value.endswith("}"):
+            content = value[1:-1]  # Remove braces
+            if content:
+                return [item.strip() for item in content.split(",")]
+            else:
+                return []
+        return []
+    else:
+        return value
+
+
+async def migrate_verification_tokens():
+    """Main migration function"""
+    prisma = Prisma()
+    await prisma.connect()
+
+    try:
+        # Read CSV file
+        csv_file_path = CSV_FILE_PATH
+
+        with open(csv_file_path, "r", encoding="utf-8") as file:
+            csv_reader = csv.DictReader(file)
+
+            processed_count = 0
+            error_count = 0
+
+            for row in csv_reader:
+                try:
+                    # Replace 'default-team' with the specified UUID
+                    team_id = row.get("team_id")
+                    if team_id == "NULL" or team_id == "":
+                        team_id = None
+
+                    # Prepare data for insertion
+                    verification_token_data = {
+                        "token": row["token"],
+                        "key_name": await parse_csv_value(row["key_name"], "string"),
+                        "key_alias": await parse_csv_value(row["key_alias"], "string"),
+                        "soft_budget_cooldown": await parse_csv_value(
+                            row["soft_budget_cooldown"], "boolean"
+                        ),
+                        "spend": await parse_csv_value(row["spend"], "float"),
+                        "expires": await parse_csv_value(row["expires"], "datetime"),
+                        "models": await parse_csv_value(row["models"], "string_array"),
+                        "aliases": await parse_csv_value(row["aliases"], "json"),
+                        "config": await parse_csv_value(row["config"], "json"),
+                        "user_id": await parse_csv_value(row["user_id"], "string"),
+                        "team_id": team_id,
+                        "permissions": await parse_csv_value(
+                            row["permissions"], "json"
+                        ),
+                        "max_parallel_requests": await parse_csv_value(
+                            row["max_parallel_requests"], "int"
+                        ),
+                        "metadata": await parse_csv_value(row["metadata"], "json"),
+                        "tpm_limit": await parse_csv_value(row["tpm_limit"], "bigint"),
+                        "rpm_limit": await parse_csv_value(row["rpm_limit"], "bigint"),
+                        "max_budget": await parse_csv_value(row["max_budget"], "float"),
+                        "budget_duration": await parse_csv_value(
+                            row["budget_duration"], "string"
+                        ),
+                        "budget_reset_at": await parse_csv_value(
+                            row["budget_reset_at"], "datetime"
+                        ),
+                        "allowed_cache_controls": await parse_csv_value(
+                            row["allowed_cache_controls"], "string_array"
+                        ),
+                        "model_spend": await parse_csv_value(
+                            row["model_spend"], "json"
+                        ),
+                        "model_max_budget": await parse_csv_value(
+                            row["model_max_budget"], "json"
+                        ),
+                        "budget_id": await parse_csv_value(row["budget_id"], "string"),
+                        "blocked": await parse_csv_value(row["blocked"], "boolean"),
+                        "created_at": await parse_csv_value(
+                            row["created_at"], "datetime"
+                        ),
+                        "updated_at": await parse_csv_value(
+                            row["updated_at"], "datetime"
+                        ),
+                        "allowed_routes": await parse_csv_value(
+                            row["allowed_routes"], "string_array"
+                        ),
+                        "object_permission_id": await parse_csv_value(
+                            row["object_permission_id"], "string"
+                        ),
+                        "created_by": await parse_csv_value(
+                            row["created_by"], "string"
+                        ),
+                        "updated_by": await parse_csv_value(
+                            row["updated_by"], "string"
+                        ),
+                        "organization_id": await parse_csv_value(
+                            row["organization_id"], "string"
+                        ),
+                    }
+
+                    # Remove None values to use database defaults
+                    verification_token_data = {
+                        k: v
+                        for k, v in verification_token_data.items()
+                        if v is not None
+                    }
+
+                    # Check if token already exists
+                    existing_token = await prisma.litellm_verificationtoken.find_unique(
+                        where={"token": verification_token_data["token"]}
+                    )
+
+                    if existing_token:
+                        print(
+                            f"Token {verification_token_data['token']} already exists, skipping..."
+                        )
+                        continue
+
+                    # Insert the record
+                    await prisma.litellm_verificationtoken.create(
+                        data=verification_token_data
+                    )
+
+                    processed_count += 1
+                    print(
+                        f"Successfully migrated token: {verification_token_data['token']}"
+                    )
+
+                except Exception as e:
+                    error_count += 1
+                    print(
+                        f"Error processing row with token {row.get('token', 'unknown')}: {str(e)}"
+                    )
+                    continue
+
+            print(f"\nMigration completed!")
+            print(f"Successfully processed: {processed_count} records")
+            print(f"Errors encountered: {error_count} records")
+
+    except Exception as e:
+        print(f"Migration failed: {str(e)}")
+
+    finally:
+        await prisma.disconnect()
+
+
+if __name__ == "__main__":
+    asyncio.run(migrate_verification_tokens())

deploy/charts/litellm-helm/Chart.yaml

@@ -18,7 +18,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.4.4
+version: 0.4.6
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

deploy/charts/litellm-helm/README.md

@@ -24,7 +24,7 @@ If `db.useStackgresOperator` is used (not yet implemented):
 | `replicaCount`                                             | The number of LiteLLM Proxy pods to be deployed                                                                                                                                       | `1`  |
 | `masterkeySecretName`                                      | The name of the Kubernetes Secret that contains the Master API Key for LiteLLM.  If not specified, use the generated secret name.                                                                                                         | N/A  |
 | `masterkeySecretKey`                                      | The key within the Kubernetes Secret that contains the Master API Key for LiteLLM.  If not specified, use `masterkey` as the key.                                                                                                         | N/A  |
-| `masterkey`                                                | The Master API Key for LiteLLM.  If not specified, a random key is generated.                                                                                                         | N/A  |
+| `masterkey`                                                | The Master API Key for LiteLLM.  If not specified, a random key in the `sk-...` format is generated.                                                                                   | N/A  |
 | `environmentSecrets`                                       | An optional array of Secret object names.  The keys and values in these secrets will be presented to the LiteLLM proxy pod as environment variables.  See below for an example Secret object.  | `[]`  |
 | `environmentConfigMaps`                                       | An optional array of ConfigMap object names.  The keys and values in these configmaps will be presented to the LiteLLM proxy pod as environment variables.  See below for an example Secret object.  | `[]`  |
 | `image.repository`                                         | LiteLLM Proxy image repository                                                                                                                                                        | `ghcr.io/berriai/litellm`  |
@@ -36,11 +36,50 @@ If `db.useStackgresOperator` is used (not yet implemented):
 | `service.port`                                             | TCP port that the Kubernetes Service will listen on.  Also the TCP port within the Pod that the proxy will listen on.                                                                 | `4000`  |
 | `service.loadBalancerClass`                                | Optional LoadBalancer implementation class (only used when `service.type` is `LoadBalancer`)                                                                                                          | `""`  |
 | `ingress.*`                                                | See [values.yaml](./values.yaml) for example settings                                                                                                                                 | N/A  |
-| `proxy_config.*`                                           | See [values.yaml](./values.yaml) for default settings.  See [example_config_yaml](../../../litellm/proxy/example_config_yaml/) for configuration examples.                            | N/A  |
-| `extraContainers[]`                                        | An array of additional containers to be deployed as sidecars alongside the LiteLLM Proxy.                                                                                             | `[]`  |
+| `proxyConfigMap.create`     | When `true`, render a ConfigMap from `.Values.proxy_config` and mount it.                                                                                                                                      | `true` |
+| `proxyConfigMap.name`       | When `create=false`, name of the existing ConfigMap to mount.                                                                                                                                                  | `""`  |
+| `proxyConfigMap.key`        | Key in the ConfigMap that contains the proxy config file.                                                                                                                                                      | `"config.yaml"` |
+| `proxy_config.*`            | See [values.yaml](./values.yaml) for default settings. Rendered into the ConfigMap’s `config.yaml` only when `proxyConfigMap.create=true`. See [example_config_yaml](../../../litellm/proxy/example_config_yaml/) for configuration examples. | `N/A` |
+| `extraContainers[]`         | An array of additional containers to be deployed as sidecars alongside the LiteLLM Proxy.
+| `pdb.enabled`                   | Enable a PodDisruptionBudget for the LiteLLM proxy Deployment                                                                 | `false` |
+| `pdb.minAvailable`             | Minimum number/percentage of pods that must be available during **voluntary** disruptions (choose **one** of minAvailable/maxUnavailable) | `null`  |
+| `pdb.maxUnavailable`           | Maximum number/percentage of pods that can be unavailable during **voluntary** disruptions (choose **one** of minAvailable/maxUnavailable) | `null`  |
+| `pdb.annotations`              | Extra metadata annotations to add to the PDB                                                                                   | `{}`    |
+| `pdb.labels`                   | Extra metadata labels to add to the PDB                                                                                        | `{}`    |
+
+#### Example `proxy_config` ConfigMap from values (default):
+
+
+```
+proxyConfigMap:
+  create: true
+  key: "config.yaml"
+
+proxy_config:
+  general_settings:
+    master_key: os.environ/PROXY_MASTER_KEY
+  model_list:
+    - model_name: gpt-3.5-turbo
+      litellm_params:
+        model: gpt-3.5-turbo
+        api_key: eXaMpLeOnLy
+```
+
+#### Example using existing `proxyConfigMap` instead of creating it:
+
+
+```
+proxyConfigMap:
+  create: false
+  name: my-litellm-config
+  key: config.yaml
+
+# proxy_config is ignored in this mode
+```
 
 #### Example `environmentSecrets` Secret 
 
+
 ```
 apiVersion: v1
 kind: Secret
@@ -110,6 +149,22 @@ data:
 
 Source: [GitHub Gist from troyharvey](https://gist.github.com/troyharvey/4506472732157221e04c6b15e3b3f094)
 
+### Migration Job Settings
+
+The migration job supports both ArgoCD and Helm hooks to ensure database migrations run at the appropriate time during deployments.
+
+| Name                                                       | Description                                                                                                                                                                           | Value |
+| ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- |
+| `migrationJob.enabled`                                     | Enable or disable the schema migration Job                                                                                                                                           | `true`  |
+| `migrationJob.backoffLimit`                                | Backoff limit for Job restarts                                                                                                                                                       | `4`  |
+| `migrationJob.ttlSecondsAfterFinished`                     | TTL for completed migration jobs                                                                                                                                                      | `120`  |
+| `migrationJob.annotations`                                 | Additional annotations for the migration job pod                                                                                                                                      | `{}`  |
+| `migrationJob.extraContainers`                             | Additional containers to run alongside the migration job                                                                                                                              | `[]`  |
+| `migrationJob.hooks.argocd.enabled`                       | Enable ArgoCD hooks for the migration job (uses PreSync hook with BeforeHookCreation delete policy)                                                                                  | `true`  |
+| `migrationJob.hooks.helm.enabled`                         | Enable Helm hooks for the migration job (uses pre-install,pre-upgrade hooks with before-hook-creation delete policy)                                                                 | `false`  |
+| `migrationJob.hooks.helm.weight`                          | Helm hook execution order (lower weights executed first). Optional - defaults to "1" if not specified.                                                                               | N/A  |
+
+
 ## Accessing the Admin UI
 When browsing to the URL published per the settings in `ingress.*`, you will
 be prompted for **Admin Configuration**.  The **Proxy Endpoint** is the internal
@@ -119,7 +174,7 @@ service, the **Proxy Endpoint** should be set to `http://<RELEASE>-litellm:4000`
 
 The **Proxy Key** is the value specified for `masterkey` or, if a `masterkey`
 was not provided to the helm command line, the `masterkey` is a randomly
-generated string stored in the `<RELEASE>-litellm-masterkey` Kubernetes Secret.
+generated string in the `sk-...` format stored in the `<RELEASE>-litellm-masterkey` Kubernetes Secret.
 
 ```bash
 kubectl -n litellm get secret <RELEASE>-litellm-masterkey -o jsonpath="{.data.masterkey}"

deploy/charts/litellm-helm/templates/NOTES.txt

@@ -20,3 +20,4 @@
   echo "Visit http://127.0.0.1:8080 to use your application"
   kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT
 {{- end }}
+PDB: {{ if .Values.pdb.enabled }}enabled{{ else }}disabled{{ end }}. Configure via .Values.pdb.*

deploy/charts/litellm-helm/templates/configmap-litellm.yaml

@@ -1,7 +1,9 @@
+{{- if .Values.proxyConfigMap.create }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
   name: {{ include "litellm.fullname" . }}-config
 data:
   config.yaml: |
-{{ .Values.proxy_config | toYaml | indent 6 }}
+{{ .Values.proxy_config | toYaml | indent 6 }}
+{{- end }}

deploy/charts/litellm-helm/templates/deployment.yaml

@@ -1,6 +1,8 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
+  annotations:
+  {{- toYaml .Values.deploymentAnnotations | nindent 4 }}
   name: {{ include "litellm.fullname" . }}
   labels:
     {{- include "litellm.labels" . | nindent 4 }}
@@ -14,7 +16,9 @@ spec:
   template:
     metadata:
       annotations:
+        {{- if .Values.proxyConfigMap.create }}
         checksum/config: {{ include (print $.Template.BasePath "/configmap-litellm.yaml") . | sha256sum }}
+        {{- end }}
         {{- with .Values.podAnnotations }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
@@ -69,7 +73,14 @@ spec:
                   name: {{ .Values.db.secret.name }}
                   key: {{ .Values.db.secret.passwordKey }}
             - name: DATABASE_HOST
+              {{- if .Values.db.secret.endpointKey }}
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.db.secret.name }}
+                  key: {{ .Values.db.secret.endpointKey }}
+              {{- else }}
               value: {{ .Values.db.endpoint }}
+              {{- end }}
             - name: DATABASE_NAME
               value: {{ .Values.db.database }}
             - name: DATABASE_URL
@@ -97,6 +108,12 @@ spec:
               value: {{ $val | quote }}
             {{- end }}
             {{- end }}
+            {{- if .Values.separateHealthApp }}
+            - name: SEPARATE_HEALTH_APP
+              value: "1"
+            - name: SEPARATE_HEALTH_PORT
+              value: {{ .Values.separateHealthPort | default "8081" | quote }}
+            {{- end }}
             {{- with .Values.extraEnvVars }}
               {{- toYaml . | nindent 12 }}
             {{- end }}
@@ -116,19 +133,23 @@ spec:
             - name: http
               containerPort: {{ .Values.service.port }}
               protocol: TCP
+            {{- if .Values.separateHealthApp }}
+            - name: health
+              containerPort: {{ .Values.separateHealthPort | default 8081 }}
+              protocol: TCP
+            {{- end }}
           livenessProbe:
             httpGet:
               path: /health/liveliness
-              port: http
+              port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
           readinessProbe:
             httpGet:
               path: /health/readiness
-              port: http
-          # Give the container time to start up.  Up to 5 minutes (10 * 30 seconds)
+              port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
           startupProbe:
             httpGet:
               path: /health/readiness
-              port: http
+              port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
             failureThreshold: 30
             periodSeconds: 10
           resources:
@@ -164,9 +185,13 @@ spec:
         {{- end }}
         - name: litellm-config
           configMap:
+            {{- if .Values.proxyConfigMap.create }}
             name: {{ include "litellm.fullname" . }}-config
+            {{- else }}
+            name: {{ .Values.proxyConfigMap.name }}
+            {{- end }}
             items:
-              - key: "config.yaml"
+              - key: {{ .Values.proxyConfigMap.key | default "config.yaml" }}
                 path: "config.yaml"
       {{- with .Values.volumes }}
         {{- toYaml . | nindent 8 }}

deploy/charts/litellm-helm/templates/migrations-job.yaml

@@ -1,16 +1,27 @@
 {{- if .Values.migrationJob.enabled }}
-# This job runs the prisma migrations for the LiteLLM DB.
+# This job runs the Prisma migrations for the LiteLLM DB.
 apiVersion: batch/v1
 kind: Job
 metadata:
   name: {{ include "litellm.fullname" . }}-migrations
+  labels:
+    {{- include "litellm.labels" . | nindent 4 }}
   annotations:
+    {{- if .Values.migrationJob.hooks.argocd.enabled }}
     argocd.argoproj.io/hook: PreSync
-    argocd.argoproj.io/hook-delete-policy: BeforeHookCreation # delete old migration on a new deploy in case the migration needs to make updates
+    argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+    {{- end }}
+    {{- if .Values.migrationJob.hooks.helm.enabled }}
+    helm.sh/hook: "pre-install,pre-upgrade"
+    helm.sh/hook-delete-policy: "before-hook-creation"
+    helm.sh/hook-weight: {{ .Values.migrationJob.hooks.helm.weight | default "1" | quote }}
+    {{- end }}
     checksum/config: {{ toYaml .Values | sha256sum }}
 spec:
   template:
     metadata:
+      labels:
+        {{- include "litellm.labels" . | nindent 8 }}
       annotations:
         {{- with .Values.migrationJob.annotations }}
         {{- toYaml . | nindent 8 }}
@@ -38,21 +49,44 @@ spec:
                   name: {{ .Values.db.secret.name }}
                   key: {{ .Values.db.secret.passwordKey }}
             - name: DATABASE_HOST
+              {{- if .Values.db.secret.endpointKey }}
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.db.secret.name }}
+                  key: {{ .Values.db.secret.endpointKey }}
+              {{- else }}
               value: {{ .Values.db.endpoint }}
+              {{- end }}
             - name: DATABASE_NAME
               value: {{ .Values.db.database }}
             - name: DATABASE_URL
               value: {{ .Values.db.url | quote }}
-            {{- else }}
+            {{- else if .Values.db.deployStandalone }}
             - name: DATABASE_URL
               value: postgresql://{{ .Values.postgresql.auth.username }}:{{ .Values.postgresql.auth.password }}@{{ .Release.Name }}-postgresql/{{ .Values.postgresql.auth.database }}
             {{- end }}
+            {{- if .Values.envVars }}
+            {{- range $key, $val := .Values.envVars }}
+            - name: {{ $key }}
+              value: {{ $val | quote }}
+            {{- end }}
+            {{- end }}
+            {{- with .Values.extraEnvVars }}
+              {{- toYaml . | nindent 12 }}
+            {{- end }}
             - name: DISABLE_SCHEMA_UPDATE
               value: "false" # always run the migration from the Helm PreSync hook, override the value set
           {{- with .Values.volumeMounts }}
           volumeMounts:
             {{- toYaml . | nindent 12 }}
           {{- end }}
+          {{- with .Values.migrationJob.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+      {{- with .Values.migrationJob.extraContainers }}
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
       {{- with .Values.volumes }}
       volumes:
         {{- toYaml . | nindent 8 }}

deploy/charts/litellm-helm/templates/poddisruptionbudget.yaml

@@ -0,0 +1,33 @@
+{{- /*
+PodDisruptionBudget for LiteLLM proxy
+Controlled via .Values.pdb.enabled and .Values.pdb.{minAvailable|maxUnavailable}
+Only one of minAvailable / maxUnavailable should be set. If both are set, minAvailable wins.
+*/ -}}
+{{- if .Values.pdb.enabled }}
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: {{ include "litellm.fullname" . }}
+  labels:
+    {{- include "litellm.labels" . | nindent 4 }}
+  {{- with .Values.pdb.labels }}
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  {{- with .Values.pdb.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  selector:
+    matchLabels:
+      {{- /* Match the Deployment selector to target the same pod set */ -}}
+      {{- include "litellm.selectorLabels" . | nindent 6 }}
+  {{- if .Values.pdb.minAvailable }}
+  minAvailable: {{ .Values.pdb.minAvailable }}
+  {{- else if .Values.pdb.maxUnavailable }}
+  maxUnavailable: {{ .Values.pdb.maxUnavailable }}
+  {{- else }}
+  # Safe default if enabled but not configured
+  maxUnavailable: 1
+  {{- end }}
+{{- end }}

deploy/charts/litellm-helm/templates/secret-masterkey.yaml

@@ -1,5 +1,5 @@
 {{- if not .Values.masterkeySecretName }}
-{{ $masterkey := (.Values.masterkey | default (randAlphaNum 17)) }}
+{{ $masterkey := (.Values.masterkey | default (printf "sk-%s" (randAlphaNum 18))) }}
 apiVersion: v1
 kind: Secret
 metadata:

deploy/charts/litellm-helm/tests/deployment_tests.yaml

@@ -115,3 +115,25 @@ tests:
           content:
             name: EXTRA_ENV_VAR
             value: EXTRA_ENV_VAR_VALUE
+  - it: should mount existing configmap when create=false
+    template: deployment.yaml
+    set:
+      proxyConfigMap:
+        create: false
+        name: my-litellm-config
+        key: custom.yaml
+    asserts:
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: litellm-config
+            configMap:
+              name: my-litellm-config
+              items:
+                - key: custom.yaml
+                  path: config.yaml
+      - contains:
+          path: spec.template.spec.containers[0].volumeMounts
+          content:
+            name: litellm-config
+            mountPath: /etc/litellm/

deploy/charts/litellm-helm/tests/masterkey-secret_tests.yaml

@@ -2,13 +2,19 @@ suite: test masterkey secret
 templates:
   - secret-masterkey.yaml
 tests:
-  - it: should create a secret if masterkeySecretName is not set
+  - it: should create a secret if masterkeySecretName is not set. should start with sk-xxxx (base64 encoded as c2st*)
     template: secret-masterkey.yaml
     set:
       masterkeySecretName: ""
     asserts:
       - isKind:
           of: Secret
+      - matchRegex:
+          path: data.masterkey
+          pattern: ^c2st
+          # Note: The masterkey is generated as "sk-<18-random-chars>" in plain text,
+          # but stored as base64 encoded in Kubernetes secret (requirement).
+          # "sk-" base64 encodes to "c2st", so we check for "^c2st" pattern.
   - it: should not create a secret if masterkeySecretName is set
     template: secret-masterkey.yaml
     set:

deploy/charts/litellm-helm/tests/migrations-job_tests.yaml

@@ -0,0 +1,127 @@
+suite: test migrations job
+templates:
+  - migrations-job.yaml
+tests:
+  - it: should work with envVars
+    template: migrations-job.yaml
+    set:
+      envVars:
+        TEST_ENV_VAR: "test_value"
+        ANOTHER_VAR: "another_value"
+      migrationJob:
+        enabled: true
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: TEST_ENV_VAR
+            value: "test_value"
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: ANOTHER_VAR
+            value: "another_value"
+
+  - it: should work with extraEnvVars
+    template: migrations-job.yaml
+    set:
+      extraEnvVars:
+        - name: EXTRA_ENV_VAR
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.labels['env']
+        - name: SIMPLE_EXTRA_VAR
+          value: "simple_value"
+      migrationJob:
+        enabled: true
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: EXTRA_ENV_VAR
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.labels['env']
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: SIMPLE_EXTRA_VAR
+            value: "simple_value"
+
+  - it: should work with both envVars and extraEnvVars
+    template: migrations-job.yaml
+    set:
+      envVars:
+        ENV_VAR: "env_var_value"
+      extraEnvVars:
+        - name: EXTRA_ENV_VAR
+          value: "extra_env_var_value"
+      migrationJob:
+        enabled: true
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: ENV_VAR
+            value: "env_var_value"
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: EXTRA_ENV_VAR
+            value: "extra_env_var_value"
+
+  - it: should not render when migrations job is disabled
+    template: migrations-job.yaml
+    set:
+      migrationJob:
+        enabled: false
+    asserts:
+      - hasDocuments:
+          count: 0
+
+  - it: should still include default env vars
+    template: migrations-job.yaml
+    set:
+      envVars:
+        CUSTOM_VAR: "custom_value"
+      migrationJob:
+        enabled: true
+      db:
+        useExisting: true
+        endpoint: "test-db"
+        database: "testdb"
+        url: "postgresql://user:pass@test-db:5432/testdb"
+        secret:
+          name: "test-secret"
+          usernameKey: "username"
+          passwordKey: "password"
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: DISABLE_SCHEMA_UPDATE
+            value: "false"
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: DATABASE_HOST
+            value: "test-db"
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: CUSTOM_VAR
+            value: "custom_value"
+
+  - it: should not include DATABASE_URL when deployStandalone is false
+    template: migrations-job.yaml
+    set:
+      migrationJob:
+        enabled: true
+      db:
+        deployStandalone: false
+        useExisting: false
+    asserts:
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: DATABASE_URL

deploy/charts/litellm-helm/tests/pdb_tests.yaml

@@ -0,0 +1,45 @@
+suite: "pdb enabled"
+templates:
+  - poddisruptionbudget.yaml
+tests:
+  - it: "renders a PDB with maxUnavailable=1"
+    set:
+      pdb.enabled: true
+      pdb.maxUnavailable: 1
+    asserts:
+      - hasDocuments: { count: 1 }
+      - isKind: { of: PodDisruptionBudget }
+      - equal: { path: apiVersion, value: policy/v1 }
+      - equal: { path: spec.maxUnavailable, value: 1 }
+      - equal:
+          path: spec.selector.matchLabels
+          value:
+            app.kubernetes.io/name: litellm
+            app.kubernetes.io/instance: RELEASE-NAME
+
+---
+suite: "pdb disabled"
+templates:
+  - poddisruptionbudget.yaml
+tests:
+  - it: "does not render when disabled"
+    set:
+      pdb.enabled: false
+    asserts:
+      - hasDocuments: { count: 0 }
+
+---
+suite: "pdb minAvailable precedence"
+templates:
+  - poddisruptionbudget.yaml
+tests:
+  - it: "uses minAvailable when both are set"
+    set:
+      pdb.enabled: true
+      pdb.minAvailable: "50%"
+      pdb.maxUnavailable: 1
+    asserts:
+      - isKind: { of: PodDisruptionBudget }
+      - equal: { path: apiVersion, value: policy/v1 }
+      - equal: { path: spec.minAvailable, value: "50%" }
+      - isNull: { path: spec.maxUnavailable }

deploy/charts/litellm-helm/values.yaml

@@ -27,6 +27,9 @@ serviceAccount:
   # If not set and create is true, a name is generated using the fullname template
   name: ""
 
+# annotations for litellm deployment
+deploymentAnnotations: {}
+# annotations for litellm pods
 podAnnotations: {}
 podLabels: {}
 
@@ -60,6 +63,12 @@ service:
   # optionally specify loadBalancerClass
   # loadBalancerClass: tailscale
 
+# Separate health app configuration
+# When enabled, health checks will use a separate port and the application
+# will receive SEPARATE_HEALTH_APP=1 and SEPARATE_HEALTH_PORT from environment variables
+separateHealthApp: false
+separateHealthPort: 8081
+
 ingress:
   enabled: false
   className: "nginx"
@@ -84,6 +93,14 @@ masterkeySecretName: ""
 # if set, use this secret key for the master key; otherwise, use the default key
 masterkeySecretKey: ""
 
+proxyConfigMap:
+  # when true, creates a new configmap
+  create: true
+  # if create is false and name is set, use existing ConfigMap
+  # create: false
+  # name: ""
+  # key: "config.yaml"
+
 # The elements within proxy_config are rendered as config.yaml for the proxy
 #  Examples: https://github.com/BerriAI/litellm/tree/main/litellm/proxy/example_config_yaml
 #  Reference: https://docs.litellm.ai/docs/proxy/configs
@@ -152,6 +169,8 @@ db:
     name: postgres
     usernameKey: username
     passwordKey: password
+    # Optional: when set, DATABASE_HOST will be sourced from this secret key instead of db.endpoint
+    endpointKey: ""
 
   # Use the Stackgres Helm chart to deploy an instance of a Stackgres cluster.
   #  The Stackgres Operator must already be installed within the target
@@ -197,6 +216,18 @@ migrationJob:
   disableSchemaUpdate: false # Skip schema migrations for specific environments. When True, the job will exit with code 0.
   annotations: {}
   ttlSecondsAfterFinished: 120
+  resources: {}
+  #  requests:
+  #    cpu: 100m
+  #    memory: 100Mi
+  extraContainers: []
+  
+  # Hook configuration
+  hooks:
+    argocd:
+      enabled: true
+    helm:
+      enabled: false
 
 # Additional environment variables to be added to the deployment as a map of key-value pairs
 envVars: {
@@ -209,4 +240,11 @@ extraEnvVars: {
     #   value: EXTRA_ENV_VAR_VALUE
 }
 
-
+# Pod Disruption Budget
+pdb:
+  enabled: false
+  # Set exactly one of the following. If both are set, minAvailable takes precedence.
+  minAvailable: null     # e.g. "50%" or 1
+  maxUnavailable: null   # e.g. 1 or "20%"
+  annotations: {}
+  labels: {}

docker-compose.yml

@@ -1,68 +1,66 @@
-version: "3.11"
-services:
-  litellm:
-    build:
-      context: .
-      args:
-        target: runtime
-    image: ghcr.io/berriai/litellm:main-stable
-    #########################################
-    ## Uncomment these lines to start proxy with a config.yaml file ##
-    # volumes:
-    #  - ./config.yaml:/app/config.yaml <<- this is missing in the docker-compose file currently
-    # command:
-    #  - "--config=/app/config.yaml"
-    ##############################################
-    ports:
-      - "4000:4000" # Map the container port to the host, change the host port if necessary
-    environment:
-      DATABASE_URL: "postgresql://llmproxy:dbpassword9090@db:5432/litellm"
-      STORE_MODEL_IN_DB: "True" # allows adding models to proxy via UI
-    env_file:
-      - .env # Load local .env file
-    depends_on:
-      - db  # Indicates that this service depends on the 'db' service, ensuring 'db' starts first
-    healthcheck:  # Defines the health check configuration for the container
-      test: [ "CMD-SHELL", "wget --no-verbose --tries=1 http://localhost:4000/health/liveliness || exit 1" ]  # Command to execute for health check
-      interval: 30s  # Perform health check every 30 seconds
-      timeout: 10s   # Health check command times out after 10 seconds
-      retries: 3     # Retry up to 3 times if health check fails
-      start_period: 40s  # Wait 40 seconds after container start before beginning health checks
-
-  db:
-    image: postgres:16
-    restart: always
-    container_name: litellm_db
-    environment:
-      POSTGRES_DB: litellm
-      POSTGRES_USER: llmproxy
-      POSTGRES_PASSWORD: dbpassword9090
-    ports:
-      - "5432:5432"
-    volumes:
-      - postgres_data:/var/lib/postgresql/data # Persists Postgres data across container restarts
-    healthcheck:
-      test: ["CMD-SHELL", "pg_isready -d litellm -U llmproxy"]
-      interval: 1s
-      timeout: 5s
-      retries: 10
-
-  prometheus:
-    image: prom/prometheus
-    volumes:
-      - prometheus_data:/prometheus
-      - ./prometheus.yml:/etc/prometheus/prometheus.yml
-    ports:
-      - "9090:9090"
-    command:
-      - "--config.file=/etc/prometheus/prometheus.yml"
-      - "--storage.tsdb.path=/prometheus"
-      - "--storage.tsdb.retention.time=15d"
-    restart: always
-
-volumes:
-  prometheus_data:
-    driver: local
-  postgres_data:
-    name: litellm_postgres_data # Named volume for Postgres data persistence
-
+services:
+  litellm:
+    build:
+      context: .
+      args:
+        target: runtime
+    image: ghcr.io/berriai/litellm:main-stable
+    #########################################
+    ## Uncomment these lines to start proxy with a config.yaml file ##
+    # volumes:
+    #  - ./config.yaml:/app/config.yaml <<- this is missing in the docker-compose file currently
+    # command:
+    #  - "--config=/app/config.yaml"
+    ##############################################
+    ports:
+      - "4000:4000" # Map the container port to the host, change the host port if necessary
+    environment:
+      DATABASE_URL: "postgresql://llmproxy:dbpassword9090@db:5432/litellm"
+      STORE_MODEL_IN_DB: "True" # allows adding models to proxy via UI
+    env_file:
+      - .env # Load local .env file
+    depends_on:
+      - db  # Indicates that this service depends on the 'db' service, ensuring 'db' starts first
+    healthcheck:  # Defines the health check configuration for the container
+      test: [ "CMD-SHELL", "wget --no-verbose --tries=1 http://localhost:4000/health/liveliness || exit 1" ]  # Command to execute for health check
+      interval: 30s  # Perform health check every 30 seconds
+      timeout: 10s   # Health check command times out after 10 seconds
+      retries: 3     # Retry up to 3 times if health check fails
+      start_period: 40s  # Wait 40 seconds after container start before beginning health checks
+
+  db:
+    image: postgres:16
+    restart: always
+    container_name: litellm_db
+    environment:
+      POSTGRES_DB: litellm
+      POSTGRES_USER: llmproxy
+      POSTGRES_PASSWORD: dbpassword9090
+    ports:
+      - "5432:5432"
+    volumes:
+      - postgres_data:/var/lib/postgresql/data # Persists Postgres data across container restarts
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -d litellm -U llmproxy"]
+      interval: 1s
+      timeout: 5s
+      retries: 10
+
+  prometheus:
+    image: prom/prometheus
+    volumes:
+      - prometheus_data:/prometheus
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml
+    ports:
+      - "9090:9090"
+    command:
+      - "--config.file=/etc/prometheus/prometheus.yml"
+      - "--storage.tsdb.path=/prometheus"
+      - "--storage.tsdb.retention.time=15d"
+    restart: always
+
+volumes:
+  prometheus_data:
+    driver: local
+  postgres_data:
+    name: litellm_postgres_data # Named volume for Postgres data persistence

docker/Dockerfile.database

@@ -57,6 +57,9 @@ COPY --from=builder /wheels/ /wheels/
 # Install the built wheel using pip; again using a wildcard if it's the only file
 RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
 
+# Install semantic_router and aurelio-sdk using script
+RUN chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
+
 # ensure pyjwt is used, not jwt
 RUN pip uninstall jwt -y
 RUN pip uninstall PyJWT -y
@@ -71,8 +74,12 @@ RUN chmod +x docker/entrypoint.sh
 RUN chmod +x docker/prod_entrypoint.sh
 EXPOSE 4000/tcp
 
+RUN apk add --no-cache supervisor
+COPY docker/supervisord.conf /etc/supervisord.conf
+
 # # Set your entrypoint and command
 
+
 ENTRYPOINT ["docker/prod_entrypoint.sh"]
 
 # Append "--detailed_debug" to the end of CMD to view detailed debug logs 

docker/Dockerfile.dev

@@ -0,0 +1,87 @@
+# Base image for building
+ARG LITELLM_BUILD_IMAGE=python:3.11-slim
+
+# Runtime image
+ARG LITELLM_RUNTIME_IMAGE=python:3.11-slim
+
+# Builder stage
+FROM $LITELLM_BUILD_IMAGE AS builder
+
+# Set the working directory to /app
+WORKDIR /app
+
+USER root
+
+# Install build dependencies in one layer
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    gcc \
+    python3-dev \
+    libssl-dev \
+    pkg-config \
+    && rm -rf /var/lib/apt/lists/* \
+    && pip install --upgrade pip build
+
+# Copy requirements first for better layer caching
+COPY requirements.txt .
+
+# Install Python dependencies with cache mount for faster rebuilds
+RUN --mount=type=cache,target=/root/.cache/pip \
+    pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt
+
+# Fix JWT dependency conflicts early
+RUN pip uninstall jwt -y || true && \
+    pip uninstall PyJWT -y || true && \
+    pip install PyJWT==2.9.0 --no-cache-dir
+
+# Copy only necessary files for build
+COPY pyproject.toml README.md schema.prisma poetry.lock ./
+COPY litellm/ ./litellm/
+COPY enterprise/ ./enterprise/
+COPY docker/ ./docker/
+
+# Build Admin UI once
+RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
+
+# Build the package
+RUN rm -rf dist/* && python -m build
+
+# Install the built package
+RUN pip install dist/*.whl
+
+# Runtime stage
+FROM $LITELLM_RUNTIME_IMAGE AS runtime
+
+# Ensure runtime stage runs as root
+USER root
+
+# Install only runtime dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libssl3 \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+# Copy only necessary runtime files
+COPY docker/entrypoint.sh docker/prod_entrypoint.sh ./docker/
+COPY litellm/ ./litellm/
+COPY pyproject.toml README.md schema.prisma poetry.lock ./
+
+# Copy pre-built wheels and install everything at once
+COPY --from=builder /wheels/ /wheels/
+COPY --from=builder /app/dist/*.whl .
+
+# Install all dependencies in one step with no-cache for smaller image
+RUN pip install --no-cache-dir *.whl /wheels/* --no-index --find-links=/wheels/ && \
+    rm -f *.whl && \
+    rm -rf /wheels
+
+# Generate prisma client and set permissions
+RUN prisma generate && \
+    chmod +x docker/entrypoint.sh docker/prod_entrypoint.sh
+
+EXPOSE 4000/tcp
+
+ENTRYPOINT ["docker/prod_entrypoint.sh"]
+
+# Append "--detailed_debug" to the end of CMD to view detailed debug logs 
+CMD ["--port", "4000"]

docker/Dockerfile.non_root

@@ -1,94 +1,106 @@
-# Base image for building
-ARG LITELLM_BUILD_IMAGE=python:3.13.1-slim
+# Base images
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/python:latest-dev
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/python:latest-dev
 
-# Runtime image
-ARG LITELLM_RUNTIME_IMAGE=python:3.13.1-slim
-# Builder stage
+# -----------------
+# Builder Stage
+# -----------------
 FROM $LITELLM_BUILD_IMAGE AS builder
-
-# Set the working directory to /app
 WORKDIR /app
 
-# Set the shell to bash
-SHELL ["/bin/bash", "-o", "pipefail", "-c"]
-
 # Install build dependencies
-RUN apt-get clean && apt-get update && \
-    apt-get install -y gcc g++ python3-dev && \
-    rm -rf /var/lib/apt/lists/*
+USER root
+RUN apk add --no-cache build-base bash \
+  && pip install --no-cache-dir --upgrade pip build
 
-RUN pip install --no-cache-dir --upgrade pip && \
-    pip install --no-cache-dir build
-
-# Copy the current directory contents into the container at /app
+# Copy project files
 COPY . .
 
 # Build Admin UI
 RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
 
-# Build the package
-RUN rm -rf dist/* && python -m build
+# Build package and wheel dependencies
+RUN rm -rf dist/* && python -m build && \
+  pip install dist/*.whl && \
+  pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt
 
-# There should be only one wheel file now, assume the build only creates one
-RUN ls -1 dist/*.whl | head -1
-
-# Install the package
-RUN pip install dist/*.whl
-
-# install dependencies as wheels
-RUN pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt
-
-# Runtime stage
+# -----------------
+# Runtime Stage
+# -----------------
 FROM $LITELLM_RUNTIME_IMAGE AS runtime
-
-# Update dependencies and clean up - handles debian security issue
-RUN apt-get update && apt-get upgrade -y && rm -rf /var/lib/apt/lists/*
-
 WORKDIR /app
-# Copy the current directory contents into the container at /app
-COPY . .
-RUN ls -la /app
 
-# Copy the built wheel from the builder stage to the runtime stage; assumes only one wheel file is present
+# Install runtime dependencies
+USER root
+RUN apk upgrade --no-cache && \
+  apk add --no-cache bash libstdc++ ca-certificates openssl supervisor
+
+# Copy only necessary artifacts from builder stage for runtime
+COPY . .
+COPY --from=builder /app/docker/entrypoint.sh /app/docker/prod_entrypoint.sh /app/docker/
+COPY --from=builder /app/docker/supervisord.conf /etc/supervisord.conf
+COPY --from=builder /app/schema.prisma /app/schema.prisma
 COPY --from=builder /app/dist/*.whl .
 COPY --from=builder /wheels/ /wheels/
 
-# Install the built wheel using pip; again using a wildcard if it's the only file
-RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
+# Install package from wheel and dependencies
+RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ \
+  && rm -f *.whl \
+  && rm -rf /wheels
 
-# ensure pyjwt is used, not jwt
+# Install semantic_router and aurelio-sdk using script
+RUN chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
+
+# Ensure correct JWT library is used (pyjwt not jwt)
 RUN pip uninstall jwt -y && \
-    pip uninstall PyJWT -y && \
-    pip install PyJWT==2.9.0 --no-cache-dir
+  pip uninstall PyJWT -y && \
+  pip install PyJWT==2.9.0 --no-cache-dir
 
-# Build Admin UI
-RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
-
-### Prisma Handling for Non-Root #################################################
-# Prisma allows you to specify the binary cache directory to use
+# --- Prisma Handling for Non-Root User ---
+# Set Prisma cache directories
 ENV PRISMA_BINARY_CACHE_DIR=/nonexistent
+ENV NPM_CONFIG_CACHE=/.npm
 
-RUN pip install --no-cache-dir nodejs-bin prisma
+# Install prisma and make entrypoints executable
+RUN pip install --no-cache-dir prisma && \
+  chmod +x docker/entrypoint.sh && \
+  chmod +x docker/prod_entrypoint.sh
 
-# Make a /non-existent folder and assign chown to nobody
-RUN mkdir -p /nonexistent && \
-    chown -R nobody:nogroup /app && \
-    chown -R nobody:nogroup /nonexistent && \
-    chown -R nobody:nogroup /usr/local/lib/python3.13/site-packages/prisma/
+# Create directories and set permissions for non-root user
+RUN mkdir -p /nonexistent /.npm && \
+  chown -R nobody:nogroup /app && \
+  chown -R nobody:nogroup /nonexistent /.npm && \
+  PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \
+  chown -R nobody:nogroup $PRISMA_PATH && \
+  LITELLM_PKG_MIGRATIONS_PATH="$(python -c 'import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))' 2>/dev/null || echo '')/migrations" && \
+  [ -n "$LITELLM_PKG_MIGRATIONS_PATH" ] && chown -R nobody:nogroup $LITELLM_PKG_MIGRATIONS_PATH
 
-RUN chmod +x docker/entrypoint.sh
-RUN chmod +x docker/prod_entrypoint.sh
+# --- OpenShift Compatibility: Apply Red Hat recommended pattern ---
+# Get paths for directories that need write access at runtime
+RUN PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \
+  LITELLM_PROXY_EXTRAS_PATH=$(python -c "import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))" 2>/dev/null || echo "") && \
+  # Set group ownership to 0 (root group) for OpenShift compatibility && \
+  chgrp -R 0 $PRISMA_PATH && \
+  [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chgrp -R 0 $LITELLM_PROXY_EXTRAS_PATH || true && \
+  # Mirror owner permissions to group (g=u) as recommended by Red Hat && \
+  chmod -R g=u $PRISMA_PATH && \
+  [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g=u $LITELLM_PROXY_EXTRAS_PATH || true && \
+  # Ensure directories are writable by group && \
+  chmod -R g+w $PRISMA_PATH && \
+  [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g+w $LITELLM_PROXY_EXTRAS_PATH || true
 
-# Run Prisma generate as user = nobody
+# Switch to non-root user
 USER nobody
 
+# Set HOME for prisma generate to have a writable directory
+ENV HOME=/app
 RUN prisma generate
-### End of Prisma Handling for Non-Root #########################################
+# --- End of Prisma Handling ---
 
 EXPOSE 4000/tcp
 
-# # Set your entrypoint and command
-ENTRYPOINT ["docker/prod_entrypoint.sh"]
+# Set entrypoint and command
+ENTRYPOINT ["/app/docker/prod_entrypoint.sh"]
 
 # Append "--detailed_debug" to the end of CMD to view detailed debug logs
 # CMD ["--port", "4000", "--detailed_debug"]

docker/README.md

@@ -1,3 +1,65 @@
-# LiteLLM Docker 
+# Docker Development Guide
 
-This is a minimal Docker Compose setup for self-hosting LiteLLM.
+This guide provides instructions for building and running the LiteLLM application using Docker and Docker Compose.
+
+## Prerequisites
+
+- Docker
+- Docker Compose
+
+## Building and Running the Application
+
+To build and run the application, you will use the `docker-compose.yml` file located in the root of the project. This file is configured to use the `Dockerfile.non_root` for a secure, non-root container environment.
+
+### 1. Set the Master Key
+
+The application requires a `MASTER_KEY` for signing and validating tokens. You must set this key as an environment variable before running the application.
+
+Create a `.env` file in the root of the project and add the following line:
+
+```
+MASTER_KEY=your-secret-key
+```
+
+Replace `your-secret-key` with a strong, randomly generated secret.
+
+### 2. Build and Run the Containers
+
+Once you have set the `MASTER_KEY`, you can build and run the containers using the following command:
+
+```bash
+docker compose up -d --build
+```
+
+This command will:
+
+-   Build the Docker image using `Dockerfile.non_root`.
+-   Start the `litellm`, `litellm_db`, and `prometheus` services in detached mode (`-d`).
+-   The `--build` flag ensures that the image is rebuilt if there are any changes to the Dockerfile or the application code.
+
+### 3. Verifying the Application is Running
+
+You can check the status of the running containers with the following command:
+
+```bash
+docker compose ps
+```
+
+To view the logs of the `litellm` container, run:
+
+```bash
+docker compose logs -f litellm
+```
+
+### 4. Stopping the Application
+
+To stop the running containers, use the following command:
+
+```bash
+docker compose down
+```
+
+## Troubleshooting
+
+-   **`build_admin_ui.sh: not found`**: This error can occur if the Docker build context is not set correctly. Ensure that you are running the `docker-compose` command from the root of the project.
+-   **`Master key is not initialized`**: This error means the `MASTER_key` environment variable is not set. Make sure you have created a `.env` file in the project root with the `MASTER_KEY` defined.

docker/build_from_pip/Dockerfile.build_from_pip

@@ -13,10 +13,16 @@ RUN apk update && \
 RUN python -m venv ${HOME}/venv
 RUN ${HOME}/venv/bin/pip install --no-cache-dir --upgrade pip
 
-COPY requirements.txt .
+COPY docker/build_from_pip/requirements.txt .
 RUN --mount=type=cache,target=${HOME}/.cache/pip \
     ${HOME}/venv/bin/pip install -r requirements.txt
 
+# Copy Prisma schema file
+COPY schema.prisma .
+
+# Generate prisma client
+RUN prisma generate
+
 EXPOSE 4000/tcp
 
 ENTRYPOINT ["litellm"]

docker/build_from_pip/requirements.txt

@@ -2,4 +2,5 @@ litellm[proxy]==1.67.4.dev1 # Specify the litellm version you want to use
 prometheus_client
 langfuse
 prisma
+openai==1.99.9
 ddtrace==2.19.0 # for advanced DD tracing / profiling

docker/install_auto_router.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+pip install semantic_router==0.1.11 --no-deps
+pip install aurelio-sdk==0.0.19

docker/prod_entrypoint.sh

@@ -1,5 +1,10 @@
 #!/bin/sh
 
+if [ "$SEPARATE_HEALTH_APP" = "1" ]; then
+    export LITELLM_ARGS="$@"
+    exec supervisord -c /etc/supervisord.conf
+fi
+
 if [ "$USE_DDTRACE" = "true" ]; then
     export DD_TRACE_OPENAI_ENABLED="False"
     exec ddtrace-run litellm "$@"

docker/supervisord.conf

@@ -0,0 +1,42 @@
+[supervisord]
+nodaemon=true
+loglevel=info
+
+[group:litellm]
+programs=main,health
+
+[program:main]
+command=sh -c 'if [ "$USE_DDTRACE" = "true" ]; then export DD_TRACE_OPENAI_ENABLED="False"; exec ddtrace-run python -m litellm.proxy.proxy_cli --host 0.0.0.0 --port=4000 $LITELLM_ARGS; else exec python -m litellm.proxy.proxy_cli --host 0.0.0.0 --port=4000 $LITELLM_ARGS; fi'
+autostart=true
+autorestart=true
+startretries=3
+priority=1
+exitcodes=0
+stopasgroup=true
+killasgroup=true
+stdout_logfile=/dev/stdout
+stderr_logfile=/dev/stderr
+stdout_logfile_maxbytes = 0
+stderr_logfile_maxbytes = 0
+environment=PYTHONUNBUFFERED=true
+
+[program:health]
+command=sh -c '[ "$SEPARATE_HEALTH_APP" = "1" ] && exec uvicorn litellm.proxy.health_endpoints.health_app_factory:build_health_app --factory --host 0.0.0.0 --port=${SEPARATE_HEALTH_PORT:-4001} || exit 0'
+autostart=true
+autorestart=true
+startretries=3
+priority=2
+exitcodes=0
+stopasgroup=true
+killasgroup=true
+stdout_logfile=/dev/stdout
+stderr_logfile=/dev/stderr
+stdout_logfile_maxbytes = 0
+stderr_logfile_maxbytes = 0
+environment=PYTHONUNBUFFERED=true
+
+[eventlistener:process_monitor]
+command=python -c "from supervisor import childutils; import os, signal; [os.kill(os.getppid(), signal.SIGTERM) for h,p in iter(lambda: childutils.listener.wait(), None) if h['eventname'] in ['PROCESS_STATE_FATAL', 'PROCESS_STATE_EXITED'] and dict([x.split(':') for x in p.split(' ')])['processname'] in ['main', 'health'] or childutils.listener.ok()]"
+events=PROCESS_STATE_EXITED,PROCESS_STATE_FATAL
+autostart=true
+autorestart=true

docs/my-website/.gitignore

@@ -10,6 +10,7 @@
 
 # Misc
 .DS_Store
+.env
 .env.local
 .env.development.local
 .env.test.local

docs/my-website/docs/adding_provider/new_rerank_provider.md

@@ -17,7 +17,7 @@ class YourProviderRerankConfig(BaseRerankConfig):
             # ... other supported params
         ]
 
-    def transform_rerank_request(self, model: str, optional_rerank_params: OptionalRerankParams, headers: dict) -> dict:
+    def transform_rerank_request(self, model: str, optional_rerank_params: Dict, headers: dict) -> dict:
         # Transform request to RerankRequest spec
         return rerank_request.model_dump(exclude_none=True)
 

docs/my-website/docs/anthropic_unified.md

@@ -1,7 +1,7 @@
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
 
-# /v1/messages [BETA] 
+# /v1/messages
 
 Use LiteLLM to call all your LLM APIs in the Anthropic `v1/messages` format. 
 
@@ -14,20 +14,20 @@ Use LiteLLM to call all your LLM APIs in the Anthropic `v1/messages` format.
 | Logging | ✅ | works across all integrations |
 | End-user Tracking | ✅ | |
 | Streaming | ✅ | |
-| Fallbacks | ✅ | between anthropic models |
-| Loadbalancing | ✅ | between anthropic models |
-| Support llm providers | - `anthropic` <br/> - `bedrock` (only Anthropic models) | |
-
-Planned improvement:
-- Vertex AI Anthropic support
+| Fallbacks | ✅ | between supported models |
+| Loadbalancing | ✅ | between supported models |
+| Support llm providers | **All LiteLLM supported providers** | `openai`, `anthropic`, `bedrock`, `vertex_ai`, `gemini`, `azure`, `azure_ai`, etc. |
 
 ## Usage 
 ---
 
 ### LiteLLM Python SDK 
 
+<Tabs>
+<TabItem value="anthropic" label="Anthropic">
+
 #### Non-streaming example
-```python showLineNumbers title="Example using LiteLLM Python SDK"
+```python showLineNumbers title="Anthropic Example using LiteLLM Python SDK"
 import litellm
 response = await litellm.anthropic.messages.acreate(
     messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
@@ -37,6 +37,179 @@ response = await litellm.anthropic.messages.acreate(
 )
 ```
 
+#### Streaming example
+```python showLineNumbers title="Anthropic Streaming Example using LiteLLM Python SDK"
+import litellm
+response = await litellm.anthropic.messages.acreate(
+    messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
+    api_key=api_key,
+    model="anthropic/claude-3-haiku-20240307",
+    max_tokens=100,
+    stream=True,
+)
+async for chunk in response:
+    print(chunk)
+```
+
+</TabItem>
+
+<TabItem value="openai" label="OpenAI">
+
+#### Non-streaming example
+```python showLineNumbers title="OpenAI Example using LiteLLM Python SDK"
+import litellm
+import os
+
+# Set API key
+os.environ["OPENAI_API_KEY"] = "your-openai-api-key"
+
+response = await litellm.anthropic.messages.acreate(
+    messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
+    model="openai/gpt-4",
+    max_tokens=100,
+)
+```
+
+#### Streaming example
+```python showLineNumbers title="OpenAI Streaming Example using LiteLLM Python SDK"
+import litellm
+import os
+
+# Set API key
+os.environ["OPENAI_API_KEY"] = "your-openai-api-key"
+
+response = await litellm.anthropic.messages.acreate(
+    messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
+    model="openai/gpt-4",
+    max_tokens=100,
+    stream=True,
+)
+async for chunk in response:
+    print(chunk)
+```
+
+</TabItem>
+
+<TabItem value="gemini" label="Google AI Studio">
+
+#### Non-streaming example
+```python showLineNumbers title="Google Gemini Example using LiteLLM Python SDK"
+import litellm
+import os
+
+# Set API key
+os.environ["GEMINI_API_KEY"] = "your-gemini-api-key"
+
+response = await litellm.anthropic.messages.acreate(
+    messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
+    model="gemini/gemini-2.0-flash-exp",
+    max_tokens=100,
+)
+```
+
+#### Streaming example
+```python showLineNumbers title="Google Gemini Streaming Example using LiteLLM Python SDK"
+import litellm
+import os
+
+# Set API key
+os.environ["GEMINI_API_KEY"] = "your-gemini-api-key"
+
+response = await litellm.anthropic.messages.acreate(
+    messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
+    model="gemini/gemini-2.0-flash-exp",
+    max_tokens=100,
+    stream=True,
+)
+async for chunk in response:
+    print(chunk)
+```
+
+</TabItem>
+
+<TabItem value="vertex" label="Vertex AI">
+
+#### Non-streaming example
+```python showLineNumbers title="Vertex AI Example using LiteLLM Python SDK"
+import litellm
+import os
+
+# Set credentials - Vertex AI uses application default credentials
+# Run 'gcloud auth application-default login' to authenticate
+os.environ["VERTEXAI_PROJECT"] = "your-gcp-project-id"
+os.environ["VERTEXAI_LOCATION"] = "us-central1"
+
+response = await litellm.anthropic.messages.acreate(
+    messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
+    model="vertex_ai/gemini-2.0-flash-exp",
+    max_tokens=100,
+)
+```
+
+#### Streaming example
+```python showLineNumbers title="Vertex AI Streaming Example using LiteLLM Python SDK"
+import litellm
+import os
+
+# Set credentials - Vertex AI uses application default credentials
+# Run 'gcloud auth application-default login' to authenticate
+os.environ["VERTEXAI_PROJECT"] = "your-gcp-project-id"
+os.environ["VERTEXAI_LOCATION"] = "us-central1"
+
+response = await litellm.anthropic.messages.acreate(
+    messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
+    model="vertex_ai/gemini-2.0-flash-exp",
+    max_tokens=100,
+    stream=True,
+)
+async for chunk in response:
+    print(chunk)
+```
+
+</TabItem>
+
+<TabItem value="bedrock" label="AWS Bedrock">
+
+#### Non-streaming example
+```python showLineNumbers title="AWS Bedrock Example using LiteLLM Python SDK"
+import litellm
+import os
+
+# Set AWS credentials
+os.environ["AWS_ACCESS_KEY_ID"] = "your-access-key-id"
+os.environ["AWS_SECRET_ACCESS_KEY"] = "your-secret-access-key"
+os.environ["AWS_REGION_NAME"] = "us-west-2"  # or your AWS region
+
+response = await litellm.anthropic.messages.acreate(
+    messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
+    model="bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
+    max_tokens=100,
+)
+```
+
+#### Streaming example
+```python showLineNumbers title="AWS Bedrock Streaming Example using LiteLLM Python SDK"
+import litellm
+import os
+
+# Set AWS credentials
+os.environ["AWS_ACCESS_KEY_ID"] = "your-access-key-id"
+os.environ["AWS_SECRET_ACCESS_KEY"] = "your-secret-access-key"
+os.environ["AWS_REGION_NAME"] = "us-west-2"  # or your AWS region
+
+response = await litellm.anthropic.messages.acreate(
+    messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
+    model="bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
+    max_tokens=100,
+    stream=True,
+)
+async for chunk in response:
+    print(chunk)
+```
+
+</TabItem>
+</Tabs>
+
 Example response:
 ```json
 {
@@ -61,22 +234,10 @@ Example response:
 }
 ```
 
-#### Streaming example
-```python showLineNumbers title="Example using LiteLLM Python SDK"
-import litellm
-response = await litellm.anthropic.messages.acreate(
-    messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
-    api_key=api_key,
-    model="anthropic/claude-3-haiku-20240307",
-    max_tokens=100,
-    stream=True,
-)
-async for chunk in response:
-    print(chunk)
-```
-
 ### LiteLLM Proxy Server 
 
+<Tabs>
+<TabItem value="anthropic-proxy" label="Anthropic">
 
 1. Setup config.yaml
 
@@ -85,6 +246,7 @@ model_list:
     - model_name: anthropic-claude
       litellm_params:
         model: claude-3-7-sonnet-latest
+        api_key: os.environ/ANTHROPIC_API_KEY
 ```
 
 2. Start proxy 
@@ -95,10 +257,7 @@ litellm --config /path/to/config.yaml
 
 3. Test it! 
 
-<Tabs>
-<TabItem label="Anthropic Python SDK" value="python">
-
-```python showLineNumbers title="Example using LiteLLM Proxy Server"
+```python showLineNumbers title="Anthropic Example using LiteLLM Proxy Server"
 import anthropic
 
 # point anthropic sdk to litellm proxy 
@@ -113,8 +272,165 @@ response = client.messages.create(
     max_tokens=100,
 )
 ```
+
 </TabItem>
-<TabItem label="curl" value="curl">
+
+<TabItem value="openai-proxy" label="OpenAI">
+
+1. Setup config.yaml
+
+```yaml
+model_list:
+    - model_name: openai-gpt4
+      litellm_params:
+        model: openai/gpt-4
+        api_key: os.environ/OPENAI_API_KEY
+```
+
+2. Start proxy 
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+3. Test it! 
+
+```python showLineNumbers title="OpenAI Example using LiteLLM Proxy Server"
+import anthropic
+
+# point anthropic sdk to litellm proxy 
+client = anthropic.Anthropic(
+    base_url="http://0.0.0.0:4000",
+    api_key="sk-1234",
+)
+
+response = client.messages.create(
+    messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
+    model="openai-gpt4",
+    max_tokens=100,
+)
+```
+
+</TabItem>
+
+<TabItem value="gemini-proxy" label="Google AI Studio">
+
+1. Setup config.yaml
+
+```yaml
+model_list:
+    - model_name: gemini-2-flash
+      litellm_params:
+        model: gemini/gemini-2.0-flash-exp
+        api_key: os.environ/GEMINI_API_KEY
+```
+
+2. Start proxy 
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+3. Test it! 
+
+```python showLineNumbers title="Google Gemini Example using LiteLLM Proxy Server"
+import anthropic
+
+# point anthropic sdk to litellm proxy 
+client = anthropic.Anthropic(
+    base_url="http://0.0.0.0:4000",
+    api_key="sk-1234",
+)
+
+response = client.messages.create(
+    messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
+    model="gemini-2-flash",
+    max_tokens=100,
+)
+```
+
+</TabItem>
+
+<TabItem value="vertex-proxy" label="Vertex AI">
+
+1. Setup config.yaml
+
+```yaml
+model_list:
+    - model_name: vertex-gemini
+      litellm_params:
+        model: vertex_ai/gemini-2.0-flash-exp
+        vertex_project: your-gcp-project-id
+        vertex_location: us-central1
+```
+
+2. Start proxy 
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+3. Test it! 
+
+```python showLineNumbers title="Vertex AI Example using LiteLLM Proxy Server"
+import anthropic
+
+# point anthropic sdk to litellm proxy 
+client = anthropic.Anthropic(
+    base_url="http://0.0.0.0:4000",
+    api_key="sk-1234",
+)
+
+response = client.messages.create(
+    messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
+    model="vertex-gemini",
+    max_tokens=100,
+)
+```
+
+</TabItem>
+
+<TabItem value="bedrock-proxy" label="AWS Bedrock">
+
+1. Setup config.yaml
+
+```yaml
+model_list:
+    - model_name: bedrock-claude
+      litellm_params:
+        model: bedrock/anthropic.claude-3-sonnet-20240229-v1:0
+        aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
+        aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
+        aws_region_name: us-west-2
+```
+
+2. Start proxy 
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+3. Test it! 
+
+```python showLineNumbers title="AWS Bedrock Example using LiteLLM Proxy Server"
+import anthropic
+
+# point anthropic sdk to litellm proxy 
+client = anthropic.Anthropic(
+    base_url="http://0.0.0.0:4000",
+    api_key="sk-1234",
+)
+
+response = client.messages.create(
+    messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}],
+    model="bedrock-claude",
+    max_tokens=100,
+)
+```
+
+</TabItem>
+
+<TabItem value="curl-proxy" label="curl">
 
 ```bash showLineNumbers title="Example using LiteLLM Proxy Server"
 curl -L -X POST 'http://0.0.0.0:4000/v1/messages' \
@@ -136,7 +452,6 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/messages' \
 </TabItem>
 </Tabs>
 
-
 ## Request Format
 ---
 
@@ -189,7 +504,7 @@ Request body will be in the Anthropic messages API format. **litellm follows the
 - **system** (string or array):  
   A system prompt providing context or specific instructions to the model.
 - **temperature** (number):  
-  Controls randomness in the model’s responses. Valid range: `0 < temperature < 1`.
+  Controls randomness in the model's responses. Valid range: `0 < temperature < 1`.
 - **thinking** (object):  
   Configuration for enabling extended thinking. If enabled, it includes:
   - **budget_tokens** (integer):  
@@ -201,7 +516,7 @@ Request body will be in the Anthropic messages API format. **litellm follows the
 - **tools** (array of objects):  
   Definitions for tools available to the model. Each tool includes:
   - **name** (string):  
-    The tool’s name.
+    The tool's name.
   - **description** (string):  
     A detailed description of the tool.
   - **input_schema** (object):  

docs/my-website/docs/assistants.md

@@ -279,7 +279,7 @@ with run as run:
 curl -X POST 'http://0.0.0.0:4000/threads/{thread_id}/runs' \
 -H 'Authorization: Bearer sk-1234' \
 -H 'Content-Type: application/json' \
--D '{
+-d '{
       "assistant_id": "asst_6xVZQFFy1Kw87NbnYeNebxTf",
       "stream": true
 }'

docs/my-website/docs/audio_transcription.md

@@ -3,13 +3,22 @@ import TabItem from '@theme/TabItem';
 
 # /audio/transcriptions
 
-Use this to loadbalance across Azure + OpenAI. 
+## Overview 
+
+| Feature | Supported | Notes | 
+|-------|-------|-------|
+| Cost Tracking | ✅ |  |
+| Logging | ✅ | works across all integrations |
+| End-user Tracking | ✅ | |
+| Fallbacks | ✅ | between supported models |
+| Loadbalancing | ✅ | between supported models |
+| Support llm providers | `openai`, `azure`, `vertex_ai`, `gemini`, `deepgram`, `groq`, `fireworks_ai` | |
 
 ## Quick Start
 
 ### LiteLLM Python SDK
 
-```python showLineNumbers
+```python showLineNumbers title="Python SDK Example"
 from litellm import transcription
 import os 
 
@@ -30,7 +39,7 @@ print(f"response: {response}")
 <Tabs>
 <TabItem value="openai" label="OpenAI">
 
-```yaml showLineNumbers
+```yaml showLineNumbers title="OpenAI Configuration"
 model_list:
 - model_name: whisper
   litellm_params:
@@ -45,7 +54,7 @@ general_settings:
 </TabItem>
 <TabItem value="openai+azure" label="OpenAI + Azure">
 
-```yaml showLineNumbers
+```yaml showLineNumbers title="OpenAI + Azure Configuration"
 model_list:
 - model_name: whisper
   litellm_params:
@@ -71,7 +80,7 @@ general_settings:
 
 ### Start proxy 
 
-```bash
+```bash showLineNumbers title="Start Proxy Server"
 litellm --config /path/to/config.yaml 
 
 # RUNNING on http://0.0.0.0:8000
@@ -82,7 +91,7 @@ litellm --config /path/to/config.yaml
 <Tabs>
 <TabItem value="curl" label="Curl">
 
-```bash
+```bash showLineNumbers title="Test with cURL"
 curl --location 'http://0.0.0.0:8000/v1/audio/transcriptions' \
 --header 'Authorization: Bearer sk-1234' \
 --form 'file=@"/Users/krrishdholakia/Downloads/gettysburg.wav"' \
@@ -92,7 +101,7 @@ curl --location 'http://0.0.0.0:8000/v1/audio/transcriptions' \
 </TabItem>
 <TabItem value="openai" label="OpenAI Python SDK">
 
-```python showLineNumbers
+```python showLineNumbers title="Test with OpenAI Python SDK"
 from openai import OpenAI
 client = openai.OpenAI(
     api_key="sk-1234",
@@ -115,4 +124,82 @@ transcript = client.audio.transcriptions.create(
 - Azure
 - [Fireworks AI](./providers/fireworks_ai.md#audio-transcription)
 - [Groq](./providers/groq.md#speech-to-text---whisper)
-- [Deepgram](./providers/deepgram.md)
+- [Deepgram](./providers/deepgram.md)
+
+---
+
+## Fallbacks
+
+You can configure fallbacks for audio transcription to automatically retry with different models if the primary model fails.
+
+<Tabs>
+<TabItem value="curl" label="Curl">
+
+```bash showLineNumbers title="Test with cURL and Fallbacks"
+curl --location 'http://0.0.0.0:4000/v1/audio/transcriptions' \
+--header 'Authorization: Bearer sk-1234' \
+--form 'file=@"gettysburg.wav"' \
+--form 'model="groq/whisper-large-v3"' \
+--form 'fallbacks[]="openai/whisper-1"'
+```
+
+</TabItem>
+<TabItem value="openai" label="OpenAI Python SDK">
+
+```python showLineNumbers title="Test with OpenAI Python SDK and Fallbacks"
+from openai import OpenAI
+client = OpenAI(
+    api_key="sk-1234",
+    base_url="http://0.0.0.0:4000"
+)
+
+audio_file = open("gettysburg.wav", "rb")
+transcript = client.audio.transcriptions.create(
+    model="groq/whisper-large-v3",
+    file=audio_file,
+    extra_body={
+        "fallbacks": ["openai/whisper-1"]
+    }
+)
+```
+</TabItem>
+</Tabs>
+
+### Testing Fallbacks
+
+You can test your fallback configuration using `mock_testing_fallbacks=true` to simulate failures:
+
+<Tabs>
+<TabItem value="curl" label="Curl">
+
+```bash showLineNumbers title="Test Fallbacks with Mock Testing"
+curl --location 'http://0.0.0.0:4000/v1/audio/transcriptions' \
+--header 'Authorization: Bearer sk-1234' \
+--form 'file=@"gettysburg.wav"' \
+--form 'model="groq/whisper-large-v3"' \
+--form 'fallbacks[]="openai/whisper-1"' \
+--form 'mock_testing_fallbacks=true'
+```
+
+</TabItem>
+<TabItem value="openai" label="OpenAI Python SDK">
+
+```python showLineNumbers title="Test Fallbacks with Mock Testing"
+from openai import OpenAI
+client = OpenAI(
+    api_key="sk-1234",
+    base_url="http://0.0.0.0:4000"
+)
+
+audio_file = open("gettysburg.wav", "rb")
+transcript = client.audio.transcriptions.create(
+    model="groq/whisper-large-v3",
+    file=audio_file,
+    extra_body={
+        "fallbacks": ["openai/whisper-1"],
+        "mock_testing_fallbacks": True
+    }
+)
+```
+</TabItem>
+</Tabs>

docs/my-website/docs/batches.md

@@ -7,7 +7,7 @@ Covers Batches, Files
 
 | Feature | Supported | Notes | 
 |-------|-------|-------|
-| Supported Providers | OpenAI, Azure, Vertex | - |
+| Supported Providers | OpenAI, Azure, Vertex, Bedrock | - |
 | ✨ Cost Tracking | ✅ | LiteLLM Enterprise only |
 | Logging | ✅ | Works across all logging integrations |
 
@@ -116,10 +116,38 @@ print("response from litellm.create_batch=", create_batch_response)
 **Retrieve the Specific Batch and File Content**
 
 ```python
+    # Maximum wait time before we give up
+    MAX_WAIT_TIME = 300  
+
+    # Time to wait between each status check
+    POLL_INTERVAL = 5
+    
+    #Time waited till now 
+    waited = 0
+
+    # Wait for the batch to finish processing before trying to retrieve output
+    # This loop checks the batch status every few seconds (polling)
+
+    while True:
+        retrieved_batch = await litellm.aretrieve_batch(
+            batch_id=create_batch_response.id,
+            custom_llm_provider="openai"
+        )
+        
+        status = retrieved_batch.status
+        print(f"⏳ Batch status: {status}")
+        
+        if status == "completed" and retrieved_batch.output_file_id:
+            print("✅ Batch complete. Output file ID:", retrieved_batch.output_file_id)
+            break
+        elif status in ["failed", "cancelled", "expired"]:
+            raise RuntimeError(f"❌ Batch failed with status: {status}")
+        
+        await asyncio.sleep(POLL_INTERVAL)
+        waited += POLL_INTERVAL
+        if waited > MAX_WAIT_TIME:
+            raise TimeoutError("❌ Timed out waiting for batch to complete.")
 
-retrieved_batch = await litellm.aretrieve_batch(
-    batch_id=create_batch_response.id, custom_llm_provider="openai"
-)
 print("retrieved batch=", retrieved_batch)
 # just assert that we retrieved a non None batch
 
@@ -150,6 +178,7 @@ print("list_batches_response=", list_batches_response)
 ### [Azure OpenAI](./providers/azure#azure-batches-api)
 ### [OpenAI](#quick-start)
 ### [Vertex AI](./providers/vertex#batch-apis)
+### [Bedrock](./providers/bedrock_batches)
 
 
 ## How Cost Tracking for Batches API Works

docs/my-website/docs/benchmarks.md

@@ -18,13 +18,17 @@ model_list:
 
 ### 1 Instance LiteLLM Proxy
 
-In these tests the median latency of directly calling the fake-openai-endpoint is 60ms.
+In these tests the baseline latency characteristics are measured against a fake-openai-endpoint.
 
-| Metric | Litellm Proxy (1 Instance) |
-|--------|------------------------|
-| RPS | 475 |
-| Median Latency (ms) | 100 |
-| Latency overhead added by LiteLLM Proxy | 40ms |
+#### Performance Metrics
+
+| Metric | Value |
+|--------|-------|
+| **Requests per Second (RPS)** | 475 |
+| **End-to-End Latency P50 (ms)** | 100 |
+| **LiteLLM Overhead P50 (ms)** | 3 |
+| **LiteLLM Overhead P90 (ms)** | 17 |
+| **LiteLLM Overhead P99 (ms)** | 31 |
 
 <!-- <Image img={require('../img/1_instance_proxy.png')} /> -->
 
@@ -33,7 +37,8 @@ In these tests the median latency of directly calling the fake-openai-endpoint i
 <Image img={require('../img/instances_vs_rps.png')} /> -->
 
 #### Key Findings
-- Single instance: 475 RPS @ 100ms latency
+- Single instance: 475 RPS @ 100ms median latency
+- LiteLLM adds 3ms P50 overhead, 17ms P90 overhead, 31ms P99 overhead
 - 2 LiteLLM instances: 950 RPS @ 100ms latency
 - 4 LiteLLM instances: 1900 RPS @ 100ms latency
 
@@ -54,6 +59,62 @@ Each machine deploying LiteLLM had the following specs:
 - 2 CPU
 - 4GB RAM
 
+## How to measure LiteLLM Overhead
+
+All responses from litellm will include the `x-litellm-overhead-duration-ms` header, this is the latency overhead in milliseconds added by LiteLLM Proxy.
+
+
+If you want to measure this on locust you can use the following code:
+
+```python showLineNumbers title="Locust Code for measuring LiteLLM Overhead"
+import os
+import uuid
+from locust import HttpUser, task, between, events
+
+# Custom metric to track LiteLLM overhead duration
+overhead_durations = []
+
+@events.request.add_listener
+def on_request(request_type, name, response_time, response_length, response, context, exception, start_time, url, **kwargs):
+    if response and hasattr(response, 'headers'):
+        overhead_duration = response.headers.get('x-litellm-overhead-duration-ms')
+        if overhead_duration:
+            try:
+                duration_ms = float(overhead_duration)
+                overhead_durations.append(duration_ms)
+                # Report as custom metric
+                events.request.fire(
+                    request_type="Custom",
+                    name="LiteLLM Overhead Duration (ms)",
+                    response_time=duration_ms,
+                    response_length=0,
+                )
+            except (ValueError, TypeError):
+                pass
+
+class MyUser(HttpUser):
+    wait_time = between(0.5, 1)  # Random wait time between requests
+
+    def on_start(self):
+        self.api_key = os.getenv('API_KEY', 'sk-1234567890')
+        self.client.headers.update({'Authorization': f'Bearer {self.api_key}'})
+
+    @task
+    def litellm_completion(self):
+        # no cache hits with this
+        payload = {
+            "model": "db-openai-endpoint",
+            "messages": [{"role": "user", "content": f"{uuid.uuid4()} This is a test there will be no cache hits and we'll fill up the context" * 150}],
+            "user": "my-new-end-user-1"
+        }
+        response = self.client.post("chat/completions", json=payload)
+        
+        if response.status_code != 200:
+            # log the errors in error.txt
+            with open("error.txt", "a") as error_log:
+                error_log.write(response.text + "\n")
+```
+
 
 
 ## Logging Callbacks

docs/my-website/docs/caching/all_caches.md

@@ -1,7 +1,7 @@
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
 
-# Caching - In-Memory, Redis, s3, Redis Semantic Cache, Disk
+# Caching - In-Memory, Redis, s3, gcs, Redis Semantic Cache, Disk
 
 [**See Code**](https://github.com/BerriAI/litellm/blob/main/litellm/caching/caching.py)
 
@@ -14,7 +14,7 @@ import TabItem from '@theme/TabItem';
 
 :::
 
-## Initialize Cache - In Memory, Redis, s3 Bucket, Redis Semantic, Disk Cache, Qdrant Semantic
+## Initialize Cache - In Memory, Redis, s3 Bucket, gcs Bucket, Redis Semantic, Disk Cache, Qdrant Semantic
 
 
 <Tabs>
@@ -28,6 +28,8 @@ pip install redis
 
 For the hosted version you can setup your own Redis DB here: https://redis.io/try-free/
 
+**Basic Redis Cache**
+
 ```python
 import litellm
 from litellm import completion
@@ -48,6 +50,91 @@ response2 = completion(
 # response1 == response2, response 1 is cached
 ```
 
+**GCP IAM Redis Authentication**
+
+For GCP Memorystore Redis with IAM authentication:
+
+```shell
+pip install google-cloud-iam
+```
+
+```python
+import litellm
+from litellm import completion
+# For Redis Cluster with GCP IAM
+from litellm.caching.redis_cluster_cache import RedisClusterCache
+
+litellm.cache = RedisClusterCache(
+    startup_nodes=[
+        {"host": "10.128.0.2", "port": 6379},
+        {"host": "10.128.0.2", "port": 11008},
+    ],
+    gcp_service_account="projects/-/serviceAccounts/your-sa@project.iam.gserviceaccount.com",
+    ssl=True,
+    ssl_cert_reqs=None,
+    ssl_check_hostname=False,
+)
+
+# Make completion calls
+response1 = completion(
+    model="gpt-3.5-turbo",
+    messages=[{"role": "user", "content": "Tell me a joke."}]
+)
+response2 = completion(
+    model="gpt-3.5-turbo",
+    messages=[{"role": "user", "content": "Tell me a joke."}]
+)
+
+# response1 == response2, response 1 is cached
+```
+
+**Environment Variables for GCP IAM Redis**
+
+You can also set these as environment variables:
+
+```shell
+export REDIS_HOST="10.128.0.2"
+export REDIS_PORT="6379"
+export REDIS_GCP_SERVICE_ACCOUNT="projects/-/serviceAccounts/your-sa@project.iam.gserviceaccount.com"
+export REDIS_SSL="False"
+```
+
+Then simply initialize:
+
+```python
+litellm.cache = Cache(type="redis")
+```
+
+</TabItem>
+
+<TabItem value="gcs" label="gcs-cache">
+
+Set environment variables
+
+```shell
+GCS_BUCKET_NAME="my-cache-bucket"
+GCS_PATH_SERVICE_ACCOUNT="/path/to/service_account.json"
+```
+
+```python
+import litellm
+from litellm import completion
+from litellm.caching.caching import Cache
+
+litellm.cache = Cache(type="gcs", gcs_bucket_name="my-cache-bucket", gcs_path_service_account="/path/to/service_account.json")
+
+response1 = completion(
+    model="gpt-3.5-turbo",
+    messages=[{"role": "user", "content": "Tell me a joke."}]
+)
+response2 = completion(
+    model="gpt-3.5-turbo",
+    messages=[{"role": "user", "content": "Tell me a joke."}]
+)
+
+# response1 == response2, response 1 is cached
+```
+
 </TabItem>
 
 
@@ -88,6 +175,37 @@ response2 = completion(
 
 </TabItem>
 
+<TabItem value="azureblob" label="azure-blob-cache">
+
+Install azure-storage-blob and azure-identity
+```shell
+pip install azure-storage-blob azure-identity
+```
+
+```python
+import litellm
+from litellm import completion
+from litellm.caching.caching import Cache
+from azure.identity import DefaultAzureCredential
+
+# pass Azure Blob Storage account URL and container name
+litellm.cache = Cache(type="azure-blob", azure_account_url="https://example.blob.core.windows.net", azure_blob_container="litellm")
+
+# Make completion calls
+response1 = completion(
+    model="gpt-3.5-turbo",
+    messages=[{"role": "user", "content": "Tell me a joke."}]
+)
+response2 = completion(
+    model="gpt-3.5-turbo",
+    messages=[{"role": "user", "content": "Tell me a joke."}]
+)
+
+# response1 == response2, response 1 is cached
+```
+
+</TabItem>
+
 
 <TabItem value="redis-sem" label="redis-semantic cache">
 
@@ -236,10 +354,10 @@ response2 = completion(
 
 ### Quick Start
 
-Install diskcache:
+Install the disk caching extra:
 
 ```shell
-pip install diskcache
+pip install "litellm[caching]"
 ```
 
 Then you can use the disk cache as follows.
@@ -374,7 +492,7 @@ Advanced Params
 
 ```python
 litellm.enable_cache(
-    type: Optional[Literal["local", "redis", "s3", "disk"]] = "local",
+    type: Optional[Literal["local", "redis", "s3", "gcs", "disk"]] = "local",
     host: Optional[str] = None,
     port: Optional[str] = None,
     password: Optional[str] = None,
@@ -398,7 +516,7 @@ Update the Cache params
 
 ```python
 litellm.update_cache(
-    type: Optional[Literal["local", "redis", "s3", "disk"]] = "local",
+    type: Optional[Literal["local", "redis", "s3", "gcs", "disk"]] = "local",
     host: Optional[str] = None,
     port: Optional[str] = None,
     password: Optional[str] = None,
@@ -459,7 +577,7 @@ cache.get_cache = get_cache
 ```python
 def __init__(
     self,
-    type: Optional[Literal["local", "redis", "redis-semantic", "s3", "disk"]] = "local",
+    type: Optional[Literal["local", "redis", "redis-semantic", "s3", "gcs", "disk"]] = "local",
     supported_call_types: Optional[
         List[Literal["completion", "acompletion", "embedding", "aembedding", "atranscription", "transcription"]]
     ] = ["completion", "acompletion", "embedding", "aembedding", "atranscription", "transcription"],
@@ -473,6 +591,13 @@ def __init__(
     namespace: Optional[str] = None,
     default_in_redis_ttl: Optional[float] = None,
     redis_flush_size=None,
+    
+    # GCP IAM Redis authentication params
+    gcp_service_account: Optional[str] = None,
+    gcp_ssl_ca_certs: Optional[str] = None,
+    ssl: Optional[bool] = None,
+    ssl_cert_reqs: Optional[Union[str, None]] = None,
+    ssl_check_hostname: Optional[bool] = None,
 
     # redis semantic cache params
     similarity_threshold: Optional[float] = None,

docs/my-website/docs/completion/computer_use.md

@@ -0,0 +1,446 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Computer Use
+
+Computer use allows models to interact with computer interfaces by taking screenshots and performing actions like clicking, typing, and scrolling. This enables AI models to autonomously operate desktop environments.
+
+**Supported Providers:**
+- Anthropic API (`anthropic/`)
+- Bedrock (Anthropic) (`bedrock/`)
+- Vertex AI (Anthropic) (`vertex_ai/`)
+
+**Supported Tool Types:**
+- `computer` - Computer interaction tool with display parameters
+- `bash` - Bash shell tool  
+- `text_editor` - Text editor tool
+- `web_search` - Web search tool
+
+LiteLLM will standardize the computer use tools across all supported providers.
+
+## Quick Start
+
+<Tabs>
+<TabItem value="sdk" label="LiteLLM Python SDK">
+
+```python
+import os 
+from litellm import completion
+
+os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
+
+# Computer use tool
+    tools = [
+        {
+            "type": "computer_20241022",
+            "name": "computer",
+            "display_height_px": 768,
+            "display_width_px": 1024,
+            "display_number": 0,
+        }
+    ]
+    
+    messages = [
+        {
+            "role": "user", 
+            "content": [
+                {
+                    "type": "text",
+                "text": "Take a screenshot and tell me what you see"
+            },
+            {
+                "type": "image_url",
+                "image_url": {
+                    "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg=="
+                }
+            }
+        ]
+    }
+]
+
+response = completion(
+    model="anthropic/claude-3-5-sonnet-latest",
+    messages=messages,
+    tools=tools,
+)
+
+print(response)
+```
+
+</TabItem>
+<TabItem value="proxy" label="LiteLLM Proxy Server">
+
+1. Define computer use models on config.yaml
+
+```yaml
+model_list:
+  - model_name: claude-3-5-sonnet-latest # Anthropic claude-3-5-sonnet-latest
+    litellm_params:
+      model: anthropic/claude-3-5-sonnet-latest
+      api_key: os.environ/ANTHROPIC_API_KEY
+  - model_name: claude-bedrock         # Bedrock Anthropic model
+    litellm_params:
+      model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
+      aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
+      aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
+      aws_region_name: us-west-2
+    model_info:
+      supports_computer_use: True        # set supports_computer_use to True so /model/info returns this attribute as True
+```
+
+2. Run proxy server
+
+```bash
+litellm --config config.yaml
+```
+
+3. Test it using the OpenAI Python SDK
+
+```python
+import os 
+from openai import OpenAI
+
+client = OpenAI(
+    api_key="sk-1234", # your litellm proxy api key
+    base_url="http://0.0.0.0:4000"
+)
+
+response = client.chat.completions.create(
+    model="claude-3-5-sonnet-latest",
+    messages=[
+        {
+            "role": "user", 
+            "content": [
+                {
+                    "type": "text",
+                    "text": "Take a screenshot and tell me what you see"
+                },
+                {
+                    "type": "image_url",
+                    "image_url": {
+                        "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg=="
+                    }
+                }
+            ]
+        }
+    ],
+    tools=[
+        {
+            "type": "computer_20241022",
+            "name": "computer",
+            "display_height_px": 768,
+            "display_width_px": 1024,
+            "display_number": 0,
+        }
+    ]
+)
+
+print(response)
+```
+
+</TabItem>
+</Tabs>
+
+## Checking if a model supports `computer use`
+
+<Tabs>
+<TabItem label="LiteLLM Python SDK" value="Python">
+
+Use `litellm.supports_computer_use(model="")` -> returns `True` if model supports computer use and `False` if not
+
+```python
+import litellm
+
+assert litellm.supports_computer_use(model="anthropic/claude-3-5-sonnet-latest") == True
+assert litellm.supports_computer_use(model="anthropic/claude-3-7-sonnet-20250219") == True
+assert litellm.supports_computer_use(model="bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0") == True
+assert litellm.supports_computer_use(model="vertex_ai/claude-3-5-sonnet") == True
+assert litellm.supports_computer_use(model="openai/gpt-4") == False
+```
+</TabItem>
+
+<TabItem label="LiteLLM Proxy Server" value="proxy">
+
+1. Define computer use models on config.yaml
+
+```yaml
+model_list:
+  - model_name: claude-3-5-sonnet-latest # Anthropic claude-3-5-sonnet-latest
+    litellm_params:
+      model: anthropic/claude-3-5-sonnet-latest
+      api_key: os.environ/ANTHROPIC_API_KEY
+  - model_name: claude-bedrock         # Bedrock Anthropic model
+    litellm_params:
+      model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
+      aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
+      aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
+      aws_region_name: us-west-2
+    model_info:
+      supports_computer_use: True        # set supports_computer_use to True so /model/info returns this attribute as True
+```
+
+2. Run proxy server
+
+```bash
+litellm --config config.yaml
+```
+
+3. Call `/model_group/info` to check if your model supports `computer use`
+
+```shell
+curl -X 'GET' \
+  'http://localhost:4000/model_group/info' \
+  -H 'accept: application/json' \
+  -H 'x-api-key: sk-1234'
+```
+
+Expected Response 
+
+```json
+{
+  "data": [
+    {
+      "model_group": "claude-3-5-sonnet-latest",
+      "providers": ["anthropic"],
+      "max_input_tokens": 200000,
+      "max_output_tokens": 8192,
+      "mode": "chat",
+      "supports_computer_use": true, # 👈 supports_computer_use is true
+      "supports_vision": true,
+      "supports_function_calling": true
+    },
+    {
+      "model_group": "claude-bedrock",
+      "providers": ["bedrock"],
+      "max_input_tokens": 200000,
+      "max_output_tokens": 8192,
+      "mode": "chat",
+      "supports_computer_use": true, # 👈 supports_computer_use is true
+      "supports_vision": true,
+      "supports_function_calling": true
+    }
+  ]
+}
+```
+
+</TabItem>
+</Tabs>
+
+## Different Tool Types
+
+Computer use supports several different tool types for various interaction modes:
+
+<Tabs>
+<TabItem value="computer" label="Computer Tool">
+
+The `computer_20241022` tool provides direct screen interaction capabilities.
+
+```python
+import os 
+from litellm import completion
+
+os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
+
+tools = [
+    {
+        "type": "computer_20241022", 
+        "name": "computer",
+        "display_height_px": 768,
+        "display_width_px": 1024,
+        "display_number": 0,
+    }
+]
+
+messages = [
+    {
+        "role": "user",
+        "content": [
+            {
+                "type": "text", 
+                "text": "Click on the search button in the screenshot"
+            },
+            {
+                "type": "image_url",
+                "image_url": {
+                    "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg=="
+                }
+            }
+        ]
+    }
+]
+
+response = completion(
+    model="anthropic/claude-3-5-sonnet-latest",
+    messages=messages,
+    tools=tools,
+)
+
+print(response)
+```
+
+</TabItem>
+<TabItem value="bash" label="Bash Tool">
+
+The `bash_20241022` tool provides command line interface access.
+
+```python
+import os 
+from litellm import completion
+
+os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
+
+tools = [
+    {
+        "type": "bash_20241022",
+        "name": "bash"
+    }
+]
+
+messages = [
+    {
+        "role": "user",
+        "content": "List the files in the current directory using bash"
+    }
+]
+
+response = completion(
+    model="anthropic/claude-3-5-sonnet-latest",
+    messages=messages,
+    tools=tools,
+)
+
+print(response)
+```
+
+</TabItem>
+<TabItem value="text_editor" label="Text Editor Tool">
+
+The `text_editor_20250124` tool provides text file editing capabilities.
+
+```python
+import os 
+from litellm import completion
+
+os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
+
+tools = [
+    {
+        "type": "text_editor_20250124",
+        "name": "str_replace_editor"
+    }
+]
+
+messages = [
+    {
+        "role": "user",
+        "content": "Create a simple Python hello world script"
+    }
+]
+
+response = completion(
+    model="anthropic/claude-3-5-sonnet-latest",
+    messages=messages,
+    tools=tools,
+)
+
+print(response)
+```
+
+</TabItem>
+</Tabs>
+
+## Advanced Usage with Multiple Tools
+
+You can combine different computer use tools in a single request:
+
+```python
+import os 
+from litellm import completion
+
+os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
+
+tools = [
+    {
+        "type": "computer_20241022",
+        "name": "computer", 
+        "display_height_px": 768,
+        "display_width_px": 1024,
+        "display_number": 0,
+    },
+    {
+        "type": "bash_20241022",
+        "name": "bash"
+    },
+    {
+        "type": "text_editor_20250124", 
+        "name": "str_replace_editor"
+    }
+]
+
+messages = [
+    {
+        "role": "user",
+        "content": [
+            {
+                "type": "text",
+                "text": "Take a screenshot, then create a file describing what you see, and finally use bash to show the file contents"
+                },
+                {
+                    "type": "image_url",
+                    "image_url": {
+                        "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg=="
+                    }
+                }
+            ]
+        }
+    ]
+    
+response = completion(
+    model="anthropic/claude-3-5-sonnet-latest",
+            messages=messages,
+            tools=tools,
+)
+
+print(response)
+```
+
+## Spec
+
+### Computer Tool (`computer_20241022`)
+
+```json
+{
+  "type": "computer_20241022",
+  "name": "computer",
+  "display_height_px": 768,    // Required: Screen height in pixels  
+  "display_width_px": 1024,    // Required: Screen width in pixels
+  "display_number": 0          // Optional: Display number (default: 0)
+}
+```
+
+### Bash Tool (`bash_20241022`)
+
+```json
+{
+  "type": "bash_20241022", 
+  "name": "bash"              // Required: Tool name
+}
+```
+
+### Text Editor Tool (`text_editor_20250124`)
+
+```json
+{
+  "type": "text_editor_20250124",
+  "name": "str_replace_editor"  // Required: Tool name
+}
+```
+
+### Web Search Tool (`web_search_20250305`)
+
+```json
+{
+  "type": "web_search_20250305",
+  "name": "web_search"         // Required: Tool name
+}
+```

docs/my-website/docs/completion/document_understanding.md

@@ -9,6 +9,8 @@ Works for:
 - Vertex AI models (Gemini + Anthropic)
 - Bedrock Models
 - Anthropic API Models
+- OpenAI API Models
+- Mistral (Only using file ID of already uploaded file, similar to OpenAI file_id input)
 
 ## Quick Start
 
@@ -278,6 +280,71 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
 </Tabs>
 
 
+## Mistral Example
+
+Here is a sample payload for using the Mistral model for document understanding:
+
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+from litellm.utils import completion
+
+# pdf file_id received from files endpoint
+file_id = "fa778e5e-46ec-4562-8418-36623fe25a71"
+
+# model
+model = "mistral/mistral-large-latest"
+
+file_content = [
+    {"type": "text", "text": "What's this file about?"},
+    {
+        "type": "file",
+        "file": {
+            "file_id": file_id,
+        }
+    },
+]
+
+response = completion(
+    model=model,
+    messages=[{"role": "user", "content": file_content}],
+)
+assert response is not None
+```
+
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+```bash
+curl -X POST 'http://0.0.0.0:4000/chat/completions' \
+-H 'Content-Type: application/json' \
+-H 'Authorization: Bearer sk-1234' \
+-d '{
+    "model": "mistral/mistral-large-latest",
+    "messages": [
+        {
+            "role": "user",
+            "content": [
+                {
+                    "type": "text",
+                    "text": "What is the content of the file?"
+                },
+                {
+                    "type": "file",
+                    "file": {
+                        "file_id": "fa778e5e-46ec-4562-8418-36623fe25a71"
+                    }
+                }
+            ]
+        }
+    ]
+}
+```
+</TabItem>
+</Tabs>
+
 ## Checking if a model supports pdf input
 
 <Tabs>

docs/my-website/docs/completion/http_handler_config.md

@@ -0,0 +1,145 @@
+# Custom HTTP Handler
+
+Configure custom aiohttp sessions for better performance and control in LiteLLM completions.
+
+## Overview
+
+You can now inject custom `aiohttp.ClientSession` instances into LiteLLM for:
+- Custom connection pooling and timeouts
+- Corporate proxy and SSL configurations  
+- Performance optimization
+- Request monitoring
+
+## Basic Usage
+
+### Default (No Changes Required)
+```python
+import litellm
+
+# Works exactly as before
+response = await litellm.acompletion(
+    model="gpt-3.5-turbo",
+    messages=[{"role": "user", "content": "Hello!"}]
+)
+```
+
+### Custom Session
+```python
+import aiohttp
+import litellm
+from litellm.llms.custom_httpx.aiohttp_handler import BaseLLMAIOHTTPHandler
+
+# Create optimized session
+session = aiohttp.ClientSession(
+    timeout=aiohttp.ClientTimeout(total=180),
+    connector=aiohttp.TCPConnector(limit=300, limit_per_host=75)
+)
+
+# Replace global handler
+litellm.base_llm_aiohttp_handler = BaseLLMAIOHTTPHandler(client_session=session)
+
+# All completions now use your session
+response = await litellm.acompletion(model="gpt-3.5-turbo", messages=[...])
+```
+
+## Common Patterns
+
+### FastAPI Integration
+```python
+from contextlib import asynccontextmanager
+from fastapi import FastAPI
+import aiohttp
+import litellm
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    # Startup
+    session = aiohttp.ClientSession(
+        timeout=aiohttp.ClientTimeout(total=180),
+        connector=aiohttp.TCPConnector(limit=300)
+    )
+    litellm.base_llm_aiohttp_handler = BaseLLMAIOHTTPHandler(
+        client_session=session
+    )
+    yield
+    # Shutdown
+    await session.close()
+
+app = FastAPI(lifespan=lifespan)
+
+@app.post("/chat")
+async def chat(messages: list[dict]):
+    return await litellm.acompletion(model="gpt-3.5-turbo", messages=messages)
+```
+
+### Corporate Proxy
+```python
+import ssl
+
+# Custom SSL context
+ssl_context = ssl.create_default_context()
+ssl_context.load_cert_chain('cert.pem', 'key.pem')
+
+# Proxy session
+session = aiohttp.ClientSession(
+    connector=aiohttp.TCPConnector(ssl=ssl_context),
+    trust_env=True  # Use environment proxy settings
+)
+
+litellm.base_llm_aiohttp_handler = BaseLLMAIOHTTPHandler(client_session=session)
+```
+
+### High Performance
+```python
+# Optimized for high throughput
+session = aiohttp.ClientSession(
+    timeout=aiohttp.ClientTimeout(total=300),
+    connector=aiohttp.TCPConnector(
+        limit=1000,             # High connection limit
+        limit_per_host=200,     # Per host limit
+        ttl_dns_cache=600,      # DNS cache
+        keepalive_timeout=60,   # Keep connections alive
+        enable_cleanup_closed=True
+    )
+)
+
+litellm.base_llm_aiohttp_handler = BaseLLMAIOHTTPHandler(client_session=session)
+```
+
+## Constructor Options
+
+```python
+BaseLLMAIOHTTPHandler(
+    client_session=None,    # Custom aiohttp.ClientSession
+    transport=None,         # Advanced transport control
+    connector=None,         # Custom aiohttp.BaseConnector
+)
+```
+
+## Resource Management
+
+- **User sessions**: You manage the lifecycle (call `await session.close()`)
+- **Auto-created sessions**: Automatically cleaned up by the handler
+- **100% backward compatible**: Existing code works unchanged
+
+## Configuration Tips
+
+### Development
+```python
+session = aiohttp.ClientSession(
+    timeout=aiohttp.ClientTimeout(total=60),
+    connector=aiohttp.TCPConnector(limit=50)
+)
+```
+
+### Production
+```python
+session = aiohttp.ClientSession(
+    timeout=aiohttp.ClientTimeout(total=300),
+    connector=aiohttp.TCPConnector(
+        limit=1000,
+        limit_per_host=200,
+        keepalive_timeout=60
+    )
+)
+```

docs/my-website/docs/completion/image_generation_chat.md

@@ -0,0 +1,232 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Image Generation in Chat Completions, Responses API
+
+This guide covers how to generate images when using the `chat/completions`. Note - if you want this on Responses API please file a Feature Request [here](https://github.com/BerriAI/litellm/issues/new).
+
+:::info
+
+Requires LiteLLM v1.76.1+
+
+:::
+
+Supported Providers:
+- Google AI Studio (`gemini`)
+- Vertex AI (`vertex_ai/`)
+
+LiteLLM will standardize the `image` response in the assistant message for models that support image generation during chat completions.
+
+```python title="Example response from litellm"
+"message": {
+    ...
+    "content": "Here's the image you requested:",
+    "image": {
+        "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...",
+        "detail": "auto"
+    }
+}
+```
+
+## Quick Start 
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python showLineNumbers title="Image generation with chat completion"
+from litellm import completion
+import os 
+
+os.environ["GEMINI_API_KEY"] = "your-api-key"
+
+response = completion(
+    model="gemini/gemini-2.5-flash-image-preview",
+    messages=[
+        {"role": "user", "content": "Generate an image of a banana wearing a costume that says LiteLLM"}
+    ],
+)
+
+print(response.choices[0].message.content)  # Text response
+print(response.choices[0].message.image)    # Image data
+```
+
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+1. Setup config.yaml
+
+```yaml showLineNumbers title="config.yaml"
+model_list:
+  - model_name: gemini-image-gen
+    litellm_params:
+      model: gemini/gemini-2.5-flash-image-preview
+      api_key: os.environ/GEMINI_API_KEY
+```
+
+2. Run proxy server
+
+```bash showLineNumbers title="Start the proxy"
+litellm --config config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+3. Test it!
+
+```bash showLineNumbers title="Make request"
+curl http://0.0.0.0:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer $LITELLM_KEY" \
+  -d '{
+    "model": "gemini-image-gen",
+    "messages": [
+      {
+        "role": "user",
+        "content": "Generate an image of a banana wearing a costume that says LiteLLM"
+      }
+    ]
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+**Expected Response**
+
+```bash
+{
+    "id": "chatcmpl-3b66124d79a708e10c603496b363574c",
+    "choices": [
+        {
+            "finish_reason": "stop",
+            "index": 0,
+            "message": {
+                "content": "Here's the image you requested:",
+                "role": "assistant",
+                "image": {
+                    "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...",
+                    "detail": "auto"
+                }
+            }
+        }
+    ],
+    "created": 1723323084,
+    "model": "gemini/gemini-2.5-flash-image-preview",
+    "object": "chat.completion",
+    "usage": {
+        "completion_tokens": 12,
+        "prompt_tokens": 16,
+        "total_tokens": 28
+    }
+}
+```
+
+## Streaming Support
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python showLineNumbers title="Streaming image generation"
+from litellm import completion
+import os 
+
+os.environ["GEMINI_API_KEY"] = "your-api-key"
+
+response = completion(
+    model="gemini/gemini-2.5-flash-image-preview",
+    messages=[
+        {"role": "user", "content": "Generate an image of a banana wearing a costume that says LiteLLM"}
+    ],
+    stream=True,
+)
+
+for chunk in response:
+    if hasattr(chunk.choices[0].delta, "image") and chunk.choices[0].delta.image is not None:
+        print("Generated image:", chunk.choices[0].delta.image["url"])
+        break
+```
+
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+```bash showLineNumbers title="Streaming request"
+curl http://0.0.0.0:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer $LITELLM_KEY" \
+  -d '{
+    "model": "gemini-image-gen",
+    "messages": [
+      {
+        "role": "user",
+        "content": "Generate an image of a banana wearing a costume that says LiteLLM"
+      }
+    ],
+    "stream": true
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+**Expected Streaming Response**
+
+```bash
+data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1723323084,"model":"gemini/gemini-2.5-flash-image-preview","choices":[{"index":0,"delta":{"role":"assistant"},"finish_reason":null}]}
+
+data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1723323084,"model":"gemini/gemini-2.5-flash-image-preview","choices":[{"index":0,"delta":{"content":"Here's the image you requested:"},"finish_reason":null}]}
+
+data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1723323084,"model":"gemini/gemini-2.5-flash-image-preview","choices":[{"index":0,"delta":{"image":{"url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...","detail":"auto"}},"finish_reason":null}]}
+
+data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1723323084,"model":"gemini/gemini-2.5-flash-image-preview","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}
+
+data: [DONE]
+```
+
+## Async Support
+
+```python showLineNumbers title="Async image generation"
+from litellm import acompletion
+import asyncio
+import os 
+
+os.environ["GEMINI_API_KEY"] = "your-api-key"
+
+async def generate_image():
+    response = await acompletion(
+        model="gemini/gemini-2.5-flash-image-preview",
+        messages=[
+            {"role": "user", "content": "Generate an image of a banana wearing a costume that says LiteLLM"}
+        ],
+    )
+    
+    print(response.choices[0].message.content)  # Text response
+    print(response.choices[0].message.image)    # Image data
+    
+    return response
+
+# Run the async function
+asyncio.run(generate_image())
+```
+
+## Supported Models
+
+| Provider | Model | 
+|----------|--------|
+| Google AI Studio | `gemini/gemini-2.5-flash-image-preview` |
+| Vertex AI | `vertex_ai/gemini-2.5-flash-image-preview` |
+
+## Spec 
+
+The `image` field in the response follows this structure:
+
+```python
+"image": {
+    "url": "data:image/png;base64,<base64_encoded_image>",
+    "detail": "auto"
+}
+```
+
+- `url` - str: Base64 encoded image data in data URI format
+- `detail` - str: Image detail level (always "auto" for generated images)
+
+The image is returned as a base64-encoded data URI that can be directly used in HTML `<img>` tags or saved to a file.

docs/my-website/docs/completion/input.md

@@ -39,31 +39,34 @@ This is a list of openai params we translate across providers.
 
 Use `litellm.get_supported_openai_params()` for an updated list of params for each model + provider 
 
-| Provider | temperature | max_completion_tokens | max_tokens | top_p | stream | stream_options | stop | n | presence_penalty | frequency_penalty | functions | function_call | logit_bias | user | response_format | seed | tools | tool_choice | logprobs | top_logprobs | extra_headers |
-|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
-|Anthropic| ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ |  |  |   |  |  |  |✅ | ✅ | | ✅ | ✅ |  |  | ✅ |
-|OpenAI| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ | ✅ |
-|Azure OpenAI| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ |✅ | ✅ | ✅ | ✅ | ✅ |
-|xAI| ✅ |  | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |  |  | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |  |
-|Replicate | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | |  |   |  |   |
-|Anyscale | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
-|Cohere| ✅ | ✅ | ✅ | ✅ |  ✅ | ✅ | ✅ | ✅ | ✅ |   |   |
-|Huggingface| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |  |   |    |
-|Openrouter| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | ✅ |✅ | | | |
-|AI21| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |  |  |
-|VertexAI| ✅ | ✅ | ✅ | | ✅ | ✅ |  |  |  |  |   | | | | ✅ | ✅ | | |
-|Bedrock| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |  |  |   |  |   | | | | | ✅ (model dependent) | |
-|Sagemaker| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |  |  |   |
-|TogetherAI| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |  |  |  |  |  | ✅ |  |  | ✅ |  | ✅ | ✅ |  |  |  |
-|Sambanova| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |  |  |  |  |  |  |  | ✅ |  | ✅ | ✅ |  |  |  |
-|AlephAlpha| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |   |  |   |
-|NLP Cloud| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | |  |  |   |
-|Petals| ✅ | ✅ |  | ✅ | ✅ | |  |   |  |   |
-|Ollama| ✅ | ✅ | ✅ |✅ | ✅ | ✅ |  |   | ✅ |  |   | | | ✅ |  | |✅| | | | | | |
-|Databricks| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |  |   | |  |   | | | | | |
-|ClarifAI| ✅ | ✅ | ✅ | |✅ | ✅ |  |   | |  |   | | | | | |
-|Github| ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | ✅ |✅ (model dependent)|✅ (model dependent)| | |
-|Novita AI| ✅ | ✅ |  | ✅ | ✅ | ✅ |  | ✅ | ✅ | ✅ | ✅ |  |  | ✅ |  |  |  |  |  |  |  |
+| Provider | temperature | max_completion_tokens | max_tokens | top_p | stream | stream_options | stop | n | presence_penalty | frequency_penalty | functions | function_call | logit_bias | user | response_format | seed| tools | tool_choice | logprobs | top_logprobs | extra_headers |
+|--------------|-------------|------------------------|------------|-------|--------|----------------|------|-----|------------------|-------------------|-----------|----------------|-------------|------|------------------|-------------------|--------|--------------|----------|---------------|----------------------|
+| Anthropic| ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | || | || | ✅ | ✅ | | ✅ | ✅ || | ✅|
+| OpenAI | ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | ✅| ✅ | ✅| ✅| ✅ | ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | ✅| ✅|
+| Azure OpenAI | ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | ✅| ✅ | ✅| ✅| ✅ | ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | ✅| ✅|
+| xAI| ✅|| ✅ | ✅| ✅ | ✅ | ✅ | ✅| ✅ | ✅| || ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | ✅||
+| Replicate| ✅| ✅ | ✅ | ✅| ✅ | ✅ || || | || ||| |||| ||
+| Anyscale | ✅| ✅ | ✅ | ✅| ✅ | ✅ || || | || ||| |||| ||
+| Cohere | ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | ✅|| | || ||| |||| ||
+| Huggingface| ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | || | || ||| |||| ||
+| Openrouter | ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | ✅| ✅ | ✅| ✅|| ||| ✅| ✅ ||| ||
+| AI21 | ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | ✅|| | || ||| |||| ||
+| VertexAI | ✅| ✅ | ✅ | | ✅ | ✅ || || | || || ✅ | ✅|||| ||
+| Bedrock| ✅| ✅ | ✅ | ✅| ✅ | ✅ || || | || || ✅ (model dependent) | |||| ||
+| Sagemaker| ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | || | || ||| |||| ||
+| TogetherAI | ✅| ✅ | ✅ | ✅| ✅ | ✅ || || | ✅|| || ✅ | | ✅ | ✅ || ||
+| Sambanova| ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | || | || || ✅ | | ✅ | ✅ || ||
+| AlephAlpha | ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | || | || ||| |||| ||
+| NLP Cloud| ✅| ✅ | ✅ | ✅| ✅ | ✅ || || | || ||| |||| ||
+| Petals | ✅| ✅ || ✅| ✅ ||| || | || ||| |||| ||
+| Ollama | ✅| ✅ | ✅ | ✅| ✅ | ✅ || ✅|| | || ✅||| | ✅ ||| ||
+| Databricks | ✅| ✅ | ✅ | ✅| ✅ | ✅ || || | || ||| |||| ||
+| ClarifAI | ✅| ✅ | ✅ | | ✅ | ✅ || || | || ||| |||| ||
+| Github | ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | ✅| ✅ | ✅| ✅|| || ✅ | ✅ (model dependent) | ✅ (model dependent) || ||
+| Novita AI| ✅| ✅ || ✅| ✅ | ✅ | ✅ | ✅| ✅ | ✅| || ✅||| |||| ||
+| Bytez | ✅| ✅ || ✅| ✅ | | | ✅|| || || || || || ||
+| OVHCloud AI Endpoints | ✅ | | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | | | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | |
+
 :::note
 
 By default, LiteLLM raises an exception if the openai param being passed in isn't supported. 
@@ -104,6 +107,7 @@ def completion(
     parallel_tool_calls: Optional[bool] = None,
     logprobs: Optional[bool] = None,
     top_logprobs: Optional[int] = None,
+    safety_identifier: Optional[str] = None,
     deployment_id=None,
     # soon to be deprecated params by OpenAI
     functions: Optional[List] = None,
@@ -194,6 +198,8 @@ def completion(
         
 - `top_logprobs`: *int (optional)* - An integer between 0 and 5 specifying the number of most likely tokens to return at each token position, each with an associated log probability. `logprobs` must be set to true if this parameter is used.
 
+- `safety_identifier`: *string (optional)* - A unique identifier for tracking and managing safety-related requests. This parameter helps with safety monitoring and compliance tracking.
+
 - `headers`: *dict (optional)* - A dictionary of headers to be sent with the request.
 
 - `extra_headers`: *dict (optional)* - Alternative to `headers`, used to send extra headers in LLM API request. 

docs/my-website/docs/completion/knowledgebase.md

@@ -17,6 +17,9 @@ LiteLLM integrates with vector stores, allowing your models to access your organ
 
 ## Supported Vector Stores
 - [Bedrock Knowledge Bases](https://aws.amazon.com/bedrock/knowledge-bases/)
+- [OpenAI Vector Stores](https://platform.openai.com/docs/api-reference/vector-stores/search)
+- [Azure Vector Stores](https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/file-search?tabs=python#vector-stores)
+- [Vertex AI RAG API](https://cloud.google.com/vertex-ai/generative-ai/docs/rag-overview)
 
 ## Quick Start
 
@@ -157,6 +160,129 @@ print(response.choices[0].message.content)
 </TabItem>
 </Tabs>
 
+## Provider Specific Guides
+
+This section covers how to add your vector stores to LiteLLM. If you want support for a new provider, please file an issue [here](https://github.com/BerriAI/litellm/issues).
+
+### Bedrock Knowledge Bases
+
+**1. Set up your Bedrock Knowledge Base**
+
+Ensure you have a Bedrock Knowledge Base created in your AWS account with the appropriate permissions configured.
+
+**2. Add to LiteLLM UI**
+
+1. Navigate to **Tools > Vector Stores > "Add new vector store"**
+2. Select **"Bedrock"** as the provider
+3. Enter your Bedrock Knowledge Base ID in the **"Vector Store ID"** field
+
+<Image 
+  img={require('../../img/kb_2.png')}
+  style={{width: '60%', display: 'block'}}
+/>
+
+
+### Vertex AI RAG Engine
+
+**1. Get your Vertex AI RAG Engine ID**
+
+1. Navigate to your RAG Engine Corpus in the [Google Cloud Console](https://console.cloud.google.com/vertex-ai/rag/corpus)
+2. Select the **RAG Engine** you want to integrate with LiteLLM
+
+<div style={{margin: '20px 0', padding: '10px', border: '1px solid #ddd', borderRadius: '8px', display: 'inline-block', boxShadow: '0 2px 8px rgba(0,0,0,0.1)'}}>
+<Image 
+  img={require('../../img/kb_vertex1.png')}
+  style={{width: '60%', display: 'block'}}
+/>
+</div>
+
+3. Click the **"Details"** button and copy the UUID for the RAG Engine
+4. The ID should look like: `6917529027641081856`
+
+<div style={{margin: '20px 0', padding: '10px', border: '1px solid #ddd', borderRadius: '8px', display: 'inline-block', boxShadow: '0 2px 8px rgba(0,0,0,0.1)'}}>
+<Image 
+  img={require('../../img/kb_vertex2.png')}
+  style={{width: '60%', display: 'block'}}
+/>
+</div>
+
+**2. Add to LiteLLM UI**
+
+1. Navigate to **Tools > Vector Stores > "Add new vector store"**
+2. Select **"Vertex AI RAG Engine"** as the provider
+3. Enter your Vertex AI RAG Engine ID in the **"Vector Store ID"** field
+
+<div style={{margin: '20px 0', padding: '10px', border: '1px solid #ddd', borderRadius: '8px', display: 'inline-block', boxShadow: '0 2px 8px rgba(0,0,0,0.1)'}}>
+<Image 
+  img={require('../../img/kb_vertex3.png')}
+  style={{width: '60%', display: 'block'}}
+/>
+</div>
+
+### PG Vector
+
+**1. Deploy the litellm-pg-vector-store connector**
+
+LiteLLM provides a server that exposes OpenAI-compatible `vector_store` endpoints for PG Vector. The LiteLLM Proxy server connects to your deployed service and uses it as a vector store when querying.
+
+1. Follow the deployment instructions for the litellm-pg-vector-store connector [here](https://github.com/BerriAI/litellm-pgvector)
+2. For detailed configuration options, see the [configuration guide](https://github.com/BerriAI/litellm-pgvector?tab=readme-ov-file#configuration)
+
+**Example .env configuration for deploying litellm-pg-vector-store:**
+
+```env
+DATABASE_URL="postgresql://neondb_owner:xxxx"
+SERVER_API_KEY="sk-1234"
+HOST="0.0.0.0"
+PORT=8001
+EMBEDDING__MODEL="text-embedding-ada-002"
+EMBEDDING__BASE_URL="http://localhost:4000"
+EMBEDDING__API_KEY="sk-1234"
+EMBEDDING__DIMENSIONS=1536
+DB_FIELDS__ID_FIELD="id"
+DB_FIELDS__CONTENT_FIELD="content"
+DB_FIELDS__METADATA_FIELD="metadata"
+DB_FIELDS__EMBEDDING_FIELD="embedding"
+DB_FIELDS__VECTOR_STORE_ID_FIELD="vector_store_id"
+DB_FIELDS__CREATED_AT_FIELD="created_at"
+```
+
+**2. Add to LiteLLM UI**
+
+Once your litellm-pg-vector-store is deployed:
+
+1. Navigate to **Tools > Vector Stores > "Add new vector store"**
+2. Select **"PG Vector"** as the provider
+3. Enter your **API Base URL** and **API Key** for your `litellm-pg-vector-store` container
+   - The API Key field corresponds to the `SERVER_API_KEY` from your .env configuration
+
+<div style={{margin: '20px 0', padding: '10px', border: '1px solid #ddd', borderRadius: '8px', display: 'inline-block', boxShadow: '0 2px 8px rgba(0,0,0,0.1)'}}>
+<Image 
+  img={require('../../img/kb_pg1.png')}
+  style={{width: '60%', display: 'block'}}
+/>
+</div>
+
+### OpenAI Vector Stores
+
+**1. Set up your OpenAI Vector Store**
+
+1. Create your Vector Store on the [OpenAI platform](https://platform.openai.com/storage/vector_stores)
+2. Note your Vector Store ID (format: `vs_687ae3b2439881918b433cb99d10662e`)
+
+**2. Add to LiteLLM UI**
+
+1. Navigate to **Tools > Vector Stores > "Add new vector store"**
+2. Select **"OpenAI"** as the provider
+3. Enter your **Vector Store ID** in the corresponding field
+4. Enter your **OpenAI API Key** in the API Key field
+
+<div style={{margin: '20px 0', padding: '10px', border: '1px solid #ddd', borderRadius: '8px', display: 'inline-block', boxShadow: '0 2px 8px rgba(0,0,0,0.1)'}}>
+<Image 
+  img={require('../../img/kb_openai1.png')}
+  style={{width: '60%', display: 'block'}}
+/>
+</div>
 
 
 

docs/my-website/docs/completion/provider_specific_params.md

@@ -423,7 +423,7 @@ model_list:
 curl -X POST 'http://0.0.0.0:4000/chat/completions' \
 -H 'Content-Type: application/json' \
 -H 'Authorization: Bearer sk-1234' \
--D '{
+-d '{
   "model": "llama-3-8b-instruct",
   "messages": [
     {
@@ -431,6 +431,56 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
       "content": "What'\''s the weather like in Boston today?"
     }
   ],
-  "adapater_id": "my-special-adapter-id" # 👈 PROVIDER-SPECIFIC PARAM
-  }'
-```
+  "adapater_id": "my-special-adapter-id"
+}'
+```
+
+## Provider-Specific Metadata Parameters
+
+| Provider | Parameter | Use Case |
+|----------|-----------|----------|
+| **AWS Bedrock** | `requestMetadata` | Cost attribution, logging |
+| **Gemini/Vertex AI** | `labels` | Resource labeling |
+| **Anthropic** | `metadata` | User identification |
+
+<Tabs>
+<TabItem value="bedrock" label="AWS Bedrock">
+
+```python
+import litellm
+
+response = litellm.completion(
+    model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
+    messages=[{"role": "user", "content": "Hello!"}],
+    requestMetadata={"cost_center": "engineering"}
+)
+```
+
+</TabItem>
+<TabItem value="gemini" label="Gemini/Vertex AI">
+
+```python
+import litellm
+
+response = litellm.completion(
+    model="vertex_ai/gemini-pro",
+    messages=[{"role": "user", "content": "Hello!"}],
+    labels={"environment": "production"}
+)
+```
+
+</TabItem>
+<TabItem value="anthropic" label="Anthropic">
+
+```python
+import litellm
+
+response = litellm.completion(
+    model="anthropic/claude-3-sonnet-20240229",
+    messages=[{"role": "user", "content": "Hello!"}],
+    metadata={"user_id": "user123"}
+)
+```
+
+</TabItem>
+</Tabs>

docs/my-website/docs/completion/shared_session.md

@@ -0,0 +1,213 @@
+# Shared Session Support
+
+## Overview
+
+LiteLLM now supports sharing `aiohttp.ClientSession` instances across multiple API calls to avoid creating unnecessary new sessions. This improves performance and resource utilization.
+
+## Usage
+
+### Basic Usage
+
+```python
+import asyncio
+from aiohttp import ClientSession
+from litellm import acompletion
+
+async def main():
+    # Create a shared session
+    async with ClientSession() as shared_session:
+        # Use the same session for multiple calls
+        response1 = await acompletion(
+            model="gpt-4o",
+            messages=[{"role": "user", "content": "Hello"}],
+            shared_session=shared_session
+        )
+        
+        response2 = await acompletion(
+            model="gpt-4o", 
+            messages=[{"role": "user", "content": "How are you?"}],
+            shared_session=shared_session
+        )
+        
+        # Both calls reuse the same session!
+
+asyncio.run(main())
+```
+
+### Without Shared Session (Default)
+
+```python
+import asyncio
+from litellm import acompletion
+
+async def main():
+    # Each call creates a new session
+    response1 = await acompletion(
+        model="gpt-4o",
+        messages=[{"role": "user", "content": "Hello"}]
+    )
+    
+    response2 = await acompletion(
+        model="gpt-4o",
+        messages=[{"role": "user", "content": "How are you?"}]
+    )
+    # Two separate sessions created
+
+asyncio.run(main())
+```
+
+## Benefits
+
+- **Performance**: Reuse HTTP connections across multiple calls
+- **Resource Efficiency**: Reduce memory and connection overhead
+- **Better Control**: Manage session lifecycle explicitly
+- **Debugging**: Easy to trace which calls use which sessions
+
+## Debug Logging
+
+Enable debug logging to see session reuse in action:
+
+```python
+import os
+import litellm
+
+# Enable debug logging
+os.environ['LITELLM_LOG'] = 'DEBUG'
+
+# You'll see logs like:
+# 🔄 SHARED SESSION: acompletion called with shared_session (ID: 12345)
+# ✅ SHARED SESSION: Reusing existing ClientSession (ID: 12345)
+```
+
+## Common Patterns
+
+### FastAPI Integration
+
+```python
+from fastapi import FastAPI
+import aiohttp
+import litellm
+
+app = FastAPI()
+
+@app.post("/chat")
+async def chat(messages: list[dict]):
+    # Create session per request
+    async with aiohttp.ClientSession() as session:
+        return await litellm.acompletion(
+            model="gpt-4o",
+            messages=messages,
+            shared_session=session
+        )
+```
+
+### Batch Processing
+
+```python
+import asyncio
+from aiohttp import ClientSession
+from litellm import acompletion
+
+async def process_batch(messages_list):
+    async with ClientSession() as shared_session:
+        tasks = []
+        for messages in messages_list:
+            task = acompletion(
+                model="gpt-4o",
+                messages=messages,
+                shared_session=shared_session
+            )
+            tasks.append(task)
+        
+        # All tasks use the same session
+        results = await asyncio.gather(*tasks)
+        return results
+```
+
+### Custom Session Configuration
+
+```python
+import aiohttp
+import litellm
+
+# Create optimized session
+async with aiohttp.ClientSession(
+    timeout=aiohttp.ClientTimeout(total=180),
+    connector=aiohttp.TCPConnector(limit=300, limit_per_host=75)
+) as shared_session:
+    
+    response = await litellm.acompletion(
+        model="gpt-4o",
+        messages=[{"role": "user", "content": "Hello"}],
+        shared_session=shared_session
+    )
+```
+
+## Implementation Details
+
+The `shared_session` parameter is threaded through the entire LiteLLM call chain:
+
+1. **`acompletion()`** - Accepts `shared_session` parameter
+2. **`BaseLLMHTTPHandler`** - Passes session to HTTP client creation
+3. **`AsyncHTTPHandler`** - Uses existing session if provided
+4. **`LiteLLMAiohttpTransport`** - Reuses the session for HTTP requests
+
+## Backward Compatibility
+
+- **100% backward compatible** - Existing code works unchanged
+- **Optional parameter** - `shared_session=None` by default
+- **No breaking changes** - All existing functionality preserved
+
+## Testing
+
+Test the shared session functionality:
+
+```python
+import asyncio
+from aiohttp import ClientSession
+from litellm import acompletion
+
+async def test_shared_session():
+    async with ClientSession() as session:
+        print(f"✅ Created session: {id(session)}")
+        
+        try:
+            response = await acompletion(
+                model="gpt-4o",
+                messages=[{"role": "user", "content": "Hello"}],
+                shared_session=session,
+                api_key="your-api-key"
+            )
+            print(f"Response: {response.choices[0].message.content}")
+        except Exception as e:
+            print(f"✅ Expected error: {type(e).__name__}")
+        
+        print("✅ Session control working!")
+
+asyncio.run(test_shared_session())
+```
+
+## Files Modified
+
+The shared session functionality was added to these files:
+
+- `litellm/main.py` - Added `shared_session` parameter to `acompletion()` and `completion()`
+- `litellm/llms/custom_httpx/http_handler.py` - Core session reuse logic
+- `litellm/llms/custom_httpx/llm_http_handler.py` - HTTP handler integration
+- `litellm/llms/openai/openai.py` - OpenAI provider integration
+- `litellm/llms/openai/common_utils.py` - OpenAI client creation
+- `litellm/llms/azure/chat/o_series_handler.py` - Azure O Series handler
+
+## Troubleshooting
+
+### Session Not Being Reused
+
+1. **Check debug logs**: Enable `LITELLM_LOG=DEBUG` to see session reuse messages
+2. **Verify session is not closed**: Ensure the session is still active when making calls
+3. **Check parameter passing**: Make sure `shared_session` is passed to all `acompletion()` calls
+
+### Performance Issues
+
+1. **Session configuration**: Tune `aiohttp.ClientSession` parameters for your use case
+2. **Connection limits**: Adjust `limit` and `limit_per_host` in `TCPConnector`
+3. **Timeout settings**: Configure appropriate timeouts for your environment

docs/my-website/docs/completion/usage.md

@@ -26,6 +26,7 @@ response = completion(
 
 print(response.usage)
 ```
+> **Note:** LiteLLM supports endpoint bridging—if a model does not natively support a requested endpoint, LiteLLM will automatically route the call to the correct supported endpoint (such as bridging `/chat/completions` to `/responses` or vice versa) based on the model's `mode`set in `model_prices_and_context_window`.
 
 ## Streaming Usage
 

docs/my-website/docs/completion/web_fetch.md

@@ -0,0 +1,294 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Web Fetch
+
+The web fetch tool allows LLMs to retrieve full content from specified web pages and PDF documents. This enables AI models to access real-time information from the internet and incorporate web content into their responses. 
+
+## Web Fetch vs Web Search
+
+**Web Fetch** retrieves the full content from specific web pages that you provide URLs for, while **Web Search** performs internet searches to find relevant information based on your queries.
+
+| Feature | Web Fetch | Web Search |
+|---------|-----------|------------|
+| **Purpose** | Retrieve content from specific URLs | Search the internet for information |
+| **Input** | You provide exact URLs to fetch | You provide search queries/questions |
+| **Output** | Full page content from specified URLs | Search results with relevant information |
+| **Use Cases** | - Analyzing specific articles<br/>- Comparing content from known websites<br/>- Extracting data from particular pages | - Finding current news/events<br/>- Researching topics<br/>- Getting real-time information |
+
+
+**Example Web Fetch**: "Fetch the content from https://example.com/pricing and summarize it"  
+**Example Web Search**: "What are the latest AI developments this week?"
+
+**Supported Providers:**
+- Anthropic API (`anthropic/`)
+
+**Supported Tool Types:**
+- `web_fetch_20250910` - Web content retrieval tool with usage limits, domain filtering, and citation support
+
+
+## Quick Start
+
+### LiteLLM Python SDK
+
+```python
+import os 
+from litellm import completion
+
+os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
+
+# Web fetch tool
+tools = [
+    {
+        "type": "web_fetch_20250910",
+        "name": "web_fetch",
+        "max_uses": 5,
+    }
+]
+
+messages = [
+    {
+        "role": "user", 
+        "content": "Please analyze the content at https://example.com/article and summarize the main points"
+    }
+]
+
+response = completion(
+    model="anthropic/claude-3-5-sonnet-latest",
+    messages=messages,
+    tools=tools,
+)
+
+print(response)
+```
+
+### LiteLLM Proxy
+
+1. Define web fetch models on config.yaml
+
+```yaml
+model_list:
+  - model_name: claude-3-5-sonnet-latest # Anthropic claude-3-5-sonnet-latest
+    litellm_params:
+      model: anthropic/claude-3-5-sonnet-latest
+      api_key: os.environ/ANTHROPIC_API_KEY
+```
+
+2. Run proxy server
+
+```bash
+litellm --config config.yaml
+```
+
+3. Test it using the OpenAI Python SDK
+
+```python
+import os 
+from openai import OpenAI
+
+client = OpenAI(
+    api_key="sk-1234", # your litellm proxy api key
+    base_url="http://0.0.0.0:4000"
+)
+
+response = client.chat.completions.create(
+    model="claude-3-5-sonnet-latest",
+    messages=[
+        {
+            "role": "user", 
+            "content": "Please fetch and analyze the content from https://news.ycombinator.com and tell me about the top stories"
+        }
+    ],
+    tools=[
+        {
+            "type": "web_fetch_20250910",
+            "name": "web_fetch",
+            "max_uses": 5,
+        }
+    ]
+)
+
+print(response)
+```
+
+## Supported Models
+
+Web fetch is available on the following Anthropic API models:
+
+- `claude-opus-4-1-20250805` (Claude Opus 4.1)
+- `claude-opus-4-20250514` (Claude Opus 4)
+- `claude-sonnet-4-20250514` (Claude Sonnet 4)
+- `claude-3-7-sonnet-20250219` (Claude Sonnet 3.7)
+- `claude-3-5-sonnet-latest` (Claude Sonnet 3.5 v2 - deprecated)
+- `claude-3-5-haiku-latest` (Claude Haiku 3.5)
+
+:::note
+The web fetch tool currently does not support websites dynamically rendered via JavaScript.
+:::
+
+## Usage Examples
+
+### Basic Web Content Retrieval
+
+```python
+import os 
+from litellm import completion
+
+os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
+
+tools = [
+    {
+        "type": "web_fetch_20250910",
+        "name": "web_fetch",
+        "max_uses": 3,
+    }
+]
+
+messages = [
+    {
+        "role": "user",
+        "content": "Fetch the latest news from https://techcrunch.com and summarize the top 3 articles"
+    }
+]
+
+response = completion(
+    model="anthropic/claude-3-5-sonnet-latest",
+    messages=messages,
+    tools=tools,
+)
+
+print(response)
+```
+
+### Research and Analysis
+
+```python
+import os 
+from litellm import completion
+
+os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
+
+tools = [
+    {
+        "type": "web_fetch_20250910",
+        "name": "web_fetch", 
+        "max_uses": 10,
+    }
+]
+
+messages = [
+    {
+        "role": "user",
+        "content": "Research the latest developments in AI by fetching content from multiple tech news websites and provide a comprehensive analysis"
+    }
+]
+
+response = completion(
+    model="anthropic/claude-3-5-sonnet-latest",
+    messages=messages,
+    tools=tools,
+)
+
+print(response)
+```
+
+### Content Comparison
+
+```python
+import os 
+from litellm import completion
+
+os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
+
+tools = [
+    {
+        "type": "web_fetch_20250910",
+        "name": "web_fetch",
+        "max_uses": 5,
+    }
+]
+
+messages = [
+    {
+        "role": "user",
+        "content": "Compare the pricing information from https://openai.com/pricing and https://anthropic.com/pricing and create a comparison table"
+    }
+]
+
+response = completion(
+    model="anthropic/claude-3-5-sonnet-latest",
+    messages=messages,
+    tools=tools,
+)
+
+print(response)
+```
+
+## Advanced Usage with Multiple Tools
+
+You can combine web fetch with other tools like computer use or text editor:
+
+```python
+import os 
+from litellm import completion
+
+os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
+
+tools = [
+    {
+        "type": "web_fetch_20250910",
+        "name": "web_fetch",
+        "max_uses": 5,
+    },
+    {
+        "type": "text_editor_20250124", 
+        "name": "str_replace_editor"
+    }
+]
+
+messages = [
+    {
+        "role": "user",
+        "content": "Fetch the latest AI research papers from arXiv, analyze them, and create a detailed report file with your findings"
+    }
+]
+    
+response = completion(
+    model="anthropic/claude-3-5-sonnet-latest",
+    messages=messages,
+    tools=tools,
+)
+
+print(response)
+```
+
+## Spec
+
+### Web Fetch Tool (`web_fetch_20250910`)
+
+The web fetch tool supports the following parameters:
+
+```json
+{
+  "type": "web_fetch_20250910",
+  "name": "web_fetch",
+
+  // Optional: Limit the number of fetches per request
+  "max_uses": 10,
+
+  // Optional: Only fetch from these domains
+  "allowed_domains": ["example.com", "docs.example.com"],
+
+  // Optional: Never fetch from these domains
+  "blocked_domains": ["private.example.com"],
+
+  // Optional: Enable citations for fetched content
+  "citations": {
+    "enabled": true
+  },
+
+  // Optional: Maximum content length in tokens
+  "max_content_tokens": 100000
+}
+```
+

docs/my-website/docs/completion/web_search.md

@@ -1,17 +1,32 @@
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
 
-# Using Web Search
+# Web Search
 
 Use web search with litellm
 
 | Feature | Details |
 |---------|---------|
 | Supported Endpoints | - `/chat/completions` <br/> - `/responses` |
-| Supported Providers | `openai` |
+| Supported Providers | `openai`, `xai`, `vertex_ai`, `anthropic`, `gemini`, `perplexity` |
 | LiteLLM Cost Tracking | ✅ Supported |
-| LiteLLM Version | `v1.63.15-nightly` or higher |
+| LiteLLM Version | `v1.71.0+` |
 
+## Which Search Engine is Used?
+
+Each provider uses their own search backend:
+
+| Provider | Search Engine | Notes |
+|----------|---------------|-------|
+| **OpenAI** (`gpt-4o-search-preview`) | OpenAI's internal search | Real-time web data |
+| **xAI** (`grok-3`) | xAI's search + X/Twitter | Real-time social media data |
+| **Google AI/Vertex** (`gemini-2.0-flash`) | **Google Search** | Uses actual Google search results |
+| **Anthropic** (`claude-3-5-sonnet`) | Anthropic's web search | Real-time web data |
+| **Perplexity** | Perplexity's search engine | AI-powered search and reasoning |
+
+:::info
+**Anthropic Web Search Models**: Claude models that support web search: `claude-3-5-sonnet-latest`, `claude-3-5-sonnet-20241022`, `claude-3-5-haiku-latest`, `claude-3-5-haiku-20241022`, `claude-3-7-sonnet-20250219`
+:::
 
 ## `/chat/completions` (litellm.completion)
 
@@ -31,8 +46,12 @@ response = completion(
             "content": "What was a positive news story from today?",
         }
     ],
+    web_search_options={
+        "search_context_size": "medium"  # Options: "low", "medium", "high"
+    }
 )
 ```
+
 </TabItem>
 <TabItem value="proxy" label="PROXY">
 
@@ -40,10 +59,36 @@ response = completion(
 
 ```yaml
 model_list:
+  # OpenAI
   - model_name: gpt-4o-search-preview
     litellm_params:
       model: openai/gpt-4o-search-preview
       api_key: os.environ/OPENAI_API_KEY
+  
+  # xAI
+  - model_name: grok-3
+    litellm_params:
+      model: xai/grok-3
+      api_key: os.environ/XAI_API_KEY
+  
+  # Anthropic
+  - model_name: claude-3-5-sonnet-latest
+    litellm_params:
+      model: anthropic/claude-3-5-sonnet-latest
+      api_key: os.environ/ANTHROPIC_API_KEY
+  
+  # VertexAI
+  - model_name: gemini-2-flash
+    litellm_params:
+      model: gemini-2.0-flash
+      vertex_project: your-project-id
+      vertex_location: us-central1
+  
+  # Google AI Studio
+  - model_name: gemini-2-flash-studio
+    litellm_params:
+      model: gemini/gemini-2.0-flash
+      api_key: os.environ/GOOGLE_API_KEY
 ```
 
 2. Start the proxy 
@@ -64,7 +109,7 @@ client = OpenAI(
 )
 
 response = client.chat.completions.create(
-    model="gpt-4o-search-preview",
+    model="grok-3",  # or any other web search enabled model
     messages=[
         {
             "role": "user",
@@ -81,6 +126,7 @@ response = client.chat.completions.create(
 <Tabs>
 <TabItem value="sdk" label="SDK">
 
+**OpenAI (using web_search_options)**
 ```python showLineNumbers
 from litellm import completion
 
@@ -98,6 +144,69 @@ response = completion(
     }
 )
 ```
+
+**xAI (using web_search_options)**
+```python showLineNumbers
+from litellm import completion
+
+# Customize search context size for xAI
+response = completion(
+    model="xai/grok-3",
+    messages=[
+        {
+            "role": "user",
+            "content": "What was a positive news story from today?",
+        }
+    ],
+    web_search_options={
+        "search_context_size": "high"  # Options: "low", "medium" (default), "high"
+    }
+)
+```
+
+**Anthropic (using web_search_options)**
+```python showLineNumbers
+from litellm import completion
+
+# Customize search context size for Anthropic
+response = completion(
+    model="anthropic/claude-3-5-sonnet-latest",
+    messages=[
+        {
+            "role": "user",
+            "content": "What was a positive news story from today?",
+        }
+    ],
+    web_search_options={
+        "search_context_size": "medium",  # Options: "low", "medium" (default), "high"
+        "user_location": {
+            "type": "approximate",
+            "approximate": {
+                "city": "San Francisco",
+            },
+        }
+    }
+)
+```
+
+**VertexAI/Gemini (using web_search_options)**
+```python showLineNumbers
+from litellm import completion
+
+# Customize search context size for Gemini
+response = completion(
+    model="gemini-2.0-flash",
+    messages=[
+        {
+            "role": "user",
+            "content": "What was a positive news story from today?",
+        }
+    ],
+    web_search_options={
+        "search_context_size": "low"  # Options: "low", "medium" (default), "high"
+    }
+)
+```
 </TabItem>
 <TabItem value="proxy" label="PROXY">
 
@@ -112,7 +221,7 @@ client = OpenAI(
 
 # Customize search context size
 response = client.chat.completions.create(
-    model="gpt-4o-search-preview",
+    model="grok-3",  # works with any web search enabled model
     messages=[
         {
             "role": "user",
@@ -127,6 +236,8 @@ response = client.chat.completions.create(
 </TabItem>
 </Tabs>
 
+
+
 ## `/responses` (litellm.responses)
 
 ### Quick Start
@@ -243,35 +354,130 @@ print(response.output_text)
 </TabItem>
 </Tabs>
 
+## Configuring Web Search in config.yaml
 
+You can set default web search options directly in your proxy config file:
 
+<Tabs>
+<TabItem value="default" label="Default Web Search">
 
+```yaml
+model_list:
+  # Enable web search by default for all requests to this model
+  - model_name: grok-3
+    litellm_params:
+      model: xai/grok-3
+      api_key: os.environ/XAI_API_KEY
+      web_search_options: {}  # Enables web search with default settings
+```
 
+</TabItem>
+<TabItem value="custom" label="Custom Search Context">
+
+```yaml
+model_list:
+  # Set custom web search context size
+  - model_name: grok-3
+    litellm_params:
+      model: xai/grok-3
+      api_key: os.environ/XAI_API_KEY
+      web_search_options:
+        search_context_size: "high"  # Options: "low", "medium", "high"
+  
+  # Different context size for different models
+  - model_name: gpt-4o-search-preview
+    litellm_params:
+      model: openai/gpt-4o-search-preview
+      api_key: os.environ/OPENAI_API_KEY
+      web_search_options:
+        search_context_size: "low"
+  
+  # Gemini with medium context (default)
+  - model_name: gemini-2-flash
+    litellm_params:
+      model: gemini-2.0-flash
+      vertex_project: your-project-id
+      vertex_location: us-central1
+      web_search_options:
+        search_context_size: "medium"
+```
+
+</TabItem>
+</Tabs>
+
+**Note:** When `web_search_options` is set in the config, it applies to all requests to that model. Users can still override these settings by passing `web_search_options` in their API requests.
 
 ## Checking if a model supports web search
 
 <Tabs>
 <TabItem label="SDK" value="sdk">
 
-Use `litellm.supports_web_search(model="openai/gpt-4o-search-preview")` -> returns `True` if model can perform web searches
+Use `litellm.supports_web_search(model="model_name")` -> returns `True` if model can perform web searches
 
 ```python showLineNumbers
+# Check OpenAI models
 assert litellm.supports_web_search(model="openai/gpt-4o-search-preview") == True
+
+# Check xAI models
+assert litellm.supports_web_search(model="xai/grok-3") == True
+
+# Check Anthropic models
+assert litellm.supports_web_search(model="anthropic/claude-3-5-sonnet-latest") == True
+
+# Check VertexAI models
+assert litellm.supports_web_search(model="gemini-2.0-flash") == True
+
+# Check Google AI Studio models
+assert litellm.supports_web_search(model="gemini/gemini-2.0-flash") == True
 ```
 </TabItem>
 
 <TabItem label="PROXY" value="proxy">
 
-1. Define OpenAI models in config.yaml
+1. Define models in config.yaml
 
 ```yaml
 model_list:
+  # OpenAI
   - model_name: gpt-4o-search-preview
     litellm_params:
       model: openai/gpt-4o-search-preview
       api_key: os.environ/OPENAI_API_KEY
     model_info:
       supports_web_search: True
+  
+  # xAI
+  - model_name: grok-3
+    litellm_params:
+      model: xai/grok-3
+      api_key: os.environ/XAI_API_KEY
+    model_info:
+      supports_web_search: True
+  
+  # Anthropic
+  - model_name: claude-3-5-sonnet-latest
+    litellm_params:
+      model: anthropic/claude-3-5-sonnet-latest
+      api_key: os.environ/ANTHROPIC_API_KEY
+    model_info:
+      supports_web_search: True
+  
+  # VertexAI
+  - model_name: gemini-2-flash
+    litellm_params:
+      model: gemini-2.0-flash
+      vertex_project: your-project-id
+      vertex_location: us-central1
+    model_info:
+      supports_web_search: True
+  
+  # Google AI Studio
+  - model_name: gemini-2-flash-studio
+    litellm_params:
+      model: gemini/gemini-2.0-flash
+      api_key: os.environ/GOOGLE_API_KEY
+    model_info:
+      supports_web_search: True
 ```
 
 2. Run proxy server
@@ -298,7 +504,19 @@ Expected Response
       "model_group": "gpt-4o-search-preview",
       "providers": ["openai"],
       "max_tokens": 128000,
-      "supports_web_search": true, # 👈 supports_web_search is true
+      "supports_web_search": true
+    },
+    {
+      "model_group": "grok-3",
+      "providers": ["xai"],
+      "max_tokens": 131072,
+      "supports_web_search": true
+    },
+    {
+      "model_group": "gemini-2-flash",
+      "providers": ["vertex_ai"],
+      "max_tokens": 8192,
+      "supports_web_search": true
     }
   ]
 }

docs/my-website/docs/contact.md

@@ -2,5 +2,6 @@
 
 [![](https://dcbadge.vercel.app/api/server/wuPM9dRgDw)](https://discord.gg/wuPM9dRgDw)
 
+* [Community Slack 💭](https://join.slack.com/share/enQtOTE0ODczMzk2Nzk4NC01YjUxNjY2YjBlYTFmNDRiZTM3NDFiYTM3MzVkODFiMDVjOGRjMmNmZTZkZTMzOWQzZGQyZWIwYjQ0MWExYmE3)
 * [Meet with us 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version)
 * Contact us at ishaan@berri.ai / krrish@berri.ai

docs/my-website/docs/contributing.md

@@ -13,7 +13,9 @@ git clone https://github.com/BerriAI/litellm.git
 
 Tell the proxy where the UI is located
 ```bash
-export PROXY_BASE_URL="http://localhost:3000/"
+DATABASE_URL = "postgresql://<user>:<password>@<host>:<port>/<dbname>"
+LITELLM_MASTER_KEY = "sk-1234"
+STORE_MODEL_IN_DB = "True"
 ```
 
 ```bash
@@ -25,7 +27,7 @@ python3 proxy_cli.py --config /path/to/config.yaml --port 4000
 
 Set the mode as development (this will assume the proxy is running on localhost:4000)
 ```bash
-export NODE_ENV="development" 
+npm install # install dependencies
 ```
 
 ```bash

docs/my-website/docs/data_security.md

@@ -45,7 +45,7 @@ For security inquiries, please contact us at support@berri.ai
 | **Certification** | **Status**                                                                                      |
 |-------------------|-------------------------------------------------------------------------------------------------|
 | SOC 2 Type I      | Certified. Report available upon request on Enterprise plan.                                                           |
-| SOC 2 Type II     | In progress. Certificate available by April 15th, 2025                   |
+| SOC 2 Type II     | Certified. Report available upon request on Enterprise plan.                   |
 | ISO 27001          | Certified. Report available upon request on Enterprise                              |
 
 

docs/my-website/docs/embedding/supported_embedding.md

@@ -266,7 +266,59 @@ print(response)
 | Titan Embeddings - G1 | `embedding(model="amazon.titan-embed-text-v1", input=input)` |
 | Cohere Embeddings - English | `embedding(model="cohere.embed-english-v3", input=input)` |
 | Cohere Embeddings - Multilingual | `embedding(model="cohere.embed-multilingual-v3", input=input)` |
+| TwelveLabs Marengo (Async) | `embedding(model="bedrock/async_invoke/us.twelvelabs.marengo-embed-2-7-v1:0", input=input, input_type="text")` | [Async Invoke Docs](../providers/bedrock_embedding#async-invoke-embedding) |
 
+## TwelveLabs Bedrock Embedding Models
+
+TwelveLabs Marengo models support multimodal embeddings (text, image, video, audio) and require the `input_type` parameter to specify the input format.
+
+### Usage
+
+```python
+from litellm import embedding
+import os
+
+# Set AWS credentials
+os.environ["AWS_ACCESS_KEY_ID"] = ""
+os.environ["AWS_SECRET_ACCESS_KEY"] = ""
+os.environ["AWS_REGION_NAME"] = "us-east-1"
+
+# Text embedding
+response = embedding(
+    model="bedrock/us.twelvelabs.marengo-embed-2-7-v1:0",
+    input=["Hello world from LiteLLM!"],
+    input_type="text"  # Required parameter
+)
+
+# Image embedding (base64)
+response = embedding(
+    model="bedrock/async_invoke/us.twelvelabs.marengo-embed-2-7-v1:0",
+    input=["data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQ..."],
+    input_type="image",  # Required parameter
+    output_s3_uri="s3://your-bucket/async-invoke-output/"
+)
+
+# Video embedding (S3 URL)
+response = embedding(
+    model="bedrock/async_invoke/us.twelvelabs.marengo-embed-2-7-v1:0",
+    input=["s3://your-bucket/video.mp4"],
+    input_type="video",  # Required parameter
+    output_s3_uri="s3://your-bucket/async-invoke-output/"
+)
+```
+
+### Required Parameters
+
+| Parameter | Description | Values |
+|-----------|-------------|--------|
+| `input_type` | Type of input content | `"text"`, `"image"`, `"video"`, `"audio"` |
+
+### Supported Models
+
+| Model Name | Function Call | Notes |
+|------------|---------------|-------|
+| TwelveLabs Marengo 2.7 (Sync) | `embedding(model="bedrock/us.twelvelabs.marengo-embed-2-7-v1:0", input=input, input_type="text")` | Text embeddings only |
+| TwelveLabs Marengo 2.7 (Async) | `embedding(model="bedrock/async_invoke/us.twelvelabs.marengo-embed-2-7-v1:0", input=input, input_type="text/image/video/audio")` | All input types, requires `output_s3_uri` |
 
 ## Cohere Embedding Models
 https://docs.cohere.com/reference/embed

docs/my-website/docs/enterprise.md

@@ -1,12 +1,19 @@
 import Image from '@theme/IdealImage';
 
 # Enterprise
+
+:::info
+✨ SSO is free for up to 5 users. After that, an enterprise license is required. [Get Started with Enterprise here](https://www.litellm.ai/enterprise)
+:::
+
 For companies that need SSO, user management and professional support for LiteLLM Proxy
 
 :::info
-Get free 7-day trial key [here](https://www.litellm.ai/#trial)
+Get free 7-day trial key [here](https://www.litellm.ai/enterprise#trial)
 :::
 
+## Enterprise Features
+
 Includes all enterprise features.
 
 <Image img={require('../img/enterprise_vs_oss.png')} />
@@ -18,32 +25,13 @@ This covers:
 - [**Enterprise Features**](./proxy/enterprise)
 - ✅ **Feature Prioritization**
 - ✅ **Custom Integrations**
-- ✅ **Professional Support - Dedicated discord + slack**
+- ✅ **Professional Support - Dedicated Slack/Teams channel**
 
 
-Deployment Options:
+## Self-Hosted
 
-**Self-Hosted**
-1. Manage Yourself - you can deploy our Docker Image or build a custom image from our pip package, and manage your own infrastructure. In this case, we would give you a license key + provide support via a dedicated support channel. 
+Manage Yourself - you can deploy our Docker Image or build a custom image from our pip package, and manage your own infrastructure. In this case, we would give you a license key + provide support via a dedicated support channel. 
 
-2. We Manage - you give us subscription access on your AWS/Azure/GCP account, and we manage the deployment.
-
-**Managed**
-
-You can use our cloud product where we setup a dedicated instance for you. 
-
-## Frequently Asked Questions
-
-### SLA's + Professional Support
-
-Professional Support can assist with LLM/Provider integrations, deployment, upgrade management, and LLM Provider troubleshooting.  We can’t solve your own infrastructure-related issues but we will guide you to fix them.
-
-- 1 hour for Sev0 issues - 100% production traffic is failing
-- 6 hours for Sev1 - <100% production traffic is failing
-- 24h for Sev2-Sev3 between 7am – 7pm PT (Monday through Saturday) - setup issues e.g. Redis working on our end, but not on your infrastructure.
-- 72h SLA for patching vulnerabilities in the software. 
-
-**We can offer custom SLAs** based on your needs and the severity of the issue
 
 ### What’s the cost of the Self-Managed Enterprise edition?
 
@@ -58,8 +46,72 @@ You just deploy [our docker image](https://docs.litellm.ai/docs/proxy/deploy) an
 LITELLM_LICENSE="eyJ..."
 ```
 
-No data leaves your environment. 
+**No data leaves your environment.** 
+
+
+## Hosted LiteLLM Proxy
+
+LiteLLM maintains the proxy, so you can focus on your core products. 
+
+We provide a dedicated proxy for your team, and manage the infrastructure. 
+
+### **Status**: GA 
+
+Our proxy is already used in production by customers. 
+
+See our status page for [**live reliability**](https://status.litellm.ai/)
+
+### **Benefits**
+- **No Maintenance, No Infra**: We'll maintain the proxy, and spin up any additional infrastructure (e.g.: separate server for spend logs) to make sure you can load balance + track spend across multiple LLM projects. 
+- **Reliable**: Our hosted proxy is tested on 1k requests per second, making it reliable for high load.
+- **Secure**: LiteLLM is SOC-2 Type 2 and ISO 27001 certified, to make sure your data is as secure as possible.
+
+### Supported data regions for LiteLLM Cloud
+
+You can find [supported data regions litellm here](../docs/data_security#supported-data-regions-for-litellm-cloud)
+
+
+## Frequently Asked Questions
+
+### SLA's + Professional Support
+
+Professional Support can assist with LLM/Provider integrations, deployment, upgrade management, and LLM Provider troubleshooting.  We can’t solve your own infrastructure-related issues but we will guide you to fix them.
+
+- 1 hour for Sev0 issues - 100% production traffic is failing
+- 6 hours for Sev1 - < 100% production traffic is failing
+- 24h for Sev2-Sev3 between 7am – 7pm PT (Monday through Saturday) - setup issues e.g. Redis working on our end, but not on your infrastructure.
+- 72h SLA for patching vulnerabilities in the software. 
+
+**We can offer custom SLAs** based on your needs and the severity of the issue
 
 ## Data Security / Legal / Compliance FAQs
 
-[Data Security / Legal / Compliance FAQs](./data_security.md)
+[Data Security / Legal / Compliance FAQs](./data_security.md)
+
+
+### Pricing
+
+Pricing is based on usage. We can figure out a price that works for your team, on the call. 
+
+[**Contact Us to learn more**](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
+
+
+
+## **Screenshots**
+
+### 1. Create keys
+
+<Image img={require('../img/litellm_hosted_ui_create_key.png')} />
+
+### 2. Add Models
+
+<Image img={require('../img/litellm_hosted_ui_add_models.png')}/>
+
+### 3. Track spend 
+
+<Image img={require('../img/litellm_hosted_usage_dashboard.png')} />
+
+
+### 4. Configure load balancing 
+
+<Image img={require('../img/litellm_hosted_ui_router.png')} />

docs/my-website/docs/exception_mapping.md

@@ -12,6 +12,7 @@ All exceptions can be imported from `litellm` - e.g. `from litellm import BadReq
 | 400 | UnsupportedParamsError | litellm.BadRequestError | Raised when unsupported params are passed |
 | 400         | ContextWindowExceededError| litellm.BadRequestError | Special error type for context window exceeded error messages - enables context window fallbacks |
 | 400         | ContentPolicyViolationError| litellm.BadRequestError | Special error type for content policy violation error messages - enables content policy fallbacks |
+| 400         | ImageFetchError | litellm.BadRequestError | Raised when there are errors fetching or processing images |
 | 400 | InvalidRequestError | openai.BadRequestError | Deprecated error, use BadRequestError instead |
 | 401         | AuthenticationError      | openai.AuthenticationError |
 | 403         | PermissionDeniedError    | openai.PermissionDeniedError |

docs/my-website/docs/extras/contributing_code.md

@@ -39,14 +39,14 @@ That's it, your local dev environment is ready!
 
 ## 2. Adding Testing to your PR
 
-- Add your test to the [`tests/litellm/` directory](https://github.com/BerriAI/litellm/tree/main/tests/litellm)
+- Add your test to the [`tests/test_litellm/` directory](https://github.com/BerriAI/litellm/tree/main/tests/litellm)
 
 - This directory 1:1 maps the the `litellm/` directory, and can only contain mocked tests.
 - Do not add real llm api calls to this directory.
 
-### 2.1 File Naming Convention for `tests/litellm/`
+### 2.1 File Naming Convention for `tests/test_litellm/`
 
-The `tests/litellm/` directory follows the same directory structure as `litellm/`.
+The `tests/test_litellm/` directory follows the same directory structure as `litellm/`.
 
 - `litellm/proxy/test_caching_routes.py` maps to `litellm/proxy/caching_routes.py`
 - `test_{filename}.py` maps to `litellm/{filename}.py`

docs/my-website/docs/extras/gemini_img_migration.md

@@ -0,0 +1,220 @@
+# Gemini Image Generation Migration Guide
+
+## Who is impacted by this change?
+
+Anyone using the following models with /chat/completions:
+- `gemini/gemini-2.0-flash-exp-image-generation`
+- `vertex_ai/gemini-2.0-flash-exp-image-generation`
+
+## Key Change
+
+:::info
+From v1.77.0, LiteLLM will return the List of images in `response.choices[0].message.images` instead of a single image in `response.choices[0].message.image`.
+:::
+
+Gemini models now support image generation through chat completions. Images are returned in `response.choices[0].message.images` with base64 data URLs.
+
+## Before and After
+
+### Before
+```python
+from litellm import completion
+
+response = completion(
+    model="gemini/gemini-2.0-flash-exp-image-generation",
+    messages=[{"role": "user", "content": "Generate an image of a cat"}],
+    modalities=["image", "text"],
+)
+
+
+base_64_image_data = response.choices[0].message.content
+```
+
+### After  
+```python
+from litellm import completion
+
+response = completion(
+    model="gemini/gemini-2.0-flash-exp-image-generation",
+    messages=[{"role": "user", "content": "Generate an image of a cat"}],
+    modalities=["image", "text"],
+)
+
+# Image is now available in the response
+image_url = response.choices[0].message.images[0]["image_url"]["url"]  # "data:image/png;base64,..."
+```
+
+### Why the change?
+
+Because the newer `gemini-2.5-flash-image-preview` model sends both text and image responses in the same response. This interface allows a developer to explicitly access the image or text components of the response. Before a developer would have needed to search through the message content to find the image generated by the model.
+
+**Why the change from `image` to `images`?**
+This is to be consistent with the OpenRouter API, making sure we are using simple, well-known interfaces where possible.
+
+## Usage
+
+### Using the Python SDK
+
+**Key Change:**
+```diff
+# Before
+-- base_64_image_data = response.choices[0].message.content
+
+# After
+++ image_url = response.choices[0].message.images[0]["image_url"]["url"]
+```
+
+#### Basic Image Generation
+
+```python
+from litellm import completion
+import os
+
+# Set your API key
+os.environ["GEMINI_API_KEY"] = "your-api-key"
+
+# Generate an image
+response = completion(
+    model="gemini/gemini-2.0-flash-exp-image-generation",
+    messages=[{"role": "user", "content": "Generate an image of a cat"}],
+    modalities=["image", "text"],
+)
+
+# Access the generated image
+print(response.choices[0].message.content)  # Text response (if any)
+print(response.choices[0].message.images[0])    # Image data
+```
+
+#### Response Format
+
+The image is returned in the `message.images` field:
+
+```python
+{
+    "image_url": {
+        "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...",
+        "detail": "auto"
+    },
+    "index": 0,
+    "type": "image_url"
+}
+```
+
+### Using the LiteLLM Proxy Server
+
+**Key Change:**
+```diff
+# Before
+-- "content": "base64-image-data..."
+
+# After  
+++ "images": [{
+++   "image_url": {
+++     "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...",
+++     "detail": "auto"
+++   },
+++   "index": 0,
+++   "type": "image_url"
+++ }]
+```
+
+#### Configuration Setup
+
+1. **Configure your models in `config.yaml`:**
+
+```yaml
+model_list:
+  - model_name: gemini-image-gen
+    litellm_params:
+      model: gemini/gemini-2.0-flash-exp-image-generation
+      api_key: os.environ/GEMINI_API_KEY
+  - model_name: vertex-image-gen  
+    litellm_params:
+      model: vertex_ai/gemini-2.5-flash-image-preview
+      vertex_project: your-project-id
+      vertex_location: us-central1
+
+general_settings:
+  master_key: sk-1234  # Your proxy API key
+```
+
+2. **Start the proxy server:**
+
+```bash
+litellm --config /path/to/config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+#### Making Requests
+
+**Using OpenAI SDK:**
+
+```python
+from openai import OpenAI
+
+# Point to your proxy server
+client = OpenAI(
+    api_key="sk-1234",  # Your proxy API key
+    base_url="http://0.0.0.0:4000"
+)
+
+response = client.chat.completions.create(
+    model="gemini-image-gen",
+    messages=[{"role": "user", "content": "Generate an image of a cat"}],
+    extra_body={"modalities": ["image", "text"]}
+)
+
+# Access the generated image
+print(response.choices[0].message.content)  # Text response (if any)
+print(response.choices[0].message.image)    # Image data
+```
+
+**Using curl:**
+
+```bash
+curl -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
+-H 'Content-Type: application/json' \
+-H 'Authorization: Bearer sk-1234' \
+-d '{
+  "model": "gemini-image-gen",
+  "messages": [
+    {
+      "role": "user",
+      "content": "Generate an image of a cat"
+    }
+  ],
+  "modalities": ["image", "text"]
+}'
+```
+
+**Response format from proxy:**
+
+```json
+{
+  "id": "chatcmpl-123",
+  "object": "chat.completion",
+  "created": 1704089632,
+  "model": "gemini-image-gen",
+  "choices": [
+    {
+      "index": 0,
+      "message": {
+        "role": "assistant",
+        "content": "Here's an image of a cat for you!",
+        "images": [{
+          "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...",
+          "detail": "auto"
+        }
+      },
+      "finish_reason": "stop"
+    }
+  ],
+  "usage": {
+    "prompt_tokens": 10,
+    "completion_tokens": 8,
+    "total_tokens": 18
+  }
+}
+```
+

docs/my-website/docs/fine_tuning.md

@@ -13,6 +13,8 @@ This is an Enterprise only endpoint [Get Started with Enterprise here](https://c
 | Feature | Supported | Notes | 
 |-------|-------|-------|
 | Supported Providers | OpenAI, Azure OpenAI, Vertex AI | - |
+
+#### ⚡️See an exhaustive list of supported models and providers at [models.litellm.ai](https://models.litellm.ai/)
 | Cost Tracking | 🟡 | [Let us know if you need this](https://github.com/BerriAI/litellm/issues) |
 | Logging | ✅ | Works across all logging integrations |
 

docs/my-website/docs/generateContent.md

@@ -0,0 +1,236 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Google AI generateContent
+
+Use LiteLLM to call Google AI's generateContent endpoints for text generation, multimodal interactions, and streaming responses.
+
+## Overview 
+
+| Feature | Supported | Notes | 
+|-------|-------|-------|
+| Cost Tracking | ✅ |  |
+| Logging | ✅ | works across all integrations |
+| End-user Tracking | ✅ | |
+| Streaming | ✅ | |
+| Fallbacks | ✅ | between supported models |
+| Loadbalancing | ✅ | between supported models |
+
+## Usage 
+---
+
+### LiteLLM Python SDK 
+
+<Tabs>
+<TabItem value="basic" label="Basic Usage">
+
+#### Non-streaming example
+```python showLineNumbers title="Basic Text Generation"
+from litellm.google_genai import agenerate_content
+from google.genai.types import ContentDict, PartDict
+import os
+
+# Set API key
+os.environ["GEMINI_API_KEY"] = "your-gemini-api-key"
+
+contents = ContentDict(
+    parts=[
+        PartDict(text="Hello, can you tell me a short joke?")
+    ],
+    role="user",
+)
+
+response = await agenerate_content(
+    contents=contents,
+    model="gemini/gemini-2.0-flash",
+    max_tokens=100,
+)
+print(response)
+```
+
+#### Streaming example
+```python showLineNumbers title="Streaming Text Generation"
+from litellm.google_genai import agenerate_content_stream
+from google.genai.types import ContentDict, PartDict
+import os
+
+# Set API key
+os.environ["GEMINI_API_KEY"] = "your-gemini-api-key"
+
+contents = ContentDict(
+    parts=[
+        PartDict(text="Write a long story about space exploration")
+    ],
+    role="user",
+)
+
+response = await agenerate_content_stream(
+    contents=contents,
+    model="gemini/gemini-2.0-flash",
+    max_tokens=500,
+)
+
+async for chunk in response:
+    print(chunk)
+```
+
+</TabItem>
+
+<TabItem value="sync" label="Sync Usage">
+
+#### Sync non-streaming example
+```python showLineNumbers title="Sync Text Generation"
+from litellm.google_genai import generate_content
+from google.genai.types import ContentDict, PartDict
+import os
+
+# Set API key
+os.environ["GEMINI_API_KEY"] = "your-gemini-api-key"
+
+contents = ContentDict(
+    parts=[
+        PartDict(text="Hello, can you tell me a short joke?")
+    ],
+    role="user",
+)
+
+response = generate_content(
+    contents=contents,
+    model="gemini/gemini-2.0-flash",
+    max_tokens=100,
+)
+print(response)
+```
+
+#### Sync streaming example
+```python showLineNumbers title="Sync Streaming Text Generation"
+from litellm.google_genai import generate_content_stream
+from google.genai.types import ContentDict, PartDict
+import os
+
+# Set API key
+os.environ["GEMINI_API_KEY"] = "your-gemini-api-key"
+
+contents = ContentDict(
+    parts=[
+        PartDict(text="Write a long story about space exploration")
+    ],
+    role="user",
+)
+
+response = generate_content_stream(
+    contents=contents,
+    model="gemini/gemini-2.0-flash",
+    max_tokens=500,
+)
+
+for chunk in response:
+    print(chunk)
+```
+
+</TabItem>
+</Tabs>
+
+### LiteLLM Proxy Server 
+
+1. Setup config.yaml
+
+```yaml
+model_list:
+    - model_name: gemini-flash
+      litellm_params:
+        model: gemini/gemini-2.0-flash
+        api_key: os.environ/GEMINI_API_KEY
+```
+
+2. Start proxy 
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+3. Test it! 
+
+<Tabs>
+<TabItem value="gemini-proxy" label="Google GenAI SDK">
+
+```python showLineNumbers title="Google GenAI SDK with LiteLLM Proxy"
+from google.genai import Client
+import os
+
+# Configure Google GenAI SDK to use LiteLLM proxy
+os.environ["GOOGLE_GEMINI_BASE_URL"] = "http://localhost:4000"
+os.environ["GEMINI_API_KEY"] = "sk-1234"
+
+client = Client()
+
+response = client.models.generate_content(
+    model="gemini-flash",
+    contents=[
+        {
+            "parts": [{"text": "Write a short story about AI"}],
+            "role": "user"
+        }
+    ],
+    config={"max_output_tokens": 100}
+)
+```
+
+
+</TabItem>
+
+<TabItem value="curl-proxy" label="curl">
+
+#### Generate Content
+
+```bash showLineNumbers title="generateContent via LiteLLM Proxy"
+curl -L -X POST 'http://localhost:4000/v1beta/models/gemini-flash:generateContent' \
+-H 'content-type: application/json' \
+-H 'authorization: Bearer sk-1234' \
+-d '{
+  "contents": [
+    {
+      "parts": [
+        {
+          "text": "Write a short story about AI"
+        }
+      ],
+      "role": "user"
+    }
+  ],
+  "generationConfig": {
+    "maxOutputTokens": 100
+  }
+}'
+```
+
+#### Stream Generate Content
+
+```bash showLineNumbers title="streamGenerateContent via LiteLLM Proxy"
+curl -L -X POST 'http://localhost:4000/v1beta/models/gemini-flash:streamGenerateContent' \
+-H 'content-type: application/json' \
+-H 'authorization: Bearer sk-1234' \
+-d '{
+  "contents": [
+    {
+      "parts": [
+        {
+          "text": "Write a long story about space exploration"
+        }
+      ],
+      "role": "user"
+    }
+  ],
+  "generationConfig": {
+    "maxOutputTokens": 500
+  }
+}'
+```
+
+</TabItem>
+</Tabs>
+
+
+## Related 
+
+- [Use LiteLLM with gemini-cli](../docs/tutorials/litellm_gemini_cli)

docs/my-website/docs/getting_started.md

@@ -32,7 +32,8 @@ Next Steps 👉 [Call all supported models - e.g. Claude-2, Llama2-70b, etc.](./
 More details 👉
 
 - [Completion() function details](./completion/)
-- [All supported models / providers on LiteLLM](./providers/)
+- [Overview of supported models / providers on LiteLLM](./providers/)
+- [Search all models / providers](https://models.litellm.ai/)
 - [Build your own OpenAI proxy](https://github.com/BerriAI/liteLLM-proxy/tree/main)
 
 ## streaming

docs/my-website/docs/guides/security_settings.md

@@ -1,14 +1,45 @@
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
 
-# SSL Security Settings
+# SSL, HTTP Proxy Security Settings
 
-If you're in an environment using an older TTS bundle, with an older encryption, follow this guide. 
+If you're in an environment using an older TTS bundle, with an older encryption, follow this guide. By default
+LiteLLM uses the certifi CA bundle for SSL verification, which is compatible with most modern servers.
+ However, if you need to disable SSL verification or use a custom CA bundle, you can do so by following the steps below.
 
+Be aware that environmental variables take precedence over the settings in the SDK.
 
-LiteLLM uses HTTPX for network requests, unless otherwise specified. 
+LiteLLM uses HTTPX for network requests, unless otherwise specified.
 
-1. Disable SSL verification
+## 1. Custom CA Bundle
+
+You can set a custom CA bundle file path using the `SSL_CERT_FILE` environmental variable or passing a string to the the ssl_verify setting.
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+import litellm
+litellm.ssl_verify = "client.pem"
+```
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+```yaml
+litellm_settings:
+  ssl_verify: "client.pem"
+```
+
+</TabItem>  
+<TabItem value="env_var" label="Environment Variables">
+
+```bash
+export SSL_CERT_FILE="client.pem"
+```
+</TabItem>
+</Tabs>
+
+## 2. Disable SSL verification
 
 
 <Tabs>
@@ -35,14 +66,42 @@ export SSL_VERIFY="False"
 </TabItem>
 </Tabs>
 
-2. Lower security settings
+## 3. Lower security settings
+
+The `ssl_security_level` allows setting a lower security level for SSL connections.
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+import litellm
+litellm.ssl_security_level = "DEFAULT@SECLEVEL=1"
+```
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+```yaml
+litellm_settings:
+  ssl_security_level: "DEFAULT@SECLEVEL=1"
+```
+</TabItem>
+<TabItem value="env_var" label="Environment Variables">
+
+```bash
+export SSL_SECURITY_LEVEL="DEFAULT@SECLEVEL=1"
+```
+</TabItem>
+</Tabs>
+
+## 4. Certificate authentication
+
+The `SSL_CERTIFICATE` environmental variable or `ssl_certificate` attribute allows setting a client side certificate to authenticate the client to the server.
 
 <Tabs>
 <TabItem value="sdk" label="SDK">
 
 ```python
 import litellm
-litellm.ssl_security_level = 1
 litellm.ssl_certificate = "/path/to/certificate.pem"
 ```
 </TabItem>
@@ -50,17 +109,40 @@ litellm.ssl_certificate = "/path/to/certificate.pem"
 
 ```yaml
 litellm_settings:
-  ssl_security_level: 1
   ssl_certificate: "/path/to/certificate.pem"
 ```
 </TabItem>
 <TabItem value="env_var" label="Environment Variables">
 
 ```bash
-export SSL_SECURITY_LEVEL="1"
 export SSL_CERTIFICATE="/path/to/certificate.pem"
 ```
 </TabItem>
 </Tabs>
 
+## 5. Use HTTP_PROXY environment variable
+
+Both httpx and aiohttp libraries use `urllib.request.getproxies` from environment variables. Before client initialization, you may set proxy (and optional SSL_CERT_FILE) by setting the environment variables:
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+import litellm
+litellm.aiohttp_trust_env = True
+```
+
+```bash
+export HTTPS_PROXY='http://username:password@proxy_uri:port'
+```
+</TabItem>
+
+<TabItem value="proxy" label="PROXY">
+
+```bash
+export HTTPS_PROXY='http://username:password@proxy_uri:port'
+export AIOHTTP_TRUST_ENV='True'
+```
+</TabItem>
+</Tabs>
 

docs/my-website/docs/hosted.md

@@ -1,66 +0,0 @@
-import Image from '@theme/IdealImage';
-
-# Hosted LiteLLM Proxy
-
-LiteLLM maintains the proxy, so you can focus on your core products. 
-
-## [**Get Onboarded**](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
-
-This is in alpha. Schedule a call with us, and we'll give you a hosted proxy within 30 minutes. 
-
-[**🚨 Schedule Call**](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
-
-### **Status**: Alpha 
-
-Our proxy is already used in production by customers. 
-
-See our status page for [**live reliability**](https://status.litellm.ai/)
-
-### **Benefits**
-- **No Maintenance, No Infra**: We'll maintain the proxy, and spin up any additional infrastructure (e.g.: separate server for spend logs) to make sure you can load balance + track spend across multiple LLM projects. 
-- **Reliable**: Our hosted proxy is tested on 1k requests per second, making it reliable for high load.
-- **Secure**: LiteLLM is currently undergoing SOC-2 compliance, to make sure your data is as secure as possible.
-
-## Data Privacy & Security
-
-You can find our [data privacy & security policy for cloud litellm here](../docs/data_security#litellm-cloud)
-
-## Supported data regions for LiteLLM Cloud
-
-You can find [supported data regions litellm here](../docs/data_security#supported-data-regions-for-litellm-cloud)
-
-### Pricing
-
-Pricing is based on usage. We can figure out a price that works for your team, on the call. 
-
-[**🚨 Schedule Call**](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
-
-## **Screenshots**
-
-### 1. Create keys
-
-<Image img={require('../img/litellm_hosted_ui_create_key.png')} />
-
-### 2. Add Models
-
-<Image img={require('../img/litellm_hosted_ui_add_models.png')}/>
-
-### 3. Track spend 
-
-<Image img={require('../img/litellm_hosted_usage_dashboard.png')} />
-
-
-### 4. Configure load balancing 
-
-<Image img={require('../img/litellm_hosted_ui_router.png')} />
-
-#### [**🚨 Schedule Call**](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
-
-## Feature List 
-
-- Easy way to add/remove models
-- 100% uptime even when models are added/removed
-- custom callback webhooks
-- your domain name with HTTPS
-- Ability to create/delete User API keys
-- Reasonable set monthly cost

docs/my-website/docs/image_edits.md

@@ -4,7 +4,7 @@ import TabItem from '@theme/TabItem';
 
 # /images/edits
 
-LiteLLM provides image editing functionality that maps to OpenAI's `/images/edits` API endpoint.
+LiteLLM provides image editing functionality that maps to OpenAI's `/images/edits` API endpoint. Now supports both single and multiple image editing.
 
 | Feature | Supported | Notes |
 |---------|-----------|--------|
@@ -13,11 +13,14 @@ LiteLLM provides image editing functionality that maps to OpenAI's `/images/edit
 | End-user Tracking | ✅ | |
 | Fallbacks | ✅ | Works between supported models |
 | Loadbalancing | ✅ | Works between supported models |
-| Supported operations | Create image edits | |
+| Supported operations | Create image edits | Single and multiple images supported |
 | Supported LiteLLM SDK Versions | 1.63.8+ | |
 | Supported LiteLLM Proxy Versions | 1.71.1+ | |
 | Supported LLM providers | **OpenAI** | Currently only `openai` is supported |
 
+ #### ⚡️See all supported models and providers at [models.litellm.ai](https://models.litellm.ai/)
+
+
 ## Usage
 
 ### LiteLLM Python SDK
@@ -41,6 +44,26 @@ response = litellm.image_edit(
 print(response)
 ```
 
+#### Multiple Images Edit
+```python showLineNumbers title="OpenAI Multiple Images Edit"
+import litellm
+
+# Edit multiple images with a prompt
+response = litellm.image_edit(
+    model="gpt-image-1",
+    image=[
+        open("image1.png", "rb"),
+        open("image2.png", "rb"),
+        open("image3.png", "rb")
+    ],
+    prompt="Apply vintage filter to all images",
+    n=1,
+    size="1024x1024"
+)
+
+print(response)
+```
+
 #### Image Edit with Mask
 ```python showLineNumbers title="OpenAI Image Edit with Mask"
 import litellm
@@ -80,6 +103,30 @@ response = asyncio.run(edit_image())
 print(response)
 ```
 
+#### Async Multiple Images Edit
+```python showLineNumbers title="Async OpenAI Multiple Images Edit"
+import litellm
+import asyncio
+
+async def edit_multiple_images():
+    response = await litellm.aimage_edit(
+        model="gpt-image-1",
+        image=[
+            open("portrait1.png", "rb"),
+            open("portrait2.png", "rb")
+        ],
+        prompt="Add professional lighting to the portraits",
+        n=1,
+        size="1024x1024",
+        response_format="url"
+    )
+    return response
+
+# Run the async function
+response = asyncio.run(edit_multiple_images())
+print(response)
+```
+
 #### Image Edit with Custom Parameters
 ```python showLineNumbers title="OpenAI Image Edit with Custom Parameters"
 import litellm
@@ -163,6 +210,20 @@ curl -X POST "http://localhost:4000/v1/images/edits" \
   -F "response_format=url"
 ```
 
+#### cURL Multiple Images Example
+```bash showLineNumbers title="cURL Multiple Images Edit Request"
+curl -X POST "http://localhost:4000/v1/images/edits" \
+  -H "Authorization: Bearer your-api-key" \
+  -F "model=gpt-image-1" \
+  -F "image=@image1.png" \
+  -F "image=@image2.png" \
+  -F "image=@image3.png" \
+  -F "prompt=Apply artistic filter to all images" \
+  -F "n=1" \
+  -F "size=1024x1024" \
+  -F "response_format=url"
+```
+
 </TabItem>
 </Tabs>
 

docs/my-website/docs/image_generation.md

@@ -52,7 +52,7 @@ litellm --config /path/to/config.yaml
 curl -X POST 'http://0.0.0.0:4000/v1/images/generations' \
 -H 'Content-Type: application/json' \
 -H 'Authorization: Bearer sk-1234' \
--D '{
+-d '{
     "model": "gpt-image-1",
     "prompt": "A cute baby sea otter",
     "n": 1,
@@ -154,7 +154,7 @@ Any non-openai params, will be treated as provider-specific params, and sent in
 ## OpenAI Image Generation Models
 
 ### Usage
-```python
+```python showLineNumbers
 from litellm import image_generation
 import os
 os.environ['OPENAI_API_KEY'] = ""
@@ -171,7 +171,7 @@ response = image_generation(model='gpt-image-1', prompt="cute baby otter")
 
 ### API keys
 This can be set as env variables or passed as **params to litellm.image_generation()**
-```python
+```python showLineNumbers
 import os
 os.environ['AZURE_API_KEY'] = 
 os.environ['AZURE_API_BASE'] = 
@@ -179,7 +179,7 @@ os.environ['AZURE_API_VERSION'] =
 ```
 
 ### Usage
-```python
+```python showLineNumbers
 from litellm import embedding
 response = embedding(
     model="azure/<your deployment name>",
@@ -197,6 +197,34 @@ print(response)
 | dall-e-3 | `image_generation(model="azure/<your deployment name>", prompt="cute baby otter")` |
 | dall-e-2 | `image_generation(model="azure/<your deployment name>", prompt="cute baby otter")` |
 
+## Xinference Image Generation Models
+
+Use this for Stable Diffusion models hosted on Xinference
+
+#### Usage
+
+See Xinference usage with LiteLLM [here](./providers/xinference.md#image-generation)
+
+## Recraft Image Generation Models
+
+Use this for AI-powered design and image generation with Recraft
+
+#### Usage
+
+```python showLineNumbers
+from litellm import image_generation
+import os
+
+os.environ['RECRAFT_API_KEY'] = "your-api-key"
+
+response = image_generation(
+    model="recraft/recraftv3",
+    prompt="A beautiful sunset over a calm ocean",
+)
+print(response)
+```
+
+See Recraft usage with LiteLLM [here](./providers/recraft.md#image-generation)
 
 ## OpenAI Compatible Image Generation Models
 Use this for calling `/image_generation` endpoints on OpenAI Compatible Servers, example https://github.com/xorbitsai/inference
@@ -204,7 +232,7 @@ Use this for calling `/image_generation` endpoints on OpenAI Compatible Servers,
 **Note add `openai/` prefix to model so litellm knows to route to OpenAI**
 
 ### Usage
-```python
+```python showLineNumbers
 from litellm import image_generation
 response = image_generation(
   model = "openai/<your-llm-name>",     # add `openai/` prefix to model so litellm knows to route to OpenAI
@@ -218,7 +246,7 @@ Use this for stable diffusion on bedrock
 
 
 ### Usage
-```python
+```python showLineNumbers
 import os
 from litellm import image_generation
 
@@ -239,7 +267,7 @@ print(f"response: {response}")
 
 Use this for image generation models on VertexAI
 
-```python
+```python showLineNumbers
 response = litellm.image_generation(
     prompt="An olympic size swimming pool",
     model="vertex_ai/imagegeneration@006",
@@ -248,3 +276,18 @@ response = litellm.image_generation(
 )
 print(f"response: {response}")
 ```
+
+## Supported Providers
+
+#### ⚡️See all supported models and providers at [models.litellm.ai](https://models.litellm.ai/)
+
+| Provider | Documentation Link |
+|----------|-------------------|
+| OpenAI | [OpenAI Image Generation →](./providers/openai) |
+| Azure OpenAI | [Azure OpenAI Image Generation →](./providers/azure/azure) |
+| Google AI Studio | [Google AI Studio Image Generation →](./providers/google_ai_studio/image_gen) |
+| Vertex AI | [Vertex AI Image Generation →](./providers/vertex_image) |
+| AWS Bedrock | [Bedrock Image Generation →](./providers/bedrock) |
+| Recraft | [Recraft Image Generation →](./providers/recraft#image-generation) |
+| Xinference | [Xinference Image Generation →](./providers/xinference#image-generation) |
+| Nscale | [Nscale Image Generation →](./providers/nscale#image-generation) | 

docs/my-website/docs/index.md

@@ -226,6 +226,23 @@ response = completion(
 
 </TabItem>
 
+<TabItem value="vercel" label="Vercel AI Gateway">
+
+```python
+from litellm import completion
+import os
+
+## set ENV variables. Visit https://vercel.com/docs/ai-gateway#using-the-ai-gateway-with-an-api-key for insturctions on obtaining a key
+os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "your-vercel-api-key"
+
+response = completion(
+  model="vercel_ai_gateway/openai/gpt-4o",
+  messages=[{ "content": "Hello, how are you?","role": "user"}]
+)
+```
+
+</TabItem>
+
 </Tabs>
 
 ### Response Format (OpenAI Format)
@@ -234,7 +251,7 @@ response = completion(
 {
     "id": "chatcmpl-565d891b-a42e-4c39-8d14-82a1f5208885",
     "created": 1734366691,
-    "model": "claude-3-sonnet-20240229",
+    "model": "gpt-4o-2024-08-06",
     "object": "chat.completion",
     "system_fingerprint": null,
     "choices": [
@@ -446,6 +463,24 @@ response = completion(
 
 </TabItem>
 
+<TabItem value="vercel" label="Vercel AI Gateway">
+
+```python
+from litellm import completion
+import os
+
+## set ENV variables. Visit https://vercel.com/docs/ai-gateway#using-the-ai-gateway-with-an-api-key for insturctions on obtaining a key
+os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "your-vercel-api-key"
+
+response = completion(
+  model="vercel_ai_gateway/openai/gpt-4o",
+  messages = [{ "content": "Hello, how are you?","role": "user"}],
+  stream=True,
+)
+```
+
+</TabItem>
+
 </Tabs>
 
 ### Streaming Response Format (OpenAI Format)
@@ -489,6 +524,15 @@ try:
 except OpenAIError as e:
     print(e)
 ```
+### See How LiteLLM Transforms Your Requests
+
+Want to understand how LiteLLM parses and normalizes your LLM API requests? Use the `/utils/transform_request` endpoint to see exactly how your request is transformed internally.
+
+You can try it out now directly on our Demo App!
+Go to the [LiteLLM API docs for transform_request](https://litellm-api.up.railway.app/#/llm%20utils/transform_request_utils_transform_request_post)
+
+LiteLLM will show you the normalized, provider-agnostic version of your request. This is useful for debugging, learning, and understanding how LiteLLM handles different providers and options.
+
 
 ### Logging Observability - Log LLM Input/Output ([Docs](https://docs.litellm.ai/docs/observability/callbacks))
 LiteLLM exposes pre defined callbacks to send data to Lunary, MLflow, Langfuse, Helicone, Promptlayer, Traceloop, Slack

docs/my-website/docs/integrations/index.md

@@ -0,0 +1,18 @@
+# Integrations
+
+This section covers integrations with various tools and services that can be used with LiteLLM (either Proxy or SDK).
+
+## AI Agent Frameworks
+- **[Letta](./letta.md)** - Build stateful LLM agents with persistent memory using LiteLLM Proxy
+
+## Development Tools
+- **[OpenWebUI](../tutorials/openweb_ui.md)** - Self-hosted ChatGPT-style interface
+
+## Observability & Monitoring
+- **[Langfuse](../observability/langfuse_integration.md)** - LLM observability and analytics
+- **[Prometheus](../proxy/prometheus.md)** - Metrics collection and monitoring
+- **[PagerDuty](../proxy/pagerduty.md)** - Incident response and alerting
+- **[Datadog](../observability/datadog.md)**
+
+
+Click into each section to learn more about the integrations.

docs/my-website/docs/integrations/letta.md

@@ -0,0 +1,928 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Letta Integration
+
+[Letta](https://github.com/letta-ai/letta) (formerly MemGPT) is a framework for building stateful LLM agents with persistent memory. This guide shows how to integrate both LiteLLM SDK and LiteLLM Proxy with Letta to leverage multiple LLM providers while building memory-enabled agents.
+
+## What is Letta?
+
+Letta allows you to build LLM agents that can:
+- Maintain long-term memory across conversations
+- Use function calling for tool interactions
+- Handle large context windows efficiently
+- Persist agent state and memory
+
+## Prerequisites
+
+```bash
+pip install letta litellm
+```
+
+## Quick Start
+
+<Tabs>
+<TabItem value="proxy" label="LiteLLM Proxy">
+
+### 1. Start LiteLLM Proxy
+
+First, create a configuration file for your LiteLLM proxy:
+
+```yaml
+# config.yaml
+model_list:
+  - model_name: gpt-4
+    litellm_params:
+      model: openai/gpt-4
+      api_key: os.environ/OPENAI_API_KEY
+
+  - model_name: claude-3-sonnet
+    litellm_params:
+      model: anthropic/claude-3-sonnet-20240229
+      api_key: os.environ/ANTHROPIC_API_KEY
+
+  - model_name: gpt-3.5-turbo
+    litellm_params:
+      model: azure/gpt-35-turbo
+      api_key: os.environ/AZURE_API_KEY
+      api_base: os.environ/AZURE_API_BASE
+      api_version: "2023-07-01-preview"
+```
+
+Start the proxy:
+
+```bash
+litellm --config config.yaml --port 4000
+```
+
+### 2. Configure Letta with LiteLLM Proxy
+
+Configure Letta to use your LiteLLM proxy endpoint:
+
+```python
+import letta
+from letta import create_client
+
+# Configure Letta to use LiteLLM proxy
+client = create_client()
+
+# Configure the LLM endpoint
+client.set_default_llm_config(
+    model="gpt-4",  # This should match a model from your LiteLLM config
+    model_endpoint_type="openai",
+    model_endpoint="http://localhost:4000",  # Your LiteLLM proxy URL
+    context_window=8192
+)
+
+# Configure embedding endpoint (optional)
+client.set_default_embedding_config(
+    embedding_endpoint_type="openai",
+    embedding_endpoint="http://localhost:4000",
+    embedding_model="text-embedding-ada-002"
+)
+```
+
+</TabItem>
+<TabItem value="sdk" label="LiteLLM SDK">
+
+### 1. Configure LiteLLM SDK
+
+Set up your API keys and configure LiteLLM:
+
+```python
+import os
+import litellm
+
+# Set your API keys
+os.environ["OPENAI_API_KEY"] = "your-openai-key"
+os.environ["ANTHROPIC_API_KEY"] = "your-anthropic-key"
+
+# Optional: Configure default settings
+litellm.set_verbose = True  # For debugging
+```
+
+### 2. Create Custom LLM Wrapper for Letta
+
+Create a custom LLM wrapper that uses LiteLLM SDK:
+
+```python
+import letta
+from letta import create_client
+from letta.llm_api.llm_api_base import LLMConfig
+import litellm
+from typing import List, Dict, Any
+
+class LiteLLMWrapper:
+    def __init__(self, model: str):
+        self.model = model
+    
+    def chat_completions_create(self, messages: List[Dict], **kwargs):
+        # Use LiteLLM SDK for completion
+        response = litellm.completion(
+            model=self.model,
+            messages=messages,
+            **kwargs
+        )
+        return response
+
+# Configure Letta with custom LiteLLM wrapper
+client = create_client()
+
+# Set up LLM configuration using direct SDK integration
+llm_config = LLMConfig(
+    model="gpt-4",  # or "claude-3-sonnet", "azure/gpt-35-turbo", etc.
+    model_endpoint_type="openai",
+    context_window=8192
+)
+
+client.set_default_llm_config(llm_config)
+```
+
+</TabItem>
+</Tabs>
+
+### 3. Create and Use a Letta Agent
+
+<Tabs>
+<TabItem value="proxy" label="Using LiteLLM Proxy">
+
+```python
+import letta
+from letta import create_client
+
+# Create Letta client
+client = create_client()
+
+# Create a new agent
+agent_state = client.create_agent(
+    name="my-assistant",
+    system="You are a helpful assistant with persistent memory.",
+    llm_config=client.get_default_llm_config(),
+    embedding_config=client.get_default_embedding_config()
+)
+
+# Send a message to the agent
+response = client.user_message(
+    agent_id=agent_state.id,
+    message="Hi! My name is Alice and I love reading science fiction books."
+)
+
+print(f"Agent response: {response.messages[-1].text}")
+
+# Send another message - the agent will remember previous context
+response = client.user_message(
+    agent_id=agent_state.id,
+    message="What did I tell you about my interests?"
+)
+
+print(f"Agent response: {response.messages[-1].text}")
+```
+
+</TabItem>
+<TabItem value="sdk" label="Using LiteLLM SDK">
+
+```python
+import letta
+from letta import create_client
+import litellm
+import os
+
+# Set up environment variables
+os.environ["OPENAI_API_KEY"] = "your-openai-key"
+
+# Create Letta client with LiteLLM integration
+client = create_client()
+
+# Create a new agent
+agent_state = client.create_agent(
+    name="my-assistant",
+    system="You are a helpful assistant with persistent memory.",
+    llm_config=client.get_default_llm_config(),
+    embedding_config=client.get_default_embedding_config()
+)
+
+# Send a message to the agent
+response = client.user_message(
+    agent_id=agent_state.id,
+    message="Hi! My name is Alice and I love reading science fiction books."
+)
+
+print(f"Agent response: {response.messages[-1].text}")
+
+# Send another message - the agent will remember previous context
+response = client.user_message(
+    agent_id=agent_state.id,
+    message="What did I tell you about my interests?"
+)
+
+print(f"Agent response: {response.messages[-1].text}")
+```
+
+</TabItem>
+</Tabs>
+
+## Advanced Configuration
+
+### Using Different Models for Different Agents
+
+<Tabs>
+<TabItem value="proxy" label="LiteLLM Proxy">
+
+```python
+from letta import LLMConfig, EmbeddingConfig
+
+# Create different LLM configurations pointing to your proxy
+gpt4_config = LLMConfig(
+    model="gpt-4",
+    model_endpoint_type="openai",
+    model_endpoint="http://localhost:4000",
+    context_window=8192
+)
+
+claude_config = LLMConfig(
+    model="claude-3-sonnet",
+    model_endpoint_type="openai",  # Using OpenAI-compatible endpoint
+    model_endpoint="http://localhost:4000",
+    context_window=200000
+)
+
+# Create agents with different configurations
+research_agent = client.create_agent(
+    name="research-agent",
+    system="You are a research assistant specialized in analysis.",
+    llm_config=claude_config  # Use Claude for research tasks
+)
+
+creative_agent = client.create_agent(
+    name="creative-agent", 
+    system="You are a creative writing assistant.",
+    llm_config=gpt4_config  # Use GPT-4 for creative tasks
+)
+```
+
+</TabItem>
+<TabItem value="sdk" label="LiteLLM SDK">
+
+```python
+import os
+import litellm
+from letta import LLMConfig, EmbeddingConfig
+
+# Set up API keys for different providers
+os.environ["OPENAI_API_KEY"] = "your-openai-key"
+os.environ["ANTHROPIC_API_KEY"] = "your-anthropic-key"
+
+# Create different LLM configurations for direct SDK usage
+gpt4_config = LLMConfig(
+    model="openai/gpt-4",  # Using LiteLLM model format
+    model_endpoint_type="openai",
+    context_window=8192
+)
+
+claude_config = LLMConfig(
+    model="anthropic/claude-3-sonnet-20240229",  # Using LiteLLM model format
+    model_endpoint_type="openai",
+    context_window=200000
+)
+
+# Create agents with different configurations
+research_agent = client.create_agent(
+    name="research-agent",
+    system="You are a research assistant specialized in analysis.",
+    llm_config=claude_config  # Use Claude for research tasks
+)
+
+creative_agent = client.create_agent(
+    name="creative-agent", 
+    system="You are a creative writing assistant.",
+    llm_config=gpt4_config  # Use GPT-4 for creative tasks
+)
+```
+
+</TabItem>
+</Tabs>
+
+### Function Calling with Tools
+
+<Tabs>
+<TabItem value="proxy" label="LiteLLM Proxy">
+
+```python
+# Define custom tools for your agent
+def search_web(query: str) -> str:
+    """Search the web for information"""
+    # Your web search implementation
+    return f"Search results for: {query}"
+
+def save_note(content: str) -> str:
+    """Save a note to persistent storage"""
+    # Your note saving implementation
+    return f"Note saved: {content}"
+
+# Create agent with tools (using proxy endpoint)
+agent_state = client.create_agent(
+    name="research-assistant",
+    system="You are a research assistant that can search the web and save notes.",
+    llm_config=client.get_default_llm_config(),
+    embedding_config=client.get_default_embedding_config(),
+    tools=[search_web, save_note]
+)
+
+# The agent can now use these tools
+response = client.user_message(
+    agent_id=agent_state.id,
+    message="Search for recent developments in AI and save important findings."
+)
+```
+
+</TabItem>
+<TabItem value="sdk" label="LiteLLM SDK">
+
+```python
+import litellm
+import os
+
+# Set up API keys
+os.environ["OPENAI_API_KEY"] = "your-openai-key"
+
+# Define custom tools for your agent
+def search_web(query: str) -> str:
+    """Search the web for information"""
+    # Your web search implementation
+    return f"Search results for: {query}"
+
+def save_note(content: str) -> str:
+    """Save a note to persistent storage"""
+    # Your note saving implementation
+    return f"Note saved: {content}"
+
+# Create agent with tools (using LiteLLM SDK directly)
+agent_state = client.create_agent(
+    name="research-assistant",
+    system="You are a research assistant that can search the web and save notes.",
+    llm_config=LLMConfig(
+        model="openai/gpt-4",  # Direct model specification
+        model_endpoint_type="openai",
+        context_window=8192
+    ),
+    embedding_config=client.get_default_embedding_config(),
+    tools=[search_web, save_note]
+)
+
+# The agent can now use these tools
+response = client.user_message(
+    agent_id=agent_state.id,
+    message="Search for recent developments in AI and save important findings."
+)
+```
+
+</TabItem>
+</Tabs>
+
+## Authentication
+
+<Tabs>
+<TabItem value="proxy" label="LiteLLM Proxy Authentication">
+
+If your LiteLLM proxy requires authentication:
+
+```python
+import os
+from letta import LLMConfig
+
+# Set up authenticated configuration
+llm_config = LLMConfig(
+    model="gpt-4",
+    model_endpoint_type="openai",
+    model_endpoint="http://localhost:4000",
+    model_wrapper="openai",
+    context_window=8192
+)
+
+# If using API keys with your proxy
+os.environ["OPENAI_API_KEY"] = "your-litellm-proxy-api-key"
+
+client = create_client()
+client.set_default_llm_config(llm_config)
+```
+
+For proxy with authentication enabled:
+
+```yaml
+# config.yaml with auth
+general_settings:
+  master_key: "your-master-key"
+
+model_list:
+  - model_name: gpt-4
+    litellm_params:
+      model: openai/gpt-4
+      api_key: os.environ/OPENAI_API_KEY
+```
+
+```python
+# Configure Letta with authenticated proxy
+llm_config = LLMConfig(
+    model="gpt-4",
+    model_endpoint_type="openai",
+    model_endpoint="http://localhost:4000",
+    context_window=8192,
+    api_key="your-master-key"  # Proxy master key
+)
+```
+
+</TabItem>
+<TabItem value="sdk" label="LiteLLM SDK Authentication">
+
+With LiteLLM SDK, set up your provider API keys directly:
+
+```python
+import os
+import litellm
+
+# Set up API keys for different providers
+os.environ["OPENAI_API_KEY"] = "your-openai-api-key"
+os.environ["ANTHROPIC_API_KEY"] = "your-anthropic-api-key" 
+os.environ["AZURE_API_KEY"] = "your-azure-api-key"
+os.environ["AZURE_API_BASE"] = "https://your-resource.openai.azure.com"
+os.environ["AZURE_API_VERSION"] = "2023-07-01-preview"
+
+# Optional: Configure default settings
+litellm.api_key = os.environ.get("OPENAI_API_KEY")  # Default key
+litellm.set_verbose = True  # For debugging
+
+# Use in Letta configuration
+from letta import LLMConfig
+
+llm_config = LLMConfig(
+    model="openai/gpt-4",  # Will use OPENAI_API_KEY automatically
+    model_endpoint_type="openai",
+    context_window=8192
+)
+
+# Or for Azure
+azure_config = LLMConfig(
+    model="azure/gpt-35-turbo", 
+    model_endpoint_type="openai",
+    context_window=4096
+)
+```
+
+</TabItem>
+</Tabs>
+
+## Load Balancing and Fallbacks
+
+<Tabs>
+<TabItem value="proxy" label="LiteLLM Proxy Features">
+
+LiteLLM proxy's load balancing and fallback features work seamlessly with Letta:
+
+```yaml
+# config.yaml with fallbacks
+model_list:
+  - model_name: gpt-4
+    litellm_params:
+      model: openai/gpt-4
+      api_key: os.environ/OPENAI_API_KEY
+    tpm: 40000
+    rpm: 500
+
+  - model_name: gpt-4  # Same model name for fallback
+    litellm_params:
+      model: azure/gpt-4
+      api_key: os.environ/AZURE_API_KEY
+      api_base: os.environ/AZURE_API_BASE
+      api_version: "2023-07-01-preview"
+    tpm: 80000
+    rpm: 800
+
+router_settings:
+  routing_strategy: "usage-based-routing"
+  fallbacks: [{"gpt-4": ["azure/gpt-4"]}]
+```
+
+The proxy handles all routing, load balancing, and fallbacks transparently for Letta.
+
+</TabItem>
+<TabItem value="sdk" label="LiteLLM SDK Router">
+
+With LiteLLM SDK, you can set up routing and fallbacks programmatically:
+
+```python
+import litellm
+from litellm import Router
+
+# Configure router with multiple models
+router = Router(
+    model_list=[
+        {
+            "model_name": "gpt-4",
+            "litellm_params": {
+                "model": "openai/gpt-4",
+                "api_key": os.environ["OPENAI_API_KEY"]
+            },
+            "tpm": 40000,
+            "rpm": 500
+        },
+        {
+            "model_name": "gpt-4",  # Same name for fallback
+            "litellm_params": {
+                "model": "azure/gpt-4", 
+                "api_key": os.environ["AZURE_API_KEY"],
+                "api_base": os.environ["AZURE_API_BASE"],
+                "api_version": "2023-07-01-preview"
+            },
+            "tpm": 80000,
+            "rpm": 800
+        }
+    ],
+    fallbacks=[{"gpt-4": ["azure/gpt-4"]}],
+    routing_strategy="usage-based-routing"
+)
+
+# Create custom completion function for Letta
+def custom_completion(messages, model="gpt-4", **kwargs):
+    return router.completion(
+        model=model,
+        messages=messages,
+        **kwargs
+    )
+
+# Use with Letta by monkey-patching or custom wrapper
+litellm.completion = custom_completion
+```
+
+</TabItem>
+</Tabs>
+
+## Monitoring and Observability
+
+<Tabs>
+<TabItem value="proxy" label="LiteLLM Proxy Monitoring">
+
+Enable logging to track your Letta agents' LLM usage through the proxy:
+
+```yaml
+# config.yaml with logging
+model_list:
+  # ... your models
+
+litellm_settings:
+  success_callback: ["langfuse"]  # or other observability tools
+  
+environment_variables:
+  LANGFUSE_PUBLIC_KEY: "your-key"
+  LANGFUSE_SECRET_KEY: "your-secret"
+```
+
+View metrics in the proxy dashboard:
+```bash
+# Start proxy with UI
+litellm --config config.yaml --port 4000 --detailed_debug
+```
+
+</TabItem>
+<TabItem value="sdk" label="LiteLLM SDK Monitoring">
+
+Set up observability directly in your SDK integration:
+
+```python
+import litellm
+import os
+
+# Configure observability callbacks
+os.environ["LANGFUSE_PUBLIC_KEY"] = "your-key"
+os.environ["LANGFUSE_SECRET_KEY"] = "your-secret"
+
+# Set global callbacks
+litellm.success_callback = ["langfuse"]
+litellm.failure_callback = ["langfuse"]
+
+# Optional: Set up custom logging
+litellm.set_verbose = True
+
+# Create custom completion wrapper with logging
+def logged_completion(messages, model="gpt-4", **kwargs):
+    try:
+        response = litellm.completion(
+            model=model,
+            messages=messages,
+            **kwargs
+        )
+        # Custom logging logic here if needed
+        return response
+    except Exception as e:
+        # Custom error handling
+        print(f"LLM call failed: {e}")
+        raise
+
+# Use in Letta configuration
+litellm.completion = logged_completion
+```
+
+</TabItem>
+</Tabs>
+
+## Example: Multi-Agent System
+
+<Tabs>
+<TabItem value="proxy" label="Using LiteLLM Proxy">
+
+```python
+import letta
+from letta import create_client, LLMConfig
+
+client = create_client()
+
+# Create specialized agents using proxy endpoints
+agents = {}
+
+# Research agent using Claude for analysis
+agents['researcher'] = client.create_agent(
+    name="researcher",
+    system="You are a research specialist. Analyze information thoroughly.",
+    llm_config=LLMConfig(
+        model="claude-3-sonnet",
+        model_endpoint="http://localhost:4000",
+        model_endpoint_type="openai"
+    )
+)
+
+# Writer agent using GPT-4 for content creation
+agents['writer'] = client.create_agent(
+    name="writer",
+    system="You are a content writer. Create engaging, well-structured content.",
+    llm_config=LLMConfig(
+        model="gpt-4",
+        model_endpoint="http://localhost:4000", 
+        model_endpoint_type="openai"
+    )
+)
+
+# Coordinator workflow
+def research_and_write_workflow(topic: str):
+    # Research phase
+    research_response = client.user_message(
+        agent_id=agents['researcher'].id,
+        message=f"Research the topic: {topic}. Provide key insights and data."
+    )
+    
+    research_results = research_response.messages[-1].text
+    
+    # Writing phase
+    write_response = client.user_message(
+        agent_id=agents['writer'].id,
+        message=f"Based on this research: {research_results}\n\nWrite an article about {topic}."
+    )
+    
+    return write_response.messages[-1].text
+
+# Execute workflow
+article = research_and_write_workflow("The future of AI in healthcare")
+print(article)
+```
+
+</TabItem>
+<TabItem value="sdk" label="Using LiteLLM SDK">
+
+```python
+import letta
+from letta import create_client, LLMConfig
+import litellm
+import os
+
+# Set up environment
+os.environ["OPENAI_API_KEY"] = "your-openai-key"
+os.environ["ANTHROPIC_API_KEY"] = "your-anthropic-key"
+
+client = create_client()
+
+# Create specialized agents using direct SDK models
+agents = {}
+
+# Research agent using Claude for analysis
+agents['researcher'] = client.create_agent(
+    name="researcher",
+    system="You are a research specialist. Analyze information thoroughly.",
+    llm_config=LLMConfig(
+        model="anthropic/claude-3-sonnet-20240229",
+        model_endpoint_type="openai"
+    )
+)
+
+# Writer agent using GPT-4 for content creation
+agents['writer'] = client.create_agent(
+    name="writer",
+    system="You are a content writer. Create engaging, well-structured content.",
+    llm_config=LLMConfig(
+        model="openai/gpt-4",
+        model_endpoint_type="openai"
+    )
+)
+
+# Cost-conscious agent using GPT-3.5
+agents['reviewer'] = client.create_agent(
+    name="reviewer",
+    system="You are an editor. Review and improve content quality.",
+    llm_config=LLMConfig(
+        model="openai/gpt-3.5-turbo",
+        model_endpoint_type="openai"
+    )
+)
+
+# Enhanced workflow with multiple agents
+def enhanced_workflow(topic: str):
+    # Research phase
+    research_response = client.user_message(
+        agent_id=agents['researcher'].id,
+        message=f"Research the topic: {topic}. Provide key insights and data."
+    )
+    
+    research_results = research_response.messages[-1].text
+    
+    # Writing phase
+    write_response = client.user_message(
+        agent_id=agents['writer'].id,
+        message=f"Based on this research: {research_results}\n\nWrite an article about {topic}."
+    )
+    
+    draft_article = write_response.messages[-1].text
+    
+    # Review phase
+    review_response = client.user_message(
+        agent_id=agents['reviewer'].id,
+        message=f"Please review and improve this article:\n\n{draft_article}"
+    )
+    
+    return review_response.messages[-1].text
+
+# Execute enhanced workflow
+article = enhanced_workflow("The future of AI in healthcare")
+print(article)
+```
+
+</TabItem>
+</Tabs>
+
+## Best Practices
+
+<Tabs>
+<TabItem value="proxy" label="LiteLLM Proxy Best Practices">
+
+1. **Model Selection**: Use appropriate models for different tasks:
+   - Claude for analysis and reasoning
+   - GPT-4 for creative tasks
+   - GPT-3.5-turbo for simple interactions
+
+2. **Proxy Configuration**:
+   - Set appropriate rate limits and timeouts
+   - Use fallbacks for reliability
+   - Enable authentication for production
+
+3. **Memory Management**: Letta handles memory automatically, but monitor usage with large contexts
+
+4. **Cost Optimization**: 
+   - Use the proxy's budgeting features to control costs
+   - Set up rate limiting per user/team
+   - Monitor token usage through proxy dashboard
+
+5. **Monitoring**: Enable observability to track agent performance and token usage
+
+</TabItem>
+<TabItem value="sdk" label="LiteLLM SDK Best Practices">
+
+1. **Model Selection**: Choose models based on task requirements:
+   - Use `openai/gpt-4` for complex reasoning
+   - Use `anthropic/claude-3-sonnet-20240229` for analysis
+   - Use `openai/gpt-3.5-turbo` for cost-effective simple tasks
+
+2. **Error Handling**: Implement robust error handling with retries:
+   ```python
+   import litellm
+   from litellm import completion
+   
+   # Set up retry logic
+   litellm.num_retries = 3
+   litellm.request_timeout = 60
+   
+   # Custom error handling
+   def safe_completion(**kwargs):
+       try:
+           return completion(**kwargs)
+       except Exception as e:
+           print(f"LLM call failed: {e}")
+           # Implement fallback logic
+           return completion(model="openai/gpt-3.5-turbo", **kwargs)
+   ```
+
+3. **Cost Management**:
+   - Use cheaper models for non-critical tasks
+   - Implement token counting and budgets
+   - Cache responses when appropriate
+
+4. **Performance**:
+   - Use async operations for concurrent requests
+   - Implement connection pooling
+   - Monitor response times
+
+5. **Security**:
+   - Store API keys securely (environment variables)
+   - Rotate keys regularly
+   - Implement rate limiting
+
+</TabItem>
+</Tabs>
+
+## Troubleshooting
+
+<Tabs>
+<TabItem value="proxy" label="LiteLLM Proxy Issues">
+
+### Connection Issues
+```bash
+# Test your LiteLLM proxy
+curl -X POST http://localhost:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "gpt-4",
+    "messages": [{"role": "user", "content": "Hello"}]
+  }'
+```
+
+### Configuration Debugging
+```python
+# Enable verbose logging
+import logging
+logging.basicConfig(level=logging.DEBUG)
+
+# Test Letta configuration
+client = create_client()
+print(client.get_default_llm_config())
+```
+
+### Common Proxy Issues
+- **Port conflicts**: Make sure port 4000 isn't in use
+- **Model not found**: Verify model names match your config.yaml
+- **Authentication errors**: Check master key configuration
+- **Rate limiting**: Monitor proxy logs for rate limit hits
+
+</TabItem>
+<TabItem value="sdk" label="LiteLLM SDK Issues">
+
+### API Key Issues
+```python
+import os
+import litellm
+
+# Check if API keys are set
+print("OpenAI Key:", os.environ.get("OPENAI_API_KEY", "Not set"))
+print("Anthropic Key:", os.environ.get("ANTHROPIC_API_KEY", "Not set"))
+
+# Test direct LiteLLM call
+try:
+    response = litellm.completion(
+        model="openai/gpt-3.5-turbo",
+        messages=[{"role": "user", "content": "Hello"}]
+    )
+    print("LiteLLM working:", response.choices[0].message.content)
+except Exception as e:
+    print("LiteLLM error:", e)
+```
+
+### Configuration Debugging
+```python
+# Enable verbose logging
+litellm.set_verbose = True
+
+# Test model availability
+models = ["openai/gpt-4", "anthropic/claude-3-sonnet-20240229"]
+for model in models:
+    try:
+        response = litellm.completion(
+            model=model,
+            messages=[{"role": "user", "content": "Test"}],
+            max_tokens=10
+        )
+        print(f"✓ {model} working")
+    except Exception as e:
+        print(f"✗ {model} failed: {e}")
+```
+
+### Common SDK Issues
+- **Import errors**: Ensure `pip install litellm letta` is run
+- **Model format**: Use `provider/model` format (e.g., `openai/gpt-4`)
+- **API key format**: Different providers have different key formats
+- **Rate limits**: Implement exponential backoff for retries
+
+</TabItem>
+</Tabs>
+
+## Resources
+
+- [Letta Documentation](https://docs.letta.ai/)
+- [LiteLLM Proxy Documentation](../proxy/quick_start.md)
+- [LiteLLM SDK Documentation](../completion/input.md)
+- [Function Calling Guide](../completion/function_call.md)
+- [Observability Setup](../observability/langfuse_integration.md)
+- [Router Configuration](../routing.md)