Merge branch 'BerriAI:main' into main

.circleci/requirements.txt

@@ -1,5 +1,5 @@
 # used by CI/CD testing
-openai==1.81.0
+openai==1.100.1
 python-dotenv
 tiktoken
 importlib_metadata
@@ -10,7 +10,9 @@ anthropic
 orjson==3.10.12 # fast /embedding responses
 pydantic==2.10.2
 google-cloud-aiplatform==1.43.0
+google-cloud-iam==2.19.1
 fastapi-sso==0.16.0 
 uvloop==0.21.0
 mcp==1.10.1    # for MCP server  
-semantic_router==0.1.10 # for auto-routing with litellm
+semantic_router==0.1.10 # for auto-routing with litellm
+fastuuid==0.12.0

.devcontainer/devcontainer.json

@@ -11,7 +11,12 @@
 	// },
 
 	// Features to add to the dev container. More info: https://containers.dev/features.
-	// "features": {},
+	"features": {
+		"ghcr.io/devcontainers/features/node:1": {
+			"version": "lts"
+		},
+		"ghcr.io/devcontainers/features/docker-in-docker:2": {}
+	},
 
 	// Configure tool-specific properties.
 	"customizations": {
@@ -30,7 +35,7 @@
 	
 	// Use 'forwardPorts' to make a list of ports inside the container available locally.
 	"forwardPorts": [4000],
-
+	
 	"containerEnv": {
 		"LITELLM_LOG": "DEBUG"
 	},
@@ -48,5 +53,5 @@
 	// "remoteUser": "litellm",
 
 	// Use 'postCreateCommand' to run commands after the container is created.
-	"postCreateCommand": "pipx install poetry && poetry install -E extra_proxy -E proxy"
+	"postCreateCommand": "bash ./.devcontainer/post-create.sh"
 }

.devcontainer/post-create.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -e
+
+echo "[post-create] Installing poetry via pip"
+python -m pip install --upgrade pip
+python -m pip install poetry
+
+echo "[post-create] Installing Python dependencies (poetry)"
+poetry install --with dev --extras proxy
+
+echo "[post-create] Generating Prisma client"
+poetry run prisma generate
+
+echo "[post-create] Installing npm dependencies"
+cd ui/litellm-dashboard && npm install --no-audit --no-fund
+
+echo "[post-create] Done"

.github/scripts/scan_keywords.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+import json
+import os
+import sys
+import urllib.request
+import urllib.error
+
+
+def read_event_payload() -> dict:
+    event_path = os.environ.get("GITHUB_EVENT_PATH")
+    if not event_path or not os.path.exists(event_path):
+        return {}
+    with open(event_path, "r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+def get_issue_text(event: dict) -> tuple[str, str, int, str, str]:
+    issue = event.get("issue") or {}
+    title = (issue.get("title") or "").strip()
+    body = (issue.get("body") or "").strip()
+    number = issue.get("number") or 0
+    html_url = issue.get("html_url") or ""
+    author = ((issue.get("user") or {}).get("login") or "").strip()
+    return title, body, number, html_url, author
+
+
+def detect_keywords(text: str, keywords: list[str]) -> list[str]:
+    lowered = text.lower()
+    matches = []
+    for keyword in keywords:
+        k = keyword.strip().lower()
+        if not k:
+            continue
+        if k in lowered:
+            matches.append(keyword.strip())
+    # Deduplicate while preserving order
+    seen = set()
+    unique_matches = []
+    for m in matches:
+        if m not in seen:
+            unique_matches.append(m)
+            seen.add(m)
+    return unique_matches
+
+
+def send_webhook(webhook_url: str, payload: dict) -> None:
+    if not webhook_url:
+        return
+    data = json.dumps(payload).encode("utf-8")
+    req = urllib.request.Request(
+        webhook_url,
+        data=data,
+        headers={"Content-Type": "application/json"},
+        method="POST",
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=10) as resp:
+            resp.read()
+    except urllib.error.HTTPError as e:
+        print(f"Webhook HTTP error: {e.code} {e.reason}", file=sys.stderr)
+    except urllib.error.URLError as e:
+        print(f"Webhook URL error: {e.reason}", file=sys.stderr)
+    except Exception as e:
+        print(f"Webhook unexpected error: {e}", file=sys.stderr)
+
+
+def _excerpt(text: str, max_len: int = 400) -> str:
+    if not text:
+        return ""
+    
+    # Keep original formatting
+    if len(text) <= max_len:
+        return text
+    return text[: max_len - 1] + "…"
+
+
+
+def main() -> int:
+    event = read_event_payload()
+    if not event:
+        print("::warning::No event payload found; exiting without labeling.")
+        return 0
+
+    # Read issue details
+    title, body, number, html_url, author = get_issue_text(event)
+    combined_text = f"{title}\n\n{body}".strip()
+
+    # Keywords from env or defaults
+    keywords_env = os.environ.get("KEYWORDS", "")
+    default_keywords = ["azure", "openai", "bedrock", "vertexai", "vertex ai", "anthropic"]
+    keywords = [k.strip() for k in keywords_env.split(",")] if keywords_env else default_keywords
+
+    matches = detect_keywords(combined_text, keywords)
+    found = bool(matches)
+
+    # Emit outputs
+    github_output = os.environ.get("GITHUB_OUTPUT")
+    if github_output:
+        with open(github_output, "a", encoding="utf-8") as fh:
+            fh.write(f"found={'true' if found else 'false'}\n")
+            fh.write(f"matches={','.join(matches)}\n")
+
+    # Optional webhook notification
+    webhook_url = os.environ.get("PROVIDER_ISSUE_WEBHOOK_URL", "").strip()
+    if found and webhook_url:
+        repo_full = (event.get("repository") or {}).get("full_name", "")
+        title_part = f"*{title}*" if title else "New issue"
+        author_part = f" by @{author}" if author else ""
+        body_preview = _excerpt(body)
+        preview_block = f"\n{body_preview}" if body_preview else ""
+        payload = {
+            "text": (
+                f"New issue 🚨\n"
+                f"{title_part}\n\n{preview_block}\n"
+                f"<{html_url}|View issue>\n"
+                f"Author: {author}"
+            )
+        }
+        send_webhook(webhook_url, payload)
+
+    # Print a short log line for Actions UI
+    if found:
+        print(f"Detected provider keywords: {', '.join(matches)}")
+    else:
+        print("No provider keywords detected.")
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
+
+

.github/workflows/auto_update_price_and_context_window_file.py

@@ -43,8 +43,8 @@ def write_to_file(file_path, data):
         # Print an error message if writing to file fails
         print("Error updating JSON file:", e)
 
-# Update the existing models and add the missing models
-def transform_remote_data(data):
+# Update the existing models and add the missing models for OpenRouter
+def transform_openrouter_data(data):
     transformed = {}
     for row in data:
         # Add the fields 'max_tokens' and 'input_cost_per_token'
@@ -81,6 +81,34 @@ def transform_remote_data(data):
 
     return transformed
 
+# Update the existing models and add the missing models for Vercel AI Gateway
+def transform_vercel_ai_gateway_data(data):
+    transformed = {}
+    for row in data:
+        obj = {
+            "max_tokens": row["context_window"],
+            "input_cost_per_token": float(row["pricing"]["input"]),
+            "output_cost_per_token": float(row["pricing"]["output"]),
+            'max_output_tokens': row['max_tokens'],
+            'max_input_tokens': row["context_window"],
+        }
+
+        # Handle cache pricing if available
+        if "pricing" in row:
+            if "input_cache_read" in row["pricing"] and row["pricing"]["input_cache_read"] is not None:
+                obj['cache_read_input_token_cost'] = float(f"{float(row['pricing']['input_cache_read']):e}")
+            
+            if "input_cache_write" in row["pricing"] and row["pricing"]["input_cache_write"] is not None:
+                obj['cache_creation_input_token_cost'] = float(f"{float(row['pricing']['input_cache_write']):e}")
+
+        mode = "embedding" if "embedding" in row["id"].lower() else "chat"
+        
+        obj.update({"litellm_provider": "vercel_ai_gateway", "mode": mode})
+
+        transformed[f'vercel_ai_gateway/{row["id"]}'] = obj
+
+    return transformed
+
 
 # Load local data from a specified file
 def load_local_data(file_path):
@@ -100,22 +128,32 @@ def load_local_data(file_path):
 
 def main():
     local_file_path = "model_prices_and_context_window.json"  # Path to the local data file
-    url = "https://openrouter.ai/api/v1/models"  # URL to fetch remote data
+    openrouter_url = "https://openrouter.ai/api/v1/models"  # URL to fetch OpenRouter data
+    vercel_ai_gateway_url = "https://ai-gateway.vercel.sh/v1/models"  # URL to fetch Vercel AI Gateway data
 
     # Load local data from file
     local_data = load_local_data(local_file_path)
-    # Fetch remote data asynchronously
-    remote_data = asyncio.run(fetch_data(url))
-    # Transform the fetched remote data
-    remote_data = transform_remote_data(remote_data)
+    
+    # Fetch OpenRouter data
+    openrouter_data = asyncio.run(fetch_data(openrouter_url))
+    # Transform the fetched OpenRouter data
+    openrouter_data = transform_openrouter_data(openrouter_data)
+    
+    # Fetch Vercel AI Gateway data
+    vercel_data = asyncio.run(fetch_data(vercel_ai_gateway_url))
+    # Transform the fetched Vercel AI Gateway data
+    vercel_data = transform_vercel_ai_gateway_data(vercel_data)
+    
+    # Combine both datasets
+    all_remote_data = {**openrouter_data, **vercel_data}
 
-    # If both local and remote data are available, synchronize and save
-    if local_data and remote_data:
-        sync_local_data_with_remote(local_data, remote_data)
+    # If both local and openrouter data are available, synchronize and save
+    if local_data and all_remote_data:
+        sync_local_data_with_remote(local_data, all_remote_data)
         write_to_file(local_file_path, local_data)
     else:
         print("Failed to fetch model data from either local file or URL.")
 
 # Entry point of the script
 if __name__ == "__main__":
-    main()
+    main()

.github/workflows/issue-keyword-labeler.yml

@@ -0,0 +1,64 @@
+name: Issue Keyword Labeler
+
+on:
+  issues:
+    types: 
+        - opened
+
+jobs:
+  scan-and-label:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      contents: read
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Scan for provider keywords
+        id: scan
+        env:
+          PROVIDER_ISSUE_WEBHOOK_URL: ${{ secrets.PROVIDER_ISSUE_WEBHOOK_URL }}
+          KEYWORDS: azure,openai,bedrock,vertexai,vertex ai,anthropic
+        run: python3 .github/scripts/scan_keywords.py
+
+      - name: Ensure label exists
+        if: steps.scan.outputs.found == 'true'
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const labelName = 'llm translation';
+            try {
+              await github.rest.issues.getLabel({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                name: labelName
+              });
+            } catch (error) {
+              if (error.status === 404) {
+                await github.rest.issues.createLabel({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  name: labelName,
+                  color: 'c1ff72',
+                  description: 'Issues related to LLM provider translation/mapping'
+                });
+              } else {
+                throw error;
+              }
+            }
+
+      - name: Add label to the issue
+        if: steps.scan.outputs.found == 'true'
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            await github.rest.issues.addLabels({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+              labels: ['llm translation']
+            });
+

.github/workflows/test-linting.yml

@@ -11,6 +11,9 @@ jobs:
 
     steps:
     - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+        clean: true
 
     - name: Set up Python
       uses: actions/setup-python@v4
@@ -20,13 +23,15 @@ jobs:
     - name: Install Poetry
       uses: snok/install-poetry@v1
 
+    - name: Clean Python cache
+      run: |
+        find . -type d -name "__pycache__" -exec rm -rf {} + || true
+        find . -name "*.pyc" -delete || true
+
     - name: Install dependencies
       run: |
-        pip install openai==1.81.0
         poetry install --with dev
-        pip install openai==1.81.0
-
-
+        poetry run pip install openai==1.100.1
 
     - name: Run Black formatting
       run: |
@@ -34,16 +39,29 @@ jobs:
         poetry run black .
         cd ..
 
+    - name: Debug - Check file state
+      run: |
+        echo "Current branch:"
+        git branch --show-current
+        echo "Last 3 commits:"
+        git log --oneline -3
+        echo "File content around line 43:"
+        head -50 litellm/litellm_core_utils/custom_logger_registry.py | tail -10
+        
     - name: Run Ruff linting
       run: |
         cd litellm
         poetry run ruff check .
         cd ..
 
+    - name: Print OpenAI version
+      run: |
+        poetry run python -c "import openai; print(f'OpenAI version: {openai.__version__}')"
+
     - name: Run MyPy type checking
       run: |
         cd litellm
-        poetry run mypy . --ignore-missing-imports
+        poetry run mypy . 
         cd ..
 
     - name: Check for circular imports

.github/workflows/test-litellm.yml

@@ -7,7 +7,7 @@ on:
 jobs:
   test:
     runs-on: ubuntu-latest
-    timeout-minutes: 20
+    timeout-minutes: 25
 
     steps:
     - uses: actions/checkout@v4
@@ -31,6 +31,8 @@ jobs:
         poetry run pip install "pytest-retry==1.6.3"
         poetry run pip install pytest-xdist
         poetry run pip install "google-genai==1.22.0"
+        poetry run pip install "google-cloud-aiplatform>=1.38"
+        poetry run pip install "fastapi-offline==1.7.3"
     - name: Setup litellm-enterprise as local package
       run: |
         cd enterprise
@@ -38,4 +40,4 @@ jobs:
         cd ..
     - name: Run tests
       run: |
-        poetry run pytest tests/test_litellm -x -vv -n 4 
+        poetry run pytest tests/test_litellm --tb=short -vv --maxfail=10 -n 4 

.github/workflows/test-mcp.yml

@@ -0,0 +1,48 @@
+name: LiteLLM MCP Tests (folder - tests/mcp_tests)
+
+on:
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Thank You Message
+      run: |
+        echo "### 🙏 Thank you for contributing to LiteLLM!" >> $GITHUB_STEP_SUMMARY
+        echo "Your PR is being tested now. We appreciate your help in making LiteLLM better!" >> $GITHUB_STEP_SUMMARY
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.12'
+
+    - name: Install Poetry
+      uses: snok/install-poetry@v1
+
+    - name: Install dependencies
+      run: |
+        poetry install --with dev,proxy-dev --extras "proxy semantic-router"
+        poetry run pip install "pytest==7.3.1"
+        poetry run pip install "pytest-retry==1.6.3"
+        poetry run pip install "pytest-cov==5.0.0"
+        poetry run pip install "pytest-asyncio==0.21.1"
+        poetry run pip install "respx==0.22.0"
+        poetry run pip install "pydantic==2.10.2"
+        poetry run pip install "mcp==1.10.1"
+        poetry run pip install pytest-xdist
+
+    - name: Setup litellm-enterprise as local package
+      run: |
+        cd enterprise
+        python -m pip install -e .
+        cd ..
+
+    - name: Run MCP tests
+      run: |
+        poetry run pytest tests/mcp_tests -x -vv -n 4 --cov=litellm --cov-report=xml --durations=5

.gitignore

@@ -86,6 +86,7 @@ litellm/proxy/db/migrations/0_init/migration.sql
 litellm/proxy/db/migrations/*
 litellm/proxy/migrations/*config.yaml
 litellm/proxy/migrations/*
+litellm/proxy/to_delete_loadtest_work/*
 config.yaml
 tests/litellm/litellm_core_utils/llm_cost_calc/log.txt
 tests/test_custom_dir/*
@@ -93,4 +94,8 @@ test.py
 
 litellm_config.yaml
 .cursor
-.vscode/launch.json
+.vscode/launch.json
+litellm/proxy/to_delete_loadtest_work/*
+update_model_cost_map.py
+tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py
+litellm/proxy/_experimental/out/guardrails/index.html

Dockerfile

@@ -15,7 +15,7 @@ USER root
 RUN apk add --no-cache gcc python3-dev openssl openssl-dev
 
 
-RUN pip install --upgrade pip && \
+RUN pip install --upgrade pip>=24.3.1 && \
     pip install build
 
 # Copy the current directory contents into the container at /app
@@ -41,9 +41,6 @@ RUN pip uninstall jwt -y
 RUN pip uninstall PyJWT -y
 RUN pip install PyJWT==2.9.0 --no-cache-dir
 
-# Build Admin UI
-RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
-
 # Runtime stage
 FROM $LITELLM_RUNTIME_IMAGE AS runtime
 
@@ -53,6 +50,9 @@ USER root
 # Install runtime dependencies
 RUN apk add --no-cache openssl tzdata
 
+# Upgrade pip to fix CVE-2025-8869
+RUN pip install --upgrade pip>=24.3.1
+
 WORKDIR /app
 # Copy the current directory contents into the container at /app
 COPY . .
@@ -65,8 +65,8 @@ COPY --from=builder /wheels/ /wheels/
 # Install the built wheel using pip; again using a wildcard if it's the only file
 RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
 
-# Install semantic_router without dependencies
-RUN pip install semantic_router --no-deps
+# Install semantic_router and aurelio-sdk using script
+RUN chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
 
 # Generate prisma client
 RUN prisma generate

Makefile

@@ -34,13 +34,13 @@ install-proxy-dev:
 
 # CI-compatible installations (matches GitHub workflows exactly)
 install-dev-ci:
-	pip install openai==1.81.0
+	pip install openai==1.99.5
 	poetry install --with dev
-	pip install openai==1.81.0
+	pip install openai==1.99.5
 
 install-proxy-dev-ci:
 	poetry install --with dev,proxy-dev --extras proxy
-	pip install openai==1.81.0
+	pip install openai==1.99.5
 
 install-test-deps: install-proxy-dev
 	poetry run pip install "pytest-retry==1.6.3"
@@ -48,7 +48,7 @@ install-test-deps: install-proxy-dev
 	cd enterprise && python -m pip install -e . && cd ..
 
 install-helm-unittest:
-	helm plugin install https://github.com/helm-unittest/helm-unittest --version v0.4.4
+	helm plugin install https://github.com/helm-unittest/helm-unittest --version v0.4.4 || echo "ignore error if plugin exists"
 
 # Formatting
 format: install-dev

README.md

@@ -25,7 +25,7 @@
     <a href="https://discord.gg/wuPM9dRgDw">
         <img src="https://img.shields.io/static/v1?label=Chat%20on&message=Discord&color=blue&logo=Discord&style=flat-square" alt="Discord">
     </a>
-    <a href="https://join.slack.com/share/enQtOTE0ODczMzk2Nzk4NC01YjUxNjY2YjBlYTFmNDRiZTM3NDFiYTM3MzVkODFiMDVjOGRjMmNmZTZkZTMzOWQzZGQyZWIwYjQ0MWExYmE3">
+    <a href="https://www.litellm.ai/support">
         <img src="https://img.shields.io/static/v1?label=Chat%20on&message=Slack&color=black&logo=Slack&style=flat-square" alt="Slack">
     </a>
 </h4>
@@ -37,7 +37,7 @@ LiteLLM manages:
 - Retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - [Router](https://docs.litellm.ai/docs/routing)
 - Set Budgets & Rate limits per project, api key, model [LiteLLM Proxy Server (LLM Gateway)](https://docs.litellm.ai/docs/simple_proxy)
 
-[**Jump to LiteLLM Proxy (LLM Gateway) Docs**](https://github.com/BerriAI/litellm?tab=readme-ov-file#openai-proxy---docs) <br>
+[**Jump to LiteLLM Proxy (LLM Gateway) Docs**](https://github.com/BerriAI/litellm?tab=readme-ov-file#litellm-proxy-server-llm-gateway---docs) <br>
 [**Jump to Supported LLM Providers**](https://github.com/BerriAI/litellm?tab=readme-ov-file#supported-providers-docs)
 
 🚨 **Stable Release:** Use docker images with the `-stable` tag. These have undergone 12 hour load tests, before being published. [More information about the release cycle here](https://docs.litellm.ai/docs/proxy/release_cycle)
@@ -47,7 +47,7 @@ Support for more providers. Missing a provider or LLM Platform, raise a [feature
 # Usage ([**Docs**](https://docs.litellm.ai/docs/))
 
 > [!IMPORTANT]
-> LiteLLM v1.0.0 now requires `openai>=1.0.0`. Migration guide [here](https://docs.litellm.ai/docs/migration)  
+> LiteLLM v1.0.0 now requires `openai>=1.0.0`. Migration guide [here](https://docs.litellm.ai/docs/migration)
 > LiteLLM v1.40.14+ now requires `pydantic>=2.0.0`. No changes required.
 
 <a target="_blank" href="https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/liteLLM_Getting_Started.ipynb">
@@ -132,7 +132,7 @@ print(response)
 
 ## Streaming ([Docs](https://docs.litellm.ai/docs/completion/stream))
 
-liteLLM supports streaming the model response back, pass `stream=True` to get a streaming iterator in response.  
+liteLLM supports streaming the model response back, pass `stream=True` to get a streaming iterator in response.
 Streaming is supported for all models (Bedrock, Huggingface, TogetherAI, Azure, OpenAI, etc.)
 
 ```python
@@ -234,7 +234,7 @@ $ litellm --model huggingface/bigcode/starcoder
 
 
 > [!IMPORTANT]
-> 💡 [Use LiteLLM Proxy with Langchain (Python, JS), OpenAI SDK (Python, JS) Anthropic SDK, Mistral SDK, LlamaIndex, Instructor, Curl](https://docs.litellm.ai/docs/proxy/user_keys)  
+> 💡 [Use LiteLLM Proxy with Langchain (Python, JS), OpenAI SDK (Python, JS) Anthropic SDK, Mistral SDK, LlamaIndex, Instructor, Curl](https://docs.litellm.ai/docs/proxy/user_keys)
 
 ```python
 import openai # openai v1.0.0+
@@ -266,14 +266,14 @@ echo 'LITELLM_MASTER_KEY="sk-1234"' > .env
 
 # Add the litellm salt key - you cannot change this after adding a model
 # It is used to encrypt / decrypt your LLM API Key credentials
-# We recommend - https://1password.com/password-generator/ 
+# We recommend - https://1password.com/password-generator/
 # password generator to get a random hash for litellm salt key
 echo 'LITELLM_SALT_KEY="sk-1234"' >> .env
 
 source .env
 
 # Start
-docker-compose up
+docker compose up
 ```
 
 
@@ -316,6 +316,7 @@ curl 'http://0.0.0.0:4000/key/generate' \
 | [google AI Studio - gemini](https://docs.litellm.ai/docs/providers/gemini)          | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                               |                                                                         |
 | [mistral ai api](https://docs.litellm.ai/docs/providers/mistral)                    | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 | ✅                                                                             |                                                                         |
 | [cloudflare AI Workers](https://docs.litellm.ai/docs/providers/cloudflare_workers)  | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                               |                                                                         |
+| [CompactifAI](https://docs.litellm.ai/docs/providers/compactifai)                   | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                               |                                                                         |
 | [cohere](https://docs.litellm.ai/docs/providers/cohere)                             | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 | ✅                                                                             |                                                                         |
 | [anthropic](https://docs.litellm.ai/docs/providers/anthropic)                       | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                               |                                                                         |
 | [empower](https://docs.litellm.ai/docs/providers/empower)                    | ✅                                                      | ✅                                                                              | ✅                                                                                  | ✅                                                                                |
@@ -340,26 +341,37 @@ curl 'http://0.0.0.0:4000/key/generate' \
 | [xinference [Xorbits Inference]](https://docs.litellm.ai/docs/providers/xinference) |                                                         |                                                                                 |                                                                                     |                                                                                   | ✅                                                                             |                                                                         |
 | [FriendliAI](https://docs.litellm.ai/docs/providers/friendliai)                              | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                               |                                                                         |
 | [Galadriel](https://docs.litellm.ai/docs/providers/galadriel)                              | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                               |                                                                         |
+| [GradientAI](https://docs.litellm.ai/docs/providers/gradient_ai)                              | ✅                                                       | ✅                                                                               |                                                                                   |                                                                                  |                                                                               |                                                                         |
 | [Novita AI](https://novita.ai/models/llm?utm_source=github_litellm&utm_medium=github_readme&utm_campaign=github_link)                     | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                               |                                                                         |
 | [Featherless AI](https://docs.litellm.ai/docs/providers/featherless_ai)                              | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                               |                                                                         |
 | [Nebius AI Studio](https://docs.litellm.ai/docs/providers/nebius)                             | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 | ✅                                                                             |                                                                         |
+| [Heroku](https://docs.litellm.ai/docs/providers/heroku)                             | ✅                                                       | ✅                                                                               |                                                                                    |                                                                                  |                                                                              |                                                                         |
+| [OVHCloud AI Endpoints](https://docs.litellm.ai/docs/providers/ovhcloud)                             | ✅                                                       | ✅                                                                               |                                                                                    |                                                                                  |                                                                              |                                                                         |
 
 [**Read the Docs**](https://docs.litellm.ai/docs/)
 
-## Contributing
+## Run in Developer mode
+### Services
+1. Setup .env file in root
+2. Run dependant services `docker-compose up db prometheus`
 
-Interested in contributing? Contributions to LiteLLM Python SDK, Proxy Server, and LLM integrations are both accepted and highly encouraged! 
+### Backend
+1. (In root) create virtual environment `python -m venv .venv`
+2. Activate virtual environment `source .venv/bin/activate`
+3. Install dependencies `pip install -e ".[all]"`
+4. Start proxy backend `python litellm/proxy_cli.py`
 
-**Quick start:** `git clone` → `make install-dev` → `make format` → `make lint` → `make test-unit`
-
-See our comprehensive [Contributing Guide (CONTRIBUTING.md)](CONTRIBUTING.md) for detailed instructions.
+### Frontend
+1. Navigate to `ui/litellm-dashboard`
+2. Install dependencies `npm install`
+3. Run `npm run dev` to start the dashboard
 
 # Enterprise
 For companies that need better security, user management and professional support
 
 [Talk to founders](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
 
-This covers: 
+This covers:
 - ✅ **Features under the [LiteLLM Commercial License](https://docs.litellm.ai/docs/proxy/enterprise):**
 - ✅ **Feature Prioritization**
 - ✅ **Custom Integrations**
@@ -373,6 +385,8 @@ We welcome contributions to LiteLLM! Whether you're fixing bugs, adding features
 
 ## Quick Start for Contributors
 
+This requires poetry to be installed.
+
 ```bash
 git clone https://github.com/BerriAI/litellm.git
 cd litellm
@@ -380,6 +394,7 @@ make install-dev    # Install development dependencies
 make format         # Format your code
 make lint           # Run all linting checks
 make test-unit      # Run unit tests
+make format-check   # Check formatting only
 ```
 
 For detailed contributing guidelines, see [CONTRIBUTING.md](CONTRIBUTING.md).
@@ -395,11 +410,6 @@ Our automated checks include:
 - **Circular import detection**
 - **Import safety checks**
 
-Run all checks locally:
-```bash
-make lint           # Run all linting (matches CI)
-make format-check   # Check formatting only
-```
 
 All these checks must pass before your PR can be merged.
 
@@ -408,7 +418,7 @@ All these checks must pass before your PR can be merged.
 
 - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version)
 - [Community Discord 💭](https://discord.gg/wuPM9dRgDw)
-- [Community Slack 💭](https://join.slack.com/share/enQtOTE0ODczMzk2Nzk4NC01YjUxNjY2YjBlYTFmNDRiZTM3NDFiYTM3MzVkODFiMDVjOGRjMmNmZTZkZTMzOWQzZGQyZWIwYjQ0MWExYmE3)
+- [Community Slack 💭](https://www.litellm.ai/support)
 - Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬
 - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai
 
@@ -432,18 +442,3 @@ All these checks must pass before your PR can be merged.
 </a>
 
 
-## Run in Developer mode
-### Services
-1. Setup .env file in root
-2. Run dependant services `docker-compose up db prometheus`
-
-### Backend
-1. (In root) create virtual environment `python -m venv .venv`
-2. Activate virtual environment `source .venv/bin/activate`
-3. Install dependencies `pip install -e ".[all]"`
-4. Start proxy backend `uvicorn litellm.proxy.proxy_server:app --host localhost --port 4000 --reload`
-
-### Frontend
-1. Navigate to `ui/litellm-dashboard`
-2. Install dependencies `npm install`
-3. Run `npm run dev` to start the dashboard

ci_cd/security_scans.sh

@@ -0,0 +1,166 @@
+#!/bin/bash
+
+# Security Scans Script for LiteLLM
+# This script runs comprehensive security scans including Trivy and Grype
+
+set -e
+
+echo "Starting security scans for LiteLLM..."
+
+# Function to install Trivy and required tools
+install_trivy() {
+    echo "Installing Trivy and required tools..."
+    sudo apt-get update
+    sudo apt-get install -y wget apt-transport-https gnupg lsb-release jq curl
+    wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | sudo apt-key add -
+    echo "deb https://aquasecurity.github.io/trivy-repo/deb $(lsb_release -sc) main" | sudo tee -a /etc/apt/sources.list.d/trivy.list
+    sudo apt-get update
+    sudo apt-get install trivy
+    echo "Trivy and required tools installed successfully"
+}
+
+# Function to install Grype
+install_grype() {
+    echo "Installing Grype..."
+    curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sudo sh -s -- -b /usr/local/bin
+    echo "Grype installed successfully"
+}
+
+# Function to run Trivy scans
+run_trivy_scans() {
+    echo "Running Trivy scans..."
+    
+    echo "Scanning LiteLLM Docs..."
+    trivy fs --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./docs/
+    
+    echo "Scanning LiteLLM UI..."
+    trivy fs --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./ui/
+    
+    echo "Trivy scans completed successfully"
+}
+
+# Function to build and scan Docker images with Grype
+run_grype_scans() {
+    echo "Running Grype scans..."
+    
+    # Temporarily add wheel files to .dockerignore for security scans
+    echo "Temporarily modifying .dockerignore to exclude problematic wheel files..."
+    cp .dockerignore .dockerignore.backup 2>/dev/null || touch .dockerignore.backup
+    echo "/*.whl" >> .dockerignore
+    
+    # Build and scan Dockerfile.database
+    echo "Building and scanning Dockerfile.database..."
+    docker build --no-cache -t litellm-database:latest -f ./docker/Dockerfile.database .
+    grype litellm-database:latest --fail-on critical
+    
+    # Build and scan main Dockerfile
+    echo "Building and scanning main Dockerfile..."
+    docker build --no-cache -t litellm:latest .
+    grype litellm:latest --fail-on critical
+    
+    # Restore original .dockerignore
+    echo "Restoring original .dockerignore..."
+    mv .dockerignore.backup .dockerignore
+    
+    # Scan the locally built LiteLLM image for vulnerabilities with CVSS >= 4.0
+    echo "Scanning locally built LiteLLM image for high-severity vulnerabilities..."
+    echo "Using locally built image: litellm:latest"
+    
+    # Allowlist of CVEs to be ignored in failure threshold/reporting
+    # - CVE-2025-8869: Not applicable on Python >=3.13 (PEP 706 implemented); pip fallback unused; no OS-level fix
+    # - GHSA-4xh5-x5gv-qwph: GitHub Security Advisory alias for CVE-2025-8869
+    ALLOWED_CVES=(
+        "CVE-2025-8869"
+        "GHSA-4xh5-x5gv-qwph"
+        "CVE-2025-8291" # no fix available as of Oct 11, 2025
+    )
+
+    # Build JSON array of allowlisted CVE IDs for jq
+    ALLOWED_IDS_JSON=$(printf '%s\n' "${ALLOWED_CVES[@]}" | jq -R . | jq -s .)
+
+    echo "Checking for vulnerabilities with CVSS score >= 4.0..."
+    echo "Allowlisted CVEs (ignored in threshold): ${ALLOWED_CVES[*]}"
+    echo ""
+    
+    # Show all high-severity vulnerabilities for transparency
+    TOTAL_HIGH_SEVERITY=$(grype litellm:latest -o json | jq -r '
+        .matches[]
+        | select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0)
+        | .vulnerability.id' | wc -l)
+    
+    if [ "$TOTAL_HIGH_SEVERITY" -gt 0 ]; then
+        echo "Total vulnerabilities found with CVSS >= 4.0: $TOTAL_HIGH_SEVERITY"
+        echo ""
+        echo "All high-severity vulnerabilities (including allowlisted):"
+        grype litellm:latest -o json | jq --argjson allow "$ALLOWED_IDS_JSON" -r '
+        ["Package", "Version", "Vulnerability ID", "CVSS Score", "Allowlisted"],
+        (.matches[]
+          | select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0)
+          | [.artifact.name, .artifact.version, .vulnerability.id, .vulnerability.cvss[0].metrics.baseScore, (if (.vulnerability.id as $id | $allow | index($id)) then "YES" else "NO" end)])
+        | @tsv' | column -t -s $'\t'
+        echo ""
+    fi
+
+    HIGH_SEVERITY_COUNT=$(grype litellm:latest -o json | jq --argjson allow "$ALLOWED_IDS_JSON" -r '
+        .matches[]
+        | select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0)
+        | select((.vulnerability.id as $id | $allow | index($id) | not))
+        | .vulnerability.id' | wc -l)
+    
+    if [ "$HIGH_SEVERITY_COUNT" -gt 0 ]; then
+        echo ""
+        echo "=========================================="
+        echo "ERROR: Security Scan Failed"
+        echo "=========================================="
+        echo "Found $HIGH_SEVERITY_COUNT non-allowlisted vulnerabilities with CVSS score >= 4.0 in litellm:latest"
+        echo ""
+        echo "These vulnerabilities are NOT in the allowlist and must be addressed."
+        echo "Current allowlisted CVEs: ${ALLOWED_CVES[*]}"
+        echo ""
+        echo "Detailed vulnerability report:"
+        echo ""
+        grype litellm:latest -o json | jq --argjson allow "$ALLOWED_IDS_JSON" -r '
+        ["Package", "Version", "Vulnerability ID", "CVSS Score", "Severity", "Fix Version", "Description"],
+        (.matches[]
+          | select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0)
+          | select((.vulnerability.id as $id | $allow | index($id) | not))
+          | [.artifact.name, .artifact.version, .vulnerability.id, .vulnerability.cvss[0].metrics.baseScore, .vulnerability.severity, (.vulnerability.fix.versions[0] // "No fix available"), .vulnerability.description])
+        | @tsv' | column -t -s $'\t'
+        echo ""
+        echo "=========================================="
+        echo "Action Required:"
+        echo "=========================================="
+        echo "1. If a fix is available, update the package to the fixed version"
+        echo "2. If the vulnerability is not applicable or has no fix:"
+        echo "   - Add the CVE/GHSA ID to ALLOWED_CVES array in ci_cd/security_scans.sh"
+        echo "   - Add a comment explaining why it's safe to ignore"
+        echo ""
+        echo "Note: Some vulnerabilities may have multiple IDs (CVE-XXXX and GHSA-XXXX)."
+        echo "Add all relevant IDs to the allowlist if they refer to the same issue."
+        echo "=========================================="
+        echo ""
+        exit 1
+    else
+        echo "No high-severity vulnerabilities (CVSS >= 4.0) found in litellm:latest"
+    fi
+    
+    echo "Grype scans completed successfully"
+}
+
+# Main execution
+main() {
+    echo "Installing security scanning tools..."
+    install_trivy
+    install_grype
+    
+    echo "Running filesystem vulnerability scans..."
+    run_trivy_scans
+    
+    echo "Running Docker image vulnerability scans..."
+    run_grype_scans
+    
+    echo "All security scans completed successfully!"
+}
+
+# Execute main function
+main "$@"

ci_cd/security_scans_readme.md

@@ -0,0 +1,9 @@
+# Security Scans
+
+## Scans that run:
+
+- Trivy scan on `./docs/` (HIGH/CRITICAL/MEDIUM)
+- Trivy scan on `./ui/` (HIGH/CRITICAL/MEDIUM) 
+- Grype scan on `Dockerfile.database` (fails on CRITICAL)
+- Grype scan on main `Dockerfile` (fails on CRITICAL)
+- Grype CVSS ≥ 4.0 scan on main `Dockerfile` (fails any vulnerabilities with CVSS ≥ 4.0)

cookbook/liteLLM_Baseten.ipynb

@@ -6,19 +6,21 @@
     "id": "gZx-wHJapG5w"
    },
    "source": [
-    "# Use liteLLM to call Falcon, Wizard, MPT 7B using OpenAI chatGPT Input/output\n",
+    "# LiteLLM with Baseten Model APIs\n",
     "\n",
-    "* Falcon 7B: https://app.baseten.co/explore/falcon_7b\n",
-    "* Wizard LM: https://app.baseten.co/explore/wizardlm\n",
-    "* MPT 7B Base: https://app.baseten.co/explore/mpt_7b_instruct\n",
+    "This notebook demonstrates how to use LiteLLM with Baseten's Model APIs instead of dedicated deployments.\n",
     "\n",
-    "\n",
-    "## Call all baseten llm models using OpenAI chatGPT Input/Output using liteLLM\n",
-    "Example call\n",
+    "## Example Usage\n",
     "```python\n",
-    "model = \"q841o8w\" # baseten model version ID\n",
-    "response = completion(model=model, messages=messages, custom_llm_provider=\"baseten\")\n",
-    "```"
+    "response = completion(\n",
+    "    model=\"baseten/openai/gpt-oss-120b\",\n",
+    "    messages=[{\"role\": \"user\", \"content\": \"Hello!\"}],\n",
+    "    max_tokens=1000,\n",
+    "    temperature=0.7\n",
+    ")\n",
+    "```\n",
+    "\n",
+    "## Setup"
    ]
   },
   {
@@ -29,20 +31,25 @@
    },
    "outputs": [],
    "source": [
-    "!pip install litellm==0.1.399\n",
-    "!pip install baseten urllib3"
+    "%pip install litellm"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "metadata": {
     "id": "VEukLhDzo4vw"
    },
    "outputs": [],
    "source": [
     "import os\n",
-    "from litellm import completion"
+    "from litellm import completion\n",
+    "\n",
+    "# Set your Baseten API key\n",
+    "os.environ['BASETEN_API_KEY'] = \"\" #@param {type:\"string\"}\n",
+    "\n",
+    "# Test message\n",
+    "messages = [{\"role\": \"user\", \"content\": \"What is AGI?\"}]"
    ]
   },
   {
@@ -51,19 +58,31 @@
     "id": "4STYM2OHFNlc"
    },
    "source": [
-    "## Setup"
+    "## Example 1: Basic Completion\n",
+    "\n",
+    "Simple completion with the GPT-OSS 120B model"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 21,
+   "execution_count": null,
    "metadata": {
     "id": "DorpLxw1FHbC"
    },
    "outputs": [],
    "source": [
-    "os.environ['BASETEN_API_KEY'] = \"\" #@param\n",
-    "messages = [{ \"content\": \"what does Baseten do? \",\"role\": \"user\"}]"
+    "print(\"=== Basic Completion ===\")\n",
+    "response = completion(\n",
+    "    model=\"baseten/openai/gpt-oss-120b\",\n",
+    "    messages=messages,\n",
+    "    max_tokens=1000,\n",
+    "    temperature=0.7,\n",
+    "    top_p=0.9,\n",
+    "    presence_penalty=0.1,\n",
+    "    frequency_penalty=0.1,\n",
+    ")\n",
+    "print(f\"Response: {response.choices[0].message.content}\")\n",
+    "print(f\"Usage: {response.usage}\")"
    ]
   },
   {
@@ -72,13 +91,14 @@
     "id": "syF3dTdKFSQQ"
    },
    "source": [
-    "## Calling Falcon 7B: https://app.baseten.co/explore/falcon_7b\n",
-    "### Pass Your Baseten model `Version ID` as `model`"
+    "## Example 2: Streaming Completion\n",
+    "\n",
+    "Streaming completion with usage statistics"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 18,
+   "execution_count": null,
    "metadata": {
     "colab": {
      "base_uri": "https://localhost:8080/"
@@ -86,137 +106,26 @@
     "id": "rPgSoMlsojz0",
     "outputId": "81d6dc7b-1681-4ae4-e4c8-5684eb1bd050"
    },
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "\u001b[32mINFO\u001b[0m API key set.\n",
-      "INFO:baseten:API key set.\n"
-     ]
-    },
-    {
-     "data": {
-      "text/plain": [
-       "{'choices': [{'finish_reason': 'stop',\n",
-       "   'index': 0,\n",
-       "   'message': {'role': 'assistant',\n",
-       "    'content': \"what does Baseten do? \\nI'm sorry, I cannot provide a specific answer as\"}}],\n",
-       " 'created': 1692135883.699066,\n",
-       " 'model': 'qvv0xeq'}"
-      ]
-     },
-     "execution_count": 18,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
-    "model = \"qvv0xeq\"\n",
-    "response = completion(model=model, messages=messages, custom_llm_provider=\"baseten\")\n",
-    "response"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "id": "7n21UroEGCGa"
-   },
-   "source": [
-    "## Calling Wizard LM https://app.baseten.co/explore/wizardlm\n",
-    "### Pass Your Baseten model `Version ID` as `model`"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 19,
-   "metadata": {
-    "colab": {
-     "base_uri": "https://localhost:8080/"
-    },
-    "id": "uLVWFH899lAF",
-    "outputId": "61c2bc74-673b-413e-bb40-179cf408523d"
-   },
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "\u001b[32mINFO\u001b[0m API key set.\n",
-      "INFO:baseten:API key set.\n"
-     ]
-    },
-    {
-     "data": {
-      "text/plain": [
-       "{'choices': [{'finish_reason': 'stop',\n",
-       "   'index': 0,\n",
-       "   'message': {'role': 'assistant',\n",
-       "    'content': 'As an AI language model, I do not have personal beliefs or practices, but based on the information available online, Baseten is a popular name for a traditional Ethiopian dish made with injera, a spongy flatbread, and wat, a spicy stew made with meat or vegetables. It is typically served for breakfast or dinner and is a staple in Ethiopian cuisine. The name Baseten is also used to refer to a traditional Ethiopian coffee ceremony, where coffee is brewed and served in a special ceremony with music and food.'}}],\n",
-       " 'created': 1692135900.2806294,\n",
-       " 'model': 'q841o8w'}"
-      ]
-     },
-     "execution_count": 19,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "model = \"q841o8w\"\n",
-    "response = completion(model=model, messages=messages, custom_llm_provider=\"baseten\")\n",
-    "response"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "id": "6-TFwmPAGPXq"
-   },
-   "source": [
-    "## Calling mosaicml/mpt-7b https://app.baseten.co/explore/mpt_7b_instruct\n",
-    "### Pass Your Baseten model `Version ID` as `model`"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 20,
-   "metadata": {
-    "colab": {
-     "base_uri": "https://localhost:8080/"
-    },
-    "id": "gbeYZOrUE_Bp",
-    "outputId": "838d86ea-2143-4cb3-bc80-2acc2346c37a"
-   },
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "\u001b[32mINFO\u001b[0m API key set.\n",
-      "INFO:baseten:API key set.\n"
-     ]
-    },
-    {
-     "data": {
-      "text/plain": [
-       "{'choices': [{'finish_reason': 'stop',\n",
-       "   'index': 0,\n",
-       "   'message': {'role': 'assistant',\n",
-       "    'content': \"\\n===================\\n\\nIt's a tool to build a local version of a game on your own machine to host\\non your website.\\n\\nIt's used to make game demos and show them on Twitter, Tumblr, and Facebook.\\n\\n\\n\\n## What's built\\n\\n- A directory of all your game directories, named with a version name and build number, with images linked to.\\n- Includes HTML to include in another site.\\n- Includes images for your icons and\"}}],\n",
-       " 'created': 1692135914.7472186,\n",
-       " 'model': '31dxrj3'}"
-      ]
-     },
-     "execution_count": 20,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "model = \"31dxrj3\"\n",
-    "response = completion(model=model, messages=messages, custom_llm_provider=\"baseten\")\n",
-    "response"
+    "print(\"=== Streaming Completion ===\")\n",
+    "response = completion(\n",
+    "    model=\"baseten/openai/gpt-oss-120b\",\n",
+    "    messages=[{\"role\": \"user\", \"content\": \"Write a short poem about AI\"}],\n",
+    "    stream=True,\n",
+    "    max_tokens=500,\n",
+    "    temperature=0.8,\n",
+    "    stream_options={\n",
+    "        \"include_usage\": True,\n",
+    "        \"continuous_usage_stats\": True\n",
+    "    },\n",
+    ")\n",
+    "\n",
+    "print(\"Streaming response:\")\n",
+    "for chunk in response:\n",
+    "    if chunk.choices and chunk.choices[0].delta.content:\n",
+    "        print(chunk.choices[0].delta.content, end=\"\", flush=True)\n",
+    "print(\"\\n\")"
    ]
   }
  ],
@@ -234,4 +143,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 0
-}
+}

cookbook/litellm_proxy_server/batch_api/bedrock/bedrock.py

@@ -0,0 +1,25 @@
+from openai import OpenAI
+
+client = OpenAI(
+    base_url="http://0.0.0.0:4000",
+    api_key="sk-1234",
+)
+
+BEDROCK_BATCH_MODEL = "bedrock/batch-anthropic.claude-3-5-sonnet-20240620-v1:0"
+
+# Upload file
+batch_input_file = client.files.create(
+    file=open("./bedrock_batch_completions.jsonl", "rb"),
+    purpose="batch",
+    extra_body={"target_model_names": BEDROCK_BATCH_MODEL}
+)
+print(batch_input_file)
+
+# Create batch
+batch = client.batches.create( 
+    input_file_id=batch_input_file.id,
+    endpoint="/v1/chat/completions",
+    completion_window="24h",
+    metadata={"description": "Test batch job"},
+)
+print(batch)

cookbook/litellm_proxy_server/batch_api/bedrock/bedrock_batch_completions.jsonl

@@ -0,0 +1,128 @@
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}

cookbook/litellm_proxy_server/cli_token_usage.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+"""
+Example: Using CLI token with LiteLLM SDK
+
+This example shows how to use the CLI authentication token
+in your Python scripts after running `litellm-proxy login`.
+"""
+
+from textwrap import indent
+import litellm
+LITELLM_BASE_URL = "http://localhost:4000/"
+
+
+def main():
+    """Using CLI token with LiteLLM SDK"""
+    print("🚀 Using CLI Token with LiteLLM SDK")
+    print("=" * 40)
+    #litellm._turn_on_debug()
+    
+    # Get the CLI token
+    api_key = litellm.get_litellm_gateway_api_key()
+    
+    if not api_key:
+        print("❌ No CLI token found. Please run 'litellm-proxy login' first.")
+        return
+    
+    print("✅ Found CLI token.")
+
+    available_models = litellm.get_valid_models(
+        check_provider_endpoint=True,
+        custom_llm_provider="litellm_proxy",
+        api_key=api_key,
+        api_base=LITELLM_BASE_URL
+    )
+    
+    print("✅ Available models:")
+    if available_models:
+        for i, model in enumerate(available_models, 1):
+            print(f"   {i:2d}. {model}")
+    else:
+        print("   No models available")
+    
+    # Use with LiteLLM
+    try:
+        response = litellm.completion(
+            model="litellm_proxy/gemini/gemini-2.5-flash",
+            messages=[{"role": "user", "content": "Hello from CLI token!"}],
+            api_key=api_key,
+            base_url=LITELLM_BASE_URL
+        )
+        print(f"✅ LLM Response: {response.model_dump_json(indent=4)}")
+    except Exception as e:
+        print(f"❌ Error: {e}")
+
+
+if __name__ == "__main__":
+    main()
+    
+    print("\n💡 Tips:")
+    print("1. Run 'litellm-proxy login' to authenticate first")
+    print("2. Replace 'https://your-proxy.com' with your actual proxy URL")
+    print("3. The token is stored locally at ~/.litellm/token.json")

cookbook/litellm_proxy_server/mcp/mcp_with_litellm_proxy.py

@@ -0,0 +1,36 @@
+"""
+Use LiteLLM Proxy MCP Gateway to call MCP tools.
+
+When using LiteLLM Proxy, you can use the same MCP tools across all your LLM providers.
+"""
+import openai
+
+client = openai.OpenAI(
+    api_key="sk-1234", # paste your litellm proxy api key here
+    base_url="http://localhost:4000" # paste your litellm proxy base url here
+)
+print("Making API request to Responses API with MCP tools")
+
+response = client.responses.create(
+    model="gpt-5",
+    input=[
+        {
+            "role": "user",
+            "content": "give me TLDR of what BerriAI/litellm repo is about",
+            "type": "message"
+        }
+    ],
+    tools=[
+        {
+            "type": "mcp",
+            "server_label": "litellm",
+            "server_url": "litellm_proxy",
+            "require_approval": "never"
+        }
+    ],
+    stream=True,
+    tool_choice="required"
+)
+
+for chunk in response:
+    print("response chunk: ", chunk)

cookbook/litellm_router_load_test/memory_usage/router_endpoint.py

@@ -5,7 +5,7 @@ import os
 import litellm
 from litellm import Router
 from dotenv import load_dotenv
-import uuid
+from litellm._uuid import uuid
 
 load_dotenv()
 

cookbook/litellm_router_load_test/memory_usage/router_memory_usage copy.py

@@ -12,7 +12,7 @@ sys.path.insert(
 import litellm
 from litellm import Router
 from dotenv import load_dotenv
-import uuid
+from litellm._uuid import uuid
 
 load_dotenv()
 

cookbook/litellm_router_load_test/memory_usage/router_memory_usage.py

@@ -12,7 +12,7 @@ sys.path.insert(
 import litellm
 from litellm import Router
 from dotenv import load_dotenv
-import uuid
+from litellm._uuid import uuid
 
 load_dotenv()
 

cookbook/misc/RELEASE_NOTES_GENERATION_INSTRUCTIONS.md

@@ -0,0 +1,400 @@
+# LiteLLM Release Notes Generation Instructions
+
+This document provides comprehensive instructions for AI agents to generate release notes for LiteLLM following the established format and style.
+
+## Required Inputs
+
+1. **Release Version** (e.g., `v1.77.3-stable`)
+2. **PR Diff/Changelog** - List of PRs with titles and contributors
+3. **Previous Version Commit Hash** - To compare model pricing changes
+4. **Reference Release Notes** - Use recent stable releases (v1.76.3-stable, v1.77.2-stable) as templates for consistent formatting
+
+## Step-by-Step Process
+
+### 1. Initial Setup and Analysis
+
+```bash
+# Check git diff for model pricing changes
+git diff <previous_commit_hash> HEAD -- model_prices_and_context_window.json
+```
+
+**Key Analysis Points:**
+- New models added (look for new entries)
+- Deprecated models removed (look for deleted entries)
+- Pricing updates (look for cost changes)
+- Feature support changes (tool calling, reasoning, etc.)
+
+### 2. Release Notes Structure
+
+Follow this exact structure based on recent stable releases (v1.76.3-stable, v1.77.2-stable, v1.77.5-stable):
+
+```markdown
+---
+title: "v1.77.X-stable - [Key Theme]"
+slug: "v1-77-X"
+date: YYYY-MM-DDTHH:mm:ss
+authors: [standard author block]
+hide_table_of_contents: false
+---
+
+## Deploy this version
+[Docker and pip installation tabs]
+
+## Key Highlights
+[3-5 bullet points of major features - prioritize MCP OAuth 2.0, scheduled key rotations, and major model updates]
+
+## New Models / Updated Models
+#### New Model Support
+[Model pricing table]
+
+#### Features
+[Provider-specific features organized by provider]
+
+### Bug Fixes
+[Provider-specific bug fixes organized by provider]
+
+#### New Provider Support
+[New provider integrations]
+
+## LLM API Endpoints
+#### Features
+[API-specific features organized by API type]
+
+#### Bugs
+[General bug fixes]
+
+## Management Endpoints / UI
+#### Features
+[UI and management features - group by functionality like Proxy CLI Auth, Virtual Keys, Models + Endpoints]
+
+#### Bugs
+[Management-related bug fixes]
+
+## Logging / Guardrail / Prompt Management Integrations
+#### Features
+[Organized by integration provider with proper doc links]
+
+#### Guardrails
+[Guardrail-specific features and fixes]
+
+#### Prompt Management
+[Prompt management integrations like BitBucket]
+
+## Spend Tracking, Budgets and Rate Limiting
+[Cost tracking, service tier pricing, rate limiting improvements]
+
+## MCP Gateway
+[MCP-specific features, OAuth 2.0, configuration improvements]
+
+## Performance / Loadbalancing / Reliability improvements
+[Infrastructure improvements, memory fixes, performance optimizations]
+
+## Documentation Updates
+[Documentation improvements, guides, corrections - separate section for visibility]
+
+## New Contributors
+[List of first-time contributors]
+
+## Full Changelog
+[Link to GitHub comparison]
+```
+
+### 3. Categorization Rules
+
+**Performance Improvements:**
+- RPS improvements
+- Memory optimizations
+- CPU usage optimizations
+- Timeout controls
+- Worker configuration
+- Memory leak fixes
+- Cache performance improvements
+- Database connection management
+- Dependency management (fastuuid, etc.)
+- Configuration management
+
+**New Models/Updated Models:**
+- Extract from model_prices_and_context_window.json diff
+- Create tables with: Provider, Model, Context Window, Input Cost, Output Cost, Features
+- **Structure:**
+  - `#### New Model Support` - pricing table
+  - `#### Features` - organized by provider with documentation links
+  - `### Bug Fixes` - provider-specific bug fixes
+  - `#### New Provider Support` - major new provider integrations
+- Group by provider with proper doc links: `**[Provider Name](../../docs/providers/[provider])**`
+- Use bullet points under each provider for multiple features
+- Separate features from bug fixes clearly
+
+**LLM API Endpoints:**
+- **Structure:**
+  - `#### Features` - organized by API type (Responses API, Batch API, etc.)
+  - `#### Bugs` - general bug fixes under **General** category
+- **API Categories:**
+  - Responses API
+  - Batch API  
+  - CountTokens API
+  - Images API
+  - Video Generation (if applicable)
+  - General (miscellaneous improvements)
+- Use proper documentation links for each API type
+
+**UI/Management:**
+- Authentication changes
+- Dashboard improvements
+- Team management
+- Key management
+- Proxy CLI authentication and improvements
+- Virtual key management and scheduled rotations
+- SSO configuration fixes
+- Admin settings updates
+- Management routes and endpoints
+
+**Logging / Guardrail / Prompt Management Integrations:**
+- **Structure:**
+  - `#### Features` - organized by integration provider with proper doc links
+  - `#### Guardrails` - guardrail-specific features and fixes
+  - `#### Prompt Management` - prompt management integrations
+  - `#### New Integration` - major new integrations
+- **Integration Categories:**
+  - **[DataDog](../../docs/proxy/logging#datadog)** - group all DataDog-related changes
+  - **[Langfuse](../../docs/proxy/logging#langfuse)** - Langfuse-specific features
+  - **[Prometheus](../../docs/proxy/logging#prometheus)** - monitoring improvements
+  - **[PostHog](../../docs/observability/posthog)** - observability integration
+  - **[SQS](../../docs/proxy/logging#sqs)** - SQS logging features
+  - **[Opik](../../docs/proxy/logging#opik)** - Opik integration improvements
+  - Other logging providers with proper doc links
+- **Guardrail Categories:**
+  - LakeraAI, Presidio, Noma, and other guardrail providers
+- **Prompt Management:**
+  - BitBucket, GitHub, and other prompt management integrations
+- Use bullet points under each provider for multiple features
+- Separate logging features from guardrails and prompt management clearly
+
+### 4. Documentation Linking Strategy
+
+**Link to docs when:**
+- New provider support added
+- Significant feature additions
+- API endpoint changes
+- Integration additions
+
+**Link format:** `../../docs/[category]/[specific_doc]`
+
+**Common doc paths:**
+- `../../docs/providers/[provider]` - Provider-specific docs
+- `../../docs/image_generation` - Image generation
+- `../../docs/video_generation` - Video generation (if exists)
+- `../../docs/response_api` - Responses API
+- `../../docs/proxy/logging` - Logging integrations
+- `../../docs/proxy/guardrails` - Guardrails
+- `../../docs/pass_through/[provider]` - Passthrough endpoints
+
+### 5. Model Table Generation
+
+From git diff analysis, create tables like:
+
+```markdown
+| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Features |
+| -------- | ----- | -------------- | ------------------- | -------------------- | -------- |
+| OpenRouter | `openrouter/openai/gpt-4.1` | 1M | $2.00 | $8.00 | Chat completions with vision |
+```
+
+**Extract from JSON:**
+- `max_input_tokens` → Context Window
+- `input_cost_per_token` × 1,000,000 → Input cost
+- `output_cost_per_token` × 1,000,000 → Output cost
+- `supports_*` fields → Features
+- Special pricing fields (per image, per second) for generation models
+
+### 6. PR Categorization Logic
+
+**By Keywords in PR Title:**
+- `[Perf]`, `Performance`, `RPS` → Performance Improvements
+- `[Bug]`, `[Bug Fix]`, `Fix` → Bug Fixes section
+- `[Feat]`, `[Feature]`, `Add support` → Features section
+- `[Docs]` → Documentation Updates section
+- Provider names (Gemini, OpenAI, etc.) → Group under provider
+- `MCP`, `oauth`, `Model Context Protocol` → MCP Gateway
+- `service_tier`, `priority`, `cost tracking` → Spend Tracking, Budgets and Rate Limiting
+
+**By PR Content Analysis:**
+- New model additions → New Models section
+- UI changes → Management Endpoints/UI
+- Logging/observability → Logging/Guardrail/Prompt Management Integrations
+- Rate limiting/budgets → Spend Tracking, Budgets and Rate Limiting
+- Authentication → Management Endpoints/UI
+- MCP-related changes → MCP Gateway
+- Documentation updates → Documentation Updates
+- Performance/memory fixes → Performance/Loadbalancing/Reliability improvements
+
+**Special Categorization Rules:**
+- **Service tier pricing** (OpenAI priority/flex) → Spend Tracking section (NOT provider features)
+- **Cost breakdown in logging** → Spend Tracking section
+- **MCP configuration/OAuth** → MCP Gateway (NOT General Proxy Improvements)
+- **All documentation PRs** → Documentation Updates section for visibility
+
+### 7. Writing Style Guidelines
+
+**Tone:**
+- Professional but accessible
+- Focus on user impact
+- Highlight breaking changes clearly
+- Use active voice
+
+**Formatting:**
+- Use consistent markdown formatting
+- Include PR links: `[PR #XXXXX](https://github.com/BerriAI/litellm/pull/XXXXX)`
+- Use code blocks for configuration examples
+- Bold important terms and section headers
+
+**Warnings/Notes:**
+- Add warning boxes for breaking changes
+- Include migration instructions when needed
+- Provide override options for default changes
+
+### 8. Quality Checks
+
+**Before finalizing:**
+- Verify all PR links work
+- Check documentation links are valid
+- Ensure model pricing is accurate
+- Confirm provider names are consistent
+- Review for typos and formatting issues
+- **Count PRs by section** - Provide final count like:
+  ```
+  ## MM/DD/YYYY
+  * New Models / Updated Models: XX
+  * LLM API Endpoints: XX
+  * Management Endpoints / UI: XX
+  * Logging / Guardrail / Prompt Management Integrations: XX
+  * Spend Tracking, Budgets and Rate Limiting: XX
+  * MCP Gateway: XX
+  * Performance / Loadbalancing / Reliability improvements: XX
+  * Documentation Updates: XX
+  ```
+
+### 9. Common Patterns to Follow
+
+**Performance Changes:**
+```markdown
+- **+400 RPS Performance Boost** - Description - [PR #XXXXX](link)
+```
+
+**New Models:**
+Always include pricing table and feature highlights
+
+**Breaking Changes:**
+```markdown
+:::warning
+This release has a known issue...
+:::
+```
+
+**Provider Features (New Models / Updated Models section):**
+```markdown
+#### Features
+
+- **[Provider Name](../../docs/providers/provider)**
+    - Feature description - [PR #XXXXX](link)
+    - Another feature description - [PR #YYYYY](link)
+```
+
+**API Features (LLM API Endpoints section):**
+```markdown
+#### Features
+
+- **[API Name](../../docs/api_path)**
+    - Feature description - [PR #XXXXX](link)
+    - Another feature - [PR #YYYYY](link)
+- **General**
+    - Miscellaneous improvements - [PR #ZZZZZ](link)
+```
+
+**Integration Features (Logging / Guardrail Integrations section):**
+```markdown
+#### Features
+
+- **[Integration Name](../../docs/proxy/logging#integration)**
+    - Feature description - [PR #XXXXX](link)
+    - Bug fix description - [PR #YYYYY](link)
+```
+
+**Bug Fixes Pattern:**
+```markdown
+### Bug Fixes
+
+- **[Provider/Component Name](../../docs/providers/provider)**
+    - Bug fix description - [PR #XXXXX](link)
+```
+
+### 10. Missing Documentation Check
+
+**Review for missing docs:**
+- New providers without documentation
+- New API endpoints without examples
+- Complex features without guides
+- Integration setup instructions
+
+**Flag for documentation needs:**
+- New provider integrations
+- Significant API changes
+- Complex configuration options
+- Migration requirements
+
+### 11. New Sections and Categories (Added in v1.77.5)
+
+**MCP Gateway Section:**
+- All MCP-related changes go here (not in General Proxy Improvements)
+- OAuth 2.0 flow improvements
+- MCP configuration and tools
+- Server management features
+
+**Spend Tracking, Budgets and Rate Limiting Section:**
+- Service tier pricing (OpenAI priority/flex pricing)
+- Cost tracking and breakdown features
+- Rate limiting improvements (Parallel Request Limiter v3)
+- Priority reservation fixes
+- Metadata handling for rate limiting
+
+**Documentation Updates Section:**
+- Create separate section for all documentation improvements
+- Include provider documentation fixes
+- Model reference updates
+- New guides and tutorials
+- Documentation corrections and clarifications
+- This gives documentation changes proper visibility
+
+**Management Endpoints / UI Grouping:**
+- Group related features under sub-categories:
+  - **Proxy CLI Auth** - CLI authentication improvements
+  - **Virtual Keys** - Key rotation and management
+  - **Models + Endpoints** - Provider and endpoint management
+
+**Logging Section Expansion:**
+- Rename to "Logging / Guardrail / Prompt Management Integrations"
+- Add **Prompt Management** subsection for BitBucket, GitHub integrations
+- Keep guardrails separate from logging features
+
+## Example Command Workflow
+
+```bash
+# 1. Get model changes
+git diff <commit> HEAD -- model_prices_and_context_window.json
+
+# 2. Analyze PR list for categorization
+# 3. Create release notes following template
+# 4. Link to appropriate documentation
+# 5. Review for missing documentation needs
+```
+
+## Output Requirements
+
+- Follow exact markdown structure from reference
+- Include all PR links and contributors
+- Provide accurate model pricing tables
+- Link to relevant documentation
+- Highlight breaking changes with warnings
+- Include deployment instructions
+- End with full changelog link
+
+This process ensures consistent, comprehensive release notes that help users understand changes and upgrade smoothly.

cookbook/misc/test_responses_api.py

@@ -0,0 +1,53 @@
+import base64
+from openai import OpenAI
+import time
+client = OpenAI(
+    base_url="http://0.0.0.0:4001",
+    api_key="sk-1234"
+)
+
+# Function to encode the image
+def encode_image(image_path):
+    with open(image_path, "rb") as image_file:
+        return base64.b64encode(image_file.read()).decode("utf-8")
+
+
+# Path to your image
+image_path = "litellm/proxy/logo.jpg"
+
+# Getting the Base64 string
+base64_image = encode_image(image_path)
+
+
+response = client.responses.create(
+    model="bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0",
+    input=[
+        {
+            "role": "user",
+            "content": [
+                { "type": "input_text", "text": "what color is the image"},
+                {
+                    "type": "input_image",
+                    "image_url": f"data:image/jpeg;base64,{base64_image}",
+                },
+            ],
+        }
+    ],
+)
+
+
+
+print(response.output_text)
+print("response1 id===", response.id)
+print("sleeping for 20 seconds...")
+time.sleep(20)
+print("making follow up request for existing id")
+response2 = client.responses.create(
+    model="bedrock/us.anthropic.claude-3-5-sonnet-20241022-v2:0",
+    previous_response_id=response.id,
+    input="ok, and what objects are in the image?"
+)
+
+print(response2.output_text)
+
+

cookbook/veo_video_generation.py

@@ -0,0 +1,311 @@
+#!/usr/bin/env python3
+"""
+Complete example for Veo video generation through LiteLLM proxy.
+
+This script demonstrates how to:
+1. Generate videos using Google's Veo model
+2. Poll for completion status
+3. Download the generated video file
+
+Requirements:
+- LiteLLM proxy running with Google AI Studio pass-through configured
+- Google AI Studio API key with Veo access
+"""
+
+import json
+import os
+import time
+import requests
+from typing import Optional
+
+
+class VeoVideoGenerator:
+    """Complete Veo video generation client using LiteLLM proxy."""
+    
+    def __init__(self, base_url: str = "http://localhost:4000/gemini/v1beta", 
+                 api_key: str = "sk-1234"):
+        """
+        Initialize the Veo video generator.
+        
+        Args:
+            base_url: Base URL for the LiteLLM proxy with Gemini pass-through
+            api_key: API key for LiteLLM proxy authentication
+        """
+        self.base_url = base_url
+        self.api_key = api_key
+        self.headers = {
+            "x-goog-api-key": api_key,
+            "Content-Type": "application/json"
+        }
+    
+    def generate_video(self, prompt: str) -> Optional[str]:
+        """
+        Initiate video generation with Veo.
+        
+        Args:
+            prompt: Text description of the video to generate
+            
+        Returns:
+            Operation name if successful, None otherwise
+        """
+        print(f"🎬 Generating video with prompt: '{prompt}'")
+        
+        url = f"{self.base_url}/models/veo-3.0-generate-preview:predictLongRunning"
+        payload = {
+            "instances": [{
+                "prompt": prompt
+            }]
+        }
+        
+        try:
+            response = requests.post(url, headers=self.headers, json=payload)
+            response.raise_for_status()
+            
+            data = response.json()
+            operation_name = data.get("name")
+            
+            if operation_name:
+                print(f"✅ Video generation started: {operation_name}")
+                return operation_name
+            else:
+                print("❌ No operation name returned")
+                print(f"Response: {json.dumps(data, indent=2)}")
+                return None
+                
+        except requests.RequestException as e:
+            print(f"❌ Failed to start video generation: {e}")
+            if hasattr(e, 'response') and e.response is not None:
+                try:
+                    error_data = e.response.json()
+                    print(f"Error details: {json.dumps(error_data, indent=2)}")
+                except:
+                    print(f"Error response: {e.response.text}")
+            return None
+    
+    def wait_for_completion(self, operation_name: str, max_wait_time: int = 600) -> Optional[str]:
+        """
+        Poll operation status until video generation is complete.
+        
+        Args:
+            operation_name: Name of the operation to monitor
+            max_wait_time: Maximum time to wait in seconds (default: 10 minutes)
+            
+        Returns:
+            Video URI if successful, None otherwise
+        """
+        print("⏳ Waiting for video generation to complete...")
+        
+        operation_url = f"{self.base_url}/{operation_name}"
+        start_time = time.time()
+        poll_interval = 10  # Start with 10 seconds
+        
+        while time.time() - start_time < max_wait_time:
+            try:
+                print(f"🔍 Polling status... ({int(time.time() - start_time)}s elapsed)")
+                
+                response = requests.get(operation_url, headers=self.headers)
+                response.raise_for_status()
+                
+                data = response.json()
+                
+                # Check for errors
+                if "error" in data:
+                    print("❌ Error in video generation:")
+                    print(json.dumps(data["error"], indent=2))
+                    return None
+                
+                # Check if operation is complete
+                is_done = data.get("done", False)
+                
+                if is_done:
+                    print("🎉 Video generation complete!")
+                    
+                    try:
+                        # Extract video URI from nested response
+                        video_uri = data["response"]["generateVideoResponse"]["generatedSamples"][0]["video"]["uri"]
+                        print(f"📹 Video URI: {video_uri}")
+                        return video_uri
+                    except KeyError as e:
+                        print(f"❌ Could not extract video URI: {e}")
+                        print("Full response:")
+                        print(json.dumps(data, indent=2))
+                        return None
+                
+                # Wait before next poll, with exponential backoff
+                time.sleep(poll_interval)
+                poll_interval = min(poll_interval * 1.2, 30)  # Cap at 30 seconds
+                
+            except requests.RequestException as e:
+                print(f"❌ Error polling operation status: {e}")
+                time.sleep(poll_interval)
+        
+        print(f"⏰ Timeout after {max_wait_time} seconds")
+        return None
+    
+    def download_video(self, video_uri: str, output_filename: str = "generated_video.mp4") -> bool:
+        """
+        Download the generated video file.
+        
+        Args:
+            video_uri: URI of the video to download (from Google's response)
+            output_filename: Local filename to save the video
+            
+        Returns:
+            True if download successful, False otherwise
+        """
+        print(f"⬇️  Downloading video...")
+        print(f"Original URI: {video_uri}")
+        
+        # Convert Google URI to LiteLLM proxy URI
+        # Example: files/abc123 -> /gemini/v1beta/files/abc123:download?alt=media
+        if video_uri.startswith("files/"):
+            download_path = f"{video_uri}:download?alt=media"
+        else:
+            download_path = video_uri
+            
+        litellm_download_url = f"{self.base_url}/{download_path}"
+        print(f"Download URL: {litellm_download_url}")
+        
+        try:
+            # Download with streaming and redirect handling
+            response = requests.get(
+                litellm_download_url, 
+                headers=self.headers, 
+                stream=True,
+                allow_redirects=True  # Handle redirects automatically
+            )
+            response.raise_for_status()
+            
+            # Save video file
+            with open(output_filename, 'wb') as f:
+                downloaded_size = 0
+                for chunk in response.iter_content(chunk_size=8192):
+                    if chunk:
+                        f.write(chunk)
+                        downloaded_size += len(chunk)
+                        
+                        # Progress indicator for large files
+                        if downloaded_size % (1024 * 1024) == 0:  # Every MB
+                            print(f"📦 Downloaded {downloaded_size / (1024*1024):.1f} MB...")
+            
+            # Verify file was created and has content
+            if os.path.exists(output_filename):
+                file_size = os.path.getsize(output_filename)
+                if file_size > 0:
+                    print(f"✅ Video downloaded successfully!")
+                    print(f"📁 Saved as: {output_filename}")
+                    print(f"📏 File size: {file_size / (1024*1024):.2f} MB")
+                    return True
+                else:
+                    print("❌ Downloaded file is empty")
+                    os.remove(output_filename)
+                    return False
+            else:
+                print("❌ File was not created")
+                return False
+                
+        except requests.RequestException as e:
+            print(f"❌ Download failed: {e}")
+            if hasattr(e, 'response') and e.response is not None:
+                print(f"Status code: {e.response.status_code}")
+                print(f"Response headers: {dict(e.response.headers)}")
+            return False
+    
+    def generate_and_download(self, prompt: str, output_filename: str = None) -> bool:
+        """
+        Complete workflow: generate video and download it.
+        
+        Args:
+            prompt: Text description for video generation
+            output_filename: Output filename (auto-generated if None)
+            
+        Returns:
+            True if successful, False otherwise
+        """
+        # Auto-generate filename if not provided
+        if output_filename is None:
+            timestamp = int(time.time())
+            safe_prompt = "".join(c for c in prompt[:30] if c.isalnum() or c in (' ', '-', '_')).rstrip()
+            output_filename = f"veo_video_{safe_prompt.replace(' ', '_')}_{timestamp}.mp4"
+        
+        print("=" * 60)
+        print("🎬 VEO VIDEO GENERATION WORKFLOW")
+        print("=" * 60)
+        
+        # Step 1: Generate video
+        operation_name = self.generate_video(prompt)
+        if not operation_name:
+            return False
+        
+        # Step 2: Wait for completion
+        video_uri = self.wait_for_completion(operation_name)
+        if not video_uri:
+            return False
+        
+        # Step 3: Download video
+        success = self.download_video(video_uri, output_filename)
+        
+        if success:
+            print("=" * 60)
+            print("🎉 SUCCESS! Video generation complete!")
+            print(f"📁 Video saved as: {output_filename}")
+            print("=" * 60)
+        else:
+            print("=" * 60)
+            print("❌ FAILED! Video generation or download failed")
+            print("=" * 60)
+        
+        return success
+
+
+def main():
+    """
+    Example usage of the VeoVideoGenerator.
+    
+    Configure these environment variables:
+    - LITELLM_BASE_URL: Your LiteLLM proxy URL (default: http://localhost:4000/gemini/v1beta)
+    - LITELLM_API_KEY: Your LiteLLM API key (default: sk-1234)
+    """
+    
+    # Configuration from environment or defaults
+    base_url = os.getenv("LITELLM_BASE_URL", "http://localhost:4000/gemini/v1beta")
+    api_key = os.getenv("LITELLM_API_KEY", "sk-1234")
+    
+    print("🚀 Starting Veo Video Generation Example")
+    print(f"📡 Using LiteLLM proxy at: {base_url}")
+    
+    # Initialize generator
+    generator = VeoVideoGenerator(base_url=base_url, api_key=api_key)
+    
+    # Example prompts - try different ones!
+    example_prompts = [
+        "A cat playing with a ball of yarn in a sunny garden",
+        "Ocean waves crashing against rocky cliffs at sunset",
+        "A bustling city street with people walking and cars passing by",
+        "A peaceful forest with sunlight filtering through the trees"
+    ]
+    
+    # Use first example or get from user
+    prompt = example_prompts[0]
+    print(f"🎬 Using prompt: '{prompt}'")
+    
+    # Generate and download video
+    success = generator.generate_and_download(prompt)
+    
+    if success:
+        print("\n✅ Example completed successfully!")
+        print("💡 Try modifying the prompt in the script for different videos!")
+    else:
+        print("\n❌ Example failed!")
+        print("🔧 Check your LiteLLM proxy configuration and Google AI Studio API key")
+        
+        # Troubleshooting tips
+        print("\n🔍 Troubleshooting:")
+        print("1. Ensure LiteLLM proxy is running with Google AI Studio pass-through")
+        print("2. Verify your Google AI Studio API key has Veo access")
+        print("3. Check that your prompt meets Veo's content guidelines")
+        print("4. Review the LiteLLM proxy logs for detailed error information")
+
+
+if __name__ == "__main__":
+    main()

deploy/charts/litellm-helm/Chart.yaml

@@ -18,7 +18,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.4.4
+version: 0.4.6
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

deploy/charts/litellm-helm/README.md

@@ -24,7 +24,7 @@ If `db.useStackgresOperator` is used (not yet implemented):
 | `replicaCount`                                             | The number of LiteLLM Proxy pods to be deployed                                                                                                                                       | `1`  |
 | `masterkeySecretName`                                      | The name of the Kubernetes Secret that contains the Master API Key for LiteLLM.  If not specified, use the generated secret name.                                                                                                         | N/A  |
 | `masterkeySecretKey`                                      | The key within the Kubernetes Secret that contains the Master API Key for LiteLLM.  If not specified, use `masterkey` as the key.                                                                                                         | N/A  |
-| `masterkey`                                                | The Master API Key for LiteLLM.  If not specified, a random key is generated.                                                                                                         | N/A  |
+| `masterkey`                                                | The Master API Key for LiteLLM.  If not specified, a random key in the `sk-...` format is generated.                                                                                   | N/A  |
 | `environmentSecrets`                                       | An optional array of Secret object names.  The keys and values in these secrets will be presented to the LiteLLM proxy pod as environment variables.  See below for an example Secret object.  | `[]`  |
 | `environmentConfigMaps`                                       | An optional array of ConfigMap object names.  The keys and values in these configmaps will be presented to the LiteLLM proxy pod as environment variables.  See below for an example Secret object.  | `[]`  |
 | `image.repository`                                         | LiteLLM Proxy image repository                                                                                                                                                        | `ghcr.io/berriai/litellm`  |
@@ -36,11 +36,50 @@ If `db.useStackgresOperator` is used (not yet implemented):
 | `service.port`                                             | TCP port that the Kubernetes Service will listen on.  Also the TCP port within the Pod that the proxy will listen on.                                                                 | `4000`  |
 | `service.loadBalancerClass`                                | Optional LoadBalancer implementation class (only used when `service.type` is `LoadBalancer`)                                                                                                          | `""`  |
 | `ingress.*`                                                | See [values.yaml](./values.yaml) for example settings                                                                                                                                 | N/A  |
-| `proxy_config.*`                                           | See [values.yaml](./values.yaml) for default settings.  See [example_config_yaml](../../../litellm/proxy/example_config_yaml/) for configuration examples.                            | N/A  |
-| `extraContainers[]`                                        | An array of additional containers to be deployed as sidecars alongside the LiteLLM Proxy.                                                                                             | `[]`  |
+| `proxyConfigMap.create`     | When `true`, render a ConfigMap from `.Values.proxy_config` and mount it.                                                                                                                                      | `true` |
+| `proxyConfigMap.name`       | When `create=false`, name of the existing ConfigMap to mount.                                                                                                                                                  | `""`  |
+| `proxyConfigMap.key`        | Key in the ConfigMap that contains the proxy config file.                                                                                                                                                      | `"config.yaml"` |
+| `proxy_config.*`            | See [values.yaml](./values.yaml) for default settings. Rendered into the ConfigMap’s `config.yaml` only when `proxyConfigMap.create=true`. See [example_config_yaml](../../../litellm/proxy/example_config_yaml/) for configuration examples. | `N/A` |
+| `extraContainers[]`         | An array of additional containers to be deployed as sidecars alongside the LiteLLM Proxy.
+| `pdb.enabled`                   | Enable a PodDisruptionBudget for the LiteLLM proxy Deployment                                                                 | `false` |
+| `pdb.minAvailable`             | Minimum number/percentage of pods that must be available during **voluntary** disruptions (choose **one** of minAvailable/maxUnavailable) | `null`  |
+| `pdb.maxUnavailable`           | Maximum number/percentage of pods that can be unavailable during **voluntary** disruptions (choose **one** of minAvailable/maxUnavailable) | `null`  |
+| `pdb.annotations`              | Extra metadata annotations to add to the PDB                                                                                   | `{}`    |
+| `pdb.labels`                   | Extra metadata labels to add to the PDB                                                                                        | `{}`    |
+
+#### Example `proxy_config` ConfigMap from values (default):
+
+
+```
+proxyConfigMap:
+  create: true
+  key: "config.yaml"
+
+proxy_config:
+  general_settings:
+    master_key: os.environ/PROXY_MASTER_KEY
+  model_list:
+    - model_name: gpt-3.5-turbo
+      litellm_params:
+        model: gpt-3.5-turbo
+        api_key: eXaMpLeOnLy
+```
+
+#### Example using existing `proxyConfigMap` instead of creating it:
+
+
+```
+proxyConfigMap:
+  create: false
+  name: my-litellm-config
+  key: config.yaml
+
+# proxy_config is ignored in this mode
+```
 
 #### Example `environmentSecrets` Secret 
 
+
 ```
 apiVersion: v1
 kind: Secret
@@ -110,6 +149,22 @@ data:
 
 Source: [GitHub Gist from troyharvey](https://gist.github.com/troyharvey/4506472732157221e04c6b15e3b3f094)
 
+### Migration Job Settings
+
+The migration job supports both ArgoCD and Helm hooks to ensure database migrations run at the appropriate time during deployments.
+
+| Name                                                       | Description                                                                                                                                                                           | Value |
+| ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- |
+| `migrationJob.enabled`                                     | Enable or disable the schema migration Job                                                                                                                                           | `true`  |
+| `migrationJob.backoffLimit`                                | Backoff limit for Job restarts                                                                                                                                                       | `4`  |
+| `migrationJob.ttlSecondsAfterFinished`                     | TTL for completed migration jobs                                                                                                                                                      | `120`  |
+| `migrationJob.annotations`                                 | Additional annotations for the migration job pod                                                                                                                                      | `{}`  |
+| `migrationJob.extraContainers`                             | Additional containers to run alongside the migration job                                                                                                                              | `[]`  |
+| `migrationJob.hooks.argocd.enabled`                       | Enable ArgoCD hooks for the migration job (uses PreSync hook with BeforeHookCreation delete policy)                                                                                  | `true`  |
+| `migrationJob.hooks.helm.enabled`                         | Enable Helm hooks for the migration job (uses pre-install,pre-upgrade hooks with before-hook-creation delete policy)                                                                 | `false`  |
+| `migrationJob.hooks.helm.weight`                          | Helm hook execution order (lower weights executed first). Optional - defaults to "1" if not specified.                                                                               | N/A  |
+
+
 ## Accessing the Admin UI
 When browsing to the URL published per the settings in `ingress.*`, you will
 be prompted for **Admin Configuration**.  The **Proxy Endpoint** is the internal
@@ -119,7 +174,7 @@ service, the **Proxy Endpoint** should be set to `http://<RELEASE>-litellm:4000`
 
 The **Proxy Key** is the value specified for `masterkey` or, if a `masterkey`
 was not provided to the helm command line, the `masterkey` is a randomly
-generated string stored in the `<RELEASE>-litellm-masterkey` Kubernetes Secret.
+generated string in the `sk-...` format stored in the `<RELEASE>-litellm-masterkey` Kubernetes Secret.
 
 ```bash
 kubectl -n litellm get secret <RELEASE>-litellm-masterkey -o jsonpath="{.data.masterkey}"

deploy/charts/litellm-helm/templates/NOTES.txt

@@ -20,3 +20,4 @@
   echo "Visit http://127.0.0.1:8080 to use your application"
   kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT
 {{- end }}
+PDB: {{ if .Values.pdb.enabled }}enabled{{ else }}disabled{{ end }}. Configure via .Values.pdb.*

deploy/charts/litellm-helm/templates/configmap-litellm.yaml

@@ -1,7 +1,9 @@
+{{- if .Values.proxyConfigMap.create }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
   name: {{ include "litellm.fullname" . }}-config
 data:
   config.yaml: |
-{{ .Values.proxy_config | toYaml | indent 6 }}
+{{ .Values.proxy_config | toYaml | indent 6 }}
+{{- end }}

deploy/charts/litellm-helm/templates/deployment.yaml

@@ -16,7 +16,9 @@ spec:
   template:
     metadata:
       annotations:
+        {{- if .Values.proxyConfigMap.create }}
         checksum/config: {{ include (print $.Template.BasePath "/configmap-litellm.yaml") . | sha256sum }}
+        {{- end }}
         {{- with .Values.podAnnotations }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
@@ -71,7 +73,14 @@ spec:
                   name: {{ .Values.db.secret.name }}
                   key: {{ .Values.db.secret.passwordKey }}
             - name: DATABASE_HOST
+              {{- if .Values.db.secret.endpointKey }}
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.db.secret.name }}
+                  key: {{ .Values.db.secret.endpointKey }}
+              {{- else }}
               value: {{ .Values.db.endpoint }}
+              {{- end }}
             - name: DATABASE_NAME
               value: {{ .Values.db.database }}
             - name: DATABASE_URL
@@ -99,6 +108,12 @@ spec:
               value: {{ $val | quote }}
             {{- end }}
             {{- end }}
+            {{- if .Values.separateHealthApp }}
+            - name: SEPARATE_HEALTH_APP
+              value: "1"
+            - name: SEPARATE_HEALTH_PORT
+              value: {{ .Values.separateHealthPort | default "8081" | quote }}
+            {{- end }}
             {{- with .Values.extraEnvVars }}
               {{- toYaml . | nindent 12 }}
             {{- end }}
@@ -118,19 +133,23 @@ spec:
             - name: http
               containerPort: {{ .Values.service.port }}
               protocol: TCP
+            {{- if .Values.separateHealthApp }}
+            - name: health
+              containerPort: {{ .Values.separateHealthPort | default 8081 }}
+              protocol: TCP
+            {{- end }}
           livenessProbe:
             httpGet:
               path: /health/liveliness
-              port: http
+              port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
           readinessProbe:
             httpGet:
               path: /health/readiness
-              port: http
-          # Give the container time to start up.  Up to 5 minutes (10 * 30 seconds)
+              port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
           startupProbe:
             httpGet:
               path: /health/readiness
-              port: http
+              port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
             failureThreshold: 30
             periodSeconds: 10
           resources:
@@ -166,9 +185,13 @@ spec:
         {{- end }}
         - name: litellm-config
           configMap:
+            {{- if .Values.proxyConfigMap.create }}
             name: {{ include "litellm.fullname" . }}-config
+            {{- else }}
+            name: {{ .Values.proxyConfigMap.name }}
+            {{- end }}
             items:
-              - key: "config.yaml"
+              - key: {{ .Values.proxyConfigMap.key | default "config.yaml" }}
                 path: "config.yaml"
       {{- with .Values.volumes }}
         {{- toYaml . | nindent 8 }}

deploy/charts/litellm-helm/templates/migrations-job.yaml

@@ -1,16 +1,27 @@
 {{- if .Values.migrationJob.enabled }}
-# This job runs the prisma migrations for the LiteLLM DB.
+# This job runs the Prisma migrations for the LiteLLM DB.
 apiVersion: batch/v1
 kind: Job
 metadata:
   name: {{ include "litellm.fullname" . }}-migrations
+  labels:
+    {{- include "litellm.labels" . | nindent 4 }}
   annotations:
+    {{- if .Values.migrationJob.hooks.argocd.enabled }}
     argocd.argoproj.io/hook: PreSync
-    argocd.argoproj.io/hook-delete-policy: BeforeHookCreation # delete old migration on a new deploy in case the migration needs to make updates
+    argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+    {{- end }}
+    {{- if .Values.migrationJob.hooks.helm.enabled }}
+    helm.sh/hook: "pre-install,pre-upgrade"
+    helm.sh/hook-delete-policy: "before-hook-creation"
+    helm.sh/hook-weight: {{ .Values.migrationJob.hooks.helm.weight | default "1" | quote }}
+    {{- end }}
     checksum/config: {{ toYaml .Values | sha256sum }}
 spec:
   template:
     metadata:
+      labels:
+        {{- include "litellm.labels" . | nindent 8 }}
       annotations:
         {{- with .Values.migrationJob.annotations }}
         {{- toYaml . | nindent 8 }}
@@ -38,17 +49,22 @@ spec:
                   name: {{ .Values.db.secret.name }}
                   key: {{ .Values.db.secret.passwordKey }}
             - name: DATABASE_HOST
+              {{- if .Values.db.secret.endpointKey }}
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.db.secret.name }}
+                  key: {{ .Values.db.secret.endpointKey }}
+              {{- else }}
               value: {{ .Values.db.endpoint }}
+              {{- end }}
             - name: DATABASE_NAME
               value: {{ .Values.db.database }}
             - name: DATABASE_URL
               value: {{ .Values.db.url | quote }}
-            {{- else }}
+            {{- else if .Values.db.deployStandalone }}
             - name: DATABASE_URL
               value: postgresql://{{ .Values.postgresql.auth.username }}:{{ .Values.postgresql.auth.password }}@{{ .Release.Name }}-postgresql/{{ .Values.postgresql.auth.database }}
             {{- end }}
-            - name: DISABLE_SCHEMA_UPDATE
-              value: "false" # always run the migration from the Helm PreSync hook, override the value set
             {{- if .Values.envVars }}
             {{- range $key, $val := .Values.envVars }}
             - name: {{ $key }}
@@ -58,10 +74,16 @@ spec:
             {{- with .Values.extraEnvVars }}
               {{- toYaml . | nindent 12 }}
             {{- end }}
+            - name: DISABLE_SCHEMA_UPDATE
+              value: "false" # always run the migration from the Helm PreSync hook, override the value set
           {{- with .Values.volumeMounts }}
           volumeMounts:
             {{- toYaml . | nindent 12 }}
           {{- end }}
+          {{- with .Values.migrationJob.resources }}
+          resources:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
       {{- with .Values.migrationJob.extraContainers }}
         {{- toYaml . | nindent 8 }}
       {{- end }}

deploy/charts/litellm-helm/templates/poddisruptionbudget.yaml

@@ -0,0 +1,33 @@
+{{- /*
+PodDisruptionBudget for LiteLLM proxy
+Controlled via .Values.pdb.enabled and .Values.pdb.{minAvailable|maxUnavailable}
+Only one of minAvailable / maxUnavailable should be set. If both are set, minAvailable wins.
+*/ -}}
+{{- if .Values.pdb.enabled }}
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: {{ include "litellm.fullname" . }}
+  labels:
+    {{- include "litellm.labels" . | nindent 4 }}
+  {{- with .Values.pdb.labels }}
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  {{- with .Values.pdb.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  selector:
+    matchLabels:
+      {{- /* Match the Deployment selector to target the same pod set */ -}}
+      {{- include "litellm.selectorLabels" . | nindent 6 }}
+  {{- if .Values.pdb.minAvailable }}
+  minAvailable: {{ .Values.pdb.minAvailable }}
+  {{- else if .Values.pdb.maxUnavailable }}
+  maxUnavailable: {{ .Values.pdb.maxUnavailable }}
+  {{- else }}
+  # Safe default if enabled but not configured
+  maxUnavailable: 1
+  {{- end }}
+{{- end }}

deploy/charts/litellm-helm/templates/secret-masterkey.yaml

@@ -1,5 +1,5 @@
 {{- if not .Values.masterkeySecretName }}
-{{ $masterkey := (.Values.masterkey | default (randAlphaNum 17)) }}
+{{ $masterkey := (.Values.masterkey | default (printf "sk-%s" (randAlphaNum 18))) }}
 apiVersion: v1
 kind: Secret
 metadata:

deploy/charts/litellm-helm/tests/deployment_tests.yaml

@@ -115,3 +115,25 @@ tests:
           content:
             name: EXTRA_ENV_VAR
             value: EXTRA_ENV_VAR_VALUE
+  - it: should mount existing configmap when create=false
+    template: deployment.yaml
+    set:
+      proxyConfigMap:
+        create: false
+        name: my-litellm-config
+        key: custom.yaml
+    asserts:
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: litellm-config
+            configMap:
+              name: my-litellm-config
+              items:
+                - key: custom.yaml
+                  path: config.yaml
+      - contains:
+          path: spec.template.spec.containers[0].volumeMounts
+          content:
+            name: litellm-config
+            mountPath: /etc/litellm/

deploy/charts/litellm-helm/tests/masterkey-secret_tests.yaml

@@ -2,13 +2,19 @@ suite: test masterkey secret
 templates:
   - secret-masterkey.yaml
 tests:
-  - it: should create a secret if masterkeySecretName is not set
+  - it: should create a secret if masterkeySecretName is not set. should start with sk-xxxx (base64 encoded as c2st*)
     template: secret-masterkey.yaml
     set:
       masterkeySecretName: ""
     asserts:
       - isKind:
           of: Secret
+      - matchRegex:
+          path: data.masterkey
+          pattern: ^c2st
+          # Note: The masterkey is generated as "sk-<18-random-chars>" in plain text,
+          # but stored as base64 encoded in Kubernetes secret (requirement).
+          # "sk-" base64 encodes to "c2st", so we check for "^c2st" pattern.
   - it: should not create a secret if masterkeySecretName is set
     template: secret-masterkey.yaml
     set:

deploy/charts/litellm-helm/tests/migrations-job_tests.yaml

@@ -110,4 +110,18 @@ tests:
           path: spec.template.spec.containers[0].env
           content:
             name: CUSTOM_VAR
-            value: "custom_value"
+            value: "custom_value"
+
+  - it: should not include DATABASE_URL when deployStandalone is false
+    template: migrations-job.yaml
+    set:
+      migrationJob:
+        enabled: true
+      db:
+        deployStandalone: false
+        useExisting: false
+    asserts:
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: DATABASE_URL

deploy/charts/litellm-helm/tests/pdb_tests.yaml

@@ -0,0 +1,45 @@
+suite: "pdb enabled"
+templates:
+  - poddisruptionbudget.yaml
+tests:
+  - it: "renders a PDB with maxUnavailable=1"
+    set:
+      pdb.enabled: true
+      pdb.maxUnavailable: 1
+    asserts:
+      - hasDocuments: { count: 1 }
+      - isKind: { of: PodDisruptionBudget }
+      - equal: { path: apiVersion, value: policy/v1 }
+      - equal: { path: spec.maxUnavailable, value: 1 }
+      - equal:
+          path: spec.selector.matchLabels
+          value:
+            app.kubernetes.io/name: litellm
+            app.kubernetes.io/instance: RELEASE-NAME
+
+---
+suite: "pdb disabled"
+templates:
+  - poddisruptionbudget.yaml
+tests:
+  - it: "does not render when disabled"
+    set:
+      pdb.enabled: false
+    asserts:
+      - hasDocuments: { count: 0 }
+
+---
+suite: "pdb minAvailable precedence"
+templates:
+  - poddisruptionbudget.yaml
+tests:
+  - it: "uses minAvailable when both are set"
+    set:
+      pdb.enabled: true
+      pdb.minAvailable: "50%"
+      pdb.maxUnavailable: 1
+    asserts:
+      - isKind: { of: PodDisruptionBudget }
+      - equal: { path: apiVersion, value: policy/v1 }
+      - equal: { path: spec.minAvailable, value: "50%" }
+      - isNull: { path: spec.maxUnavailable }

deploy/charts/litellm-helm/values.yaml

@@ -63,6 +63,12 @@ service:
   # optionally specify loadBalancerClass
   # loadBalancerClass: tailscale
 
+# Separate health app configuration
+# When enabled, health checks will use a separate port and the application
+# will receive SEPARATE_HEALTH_APP=1 and SEPARATE_HEALTH_PORT from environment variables
+separateHealthApp: false
+separateHealthPort: 8081
+
 ingress:
   enabled: false
   className: "nginx"
@@ -87,6 +93,14 @@ masterkeySecretName: ""
 # if set, use this secret key for the master key; otherwise, use the default key
 masterkeySecretKey: ""
 
+proxyConfigMap:
+  # when true, creates a new configmap
+  create: true
+  # if create is false and name is set, use existing ConfigMap
+  # create: false
+  # name: ""
+  # key: "config.yaml"
+
 # The elements within proxy_config are rendered as config.yaml for the proxy
 #  Examples: https://github.com/BerriAI/litellm/tree/main/litellm/proxy/example_config_yaml
 #  Reference: https://docs.litellm.ai/docs/proxy/configs
@@ -155,6 +169,8 @@ db:
     name: postgres
     usernameKey: username
     passwordKey: password
+    # Optional: when set, DATABASE_HOST will be sourced from this secret key instead of db.endpoint
+    endpointKey: ""
 
   # Use the Stackgres Helm chart to deploy an instance of a Stackgres cluster.
   #  The Stackgres Operator must already be installed within the target
@@ -200,7 +216,18 @@ migrationJob:
   disableSchemaUpdate: false # Skip schema migrations for specific environments. When True, the job will exit with code 0.
   annotations: {}
   ttlSecondsAfterFinished: 120
+  resources: {}
+  #  requests:
+  #    cpu: 100m
+  #    memory: 100Mi
   extraContainers: []
+  
+  # Hook configuration
+  hooks:
+    argocd:
+      enabled: true
+    helm:
+      enabled: false
 
 # Additional environment variables to be added to the deployment as a map of key-value pairs
 envVars: {
@@ -213,4 +240,11 @@ extraEnvVars: {
     #   value: EXTRA_ENV_VAR_VALUE
 }
 
-
+# Pod Disruption Budget
+pdb:
+  enabled: false
+  # Set exactly one of the following. If both are set, minAvailable takes precedence.
+  minAvailable: null     # e.g. "50%" or 1
+  maxUnavailable: null   # e.g. 1 or "20%"
+  annotations: {}
+  labels: {}

docker-compose.yml

@@ -1,68 +1,66 @@
-version: "3.11"
-services:
-  litellm:
-    build:
-      context: .
-      args:
-        target: runtime
-    image: ghcr.io/berriai/litellm:main-stable
-    #########################################
-    ## Uncomment these lines to start proxy with a config.yaml file ##
-    # volumes:
-    #  - ./config.yaml:/app/config.yaml <<- this is missing in the docker-compose file currently
-    # command:
-    #  - "--config=/app/config.yaml"
-    ##############################################
-    ports:
-      - "4000:4000" # Map the container port to the host, change the host port if necessary
-    environment:
-      DATABASE_URL: "postgresql://llmproxy:dbpassword9090@db:5432/litellm"
-      STORE_MODEL_IN_DB: "True" # allows adding models to proxy via UI
-    env_file:
-      - .env # Load local .env file
-    depends_on:
-      - db  # Indicates that this service depends on the 'db' service, ensuring 'db' starts first
-    healthcheck:  # Defines the health check configuration for the container
-      test: [ "CMD-SHELL", "wget --no-verbose --tries=1 http://localhost:4000/health/liveliness || exit 1" ]  # Command to execute for health check
-      interval: 30s  # Perform health check every 30 seconds
-      timeout: 10s   # Health check command times out after 10 seconds
-      retries: 3     # Retry up to 3 times if health check fails
-      start_period: 40s  # Wait 40 seconds after container start before beginning health checks
-
-  db:
-    image: postgres:16
-    restart: always
-    container_name: litellm_db
-    environment:
-      POSTGRES_DB: litellm
-      POSTGRES_USER: llmproxy
-      POSTGRES_PASSWORD: dbpassword9090
-    ports:
-      - "5432:5432"
-    volumes:
-      - postgres_data:/var/lib/postgresql/data # Persists Postgres data across container restarts
-    healthcheck:
-      test: ["CMD-SHELL", "pg_isready -d litellm -U llmproxy"]
-      interval: 1s
-      timeout: 5s
-      retries: 10
-
-  prometheus:
-    image: prom/prometheus
-    volumes:
-      - prometheus_data:/prometheus
-      - ./prometheus.yml:/etc/prometheus/prometheus.yml
-    ports:
-      - "9090:9090"
-    command:
-      - "--config.file=/etc/prometheus/prometheus.yml"
-      - "--storage.tsdb.path=/prometheus"
-      - "--storage.tsdb.retention.time=15d"
-    restart: always
-
-volumes:
-  prometheus_data:
-    driver: local
-  postgres_data:
-    name: litellm_postgres_data # Named volume for Postgres data persistence
-
+services:
+  litellm:
+    build:
+      context: .
+      args:
+        target: runtime
+    image: ghcr.io/berriai/litellm:main-stable
+    #########################################
+    ## Uncomment these lines to start proxy with a config.yaml file ##
+    # volumes:
+    #  - ./config.yaml:/app/config.yaml
+    # command:
+    #  - "--config=/app/config.yaml"
+    ##############################################
+    ports:
+      - "4000:4000" # Map the container port to the host, change the host port if necessary
+    environment:
+      DATABASE_URL: "postgresql://llmproxy:dbpassword9090@db:5432/litellm"
+      STORE_MODEL_IN_DB: "True" # allows adding models to proxy via UI
+    env_file:
+      - .env # Load local .env file
+    depends_on:
+      - db  # Indicates that this service depends on the 'db' service, ensuring 'db' starts first
+    healthcheck:  # Defines the health check configuration for the container
+      test: [ "CMD-SHELL", "wget --no-verbose --tries=1 http://localhost:4000/health/liveliness || exit 1" ]  # Command to execute for health check
+      interval: 30s  # Perform health check every 30 seconds
+      timeout: 10s   # Health check command times out after 10 seconds
+      retries: 3     # Retry up to 3 times if health check fails
+      start_period: 40s  # Wait 40 seconds after container start before beginning health checks
+
+  db:
+    image: postgres:16
+    restart: always
+    container_name: litellm_db
+    environment:
+      POSTGRES_DB: litellm
+      POSTGRES_USER: llmproxy
+      POSTGRES_PASSWORD: dbpassword9090
+    ports:
+      - "5432:5432"
+    volumes:
+      - postgres_data:/var/lib/postgresql/data # Persists Postgres data across container restarts
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -d litellm -U llmproxy"]
+      interval: 1s
+      timeout: 5s
+      retries: 10
+
+  prometheus:
+    image: prom/prometheus
+    volumes:
+      - prometheus_data:/prometheus
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml
+    ports:
+      - "9090:9090"
+    command:
+      - "--config.file=/etc/prometheus/prometheus.yml"
+      - "--storage.tsdb.path=/prometheus"
+      - "--storage.tsdb.retention.time=15d"
+    restart: always
+
+volumes:
+  prometheus_data:
+    driver: local
+  postgres_data:
+    name: litellm_postgres_data # Named volume for Postgres data persistence

docker/Dockerfile.database

@@ -57,8 +57,8 @@ COPY --from=builder /wheels/ /wheels/
 # Install the built wheel using pip; again using a wildcard if it's the only file
 RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
 
-# Install semantic_router without dependencies
-RUN pip install semantic_router --no-deps
+# Install semantic_router and aurelio-sdk using script
+RUN chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
 
 # ensure pyjwt is used, not jwt
 RUN pip uninstall jwt -y

docker/Dockerfile.non_root

@@ -11,7 +11,7 @@ WORKDIR /app
 # Install build dependencies
 USER root
 RUN apk add --no-cache build-base bash \
-    && pip install --no-cache-dir --upgrade pip build
+  && pip install --no-cache-dir --upgrade pip build
 
 # Copy project files
 COPY . .
@@ -21,8 +21,8 @@ RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
 
 # Build package and wheel dependencies
 RUN rm -rf dist/* && python -m build && \
-    pip install dist/*.whl && \
-    pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt
+  pip install dist/*.whl && \
+  pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt
 
 # -----------------
 # Runtime Stage
@@ -33,26 +33,28 @@ WORKDIR /app
 # Install runtime dependencies
 USER root
 RUN apk upgrade --no-cache && \
-    apk add --no-cache bash
+  apk add --no-cache bash libstdc++ ca-certificates openssl supervisor
 
 # Copy only necessary artifacts from builder stage for runtime
+COPY . .
 COPY --from=builder /app/docker/entrypoint.sh /app/docker/prod_entrypoint.sh /app/docker/
+COPY --from=builder /app/docker/supervisord.conf /etc/supervisord.conf
 COPY --from=builder /app/schema.prisma /app/schema.prisma
 COPY --from=builder /app/dist/*.whl .
 COPY --from=builder /wheels/ /wheels/
 
 # Install package from wheel and dependencies
 RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ \
-    && rm -f *.whl \
-    && rm -rf /wheels
+  && rm -f *.whl \
+  && rm -rf /wheels
 
-# Install semantic_router without dependencies
-RUN pip install semantic_router --no-deps
+# Install semantic_router and aurelio-sdk using script
+RUN chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
 
 # Ensure correct JWT library is used (pyjwt not jwt)
 RUN pip uninstall jwt -y && \
-    pip uninstall PyJWT -y && \
-    pip install PyJWT==2.9.0 --no-cache-dir
+  pip uninstall PyJWT -y && \
+  pip install PyJWT==2.9.0 --no-cache-dir
 
 # --- Prisma Handling for Non-Root User ---
 # Set Prisma cache directories
@@ -61,15 +63,31 @@ ENV NPM_CONFIG_CACHE=/.npm
 
 # Install prisma and make entrypoints executable
 RUN pip install --no-cache-dir prisma && \
-    chmod +x docker/entrypoint.sh && \
-    chmod +x docker/prod_entrypoint.sh
+  chmod +x docker/entrypoint.sh && \
+  chmod +x docker/prod_entrypoint.sh
 
 # Create directories and set permissions for non-root user
 RUN mkdir -p /nonexistent /.npm && \
-    chown -R nobody:nogroup /app && \
-    chown -R nobody:nogroup /nonexistent /.npm && \
-    PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \
-    chown -R nobody:nogroup $PRISMA_PATH
+  chown -R nobody:nogroup /app && \
+  chown -R nobody:nogroup /nonexistent /.npm && \
+  PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \
+  chown -R nobody:nogroup $PRISMA_PATH && \
+  LITELLM_PKG_MIGRATIONS_PATH="$(python -c 'import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))' 2>/dev/null || echo '')/migrations" && \
+  [ -n "$LITELLM_PKG_MIGRATIONS_PATH" ] && chown -R nobody:nogroup $LITELLM_PKG_MIGRATIONS_PATH
+
+# --- OpenShift Compatibility: Apply Red Hat recommended pattern ---
+# Get paths for directories that need write access at runtime
+RUN PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \
+  LITELLM_PROXY_EXTRAS_PATH=$(python -c "import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))" 2>/dev/null || echo "") && \
+  # Set group ownership to 0 (root group) for OpenShift compatibility && \
+  chgrp -R 0 $PRISMA_PATH && \
+  [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chgrp -R 0 $LITELLM_PROXY_EXTRAS_PATH || true && \
+  # Mirror owner permissions to group (g=u) as recommended by Red Hat && \
+  chmod -R g=u $PRISMA_PATH && \
+  [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g=u $LITELLM_PROXY_EXTRAS_PATH || true && \
+  # Ensure directories are writable by group && \
+  chmod -R g+w $PRISMA_PATH && \
+  [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g+w $LITELLM_PROXY_EXTRAS_PATH || true
 
 # Switch to non-root user
 USER nobody

docker/README.md

@@ -1,3 +1,65 @@
-# LiteLLM Docker 
+# Docker Development Guide
 
-This is a minimal Docker Compose setup for self-hosting LiteLLM.
+This guide provides instructions for building and running the LiteLLM application using Docker and Docker Compose.
+
+## Prerequisites
+
+- Docker
+- Docker Compose
+
+## Building and Running the Application
+
+To build and run the application, you will use the `docker-compose.yml` file located in the root of the project. This file is configured to use the `Dockerfile.non_root` for a secure, non-root container environment.
+
+### 1. Set the Master Key
+
+The application requires a `MASTER_KEY` for signing and validating tokens. You must set this key as an environment variable before running the application.
+
+Create a `.env` file in the root of the project and add the following line:
+
+```
+MASTER_KEY=your-secret-key
+```
+
+Replace `your-secret-key` with a strong, randomly generated secret.
+
+### 2. Build and Run the Containers
+
+Once you have set the `MASTER_KEY`, you can build and run the containers using the following command:
+
+```bash
+docker compose up -d --build
+```
+
+This command will:
+
+-   Build the Docker image using `Dockerfile.non_root`.
+-   Start the `litellm`, `litellm_db`, and `prometheus` services in detached mode (`-d`).
+-   The `--build` flag ensures that the image is rebuilt if there are any changes to the Dockerfile or the application code.
+
+### 3. Verifying the Application is Running
+
+You can check the status of the running containers with the following command:
+
+```bash
+docker compose ps
+```
+
+To view the logs of the `litellm` container, run:
+
+```bash
+docker compose logs -f litellm
+```
+
+### 4. Stopping the Application
+
+To stop the running containers, use the following command:
+
+```bash
+docker compose down
+```
+
+## Troubleshooting
+
+-   **`build_admin_ui.sh: not found`**: This error can occur if the Docker build context is not set correctly. Ensure that you are running the `docker-compose` command from the root of the project.
+-   **`Master key is not initialized`**: This error means the `MASTER_key` environment variable is not set. Make sure you have created a `.env` file in the project root with the `MASTER_KEY` defined.

docker/build_from_pip/requirements.txt

@@ -2,4 +2,5 @@ litellm[proxy]==1.67.4.dev1 # Specify the litellm version you want to use
 prometheus_client
 langfuse
 prisma
+openai==1.99.9
 ddtrace==2.19.0 # for advanced DD tracing / profiling

docker/install_auto_router.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+pip install semantic_router==0.1.11 --no-deps
+pip install aurelio-sdk==0.0.19

docs/my-website/docs/adding_provider/new_rerank_provider.md

@@ -17,7 +17,7 @@ class YourProviderRerankConfig(BaseRerankConfig):
             # ... other supported params
         ]
 
-    def transform_rerank_request(self, model: str, optional_rerank_params: OptionalRerankParams, headers: dict) -> dict:
+    def transform_rerank_request(self, model: str, optional_rerank_params: Dict, headers: dict) -> dict:
         # Transform request to RerankRequest spec
         return rerank_request.model_dump(exclude_none=True)
 

docs/my-website/docs/batches.md

@@ -7,7 +7,7 @@ Covers Batches, Files
 
 | Feature | Supported | Notes | 
 |-------|-------|-------|
-| Supported Providers | OpenAI, Azure, Vertex | - |
+| Supported Providers | OpenAI, Azure, Vertex, Bedrock | - |
 | ✨ Cost Tracking | ✅ | LiteLLM Enterprise only |
 | Logging | ✅ | Works across all logging integrations |
 
@@ -178,6 +178,7 @@ print("list_batches_response=", list_batches_response)
 ### [Azure OpenAI](./providers/azure#azure-batches-api)
 ### [OpenAI](#quick-start)
 ### [Vertex AI](./providers/vertex#batch-apis)
+### [Bedrock](./providers/bedrock_batches)
 
 
 ## How Cost Tracking for Batches API Works

docs/my-website/docs/benchmarks.md

@@ -16,19 +16,17 @@ model_list:
       api_key: "test"
 ```
 
-### 1 Instance LiteLLM Proxy
+### 2 Instance LiteLLM Proxy
 
 In these tests the baseline latency characteristics are measured against a fake-openai-endpoint.
 
 #### Performance Metrics
 
-| Metric | Value |
-|--------|-------|
-| **Requests per Second (RPS)** | 475 |
-| **End-to-End Latency P50 (ms)** | 100 |
-| **LiteLLM Overhead P50 (ms)** | 3 |
-| **LiteLLM Overhead P90 (ms)** | 17 |
-| **LiteLLM Overhead P99 (ms)** | 31 |
+| **Type** | **Name** | **Median (ms)** | **95%ile (ms)** | **99%ile (ms)** | **Average (ms)** | **Current RPS** |
+| --- | --- | --- | --- | --- | --- | --- |
+| POST | /chat/completions | 200 | 630 | 1200 | 262.46 | 1035.7 |
+| Custom | LiteLLM Overhead Duration (ms) | 12 | 29 | 43 | 14.74 | 1035.7 |
+|  | Aggregated | 100 | 430 | 930 | 138.6 | 2071.4 |
 
 <!-- <Image img={require('../img/1_instance_proxy.png')} /> -->
 
@@ -36,28 +34,32 @@ In these tests the baseline latency characteristics are measured against a fake-
 
 <Image img={require('../img/instances_vs_rps.png')} /> -->
 
+
+### 4 Instances
+
+| **Type** | **Name** | **Median (ms)** | **95%ile (ms)** | **99%ile (ms)** | **Average (ms)** | **Current RPS** |
+| --- | --- | --- | --- | --- | --- | --- |
+| POST | /chat/completions | 100 | 150 | 240 | 111.73 | 1170 |
+| Custom | LiteLLM Overhead Duration (ms) | 2 | 8 | 13 | 3.32 | 1170 |
+|  | Aggregated | 77 | 130 | 180 | 57.53 | 2340 |
+
 #### Key Findings
-- Single instance: 475 RPS @ 100ms median latency
-- LiteLLM adds 3ms P50 overhead, 17ms P90 overhead, 31ms P99 overhead
-- 2 LiteLLM instances: 950 RPS @ 100ms latency
-- 4 LiteLLM instances: 1900 RPS @ 100ms latency
-
-### 2 Instances
-
-**Adding 1 instance, will double the RPS and maintain the `100ms-110ms` median latency.**
-
-| Metric | Litellm Proxy (2 Instances) |
-|--------|------------------------|
-| Median Latency (ms) | 100 |
-| RPS | 950 |
-
+- Doubling from 2 to 4 LiteLLM instances halves median latency: 200 ms → 100 ms.
+- High-percentile latencies drop significantly: P95 630 ms → 150 ms, P99 1,200 ms → 240 ms.
+- Setting workers equal to CPU count gives optimal performance.
 
 ## Machine Spec used for testing
 
 Each machine deploying LiteLLM had the following specs:
 
-- 2 CPU
-- 4GB RAM
+- 4 CPU
+- 8GB RAM
+
+
+## Locust Settings
+
+- 1000 Users
+- 500 user Ramp Up
 
 ## How to measure LiteLLM Overhead
 
@@ -137,10 +139,3 @@ Using LangSmith has **no impact on latency, RPS compared to Basic Litellm Proxy*
 |--------|------------------------|---------------------|
 | RPS | 1133.2 | 1135 |
 | Median Latency (ms) | 140 | 132 |
-
-
-
-## Locust Settings
-
-- 2500 Users
-- 100 user Ramp Up

docs/my-website/docs/caching/all_caches.md

@@ -1,7 +1,7 @@
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
 
-# Caching - In-Memory, Redis, s3, Redis Semantic Cache, Disk
+# Caching - In-Memory, Redis, s3, gcs, Redis Semantic Cache, Disk
 
 [**See Code**](https://github.com/BerriAI/litellm/blob/main/litellm/caching/caching.py)
 
@@ -14,7 +14,7 @@ import TabItem from '@theme/TabItem';
 
 :::
 
-## Initialize Cache - In Memory, Redis, s3 Bucket, Redis Semantic, Disk Cache, Qdrant Semantic
+## Initialize Cache - In Memory, Redis, s3 Bucket, gcs Bucket, Redis Semantic, Disk Cache, Qdrant Semantic
 
 
 <Tabs>
@@ -28,6 +28,8 @@ pip install redis
 
 For the hosted version you can setup your own Redis DB here: https://redis.io/try-free/
 
+**Basic Redis Cache**
+
 ```python
 import litellm
 from litellm import completion
@@ -48,6 +50,91 @@ response2 = completion(
 # response1 == response2, response 1 is cached
 ```
 
+**GCP IAM Redis Authentication**
+
+For GCP Memorystore Redis with IAM authentication:
+
+```shell
+pip install google-cloud-iam
+```
+
+```python
+import litellm
+from litellm import completion
+# For Redis Cluster with GCP IAM
+from litellm.caching.redis_cluster_cache import RedisClusterCache
+
+litellm.cache = RedisClusterCache(
+    startup_nodes=[
+        {"host": "10.128.0.2", "port": 6379},
+        {"host": "10.128.0.2", "port": 11008},
+    ],
+    gcp_service_account="projects/-/serviceAccounts/your-sa@project.iam.gserviceaccount.com",
+    ssl=True,
+    ssl_cert_reqs=None,
+    ssl_check_hostname=False,
+)
+
+# Make completion calls
+response1 = completion(
+    model="gpt-3.5-turbo",
+    messages=[{"role": "user", "content": "Tell me a joke."}]
+)
+response2 = completion(
+    model="gpt-3.5-turbo",
+    messages=[{"role": "user", "content": "Tell me a joke."}]
+)
+
+# response1 == response2, response 1 is cached
+```
+
+**Environment Variables for GCP IAM Redis**
+
+You can also set these as environment variables:
+
+```shell
+export REDIS_HOST="10.128.0.2"
+export REDIS_PORT="6379"
+export REDIS_GCP_SERVICE_ACCOUNT="projects/-/serviceAccounts/your-sa@project.iam.gserviceaccount.com"
+export REDIS_SSL="False"
+```
+
+Then simply initialize:
+
+```python
+litellm.cache = Cache(type="redis")
+```
+
+</TabItem>
+
+<TabItem value="gcs" label="gcs-cache">
+
+Set environment variables
+
+```shell
+GCS_BUCKET_NAME="my-cache-bucket"
+GCS_PATH_SERVICE_ACCOUNT="/path/to/service_account.json"
+```
+
+```python
+import litellm
+from litellm import completion
+from litellm.caching.caching import Cache
+
+litellm.cache = Cache(type="gcs", gcs_bucket_name="my-cache-bucket", gcs_path_service_account="/path/to/service_account.json")
+
+response1 = completion(
+    model="gpt-3.5-turbo",
+    messages=[{"role": "user", "content": "Tell me a joke."}]
+)
+response2 = completion(
+    model="gpt-3.5-turbo",
+    messages=[{"role": "user", "content": "Tell me a joke."}]
+)
+
+# response1 == response2, response 1 is cached
+```
+
 </TabItem>
 
 
@@ -405,7 +492,7 @@ Advanced Params
 
 ```python
 litellm.enable_cache(
-    type: Optional[Literal["local", "redis", "s3", "disk"]] = "local",
+    type: Optional[Literal["local", "redis", "s3", "gcs", "disk"]] = "local",
     host: Optional[str] = None,
     port: Optional[str] = None,
     password: Optional[str] = None,
@@ -429,7 +516,7 @@ Update the Cache params
 
 ```python
 litellm.update_cache(
-    type: Optional[Literal["local", "redis", "s3", "disk"]] = "local",
+    type: Optional[Literal["local", "redis", "s3", "gcs", "disk"]] = "local",
     host: Optional[str] = None,
     port: Optional[str] = None,
     password: Optional[str] = None,
@@ -490,7 +577,7 @@ cache.get_cache = get_cache
 ```python
 def __init__(
     self,
-    type: Optional[Literal["local", "redis", "redis-semantic", "s3", "disk"]] = "local",
+    type: Optional[Literal["local", "redis", "redis-semantic", "s3", "gcs", "disk"]] = "local",
     supported_call_types: Optional[
         List[Literal["completion", "acompletion", "embedding", "aembedding", "atranscription", "transcription"]]
     ] = ["completion", "acompletion", "embedding", "aembedding", "atranscription", "transcription"],
@@ -504,6 +591,13 @@ def __init__(
     namespace: Optional[str] = None,
     default_in_redis_ttl: Optional[float] = None,
     redis_flush_size=None,
+    
+    # GCP IAM Redis authentication params
+    gcp_service_account: Optional[str] = None,
+    gcp_ssl_ca_certs: Optional[str] = None,
+    ssl: Optional[bool] = None,
+    ssl_cert_reqs: Optional[Union[str, None]] = None,
+    ssl_check_hostname: Optional[bool] = None,
 
     # redis semantic cache params
     similarity_threshold: Optional[float] = None,

docs/my-website/docs/completion/computer_use.md

@@ -0,0 +1,446 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Computer Use
+
+Computer use allows models to interact with computer interfaces by taking screenshots and performing actions like clicking, typing, and scrolling. This enables AI models to autonomously operate desktop environments.
+
+**Supported Providers:**
+- Anthropic API (`anthropic/`)
+- Bedrock (Anthropic) (`bedrock/`)
+- Vertex AI (Anthropic) (`vertex_ai/`)
+
+**Supported Tool Types:**
+- `computer` - Computer interaction tool with display parameters
+- `bash` - Bash shell tool  
+- `text_editor` - Text editor tool
+- `web_search` - Web search tool
+
+LiteLLM will standardize the computer use tools across all supported providers.
+
+## Quick Start
+
+<Tabs>
+<TabItem value="sdk" label="LiteLLM Python SDK">
+
+```python
+import os 
+from litellm import completion
+
+os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
+
+# Computer use tool
+    tools = [
+        {
+            "type": "computer_20241022",
+            "name": "computer",
+            "display_height_px": 768,
+            "display_width_px": 1024,
+            "display_number": 0,
+        }
+    ]
+    
+    messages = [
+        {
+            "role": "user", 
+            "content": [
+                {
+                    "type": "text",
+                "text": "Take a screenshot and tell me what you see"
+            },
+            {
+                "type": "image_url",
+                "image_url": {
+                    "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg=="
+                }
+            }
+        ]
+    }
+]
+
+response = completion(
+    model="anthropic/claude-3-5-sonnet-latest",
+    messages=messages,
+    tools=tools,
+)
+
+print(response)
+```
+
+</TabItem>
+<TabItem value="proxy" label="LiteLLM Proxy Server">
+
+1. Define computer use models on config.yaml
+
+```yaml
+model_list:
+  - model_name: claude-3-5-sonnet-latest # Anthropic claude-3-5-sonnet-latest
+    litellm_params:
+      model: anthropic/claude-3-5-sonnet-latest
+      api_key: os.environ/ANTHROPIC_API_KEY
+  - model_name: claude-bedrock         # Bedrock Anthropic model
+    litellm_params:
+      model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
+      aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
+      aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
+      aws_region_name: us-west-2
+    model_info:
+      supports_computer_use: True        # set supports_computer_use to True so /model/info returns this attribute as True
+```
+
+2. Run proxy server
+
+```bash
+litellm --config config.yaml
+```
+
+3. Test it using the OpenAI Python SDK
+
+```python
+import os 
+from openai import OpenAI
+
+client = OpenAI(
+    api_key="sk-1234", # your litellm proxy api key
+    base_url="http://0.0.0.0:4000"
+)
+
+response = client.chat.completions.create(
+    model="claude-3-5-sonnet-latest",
+    messages=[
+        {
+            "role": "user", 
+            "content": [
+                {
+                    "type": "text",
+                    "text": "Take a screenshot and tell me what you see"
+                },
+                {
+                    "type": "image_url",
+                    "image_url": {
+                        "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg=="
+                    }
+                }
+            ]
+        }
+    ],
+    tools=[
+        {
+            "type": "computer_20241022",
+            "name": "computer",
+            "display_height_px": 768,
+            "display_width_px": 1024,
+            "display_number": 0,
+        }
+    ]
+)
+
+print(response)
+```
+
+</TabItem>
+</Tabs>
+
+## Checking if a model supports `computer use`
+
+<Tabs>
+<TabItem label="LiteLLM Python SDK" value="Python">
+
+Use `litellm.supports_computer_use(model="")` -> returns `True` if model supports computer use and `False` if not
+
+```python
+import litellm
+
+assert litellm.supports_computer_use(model="anthropic/claude-3-5-sonnet-latest") == True
+assert litellm.supports_computer_use(model="anthropic/claude-3-7-sonnet-20250219") == True
+assert litellm.supports_computer_use(model="bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0") == True
+assert litellm.supports_computer_use(model="vertex_ai/claude-3-5-sonnet") == True
+assert litellm.supports_computer_use(model="openai/gpt-4") == False
+```
+</TabItem>
+
+<TabItem label="LiteLLM Proxy Server" value="proxy">
+
+1. Define computer use models on config.yaml
+
+```yaml
+model_list:
+  - model_name: claude-3-5-sonnet-latest # Anthropic claude-3-5-sonnet-latest
+    litellm_params:
+      model: anthropic/claude-3-5-sonnet-latest
+      api_key: os.environ/ANTHROPIC_API_KEY
+  - model_name: claude-bedrock         # Bedrock Anthropic model
+    litellm_params:
+      model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
+      aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
+      aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
+      aws_region_name: us-west-2
+    model_info:
+      supports_computer_use: True        # set supports_computer_use to True so /model/info returns this attribute as True
+```
+
+2. Run proxy server
+
+```bash
+litellm --config config.yaml
+```
+
+3. Call `/model_group/info` to check if your model supports `computer use`
+
+```shell
+curl -X 'GET' \
+  'http://localhost:4000/model_group/info' \
+  -H 'accept: application/json' \
+  -H 'x-api-key: sk-1234'
+```
+
+Expected Response 
+
+```json
+{
+  "data": [
+    {
+      "model_group": "claude-3-5-sonnet-latest",
+      "providers": ["anthropic"],
+      "max_input_tokens": 200000,
+      "max_output_tokens": 8192,
+      "mode": "chat",
+      "supports_computer_use": true, # 👈 supports_computer_use is true
+      "supports_vision": true,
+      "supports_function_calling": true
+    },
+    {
+      "model_group": "claude-bedrock",
+      "providers": ["bedrock"],
+      "max_input_tokens": 200000,
+      "max_output_tokens": 8192,
+      "mode": "chat",
+      "supports_computer_use": true, # 👈 supports_computer_use is true
+      "supports_vision": true,
+      "supports_function_calling": true
+    }
+  ]
+}
+```
+
+</TabItem>
+</Tabs>
+
+## Different Tool Types
+
+Computer use supports several different tool types for various interaction modes:
+
+<Tabs>
+<TabItem value="computer" label="Computer Tool">
+
+The `computer_20241022` tool provides direct screen interaction capabilities.
+
+```python
+import os 
+from litellm import completion
+
+os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
+
+tools = [
+    {
+        "type": "computer_20241022", 
+        "name": "computer",
+        "display_height_px": 768,
+        "display_width_px": 1024,
+        "display_number": 0,
+    }
+]
+
+messages = [
+    {
+        "role": "user",
+        "content": [
+            {
+                "type": "text", 
+                "text": "Click on the search button in the screenshot"
+            },
+            {
+                "type": "image_url",
+                "image_url": {
+                    "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg=="
+                }
+            }
+        ]
+    }
+]
+
+response = completion(
+    model="anthropic/claude-3-5-sonnet-latest",
+    messages=messages,
+    tools=tools,
+)
+
+print(response)
+```
+
+</TabItem>
+<TabItem value="bash" label="Bash Tool">
+
+The `bash_20241022` tool provides command line interface access.
+
+```python
+import os 
+from litellm import completion
+
+os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
+
+tools = [
+    {
+        "type": "bash_20241022",
+        "name": "bash"
+    }
+]
+
+messages = [
+    {
+        "role": "user",
+        "content": "List the files in the current directory using bash"
+    }
+]
+
+response = completion(
+    model="anthropic/claude-3-5-sonnet-latest",
+    messages=messages,
+    tools=tools,
+)
+
+print(response)
+```
+
+</TabItem>
+<TabItem value="text_editor" label="Text Editor Tool">
+
+The `text_editor_20250124` tool provides text file editing capabilities.
+
+```python
+import os 
+from litellm import completion
+
+os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
+
+tools = [
+    {
+        "type": "text_editor_20250124",
+        "name": "str_replace_editor"
+    }
+]
+
+messages = [
+    {
+        "role": "user",
+        "content": "Create a simple Python hello world script"
+    }
+]
+
+response = completion(
+    model="anthropic/claude-3-5-sonnet-latest",
+    messages=messages,
+    tools=tools,
+)
+
+print(response)
+```
+
+</TabItem>
+</Tabs>
+
+## Advanced Usage with Multiple Tools
+
+You can combine different computer use tools in a single request:
+
+```python
+import os 
+from litellm import completion
+
+os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
+
+tools = [
+    {
+        "type": "computer_20241022",
+        "name": "computer", 
+        "display_height_px": 768,
+        "display_width_px": 1024,
+        "display_number": 0,
+    },
+    {
+        "type": "bash_20241022",
+        "name": "bash"
+    },
+    {
+        "type": "text_editor_20250124", 
+        "name": "str_replace_editor"
+    }
+]
+
+messages = [
+    {
+        "role": "user",
+        "content": [
+            {
+                "type": "text",
+                "text": "Take a screenshot, then create a file describing what you see, and finally use bash to show the file contents"
+                },
+                {
+                    "type": "image_url",
+                    "image_url": {
+                        "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg=="
+                    }
+                }
+            ]
+        }
+    ]
+    
+response = completion(
+    model="anthropic/claude-3-5-sonnet-latest",
+            messages=messages,
+            tools=tools,
+)
+
+print(response)
+```
+
+## Spec
+
+### Computer Tool (`computer_20241022`)
+
+```json
+{
+  "type": "computer_20241022",
+  "name": "computer",
+  "display_height_px": 768,    // Required: Screen height in pixels  
+  "display_width_px": 1024,    // Required: Screen width in pixels
+  "display_number": 0          // Optional: Display number (default: 0)
+}
+```
+
+### Bash Tool (`bash_20241022`)
+
+```json
+{
+  "type": "bash_20241022", 
+  "name": "bash"              // Required: Tool name
+}
+```
+
+### Text Editor Tool (`text_editor_20250124`)
+
+```json
+{
+  "type": "text_editor_20250124",
+  "name": "str_replace_editor"  // Required: Tool name
+}
+```
+
+### Web Search Tool (`web_search_20250305`)
+
+```json
+{
+  "type": "web_search_20250305",
+  "name": "web_search"         // Required: Tool name
+}
+```

docs/my-website/docs/completion/document_understanding.md

@@ -10,6 +10,7 @@ Works for:
 - Bedrock Models
 - Anthropic API Models
 - OpenAI API Models
+- Mistral (Only using file ID of already uploaded file, similar to OpenAI file_id input)
 
 ## Quick Start
 
@@ -279,6 +280,71 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
 </Tabs>
 
 
+## Mistral Example
+
+Here is a sample payload for using the Mistral model for document understanding:
+
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+from litellm.utils import completion
+
+# pdf file_id received from files endpoint
+file_id = "fa778e5e-46ec-4562-8418-36623fe25a71"
+
+# model
+model = "mistral/mistral-large-latest"
+
+file_content = [
+    {"type": "text", "text": "What's this file about?"},
+    {
+        "type": "file",
+        "file": {
+            "file_id": file_id,
+        }
+    },
+]
+
+response = completion(
+    model=model,
+    messages=[{"role": "user", "content": file_content}],
+)
+assert response is not None
+```
+
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+```bash
+curl -X POST 'http://0.0.0.0:4000/chat/completions' \
+-H 'Content-Type: application/json' \
+-H 'Authorization: Bearer sk-1234' \
+-d '{
+    "model": "mistral/mistral-large-latest",
+    "messages": [
+        {
+            "role": "user",
+            "content": [
+                {
+                    "type": "text",
+                    "text": "What is the content of the file?"
+                },
+                {
+                    "type": "file",
+                    "file": {
+                        "file_id": "fa778e5e-46ec-4562-8418-36623fe25a71"
+                    }
+                }
+            ]
+        }
+    ]
+}
+```
+</TabItem>
+</Tabs>
+
 ## Checking if a model supports pdf input
 
 <Tabs>

docs/my-website/docs/completion/http_handler_config.md

@@ -0,0 +1,145 @@
+# Custom HTTP Handler
+
+Configure custom aiohttp sessions for better performance and control in LiteLLM completions.
+
+## Overview
+
+You can now inject custom `aiohttp.ClientSession` instances into LiteLLM for:
+- Custom connection pooling and timeouts
+- Corporate proxy and SSL configurations  
+- Performance optimization
+- Request monitoring
+
+## Basic Usage
+
+### Default (No Changes Required)
+```python
+import litellm
+
+# Works exactly as before
+response = await litellm.acompletion(
+    model="gpt-3.5-turbo",
+    messages=[{"role": "user", "content": "Hello!"}]
+)
+```
+
+### Custom Session
+```python
+import aiohttp
+import litellm
+from litellm.llms.custom_httpx.aiohttp_handler import BaseLLMAIOHTTPHandler
+
+# Create optimized session
+session = aiohttp.ClientSession(
+    timeout=aiohttp.ClientTimeout(total=180),
+    connector=aiohttp.TCPConnector(limit=300, limit_per_host=75)
+)
+
+# Replace global handler
+litellm.base_llm_aiohttp_handler = BaseLLMAIOHTTPHandler(client_session=session)
+
+# All completions now use your session
+response = await litellm.acompletion(model="gpt-3.5-turbo", messages=[...])
+```
+
+## Common Patterns
+
+### FastAPI Integration
+```python
+from contextlib import asynccontextmanager
+from fastapi import FastAPI
+import aiohttp
+import litellm
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    # Startup
+    session = aiohttp.ClientSession(
+        timeout=aiohttp.ClientTimeout(total=180),
+        connector=aiohttp.TCPConnector(limit=300)
+    )
+    litellm.base_llm_aiohttp_handler = BaseLLMAIOHTTPHandler(
+        client_session=session
+    )
+    yield
+    # Shutdown
+    await session.close()
+
+app = FastAPI(lifespan=lifespan)
+
+@app.post("/chat")
+async def chat(messages: list[dict]):
+    return await litellm.acompletion(model="gpt-3.5-turbo", messages=messages)
+```
+
+### Corporate Proxy
+```python
+import ssl
+
+# Custom SSL context
+ssl_context = ssl.create_default_context()
+ssl_context.load_cert_chain('cert.pem', 'key.pem')
+
+# Proxy session
+session = aiohttp.ClientSession(
+    connector=aiohttp.TCPConnector(ssl=ssl_context),
+    trust_env=True  # Use environment proxy settings
+)
+
+litellm.base_llm_aiohttp_handler = BaseLLMAIOHTTPHandler(client_session=session)
+```
+
+### High Performance
+```python
+# Optimized for high throughput
+session = aiohttp.ClientSession(
+    timeout=aiohttp.ClientTimeout(total=300),
+    connector=aiohttp.TCPConnector(
+        limit=1000,             # High connection limit
+        limit_per_host=200,     # Per host limit
+        ttl_dns_cache=600,      # DNS cache
+        keepalive_timeout=60,   # Keep connections alive
+        enable_cleanup_closed=True
+    )
+)
+
+litellm.base_llm_aiohttp_handler = BaseLLMAIOHTTPHandler(client_session=session)
+```
+
+## Constructor Options
+
+```python
+BaseLLMAIOHTTPHandler(
+    client_session=None,    # Custom aiohttp.ClientSession
+    transport=None,         # Advanced transport control
+    connector=None,         # Custom aiohttp.BaseConnector
+)
+```
+
+## Resource Management
+
+- **User sessions**: You manage the lifecycle (call `await session.close()`)
+- **Auto-created sessions**: Automatically cleaned up by the handler
+- **100% backward compatible**: Existing code works unchanged
+
+## Configuration Tips
+
+### Development
+```python
+session = aiohttp.ClientSession(
+    timeout=aiohttp.ClientTimeout(total=60),
+    connector=aiohttp.TCPConnector(limit=50)
+)
+```
+
+### Production
+```python
+session = aiohttp.ClientSession(
+    timeout=aiohttp.ClientTimeout(total=300),
+    connector=aiohttp.TCPConnector(
+        limit=1000,
+        limit_per_host=200,
+        keepalive_timeout=60
+    )
+)
+```

docs/my-website/docs/completion/image_generation_chat.md

@@ -0,0 +1,232 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Image Generation in Chat Completions, Responses API
+
+This guide covers how to generate images when using the `chat/completions`. Note - if you want this on Responses API please file a Feature Request [here](https://github.com/BerriAI/litellm/issues/new).
+
+:::info
+
+Requires LiteLLM v1.76.1+
+
+:::
+
+Supported Providers:
+- Google AI Studio (`gemini`)
+- Vertex AI (`vertex_ai/`)
+
+LiteLLM will standardize the `image` response in the assistant message for models that support image generation during chat completions.
+
+```python title="Example response from litellm"
+"message": {
+    ...
+    "content": "Here's the image you requested:",
+    "image": {
+        "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...",
+        "detail": "auto"
+    }
+}
+```
+
+## Quick Start 
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python showLineNumbers title="Image generation with chat completion"
+from litellm import completion
+import os 
+
+os.environ["GEMINI_API_KEY"] = "your-api-key"
+
+response = completion(
+    model="gemini/gemini-2.5-flash-image-preview",
+    messages=[
+        {"role": "user", "content": "Generate an image of a banana wearing a costume that says LiteLLM"}
+    ],
+)
+
+print(response.choices[0].message.content)  # Text response
+print(response.choices[0].message.image)    # Image data
+```
+
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+1. Setup config.yaml
+
+```yaml showLineNumbers title="config.yaml"
+model_list:
+  - model_name: gemini-image-gen
+    litellm_params:
+      model: gemini/gemini-2.5-flash-image-preview
+      api_key: os.environ/GEMINI_API_KEY
+```
+
+2. Run proxy server
+
+```bash showLineNumbers title="Start the proxy"
+litellm --config config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+3. Test it!
+
+```bash showLineNumbers title="Make request"
+curl http://0.0.0.0:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer $LITELLM_KEY" \
+  -d '{
+    "model": "gemini-image-gen",
+    "messages": [
+      {
+        "role": "user",
+        "content": "Generate an image of a banana wearing a costume that says LiteLLM"
+      }
+    ]
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+**Expected Response**
+
+```bash
+{
+    "id": "chatcmpl-3b66124d79a708e10c603496b363574c",
+    "choices": [
+        {
+            "finish_reason": "stop",
+            "index": 0,
+            "message": {
+                "content": "Here's the image you requested:",
+                "role": "assistant",
+                "image": {
+                    "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...",
+                    "detail": "auto"
+                }
+            }
+        }
+    ],
+    "created": 1723323084,
+    "model": "gemini/gemini-2.5-flash-image-preview",
+    "object": "chat.completion",
+    "usage": {
+        "completion_tokens": 12,
+        "prompt_tokens": 16,
+        "total_tokens": 28
+    }
+}
+```
+
+## Streaming Support
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python showLineNumbers title="Streaming image generation"
+from litellm import completion
+import os 
+
+os.environ["GEMINI_API_KEY"] = "your-api-key"
+
+response = completion(
+    model="gemini/gemini-2.5-flash-image-preview",
+    messages=[
+        {"role": "user", "content": "Generate an image of a banana wearing a costume that says LiteLLM"}
+    ],
+    stream=True,
+)
+
+for chunk in response:
+    if hasattr(chunk.choices[0].delta, "image") and chunk.choices[0].delta.image is not None:
+        print("Generated image:", chunk.choices[0].delta.image["url"])
+        break
+```
+
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+```bash showLineNumbers title="Streaming request"
+curl http://0.0.0.0:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer $LITELLM_KEY" \
+  -d '{
+    "model": "gemini-image-gen",
+    "messages": [
+      {
+        "role": "user",
+        "content": "Generate an image of a banana wearing a costume that says LiteLLM"
+      }
+    ],
+    "stream": true
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+**Expected Streaming Response**
+
+```bash
+data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1723323084,"model":"gemini/gemini-2.5-flash-image-preview","choices":[{"index":0,"delta":{"role":"assistant"},"finish_reason":null}]}
+
+data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1723323084,"model":"gemini/gemini-2.5-flash-image-preview","choices":[{"index":0,"delta":{"content":"Here's the image you requested:"},"finish_reason":null}]}
+
+data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1723323084,"model":"gemini/gemini-2.5-flash-image-preview","choices":[{"index":0,"delta":{"image":{"url":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...","detail":"auto"}},"finish_reason":null}]}
+
+data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1723323084,"model":"gemini/gemini-2.5-flash-image-preview","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}
+
+data: [DONE]
+```
+
+## Async Support
+
+```python showLineNumbers title="Async image generation"
+from litellm import acompletion
+import asyncio
+import os 
+
+os.environ["GEMINI_API_KEY"] = "your-api-key"
+
+async def generate_image():
+    response = await acompletion(
+        model="gemini/gemini-2.5-flash-image-preview",
+        messages=[
+            {"role": "user", "content": "Generate an image of a banana wearing a costume that says LiteLLM"}
+        ],
+    )
+    
+    print(response.choices[0].message.content)  # Text response
+    print(response.choices[0].message.image)    # Image data
+    
+    return response
+
+# Run the async function
+asyncio.run(generate_image())
+```
+
+## Supported Models
+
+| Provider | Model | 
+|----------|--------|
+| Google AI Studio | `gemini/gemini-2.5-flash-image-preview` |
+| Vertex AI | `vertex_ai/gemini-2.5-flash-image-preview` |
+
+## Spec 
+
+The `image` field in the response follows this structure:
+
+```python
+"image": {
+    "url": "data:image/png;base64,<base64_encoded_image>",
+    "detail": "auto"
+}
+```
+
+- `url` - str: Base64 encoded image data in data URI format
+- `detail` - str: Image detail level (always "auto" for generated images)
+
+The image is returned as a base64-encoded data URI that can be directly used in HTML `<img>` tags or saved to a file.

docs/my-website/docs/completion/input.md

@@ -65,6 +65,7 @@ Use `litellm.get_supported_openai_params()` for an updated list of params for ea
 | Github | ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | ✅| ✅ | ✅| ✅|| || ✅ | ✅ (model dependent) | ✅ (model dependent) || ||
 | Novita AI| ✅| ✅ || ✅| ✅ | ✅ | ✅ | ✅| ✅ | ✅| || ✅||| |||| ||
 | Bytez | ✅| ✅ || ✅| ✅ | | | ✅|| || || || || || ||
+| OVHCloud AI Endpoints | ✅ | | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | | | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | |
 
 :::note
 
@@ -106,6 +107,7 @@ def completion(
     parallel_tool_calls: Optional[bool] = None,
     logprobs: Optional[bool] = None,
     top_logprobs: Optional[int] = None,
+    safety_identifier: Optional[str] = None,
     deployment_id=None,
     # soon to be deprecated params by OpenAI
     functions: Optional[List] = None,
@@ -178,11 +180,11 @@ def completion(
 
     - `function`: *object* - Required.
 
-- `tool_choice`: *string or object (optional)* - Controls which (if any) function is called by the model. none means the model will not call a function and instead generates a message. auto means the model can pick between generating a message or calling a function. Specifying a particular function via `{"type: "function", "function": {"name": "my_function"}}` forces the model to call that function.
+- `tool_choice`: *string or object (optional)* - Controls which (if any) function is called by the model. none means the model will not call a function and instead generates a message. auto means the model can pick between generating a message or calling a function. Specifying a particular function via `{"type": "function", "function": {"name": "my_function"}}` forces the model to call that function.
 
     - `none` is the default when no functions are present. `auto` is the default if functions are present.
 
-- `parallel_tool_calls`: *boolean (optional)* - Whether to enable parallel function calling during tool use.. OpenAI default is true.
+- `parallel_tool_calls`: *boolean (optional)* - Whether to enable parallel function calling during tool use. OpenAI default is true.
 
 - `frequency_penalty`: *number or null (optional)* - It is used to penalize new tokens based on their frequency in the text so far.
 
@@ -196,6 +198,8 @@ def completion(
         
 - `top_logprobs`: *int (optional)* - An integer between 0 and 5 specifying the number of most likely tokens to return at each token position, each with an associated log probability. `logprobs` must be set to true if this parameter is used.
 
+- `safety_identifier`: *string (optional)* - A unique identifier for tracking and managing safety-related requests. This parameter helps with safety monitoring and compliance tracking.
+
 - `headers`: *dict (optional)* - A dictionary of headers to be sent with the request.
 
 - `extra_headers`: *dict (optional)* - Alternative to `headers`, used to send extra headers in LLM API request. 

docs/my-website/docs/completion/provider_specific_params.md

@@ -423,7 +423,7 @@ model_list:
 curl -X POST 'http://0.0.0.0:4000/chat/completions' \
 -H 'Content-Type: application/json' \
 -H 'Authorization: Bearer sk-1234' \
--D '{
+-d '{
   "model": "llama-3-8b-instruct",
   "messages": [
     {
@@ -431,6 +431,56 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
       "content": "What'\''s the weather like in Boston today?"
     }
   ],
-  "adapater_id": "my-special-adapter-id" # 👈 PROVIDER-SPECIFIC PARAM
-  }'
-```
+  "adapater_id": "my-special-adapter-id"
+}'
+```
+
+## Provider-Specific Metadata Parameters
+
+| Provider | Parameter | Use Case |
+|----------|-----------|----------|
+| **AWS Bedrock** | `requestMetadata` | Cost attribution, logging |
+| **Gemini/Vertex AI** | `labels` | Resource labeling |
+| **Anthropic** | `metadata` | User identification |
+
+<Tabs>
+<TabItem value="bedrock" label="AWS Bedrock">
+
+```python
+import litellm
+
+response = litellm.completion(
+    model="bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
+    messages=[{"role": "user", "content": "Hello!"}],
+    requestMetadata={"cost_center": "engineering"}
+)
+```
+
+</TabItem>
+<TabItem value="gemini" label="Gemini/Vertex AI">
+
+```python
+import litellm
+
+response = litellm.completion(
+    model="vertex_ai/gemini-pro",
+    messages=[{"role": "user", "content": "Hello!"}],
+    labels={"environment": "production"}
+)
+```
+
+</TabItem>
+<TabItem value="anthropic" label="Anthropic">
+
+```python
+import litellm
+
+response = litellm.completion(
+    model="anthropic/claude-3-sonnet-20240229",
+    messages=[{"role": "user", "content": "Hello!"}],
+    metadata={"user_id": "user123"}
+)
+```
+
+</TabItem>
+</Tabs>

docs/my-website/docs/completion/shared_session.md

@@ -0,0 +1,213 @@
+# Shared Session Support
+
+## Overview
+
+LiteLLM now supports sharing `aiohttp.ClientSession` instances across multiple API calls to avoid creating unnecessary new sessions. This improves performance and resource utilization.
+
+## Usage
+
+### Basic Usage
+
+```python
+import asyncio
+from aiohttp import ClientSession
+from litellm import acompletion
+
+async def main():
+    # Create a shared session
+    async with ClientSession() as shared_session:
+        # Use the same session for multiple calls
+        response1 = await acompletion(
+            model="gpt-4o",
+            messages=[{"role": "user", "content": "Hello"}],
+            shared_session=shared_session
+        )
+        
+        response2 = await acompletion(
+            model="gpt-4o", 
+            messages=[{"role": "user", "content": "How are you?"}],
+            shared_session=shared_session
+        )
+        
+        # Both calls reuse the same session!
+
+asyncio.run(main())
+```
+
+### Without Shared Session (Default)
+
+```python
+import asyncio
+from litellm import acompletion
+
+async def main():
+    # Each call creates a new session
+    response1 = await acompletion(
+        model="gpt-4o",
+        messages=[{"role": "user", "content": "Hello"}]
+    )
+    
+    response2 = await acompletion(
+        model="gpt-4o",
+        messages=[{"role": "user", "content": "How are you?"}]
+    )
+    # Two separate sessions created
+
+asyncio.run(main())
+```
+
+## Benefits
+
+- **Performance**: Reuse HTTP connections across multiple calls
+- **Resource Efficiency**: Reduce memory and connection overhead
+- **Better Control**: Manage session lifecycle explicitly
+- **Debugging**: Easy to trace which calls use which sessions
+
+## Debug Logging
+
+Enable debug logging to see session reuse in action:
+
+```python
+import os
+import litellm
+
+# Enable debug logging
+os.environ['LITELLM_LOG'] = 'DEBUG'
+
+# You'll see logs like:
+# 🔄 SHARED SESSION: acompletion called with shared_session (ID: 12345)
+# ✅ SHARED SESSION: Reusing existing ClientSession (ID: 12345)
+```
+
+## Common Patterns
+
+### FastAPI Integration
+
+```python
+from fastapi import FastAPI
+import aiohttp
+import litellm
+
+app = FastAPI()
+
+@app.post("/chat")
+async def chat(messages: list[dict]):
+    # Create session per request
+    async with aiohttp.ClientSession() as session:
+        return await litellm.acompletion(
+            model="gpt-4o",
+            messages=messages,
+            shared_session=session
+        )
+```
+
+### Batch Processing
+
+```python
+import asyncio
+from aiohttp import ClientSession
+from litellm import acompletion
+
+async def process_batch(messages_list):
+    async with ClientSession() as shared_session:
+        tasks = []
+        for messages in messages_list:
+            task = acompletion(
+                model="gpt-4o",
+                messages=messages,
+                shared_session=shared_session
+            )
+            tasks.append(task)
+        
+        # All tasks use the same session
+        results = await asyncio.gather(*tasks)
+        return results
+```
+
+### Custom Session Configuration
+
+```python
+import aiohttp
+import litellm
+
+# Create optimized session
+async with aiohttp.ClientSession(
+    timeout=aiohttp.ClientTimeout(total=180),
+    connector=aiohttp.TCPConnector(limit=300, limit_per_host=75)
+) as shared_session:
+    
+    response = await litellm.acompletion(
+        model="gpt-4o",
+        messages=[{"role": "user", "content": "Hello"}],
+        shared_session=shared_session
+    )
+```
+
+## Implementation Details
+
+The `shared_session` parameter is threaded through the entire LiteLLM call chain:
+
+1. **`acompletion()`** - Accepts `shared_session` parameter
+2. **`BaseLLMHTTPHandler`** - Passes session to HTTP client creation
+3. **`AsyncHTTPHandler`** - Uses existing session if provided
+4. **`LiteLLMAiohttpTransport`** - Reuses the session for HTTP requests
+
+## Backward Compatibility
+
+- **100% backward compatible** - Existing code works unchanged
+- **Optional parameter** - `shared_session=None` by default
+- **No breaking changes** - All existing functionality preserved
+
+## Testing
+
+Test the shared session functionality:
+
+```python
+import asyncio
+from aiohttp import ClientSession
+from litellm import acompletion
+
+async def test_shared_session():
+    async with ClientSession() as session:
+        print(f"✅ Created session: {id(session)}")
+        
+        try:
+            response = await acompletion(
+                model="gpt-4o",
+                messages=[{"role": "user", "content": "Hello"}],
+                shared_session=session,
+                api_key="your-api-key"
+            )
+            print(f"Response: {response.choices[0].message.content}")
+        except Exception as e:
+            print(f"✅ Expected error: {type(e).__name__}")
+        
+        print("✅ Session control working!")
+
+asyncio.run(test_shared_session())
+```
+
+## Files Modified
+
+The shared session functionality was added to these files:
+
+- `litellm/main.py` - Added `shared_session` parameter to `acompletion()` and `completion()`
+- `litellm/llms/custom_httpx/http_handler.py` - Core session reuse logic
+- `litellm/llms/custom_httpx/llm_http_handler.py` - HTTP handler integration
+- `litellm/llms/openai/openai.py` - OpenAI provider integration
+- `litellm/llms/openai/common_utils.py` - OpenAI client creation
+- `litellm/llms/azure/chat/o_series_handler.py` - Azure O Series handler
+
+## Troubleshooting
+
+### Session Not Being Reused
+
+1. **Check debug logs**: Enable `LITELLM_LOG=DEBUG` to see session reuse messages
+2. **Verify session is not closed**: Ensure the session is still active when making calls
+3. **Check parameter passing**: Make sure `shared_session` is passed to all `acompletion()` calls
+
+### Performance Issues
+
+1. **Session configuration**: Tune `aiohttp.ClientSession` parameters for your use case
+2. **Connection limits**: Adjust `limit` and `limit_per_host` in `TCPConnector`
+3. **Timeout settings**: Configure appropriate timeouts for your environment

docs/my-website/docs/completion/usage.md

@@ -26,6 +26,7 @@ response = completion(
 
 print(response.usage)
 ```
+> **Note:** LiteLLM supports endpoint bridging—if a model does not natively support a requested endpoint, LiteLLM will automatically route the call to the correct supported endpoint (such as bridging `/chat/completions` to `/responses` or vice versa) based on the model's `mode`set in `model_prices_and_context_window`.
 
 ## Streaming Usage
 

docs/my-website/docs/completion/web_fetch.md

@@ -0,0 +1,294 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Web Fetch
+
+The web fetch tool allows LLMs to retrieve full content from specified web pages and PDF documents. This enables AI models to access real-time information from the internet and incorporate web content into their responses. 
+
+## Web Fetch vs Web Search
+
+**Web Fetch** retrieves the full content from specific web pages that you provide URLs for, while **Web Search** performs internet searches to find relevant information based on your queries.
+
+| Feature | Web Fetch | Web Search |
+|---------|-----------|------------|
+| **Purpose** | Retrieve content from specific URLs | Search the internet for information |
+| **Input** | You provide exact URLs to fetch | You provide search queries/questions |
+| **Output** | Full page content from specified URLs | Search results with relevant information |
+| **Use Cases** | - Analyzing specific articles<br/>- Comparing content from known websites<br/>- Extracting data from particular pages | - Finding current news/events<br/>- Researching topics<br/>- Getting real-time information |
+
+
+**Example Web Fetch**: "Fetch the content from https://example.com/pricing and summarize it"  
+**Example Web Search**: "What are the latest AI developments this week?"
+
+**Supported Providers:**
+- Anthropic API (`anthropic/`)
+
+**Supported Tool Types:**
+- `web_fetch_20250910` - Web content retrieval tool with usage limits, domain filtering, and citation support
+
+
+## Quick Start
+
+### LiteLLM Python SDK
+
+```python
+import os 
+from litellm import completion
+
+os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
+
+# Web fetch tool
+tools = [
+    {
+        "type": "web_fetch_20250910",
+        "name": "web_fetch",
+        "max_uses": 5,
+    }
+]
+
+messages = [
+    {
+        "role": "user", 
+        "content": "Please analyze the content at https://example.com/article and summarize the main points"
+    }
+]
+
+response = completion(
+    model="anthropic/claude-3-5-sonnet-latest",
+    messages=messages,
+    tools=tools,
+)
+
+print(response)
+```
+
+### LiteLLM Proxy
+
+1. Define web fetch models on config.yaml
+
+```yaml
+model_list:
+  - model_name: claude-3-5-sonnet-latest # Anthropic claude-3-5-sonnet-latest
+    litellm_params:
+      model: anthropic/claude-3-5-sonnet-latest
+      api_key: os.environ/ANTHROPIC_API_KEY
+```
+
+2. Run proxy server
+
+```bash
+litellm --config config.yaml
+```
+
+3. Test it using the OpenAI Python SDK
+
+```python
+import os 
+from openai import OpenAI
+
+client = OpenAI(
+    api_key="sk-1234", # your litellm proxy api key
+    base_url="http://0.0.0.0:4000"
+)
+
+response = client.chat.completions.create(
+    model="claude-3-5-sonnet-latest",
+    messages=[
+        {
+            "role": "user", 
+            "content": "Please fetch and analyze the content from https://news.ycombinator.com and tell me about the top stories"
+        }
+    ],
+    tools=[
+        {
+            "type": "web_fetch_20250910",
+            "name": "web_fetch",
+            "max_uses": 5,
+        }
+    ]
+)
+
+print(response)
+```
+
+## Supported Models
+
+Web fetch is available on the following Anthropic API models:
+
+- `claude-opus-4-1-20250805` (Claude Opus 4.1)
+- `claude-opus-4-20250514` (Claude Opus 4)
+- `claude-sonnet-4-20250514` (Claude Sonnet 4)
+- `claude-3-7-sonnet-20250219` (Claude Sonnet 3.7)
+- `claude-3-5-sonnet-latest` (Claude Sonnet 3.5 v2 - deprecated)
+- `claude-3-5-haiku-latest` (Claude Haiku 3.5)
+
+:::note
+The web fetch tool currently does not support websites dynamically rendered via JavaScript.
+:::
+
+## Usage Examples
+
+### Basic Web Content Retrieval
+
+```python
+import os 
+from litellm import completion
+
+os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
+
+tools = [
+    {
+        "type": "web_fetch_20250910",
+        "name": "web_fetch",
+        "max_uses": 3,
+    }
+]
+
+messages = [
+    {
+        "role": "user",
+        "content": "Fetch the latest news from https://techcrunch.com and summarize the top 3 articles"
+    }
+]
+
+response = completion(
+    model="anthropic/claude-3-5-sonnet-latest",
+    messages=messages,
+    tools=tools,
+)
+
+print(response)
+```
+
+### Research and Analysis
+
+```python
+import os 
+from litellm import completion
+
+os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
+
+tools = [
+    {
+        "type": "web_fetch_20250910",
+        "name": "web_fetch", 
+        "max_uses": 10,
+    }
+]
+
+messages = [
+    {
+        "role": "user",
+        "content": "Research the latest developments in AI by fetching content from multiple tech news websites and provide a comprehensive analysis"
+    }
+]
+
+response = completion(
+    model="anthropic/claude-3-5-sonnet-latest",
+    messages=messages,
+    tools=tools,
+)
+
+print(response)
+```
+
+### Content Comparison
+
+```python
+import os 
+from litellm import completion
+
+os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
+
+tools = [
+    {
+        "type": "web_fetch_20250910",
+        "name": "web_fetch",
+        "max_uses": 5,
+    }
+]
+
+messages = [
+    {
+        "role": "user",
+        "content": "Compare the pricing information from https://openai.com/pricing and https://anthropic.com/pricing and create a comparison table"
+    }
+]
+
+response = completion(
+    model="anthropic/claude-3-5-sonnet-latest",
+    messages=messages,
+    tools=tools,
+)
+
+print(response)
+```
+
+## Advanced Usage with Multiple Tools
+
+You can combine web fetch with other tools like computer use or text editor:
+
+```python
+import os 
+from litellm import completion
+
+os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
+
+tools = [
+    {
+        "type": "web_fetch_20250910",
+        "name": "web_fetch",
+        "max_uses": 5,
+    },
+    {
+        "type": "text_editor_20250124", 
+        "name": "str_replace_editor"
+    }
+]
+
+messages = [
+    {
+        "role": "user",
+        "content": "Fetch the latest AI research papers from arXiv, analyze them, and create a detailed report file with your findings"
+    }
+]
+    
+response = completion(
+    model="anthropic/claude-3-5-sonnet-latest",
+    messages=messages,
+    tools=tools,
+)
+
+print(response)
+```
+
+## Spec
+
+### Web Fetch Tool (`web_fetch_20250910`)
+
+The web fetch tool supports the following parameters:
+
+```json
+{
+  "type": "web_fetch_20250910",
+  "name": "web_fetch",
+
+  // Optional: Limit the number of fetches per request
+  "max_uses": 10,
+
+  // Optional: Only fetch from these domains
+  "allowed_domains": ["example.com", "docs.example.com"],
+
+  // Optional: Never fetch from these domains
+  "blocked_domains": ["private.example.com"],
+
+  // Optional: Enable citations for fetched content
+  "citations": {
+    "enabled": true
+  },
+
+  // Optional: Maximum content length in tokens
+  "max_content_tokens": 100000
+}
+```
+

docs/my-website/docs/completion/web_search.md

@@ -1,17 +1,32 @@
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
 
-# Using Web Search
+# Web Search
 
 Use web search with litellm
 
 | Feature | Details |
 |---------|---------|
 | Supported Endpoints | - `/chat/completions` <br/> - `/responses` |
-| Supported Providers | `openai`, `xai`, `vertex_ai`, `gemini`, `perplexity` |
+| Supported Providers | `openai`, `xai`, `vertex_ai`, `anthropic`, `gemini`, `perplexity` |
 | LiteLLM Cost Tracking | ✅ Supported |
 | LiteLLM Version | `v1.71.0+` |
 
+## Which Search Engine is Used?
+
+Each provider uses their own search backend:
+
+| Provider | Search Engine | Notes |
+|----------|---------------|-------|
+| **OpenAI** (`gpt-4o-search-preview`) | OpenAI's internal search | Real-time web data |
+| **xAI** (`grok-3`) | xAI's search + X/Twitter | Real-time social media data |
+| **Google AI/Vertex** (`gemini-2.0-flash`) | **Google Search** | Uses actual Google search results |
+| **Anthropic** (`claude-3-5-sonnet`) | Anthropic's web search | Real-time web data |
+| **Perplexity** | Perplexity's search engine | AI-powered search and reasoning |
+
+:::info
+**Anthropic Web Search Models**: Claude models that support web search: `claude-3-5-sonnet-latest`, `claude-3-5-sonnet-20241022`, `claude-3-5-haiku-latest`, `claude-3-5-haiku-20241022`, `claude-3-7-sonnet-20250219`
+:::
 
 ## `/chat/completions` (litellm.completion)
 
@@ -56,6 +71,12 @@ model_list:
       model: xai/grok-3
       api_key: os.environ/XAI_API_KEY
   
+  # Anthropic
+  - model_name: claude-3-5-sonnet-latest
+    litellm_params:
+      model: anthropic/claude-3-5-sonnet-latest
+      api_key: os.environ/ANTHROPIC_API_KEY
+  
   # VertexAI
   - model_name: gemini-2-flash
     litellm_params:
@@ -143,6 +164,31 @@ response = completion(
 )
 ```
 
+**Anthropic (using web_search_options)**
+```python showLineNumbers
+from litellm import completion
+
+# Customize search context size for Anthropic
+response = completion(
+    model="anthropic/claude-3-5-sonnet-latest",
+    messages=[
+        {
+            "role": "user",
+            "content": "What was a positive news story from today?",
+        }
+    ],
+    web_search_options={
+        "search_context_size": "medium",  # Options: "low", "medium" (default), "high"
+        "user_location": {
+            "type": "approximate",
+            "approximate": {
+                "city": "San Francisco",
+            },
+        }
+    }
+)
+```
+
 **VertexAI/Gemini (using web_search_options)**
 ```python showLineNumbers
 from litellm import completion
@@ -375,6 +421,9 @@ assert litellm.supports_web_search(model="openai/gpt-4o-search-preview") == True
 # Check xAI models
 assert litellm.supports_web_search(model="xai/grok-3") == True
 
+# Check Anthropic models
+assert litellm.supports_web_search(model="anthropic/claude-3-5-sonnet-latest") == True
+
 # Check VertexAI models
 assert litellm.supports_web_search(model="gemini-2.0-flash") == True
 
@@ -405,6 +454,14 @@ model_list:
     model_info:
       supports_web_search: True
   
+  # Anthropic
+  - model_name: claude-3-5-sonnet-latest
+    litellm_params:
+      model: anthropic/claude-3-5-sonnet-latest
+      api_key: os.environ/ANTHROPIC_API_KEY
+    model_info:
+      supports_web_search: True
+  
   # VertexAI
   - model_name: gemini-2-flash
     litellm_params:

docs/my-website/docs/contributing.md

@@ -13,7 +13,9 @@ git clone https://github.com/BerriAI/litellm.git
 
 Tell the proxy where the UI is located
 ```bash
-export PROXY_BASE_URL="http://localhost:3000/"
+DATABASE_URL = "postgresql://<user>:<password>@<host>:<port>/<dbname>"
+LITELLM_MASTER_KEY = "sk-1234"
+STORE_MODEL_IN_DB = "True"
 ```
 
 ```bash
@@ -25,7 +27,7 @@ python3 proxy_cli.py --config /path/to/config.yaml --port 4000
 
 Set the mode as development (this will assume the proxy is running on localhost:4000)
 ```bash
-export NODE_ENV="development" 
+npm install # install dependencies
 ```
 
 ```bash

docs/my-website/docs/embedding/supported_embedding.md

@@ -266,7 +266,59 @@ print(response)
 | Titan Embeddings - G1 | `embedding(model="amazon.titan-embed-text-v1", input=input)` |
 | Cohere Embeddings - English | `embedding(model="cohere.embed-english-v3", input=input)` |
 | Cohere Embeddings - Multilingual | `embedding(model="cohere.embed-multilingual-v3", input=input)` |
+| TwelveLabs Marengo (Async) | `embedding(model="bedrock/async_invoke/us.twelvelabs.marengo-embed-2-7-v1:0", input=input, input_type="text")` | [Async Invoke Docs](../providers/bedrock_embedding#async-invoke-embedding) |
 
+## TwelveLabs Bedrock Embedding Models
+
+TwelveLabs Marengo models support multimodal embeddings (text, image, video, audio) and require the `input_type` parameter to specify the input format.
+
+### Usage
+
+```python
+from litellm import embedding
+import os
+
+# Set AWS credentials
+os.environ["AWS_ACCESS_KEY_ID"] = ""
+os.environ["AWS_SECRET_ACCESS_KEY"] = ""
+os.environ["AWS_REGION_NAME"] = "us-east-1"
+
+# Text embedding
+response = embedding(
+    model="bedrock/us.twelvelabs.marengo-embed-2-7-v1:0",
+    input=["Hello world from LiteLLM!"],
+    input_type="text"  # Required parameter
+)
+
+# Image embedding (base64)
+response = embedding(
+    model="bedrock/async_invoke/us.twelvelabs.marengo-embed-2-7-v1:0",
+    input=["data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQ..."],
+    input_type="image",  # Required parameter
+    output_s3_uri="s3://your-bucket/async-invoke-output/"
+)
+
+# Video embedding (S3 URL)
+response = embedding(
+    model="bedrock/async_invoke/us.twelvelabs.marengo-embed-2-7-v1:0",
+    input=["s3://your-bucket/video.mp4"],
+    input_type="video",  # Required parameter
+    output_s3_uri="s3://your-bucket/async-invoke-output/"
+)
+```
+
+### Required Parameters
+
+| Parameter | Description | Values |
+|-----------|-------------|--------|
+| `input_type` | Type of input content | `"text"`, `"image"`, `"video"`, `"audio"` |
+
+### Supported Models
+
+| Model Name | Function Call | Notes |
+|------------|---------------|-------|
+| TwelveLabs Marengo 2.7 (Sync) | `embedding(model="bedrock/us.twelvelabs.marengo-embed-2-7-v1:0", input=input, input_type="text")` | Text embeddings only |
+| TwelveLabs Marengo 2.7 (Async) | `embedding(model="bedrock/async_invoke/us.twelvelabs.marengo-embed-2-7-v1:0", input=input, input_type="text/image/video/audio")` | All input types, requires `output_s3_uri` |
 
 ## Cohere Embedding Models
 https://docs.cohere.com/reference/embed

docs/my-website/docs/enterprise.md

@@ -1,6 +1,11 @@
 import Image from '@theme/IdealImage';
 
 # Enterprise
+
+:::info
+✨ SSO is free for up to 5 users. After that, an enterprise license is required. [Get Started with Enterprise here](https://www.litellm.ai/enterprise)
+:::
+
 For companies that need SSO, user management and professional support for LiteLLM Proxy
 
 :::info

docs/my-website/docs/exception_mapping.md

@@ -12,6 +12,7 @@ All exceptions can be imported from `litellm` - e.g. `from litellm import BadReq
 | 400 | UnsupportedParamsError | litellm.BadRequestError | Raised when unsupported params are passed |
 | 400         | ContextWindowExceededError| litellm.BadRequestError | Special error type for context window exceeded error messages - enables context window fallbacks |
 | 400         | ContentPolicyViolationError| litellm.BadRequestError | Special error type for content policy violation error messages - enables content policy fallbacks |
+| 400         | ImageFetchError | litellm.BadRequestError | Raised when there are errors fetching or processing images |
 | 400 | InvalidRequestError | openai.BadRequestError | Deprecated error, use BadRequestError instead |
 | 401         | AuthenticationError      | openai.AuthenticationError |
 | 403         | PermissionDeniedError    | openai.PermissionDeniedError |

docs/my-website/docs/extras/gemini_img_migration.md

@@ -0,0 +1,220 @@
+# Gemini Image Generation Migration Guide
+
+## Who is impacted by this change?
+
+Anyone using the following models with /chat/completions:
+- `gemini/gemini-2.0-flash-exp-image-generation`
+- `vertex_ai/gemini-2.0-flash-exp-image-generation`
+
+## Key Change
+
+:::info
+From v1.77.0, LiteLLM will return the List of images in `response.choices[0].message.images` instead of a single image in `response.choices[0].message.image`.
+:::
+
+Gemini models now support image generation through chat completions. Images are returned in `response.choices[0].message.images` with base64 data URLs.
+
+## Before and After
+
+### Before
+```python
+from litellm import completion
+
+response = completion(
+    model="gemini/gemini-2.0-flash-exp-image-generation",
+    messages=[{"role": "user", "content": "Generate an image of a cat"}],
+    modalities=["image", "text"],
+)
+
+
+base_64_image_data = response.choices[0].message.content
+```
+
+### After  
+```python
+from litellm import completion
+
+response = completion(
+    model="gemini/gemini-2.0-flash-exp-image-generation",
+    messages=[{"role": "user", "content": "Generate an image of a cat"}],
+    modalities=["image", "text"],
+)
+
+# Image is now available in the response
+image_url = response.choices[0].message.images[0]["image_url"]["url"]  # "data:image/png;base64,..."
+```
+
+### Why the change?
+
+Because the newer `gemini-2.5-flash-image-preview` model sends both text and image responses in the same response. This interface allows a developer to explicitly access the image or text components of the response. Before a developer would have needed to search through the message content to find the image generated by the model.
+
+**Why the change from `image` to `images`?**
+This is to be consistent with the OpenRouter API, making sure we are using simple, well-known interfaces where possible.
+
+## Usage
+
+### Using the Python SDK
+
+**Key Change:**
+```diff
+# Before
+-- base_64_image_data = response.choices[0].message.content
+
+# After
+++ image_url = response.choices[0].message.images[0]["image_url"]["url"]
+```
+
+#### Basic Image Generation
+
+```python
+from litellm import completion
+import os
+
+# Set your API key
+os.environ["GEMINI_API_KEY"] = "your-api-key"
+
+# Generate an image
+response = completion(
+    model="gemini/gemini-2.0-flash-exp-image-generation",
+    messages=[{"role": "user", "content": "Generate an image of a cat"}],
+    modalities=["image", "text"],
+)
+
+# Access the generated image
+print(response.choices[0].message.content)  # Text response (if any)
+print(response.choices[0].message.images[0])    # Image data
+```
+
+#### Response Format
+
+The image is returned in the `message.images` field:
+
+```python
+{
+    "image_url": {
+        "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...",
+        "detail": "auto"
+    },
+    "index": 0,
+    "type": "image_url"
+}
+```
+
+### Using the LiteLLM Proxy Server
+
+**Key Change:**
+```diff
+# Before
+-- "content": "base64-image-data..."
+
+# After  
+++ "images": [{
+++   "image_url": {
+++     "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...",
+++     "detail": "auto"
+++   },
+++   "index": 0,
+++   "type": "image_url"
+++ }]
+```
+
+#### Configuration Setup
+
+1. **Configure your models in `config.yaml`:**
+
+```yaml
+model_list:
+  - model_name: gemini-image-gen
+    litellm_params:
+      model: gemini/gemini-2.0-flash-exp-image-generation
+      api_key: os.environ/GEMINI_API_KEY
+  - model_name: vertex-image-gen  
+    litellm_params:
+      model: vertex_ai/gemini-2.5-flash-image-preview
+      vertex_project: your-project-id
+      vertex_location: us-central1
+
+general_settings:
+  master_key: sk-1234  # Your proxy API key
+```
+
+2. **Start the proxy server:**
+
+```bash
+litellm --config /path/to/config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+#### Making Requests
+
+**Using OpenAI SDK:**
+
+```python
+from openai import OpenAI
+
+# Point to your proxy server
+client = OpenAI(
+    api_key="sk-1234",  # Your proxy API key
+    base_url="http://0.0.0.0:4000"
+)
+
+response = client.chat.completions.create(
+    model="gemini-image-gen",
+    messages=[{"role": "user", "content": "Generate an image of a cat"}],
+    extra_body={"modalities": ["image", "text"]}
+)
+
+# Access the generated image
+print(response.choices[0].message.content)  # Text response (if any)
+print(response.choices[0].message.image)    # Image data
+```
+
+**Using curl:**
+
+```bash
+curl -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
+-H 'Content-Type: application/json' \
+-H 'Authorization: Bearer sk-1234' \
+-d '{
+  "model": "gemini-image-gen",
+  "messages": [
+    {
+      "role": "user",
+      "content": "Generate an image of a cat"
+    }
+  ],
+  "modalities": ["image", "text"]
+}'
+```
+
+**Response format from proxy:**
+
+```json
+{
+  "id": "chatcmpl-123",
+  "object": "chat.completion",
+  "created": 1704089632,
+  "model": "gemini-image-gen",
+  "choices": [
+    {
+      "index": 0,
+      "message": {
+        "role": "assistant",
+        "content": "Here's an image of a cat for you!",
+        "images": [{
+          "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA...",
+          "detail": "auto"
+        }
+      },
+      "finish_reason": "stop"
+    }
+  ],
+  "usage": {
+    "prompt_tokens": 10,
+    "completion_tokens": 8,
+    "total_tokens": 18
+  }
+}
+```
+

docs/my-website/docs/fine_tuning.md

@@ -13,6 +13,8 @@ This is an Enterprise only endpoint [Get Started with Enterprise here](https://c
 | Feature | Supported | Notes | 
 |-------|-------|-------|
 | Supported Providers | OpenAI, Azure OpenAI, Vertex AI | - |
+
+#### ⚡️See an exhaustive list of supported models and providers at [models.litellm.ai](https://models.litellm.ai/)
 | Cost Tracking | 🟡 | [Let us know if you need this](https://github.com/BerriAI/litellm/issues) |
 | Logging | ✅ | Works across all logging integrations |
 

docs/my-website/docs/getting_started.md

@@ -32,7 +32,8 @@ Next Steps 👉 [Call all supported models - e.g. Claude-2, Llama2-70b, etc.](./
 More details 👉
 
 - [Completion() function details](./completion/)
-- [All supported models / providers on LiteLLM](./providers/)
+- [Overview of supported models / providers on LiteLLM](./providers/)
+- [Search all models / providers](https://models.litellm.ai/)
 - [Build your own OpenAI proxy](https://github.com/BerriAI/liteLLM-proxy/tree/main)
 
 ## streaming

docs/my-website/docs/image_edits.md

@@ -4,7 +4,7 @@ import TabItem from '@theme/TabItem';
 
 # /images/edits
 
-LiteLLM provides image editing functionality that maps to OpenAI's `/images/edits` API endpoint.
+LiteLLM provides image editing functionality that maps to OpenAI's `/images/edits` API endpoint. Now supports both single and multiple image editing.
 
 | Feature | Supported | Notes |
 |---------|-----------|--------|
@@ -13,11 +13,14 @@ LiteLLM provides image editing functionality that maps to OpenAI's `/images/edit
 | End-user Tracking | ✅ | |
 | Fallbacks | ✅ | Works between supported models |
 | Loadbalancing | ✅ | Works between supported models |
-| Supported operations | Create image edits | |
+| Supported operations | Create image edits | Single and multiple images supported |
 | Supported LiteLLM SDK Versions | 1.63.8+ | |
 | Supported LiteLLM Proxy Versions | 1.71.1+ | |
 | Supported LLM providers | **OpenAI** | Currently only `openai` is supported |
 
+ #### ⚡️See all supported models and providers at [models.litellm.ai](https://models.litellm.ai/)
+
+
 ## Usage
 
 ### LiteLLM Python SDK
@@ -41,6 +44,26 @@ response = litellm.image_edit(
 print(response)
 ```
 
+#### Multiple Images Edit
+```python showLineNumbers title="OpenAI Multiple Images Edit"
+import litellm
+
+# Edit multiple images with a prompt
+response = litellm.image_edit(
+    model="gpt-image-1",
+    image=[
+        open("image1.png", "rb"),
+        open("image2.png", "rb"),
+        open("image3.png", "rb")
+    ],
+    prompt="Apply vintage filter to all images",
+    n=1,
+    size="1024x1024"
+)
+
+print(response)
+```
+
 #### Image Edit with Mask
 ```python showLineNumbers title="OpenAI Image Edit with Mask"
 import litellm
@@ -80,6 +103,30 @@ response = asyncio.run(edit_image())
 print(response)
 ```
 
+#### Async Multiple Images Edit
+```python showLineNumbers title="Async OpenAI Multiple Images Edit"
+import litellm
+import asyncio
+
+async def edit_multiple_images():
+    response = await litellm.aimage_edit(
+        model="gpt-image-1",
+        image=[
+            open("portrait1.png", "rb"),
+            open("portrait2.png", "rb")
+        ],
+        prompt="Add professional lighting to the portraits",
+        n=1,
+        size="1024x1024",
+        response_format="url"
+    )
+    return response
+
+# Run the async function
+response = asyncio.run(edit_multiple_images())
+print(response)
+```
+
 #### Image Edit with Custom Parameters
 ```python showLineNumbers title="OpenAI Image Edit with Custom Parameters"
 import litellm
@@ -163,6 +210,20 @@ curl -X POST "http://localhost:4000/v1/images/edits" \
   -F "response_format=url"
 ```
 
+#### cURL Multiple Images Example
+```bash showLineNumbers title="cURL Multiple Images Edit Request"
+curl -X POST "http://localhost:4000/v1/images/edits" \
+  -H "Authorization: Bearer your-api-key" \
+  -F "model=gpt-image-1" \
+  -F "image=@image1.png" \
+  -F "image=@image2.png" \
+  -F "image=@image3.png" \
+  -F "prompt=Apply artistic filter to all images" \
+  -F "n=1" \
+  -F "size=1024x1024" \
+  -F "response_format=url"
+```
+
 </TabItem>
 </Tabs>
 

docs/my-website/docs/image_generation.md

@@ -124,8 +124,6 @@ Any non-openai params, will be treated as provider-specific params, and sent in
 
 - `size`: *string (optional)* The size of the generated images. Must be one of `1024x1024`, `1536x1024` (landscape), `1024x1536` (portrait), or `auto` (default value) for `gpt-image-1`, one of `256x256`, `512x512`, or `1024x1024` for `dall-e-2`, and one of `1024x1024`, `1792x1024`, or `1024x1792` for `dall-e-3`.
 
-- `input_fidelity`: *string (optional)* Controls how closely the model follows the input prompt. Supported for `gpt-image-1` model. Higher fidelity may improve prompt adherence but could affect generation speed.
-
 - `timeout`: *integer* - The maximum time, in seconds, to wait for the API to respond. Defaults to 600 seconds (10 minutes).
 
 - `user`: *string (optional)* A unique identifier representing your end-user, 
@@ -281,6 +279,8 @@ print(f"response: {response}")
 
 ## Supported Providers
 
+#### ⚡️See all supported models and providers at [models.litellm.ai](https://models.litellm.ai/)
+
 | Provider | Documentation Link |
 |----------|-------------------|
 | OpenAI | [OpenAI Image Generation →](./providers/openai) |

docs/my-website/docs/index.md

@@ -226,6 +226,23 @@ response = completion(
 
 </TabItem>
 
+<TabItem value="vercel" label="Vercel AI Gateway">
+
+```python
+from litellm import completion
+import os
+
+## set ENV variables. Visit https://vercel.com/docs/ai-gateway#using-the-ai-gateway-with-an-api-key for insturctions on obtaining a key
+os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "your-vercel-api-key"
+
+response = completion(
+  model="vercel_ai_gateway/openai/gpt-4o",
+  messages=[{ "content": "Hello, how are you?","role": "user"}]
+)
+```
+
+</TabItem>
+
 </Tabs>
 
 ### Response Format (OpenAI Format)
@@ -234,7 +251,7 @@ response = completion(
 {
     "id": "chatcmpl-565d891b-a42e-4c39-8d14-82a1f5208885",
     "created": 1734366691,
-    "model": "claude-3-sonnet-20240229",
+    "model": "gpt-4o-2024-08-06",
     "object": "chat.completion",
     "system_fingerprint": null,
     "choices": [
@@ -446,6 +463,24 @@ response = completion(
 
 </TabItem>
 
+<TabItem value="vercel" label="Vercel AI Gateway">
+
+```python
+from litellm import completion
+import os
+
+## set ENV variables. Visit https://vercel.com/docs/ai-gateway#using-the-ai-gateway-with-an-api-key for insturctions on obtaining a key
+os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "your-vercel-api-key"
+
+response = completion(
+  model="vercel_ai_gateway/openai/gpt-4o",
+  messages = [{ "content": "Hello, how are you?","role": "user"}],
+  stream=True,
+)
+```
+
+</TabItem>
+
 </Tabs>
 
 ### Streaming Response Format (OpenAI Format)
@@ -489,6 +524,15 @@ try:
 except OpenAIError as e:
     print(e)
 ```
+### See How LiteLLM Transforms Your Requests
+
+Want to understand how LiteLLM parses and normalizes your LLM API requests? Use the `/utils/transform_request` endpoint to see exactly how your request is transformed internally.
+
+You can try it out now directly on our Demo App!
+Go to the [LiteLLM API docs for transform_request](https://litellm-api.up.railway.app/#/llm%20utils/transform_request_utils_transform_request_post)
+
+LiteLLM will show you the normalized, provider-agnostic version of your request. This is useful for debugging, learning, and understanding how LiteLLM handles different providers and options.
+
 
 ### Logging Observability - Log LLM Input/Output ([Docs](https://docs.litellm.ai/docs/observability/callbacks))
 LiteLLM exposes pre defined callbacks to send data to Lunary, MLflow, Langfuse, Helicone, Promptlayer, Traceloop, Slack

docs/my-website/docs/integrations/index.md

@@ -2,4 +2,17 @@
 
 This section covers integrations with various tools and services that can be used with LiteLLM (either Proxy or SDK).
 
+## AI Agent Frameworks
+- **[Letta](./letta.md)** - Build stateful LLM agents with persistent memory using LiteLLM Proxy
+
+## Development Tools
+- **[OpenWebUI](../tutorials/openweb_ui.md)** - Self-hosted ChatGPT-style interface
+
+## Observability & Monitoring
+- **[Langfuse](../observability/langfuse_integration.md)** - LLM observability and analytics
+- **[Prometheus](../proxy/prometheus.md)** - Metrics collection and monitoring
+- **[PagerDuty](../proxy/pagerduty.md)** - Incident response and alerting
+- **[Datadog](../observability/datadog.md)**
+
+
 Click into each section to learn more about the integrations.

docs/my-website/docs/integrations/letta.md

@@ -0,0 +1,928 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Letta Integration
+
+[Letta](https://github.com/letta-ai/letta) (formerly MemGPT) is a framework for building stateful LLM agents with persistent memory. This guide shows how to integrate both LiteLLM SDK and LiteLLM Proxy with Letta to leverage multiple LLM providers while building memory-enabled agents.
+
+## What is Letta?
+
+Letta allows you to build LLM agents that can:
+- Maintain long-term memory across conversations
+- Use function calling for tool interactions
+- Handle large context windows efficiently
+- Persist agent state and memory
+
+## Prerequisites
+
+```bash
+pip install letta litellm
+```
+
+## Quick Start
+
+<Tabs>
+<TabItem value="proxy" label="LiteLLM Proxy">
+
+### 1. Start LiteLLM Proxy
+
+First, create a configuration file for your LiteLLM proxy:
+
+```yaml
+# config.yaml
+model_list:
+  - model_name: gpt-4
+    litellm_params:
+      model: openai/gpt-4
+      api_key: os.environ/OPENAI_API_KEY
+
+  - model_name: claude-3-sonnet
+    litellm_params:
+      model: anthropic/claude-3-sonnet-20240229
+      api_key: os.environ/ANTHROPIC_API_KEY
+
+  - model_name: gpt-3.5-turbo
+    litellm_params:
+      model: azure/gpt-35-turbo
+      api_key: os.environ/AZURE_API_KEY
+      api_base: os.environ/AZURE_API_BASE
+      api_version: "2023-07-01-preview"
+```
+
+Start the proxy:
+
+```bash
+litellm --config config.yaml --port 4000
+```
+
+### 2. Configure Letta with LiteLLM Proxy
+
+Configure Letta to use your LiteLLM proxy endpoint:
+
+```python
+import letta
+from letta import create_client
+
+# Configure Letta to use LiteLLM proxy
+client = create_client()
+
+# Configure the LLM endpoint
+client.set_default_llm_config(
+    model="gpt-4",  # This should match a model from your LiteLLM config
+    model_endpoint_type="openai",
+    model_endpoint="http://localhost:4000",  # Your LiteLLM proxy URL
+    context_window=8192
+)
+
+# Configure embedding endpoint (optional)
+client.set_default_embedding_config(
+    embedding_endpoint_type="openai",
+    embedding_endpoint="http://localhost:4000",
+    embedding_model="text-embedding-ada-002"
+)
+```
+
+</TabItem>
+<TabItem value="sdk" label="LiteLLM SDK">
+
+### 1. Configure LiteLLM SDK
+
+Set up your API keys and configure LiteLLM:
+
+```python
+import os
+import litellm
+
+# Set your API keys
+os.environ["OPENAI_API_KEY"] = "your-openai-key"
+os.environ["ANTHROPIC_API_KEY"] = "your-anthropic-key"
+
+# Optional: Configure default settings
+litellm.set_verbose = True  # For debugging
+```
+
+### 2. Create Custom LLM Wrapper for Letta
+
+Create a custom LLM wrapper that uses LiteLLM SDK:
+
+```python
+import letta
+from letta import create_client
+from letta.llm_api.llm_api_base import LLMConfig
+import litellm
+from typing import List, Dict, Any
+
+class LiteLLMWrapper:
+    def __init__(self, model: str):
+        self.model = model
+    
+    def chat_completions_create(self, messages: List[Dict], **kwargs):
+        # Use LiteLLM SDK for completion
+        response = litellm.completion(
+            model=self.model,
+            messages=messages,
+            **kwargs
+        )
+        return response
+
+# Configure Letta with custom LiteLLM wrapper
+client = create_client()
+
+# Set up LLM configuration using direct SDK integration
+llm_config = LLMConfig(
+    model="gpt-4",  # or "claude-3-sonnet", "azure/gpt-35-turbo", etc.
+    model_endpoint_type="openai",
+    context_window=8192
+)
+
+client.set_default_llm_config(llm_config)
+```
+
+</TabItem>
+</Tabs>
+
+### 3. Create and Use a Letta Agent
+
+<Tabs>
+<TabItem value="proxy" label="Using LiteLLM Proxy">
+
+```python
+import letta
+from letta import create_client
+
+# Create Letta client
+client = create_client()
+
+# Create a new agent
+agent_state = client.create_agent(
+    name="my-assistant",
+    system="You are a helpful assistant with persistent memory.",
+    llm_config=client.get_default_llm_config(),
+    embedding_config=client.get_default_embedding_config()
+)
+
+# Send a message to the agent
+response = client.user_message(
+    agent_id=agent_state.id,
+    message="Hi! My name is Alice and I love reading science fiction books."
+)
+
+print(f"Agent response: {response.messages[-1].text}")
+
+# Send another message - the agent will remember previous context
+response = client.user_message(
+    agent_id=agent_state.id,
+    message="What did I tell you about my interests?"
+)
+
+print(f"Agent response: {response.messages[-1].text}")
+```
+
+</TabItem>
+<TabItem value="sdk" label="Using LiteLLM SDK">
+
+```python
+import letta
+from letta import create_client
+import litellm
+import os
+
+# Set up environment variables
+os.environ["OPENAI_API_KEY"] = "your-openai-key"
+
+# Create Letta client with LiteLLM integration
+client = create_client()
+
+# Create a new agent
+agent_state = client.create_agent(
+    name="my-assistant",
+    system="You are a helpful assistant with persistent memory.",
+    llm_config=client.get_default_llm_config(),
+    embedding_config=client.get_default_embedding_config()
+)
+
+# Send a message to the agent
+response = client.user_message(
+    agent_id=agent_state.id,
+    message="Hi! My name is Alice and I love reading science fiction books."
+)
+
+print(f"Agent response: {response.messages[-1].text}")
+
+# Send another message - the agent will remember previous context
+response = client.user_message(
+    agent_id=agent_state.id,
+    message="What did I tell you about my interests?"
+)
+
+print(f"Agent response: {response.messages[-1].text}")
+```
+
+</TabItem>
+</Tabs>
+
+## Advanced Configuration
+
+### Using Different Models for Different Agents
+
+<Tabs>
+<TabItem value="proxy" label="LiteLLM Proxy">
+
+```python
+from letta import LLMConfig, EmbeddingConfig
+
+# Create different LLM configurations pointing to your proxy
+gpt4_config = LLMConfig(
+    model="gpt-4",
+    model_endpoint_type="openai",
+    model_endpoint="http://localhost:4000",
+    context_window=8192
+)
+
+claude_config = LLMConfig(
+    model="claude-3-sonnet",
+    model_endpoint_type="openai",  # Using OpenAI-compatible endpoint
+    model_endpoint="http://localhost:4000",
+    context_window=200000
+)
+
+# Create agents with different configurations
+research_agent = client.create_agent(
+    name="research-agent",
+    system="You are a research assistant specialized in analysis.",
+    llm_config=claude_config  # Use Claude for research tasks
+)
+
+creative_agent = client.create_agent(
+    name="creative-agent", 
+    system="You are a creative writing assistant.",
+    llm_config=gpt4_config  # Use GPT-4 for creative tasks
+)
+```
+
+</TabItem>
+<TabItem value="sdk" label="LiteLLM SDK">
+
+```python
+import os
+import litellm
+from letta import LLMConfig, EmbeddingConfig
+
+# Set up API keys for different providers
+os.environ["OPENAI_API_KEY"] = "your-openai-key"
+os.environ["ANTHROPIC_API_KEY"] = "your-anthropic-key"
+
+# Create different LLM configurations for direct SDK usage
+gpt4_config = LLMConfig(
+    model="openai/gpt-4",  # Using LiteLLM model format
+    model_endpoint_type="openai",
+    context_window=8192
+)
+
+claude_config = LLMConfig(
+    model="anthropic/claude-3-sonnet-20240229",  # Using LiteLLM model format
+    model_endpoint_type="openai",
+    context_window=200000
+)
+
+# Create agents with different configurations
+research_agent = client.create_agent(
+    name="research-agent",
+    system="You are a research assistant specialized in analysis.",
+    llm_config=claude_config  # Use Claude for research tasks
+)
+
+creative_agent = client.create_agent(
+    name="creative-agent", 
+    system="You are a creative writing assistant.",
+    llm_config=gpt4_config  # Use GPT-4 for creative tasks
+)
+```
+
+</TabItem>
+</Tabs>
+
+### Function Calling with Tools
+
+<Tabs>
+<TabItem value="proxy" label="LiteLLM Proxy">
+
+```python
+# Define custom tools for your agent
+def search_web(query: str) -> str:
+    """Search the web for information"""
+    # Your web search implementation
+    return f"Search results for: {query}"
+
+def save_note(content: str) -> str:
+    """Save a note to persistent storage"""
+    # Your note saving implementation
+    return f"Note saved: {content}"
+
+# Create agent with tools (using proxy endpoint)
+agent_state = client.create_agent(
+    name="research-assistant",
+    system="You are a research assistant that can search the web and save notes.",
+    llm_config=client.get_default_llm_config(),
+    embedding_config=client.get_default_embedding_config(),
+    tools=[search_web, save_note]
+)
+
+# The agent can now use these tools
+response = client.user_message(
+    agent_id=agent_state.id,
+    message="Search for recent developments in AI and save important findings."
+)
+```
+
+</TabItem>
+<TabItem value="sdk" label="LiteLLM SDK">
+
+```python
+import litellm
+import os
+
+# Set up API keys
+os.environ["OPENAI_API_KEY"] = "your-openai-key"
+
+# Define custom tools for your agent
+def search_web(query: str) -> str:
+    """Search the web for information"""
+    # Your web search implementation
+    return f"Search results for: {query}"
+
+def save_note(content: str) -> str:
+    """Save a note to persistent storage"""
+    # Your note saving implementation
+    return f"Note saved: {content}"
+
+# Create agent with tools (using LiteLLM SDK directly)
+agent_state = client.create_agent(
+    name="research-assistant",
+    system="You are a research assistant that can search the web and save notes.",
+    llm_config=LLMConfig(
+        model="openai/gpt-4",  # Direct model specification
+        model_endpoint_type="openai",
+        context_window=8192
+    ),
+    embedding_config=client.get_default_embedding_config(),
+    tools=[search_web, save_note]
+)
+
+# The agent can now use these tools
+response = client.user_message(
+    agent_id=agent_state.id,
+    message="Search for recent developments in AI and save important findings."
+)
+```
+
+</TabItem>
+</Tabs>
+
+## Authentication
+
+<Tabs>
+<TabItem value="proxy" label="LiteLLM Proxy Authentication">
+
+If your LiteLLM proxy requires authentication:
+
+```python
+import os
+from letta import LLMConfig
+
+# Set up authenticated configuration
+llm_config = LLMConfig(
+    model="gpt-4",
+    model_endpoint_type="openai",
+    model_endpoint="http://localhost:4000",
+    model_wrapper="openai",
+    context_window=8192
+)
+
+# If using API keys with your proxy
+os.environ["OPENAI_API_KEY"] = "your-litellm-proxy-api-key"
+
+client = create_client()
+client.set_default_llm_config(llm_config)
+```
+
+For proxy with authentication enabled:
+
+```yaml
+# config.yaml with auth
+general_settings:
+  master_key: "your-master-key"
+
+model_list:
+  - model_name: gpt-4
+    litellm_params:
+      model: openai/gpt-4
+      api_key: os.environ/OPENAI_API_KEY
+```
+
+```python
+# Configure Letta with authenticated proxy
+llm_config = LLMConfig(
+    model="gpt-4",
+    model_endpoint_type="openai",
+    model_endpoint="http://localhost:4000",
+    context_window=8192,
+    api_key="your-master-key"  # Proxy master key
+)
+```
+
+</TabItem>
+<TabItem value="sdk" label="LiteLLM SDK Authentication">
+
+With LiteLLM SDK, set up your provider API keys directly:
+
+```python
+import os
+import litellm
+
+# Set up API keys for different providers
+os.environ["OPENAI_API_KEY"] = "your-openai-api-key"
+os.environ["ANTHROPIC_API_KEY"] = "your-anthropic-api-key" 
+os.environ["AZURE_API_KEY"] = "your-azure-api-key"
+os.environ["AZURE_API_BASE"] = "https://your-resource.openai.azure.com"
+os.environ["AZURE_API_VERSION"] = "2023-07-01-preview"
+
+# Optional: Configure default settings
+litellm.api_key = os.environ.get("OPENAI_API_KEY")  # Default key
+litellm.set_verbose = True  # For debugging
+
+# Use in Letta configuration
+from letta import LLMConfig
+
+llm_config = LLMConfig(
+    model="openai/gpt-4",  # Will use OPENAI_API_KEY automatically
+    model_endpoint_type="openai",
+    context_window=8192
+)
+
+# Or for Azure
+azure_config = LLMConfig(
+    model="azure/gpt-35-turbo", 
+    model_endpoint_type="openai",
+    context_window=4096
+)
+```
+
+</TabItem>
+</Tabs>
+
+## Load Balancing and Fallbacks
+
+<Tabs>
+<TabItem value="proxy" label="LiteLLM Proxy Features">
+
+LiteLLM proxy's load balancing and fallback features work seamlessly with Letta:
+
+```yaml
+# config.yaml with fallbacks
+model_list:
+  - model_name: gpt-4
+    litellm_params:
+      model: openai/gpt-4
+      api_key: os.environ/OPENAI_API_KEY
+    tpm: 40000
+    rpm: 500
+
+  - model_name: gpt-4  # Same model name for fallback
+    litellm_params:
+      model: azure/gpt-4
+      api_key: os.environ/AZURE_API_KEY
+      api_base: os.environ/AZURE_API_BASE
+      api_version: "2023-07-01-preview"
+    tpm: 80000
+    rpm: 800
+
+router_settings:
+  routing_strategy: "usage-based-routing"
+  fallbacks: [{"gpt-4": ["azure/gpt-4"]}]
+```
+
+The proxy handles all routing, load balancing, and fallbacks transparently for Letta.
+
+</TabItem>
+<TabItem value="sdk" label="LiteLLM SDK Router">
+
+With LiteLLM SDK, you can set up routing and fallbacks programmatically:
+
+```python
+import litellm
+from litellm import Router
+
+# Configure router with multiple models
+router = Router(
+    model_list=[
+        {
+            "model_name": "gpt-4",
+            "litellm_params": {
+                "model": "openai/gpt-4",
+                "api_key": os.environ["OPENAI_API_KEY"]
+            },
+            "tpm": 40000,
+            "rpm": 500
+        },
+        {
+            "model_name": "gpt-4",  # Same name for fallback
+            "litellm_params": {
+                "model": "azure/gpt-4", 
+                "api_key": os.environ["AZURE_API_KEY"],
+                "api_base": os.environ["AZURE_API_BASE"],
+                "api_version": "2023-07-01-preview"
+            },
+            "tpm": 80000,
+            "rpm": 800
+        }
+    ],
+    fallbacks=[{"gpt-4": ["azure/gpt-4"]}],
+    routing_strategy="usage-based-routing"
+)
+
+# Create custom completion function for Letta
+def custom_completion(messages, model="gpt-4", **kwargs):
+    return router.completion(
+        model=model,
+        messages=messages,
+        **kwargs
+    )
+
+# Use with Letta by monkey-patching or custom wrapper
+litellm.completion = custom_completion
+```
+
+</TabItem>
+</Tabs>
+
+## Monitoring and Observability
+
+<Tabs>
+<TabItem value="proxy" label="LiteLLM Proxy Monitoring">
+
+Enable logging to track your Letta agents' LLM usage through the proxy:
+
+```yaml
+# config.yaml with logging
+model_list:
+  # ... your models
+
+litellm_settings:
+  success_callback: ["langfuse"]  # or other observability tools
+  
+environment_variables:
+  LANGFUSE_PUBLIC_KEY: "your-key"
+  LANGFUSE_SECRET_KEY: "your-secret"
+```
+
+View metrics in the proxy dashboard:
+```bash
+# Start proxy with UI
+litellm --config config.yaml --port 4000 --detailed_debug
+```
+
+</TabItem>
+<TabItem value="sdk" label="LiteLLM SDK Monitoring">
+
+Set up observability directly in your SDK integration:
+
+```python
+import litellm
+import os
+
+# Configure observability callbacks
+os.environ["LANGFUSE_PUBLIC_KEY"] = "your-key"
+os.environ["LANGFUSE_SECRET_KEY"] = "your-secret"
+
+# Set global callbacks
+litellm.success_callback = ["langfuse"]
+litellm.failure_callback = ["langfuse"]
+
+# Optional: Set up custom logging
+litellm.set_verbose = True
+
+# Create custom completion wrapper with logging
+def logged_completion(messages, model="gpt-4", **kwargs):
+    try:
+        response = litellm.completion(
+            model=model,
+            messages=messages,
+            **kwargs
+        )
+        # Custom logging logic here if needed
+        return response
+    except Exception as e:
+        # Custom error handling
+        print(f"LLM call failed: {e}")
+        raise
+
+# Use in Letta configuration
+litellm.completion = logged_completion
+```
+
+</TabItem>
+</Tabs>
+
+## Example: Multi-Agent System
+
+<Tabs>
+<TabItem value="proxy" label="Using LiteLLM Proxy">
+
+```python
+import letta
+from letta import create_client, LLMConfig
+
+client = create_client()
+
+# Create specialized agents using proxy endpoints
+agents = {}
+
+# Research agent using Claude for analysis
+agents['researcher'] = client.create_agent(
+    name="researcher",
+    system="You are a research specialist. Analyze information thoroughly.",
+    llm_config=LLMConfig(
+        model="claude-3-sonnet",
+        model_endpoint="http://localhost:4000",
+        model_endpoint_type="openai"
+    )
+)
+
+# Writer agent using GPT-4 for content creation
+agents['writer'] = client.create_agent(
+    name="writer",
+    system="You are a content writer. Create engaging, well-structured content.",
+    llm_config=LLMConfig(
+        model="gpt-4",
+        model_endpoint="http://localhost:4000", 
+        model_endpoint_type="openai"
+    )
+)
+
+# Coordinator workflow
+def research_and_write_workflow(topic: str):
+    # Research phase
+    research_response = client.user_message(
+        agent_id=agents['researcher'].id,
+        message=f"Research the topic: {topic}. Provide key insights and data."
+    )
+    
+    research_results = research_response.messages[-1].text
+    
+    # Writing phase
+    write_response = client.user_message(
+        agent_id=agents['writer'].id,
+        message=f"Based on this research: {research_results}\n\nWrite an article about {topic}."
+    )
+    
+    return write_response.messages[-1].text
+
+# Execute workflow
+article = research_and_write_workflow("The future of AI in healthcare")
+print(article)
+```
+
+</TabItem>
+<TabItem value="sdk" label="Using LiteLLM SDK">
+
+```python
+import letta
+from letta import create_client, LLMConfig
+import litellm
+import os
+
+# Set up environment
+os.environ["OPENAI_API_KEY"] = "your-openai-key"
+os.environ["ANTHROPIC_API_KEY"] = "your-anthropic-key"
+
+client = create_client()
+
+# Create specialized agents using direct SDK models
+agents = {}
+
+# Research agent using Claude for analysis
+agents['researcher'] = client.create_agent(
+    name="researcher",
+    system="You are a research specialist. Analyze information thoroughly.",
+    llm_config=LLMConfig(
+        model="anthropic/claude-3-sonnet-20240229",
+        model_endpoint_type="openai"
+    )
+)
+
+# Writer agent using GPT-4 for content creation
+agents['writer'] = client.create_agent(
+    name="writer",
+    system="You are a content writer. Create engaging, well-structured content.",
+    llm_config=LLMConfig(
+        model="openai/gpt-4",
+        model_endpoint_type="openai"
+    )
+)
+
+# Cost-conscious agent using GPT-3.5
+agents['reviewer'] = client.create_agent(
+    name="reviewer",
+    system="You are an editor. Review and improve content quality.",
+    llm_config=LLMConfig(
+        model="openai/gpt-3.5-turbo",
+        model_endpoint_type="openai"
+    )
+)
+
+# Enhanced workflow with multiple agents
+def enhanced_workflow(topic: str):
+    # Research phase
+    research_response = client.user_message(
+        agent_id=agents['researcher'].id,
+        message=f"Research the topic: {topic}. Provide key insights and data."
+    )
+    
+    research_results = research_response.messages[-1].text
+    
+    # Writing phase
+    write_response = client.user_message(
+        agent_id=agents['writer'].id,
+        message=f"Based on this research: {research_results}\n\nWrite an article about {topic}."
+    )
+    
+    draft_article = write_response.messages[-1].text
+    
+    # Review phase
+    review_response = client.user_message(
+        agent_id=agents['reviewer'].id,
+        message=f"Please review and improve this article:\n\n{draft_article}"
+    )
+    
+    return review_response.messages[-1].text
+
+# Execute enhanced workflow
+article = enhanced_workflow("The future of AI in healthcare")
+print(article)
+```
+
+</TabItem>
+</Tabs>
+
+## Best Practices
+
+<Tabs>
+<TabItem value="proxy" label="LiteLLM Proxy Best Practices">
+
+1. **Model Selection**: Use appropriate models for different tasks:
+   - Claude for analysis and reasoning
+   - GPT-4 for creative tasks
+   - GPT-3.5-turbo for simple interactions
+
+2. **Proxy Configuration**:
+   - Set appropriate rate limits and timeouts
+   - Use fallbacks for reliability
+   - Enable authentication for production
+
+3. **Memory Management**: Letta handles memory automatically, but monitor usage with large contexts
+
+4. **Cost Optimization**: 
+   - Use the proxy's budgeting features to control costs
+   - Set up rate limiting per user/team
+   - Monitor token usage through proxy dashboard
+
+5. **Monitoring**: Enable observability to track agent performance and token usage
+
+</TabItem>
+<TabItem value="sdk" label="LiteLLM SDK Best Practices">
+
+1. **Model Selection**: Choose models based on task requirements:
+   - Use `openai/gpt-4` for complex reasoning
+   - Use `anthropic/claude-3-sonnet-20240229` for analysis
+   - Use `openai/gpt-3.5-turbo` for cost-effective simple tasks
+
+2. **Error Handling**: Implement robust error handling with retries:
+   ```python
+   import litellm
+   from litellm import completion
+   
+   # Set up retry logic
+   litellm.num_retries = 3
+   litellm.request_timeout = 60
+   
+   # Custom error handling
+   def safe_completion(**kwargs):
+       try:
+           return completion(**kwargs)
+       except Exception as e:
+           print(f"LLM call failed: {e}")
+           # Implement fallback logic
+           return completion(model="openai/gpt-3.5-turbo", **kwargs)
+   ```
+
+3. **Cost Management**:
+   - Use cheaper models for non-critical tasks
+   - Implement token counting and budgets
+   - Cache responses when appropriate
+
+4. **Performance**:
+   - Use async operations for concurrent requests
+   - Implement connection pooling
+   - Monitor response times
+
+5. **Security**:
+   - Store API keys securely (environment variables)
+   - Rotate keys regularly
+   - Implement rate limiting
+
+</TabItem>
+</Tabs>
+
+## Troubleshooting
+
+<Tabs>
+<TabItem value="proxy" label="LiteLLM Proxy Issues">
+
+### Connection Issues
+```bash
+# Test your LiteLLM proxy
+curl -X POST http://localhost:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "gpt-4",
+    "messages": [{"role": "user", "content": "Hello"}]
+  }'
+```
+
+### Configuration Debugging
+```python
+# Enable verbose logging
+import logging
+logging.basicConfig(level=logging.DEBUG)
+
+# Test Letta configuration
+client = create_client()
+print(client.get_default_llm_config())
+```
+
+### Common Proxy Issues
+- **Port conflicts**: Make sure port 4000 isn't in use
+- **Model not found**: Verify model names match your config.yaml
+- **Authentication errors**: Check master key configuration
+- **Rate limiting**: Monitor proxy logs for rate limit hits
+
+</TabItem>
+<TabItem value="sdk" label="LiteLLM SDK Issues">
+
+### API Key Issues
+```python
+import os
+import litellm
+
+# Check if API keys are set
+print("OpenAI Key:", os.environ.get("OPENAI_API_KEY", "Not set"))
+print("Anthropic Key:", os.environ.get("ANTHROPIC_API_KEY", "Not set"))
+
+# Test direct LiteLLM call
+try:
+    response = litellm.completion(
+        model="openai/gpt-3.5-turbo",
+        messages=[{"role": "user", "content": "Hello"}]
+    )
+    print("LiteLLM working:", response.choices[0].message.content)
+except Exception as e:
+    print("LiteLLM error:", e)
+```
+
+### Configuration Debugging
+```python
+# Enable verbose logging
+litellm.set_verbose = True
+
+# Test model availability
+models = ["openai/gpt-4", "anthropic/claude-3-sonnet-20240229"]
+for model in models:
+    try:
+        response = litellm.completion(
+            model=model,
+            messages=[{"role": "user", "content": "Test"}],
+            max_tokens=10
+        )
+        print(f"✓ {model} working")
+    except Exception as e:
+        print(f"✗ {model} failed: {e}")
+```
+
+### Common SDK Issues
+- **Import errors**: Ensure `pip install litellm letta` is run
+- **Model format**: Use `provider/model` format (e.g., `openai/gpt-4`)
+- **API key format**: Different providers have different key formats
+- **Rate limits**: Implement exponential backoff for retries
+
+</TabItem>
+</Tabs>
+
+## Resources
+
+- [Letta Documentation](https://docs.letta.ai/)
+- [LiteLLM Proxy Documentation](../proxy/quick_start.md)
+- [LiteLLM SDK Documentation](../completion/input.md)
+- [Function Calling Guide](../completion/function_call.md)
+- [Observability Setup](../observability/langfuse_integration.md)
+- [Router Configuration](../routing.md)

docs/my-website/docs/langchain/langchain.md

@@ -162,3 +162,321 @@ Get more details [here](../observability/lunary_integration.md)
 
 ## Use LangChain ChatLiteLLM + Langfuse
 Checkout this section [here](../observability/langfuse_integration#use-langchain-chatlitellm--langfuse) for more details on how to integrate Langfuse with ChatLiteLLM.
+
+## Using Tags with LangChain and LiteLLM
+
+Tags are a powerful feature in LiteLLM that allow you to categorize, filter, and track your LLM requests. When using LangChain with LiteLLM, you can pass tags through the `extra_body` parameter in the metadata.
+
+### Basic Tag Usage
+
+<Tabs>
+<TabItem value="openai" label="OpenAI">
+
+```python
+import os
+from langchain_openai import ChatOpenAI
+from langchain_core.messages import HumanMessage, SystemMessage
+
+os.environ['OPENAI_API_KEY'] = "sk-your-key-here"
+
+chat = ChatOpenAI(
+    model="gpt-4o",
+    temperature=0.7,
+    extra_body={
+        "metadata": {
+            "tags": ["production", "customer-support", "high-priority"]
+        }
+    }
+)
+
+messages = [
+    SystemMessage(content="You are a helpful customer support assistant."),
+    HumanMessage(content="How do I reset my password?")
+]
+
+response = chat.invoke(messages)
+print(response)
+```
+
+</TabItem>
+
+<TabItem value="anthropic" label="Anthropic">
+
+```python
+import os
+from langchain_openai import ChatOpenAI
+from langchain_core.messages import HumanMessage, SystemMessage
+
+os.environ['ANTHROPIC_API_KEY'] = "sk-ant-your-key-here"
+
+chat = ChatOpenAI(
+    model="claude-3-sonnet-20240229",
+    temperature=0.7,
+    extra_body={
+        "metadata": {
+            "tags": ["research", "analysis", "claude-model"]
+        }
+    }
+)
+
+messages = [
+    SystemMessage(content="You are a research analyst."),
+    HumanMessage(content="Analyze this market trend...")
+]
+
+response = chat.invoke(messages)
+print(response)
+```
+
+</TabItem>
+
+<TabItem value="litellm-proxy" label="LiteLLM Proxy">
+
+```python
+import os
+from langchain_openai import ChatOpenAI
+from langchain_core.messages import HumanMessage, SystemMessage
+
+# No API key needed when using proxy
+chat = ChatOpenAI(
+    openai_api_base="http://localhost:4000",  # Your proxy URL
+    model="gpt-4o",
+    temperature=0.7,
+    extra_body={
+        "metadata": {
+            "tags": ["proxy", "team-alpha", "feature-flagged"],
+            "generation_name": "customer-onboarding",
+            "trace_user_id": "user-12345"
+        }
+    }
+)
+
+messages = [
+    SystemMessage(content="You are an onboarding assistant."),
+    HumanMessage(content="Welcome our new customer!")
+]
+
+response = chat.invoke(messages)
+print(response)
+```
+
+</TabItem>
+</Tabs>
+
+### Advanced Tag Patterns
+
+#### Dynamic Tags Based on Context
+
+```python
+import os
+from langchain_openai import ChatOpenAI
+from langchain_core.messages import HumanMessage, SystemMessage
+
+def create_chat_with_tags(user_type: str, feature: str):
+    """Create a chat instance with dynamic tags based on context"""
+    
+    # Build tags dynamically
+    tags = ["langchain-integration"]
+    
+    if user_type == "premium":
+        tags.extend(["premium-user", "high-priority"])
+    elif user_type == "enterprise":
+        tags.extend(["enterprise", "custom-sla"])
+    else:
+        tags.append("standard-user")
+    
+    # Add feature-specific tags
+    if feature == "code-review":
+        tags.extend(["development", "code-analysis"])
+    elif feature == "content-gen":
+        tags.extend(["marketing", "content-creation"])
+    
+    return ChatOpenAI(
+        openai_api_base="http://localhost:4000",
+        model="gpt-4o",
+        temperature=0.7,
+        extra_body={
+            "metadata": {
+                "tags": tags,
+                "user_type": user_type,
+                "feature": feature,
+                "trace_user_id": f"user-{user_type}-{feature}"
+            }
+        }
+    )
+
+# Usage examples
+premium_chat = create_chat_with_tags("premium", "code-review")
+enterprise_chat = create_chat_with_tags("enterprise", "content-gen")
+
+messages = [HumanMessage(content="Help me with this task")]
+response = premium_chat.invoke(messages)
+```
+
+#### Tags for Cost Tracking and Analytics
+
+```python
+import os
+from langchain_openai import ChatOpenAI
+from langchain_core.messages import HumanMessage, SystemMessage
+
+# Tags for cost tracking
+cost_tracking_chat = ChatOpenAI(
+    openai_api_base="http://localhost:4000",
+    model="gpt-4o",
+    temperature=0.7,
+    extra_body={
+        "metadata": {
+            "tags": [
+                "cost-center-marketing",
+                "budget-q4-2024",
+                "project-launch-campaign",
+                "high-cost-model"  # Flag for expensive models
+            ],
+            "department": "marketing",
+            "project_id": "campaign-2024-q4",
+            "cost_threshold": "high"
+        }
+    }
+)
+
+messages = [
+    SystemMessage(content="You are a marketing copywriter."),
+    HumanMessage(content="Create compelling ad copy for our new product launch.")
+]
+
+response = cost_tracking_chat.invoke(messages)
+```
+
+#### Tags for A/B Testing
+
+```python
+import os
+from langchain_openai import ChatOpenAI
+from langchain_core.messages import HumanMessage, SystemMessage
+import random
+
+def create_ab_test_chat(test_variant: str = None):
+    """Create chat instance for A/B testing with appropriate tags"""
+    
+    if test_variant is None:
+        test_variant = random.choice(["variant-a", "variant-b"])
+    
+    return ChatOpenAI(
+        openai_api_base="http://localhost:4000",
+        model="gpt-4o",
+        temperature=0.7 if test_variant == "variant-a" else 0.9,  # Different temp for variants
+        extra_body={
+            "metadata": {
+                "tags": [
+                    "ab-test-experiment-1",
+                    f"variant-{test_variant}",
+                    "temperature-test",
+                    "user-experience"
+                ],
+                "experiment_id": "ab-test-001",
+                "variant": test_variant,
+                "test_group": "temperature-optimization"
+            }
+        }
+    )
+
+# Run A/B test
+variant_a_chat = create_ab_test_chat("variant-a")
+variant_b_chat = create_ab_test_chat("variant-b")
+
+test_message = [HumanMessage(content="Explain quantum computing in simple terms")]
+
+response_a = variant_a_chat.invoke(test_message)
+response_b = variant_b_chat.invoke(test_message)
+```
+
+### Tag Best Practices
+
+#### 1. **Consistent Naming Convention**
+```python
+# ✅ Good: Consistent, descriptive tags
+tags = ["production", "api-v2", "customer-support", "urgent"]
+
+# ❌ Avoid: Inconsistent or unclear tags
+tags = ["prod", "v2", "support", "urgent123"]
+```
+
+#### 2. **Hierarchical Tags**
+```python
+# ✅ Good: Hierarchical structure
+tags = ["env:production", "team:backend", "service:api", "priority:high"]
+
+# This allows for easy filtering and grouping
+```
+
+#### 3. **Include Context Information**
+```python
+extra_body={
+    "metadata": {
+        "tags": ["production", "user-onboarding"],
+        "user_id": "user-12345",
+        "session_id": "session-abc123",
+        "feature_flag": "new-onboarding-flow",
+        "environment": "production"
+    }
+}
+```
+
+#### 4. **Tag Categories**
+Consider organizing tags into categories:
+- **Environment**: `production`, `staging`, `development`
+- **Team/Service**: `backend`, `frontend`, `api`, `worker`
+- **Feature**: `authentication`, `payment`, `notification`
+- **Priority**: `critical`, `high`, `medium`, `low`
+- **User Type**: `premium`, `enterprise`, `free`
+
+### Using Tags with LiteLLM Proxy
+
+When using tags with LiteLLM Proxy, you can:
+
+1. **Filter requests** based on tags
+2. **Track costs** by tags in spend reports
+3. **Apply routing rules** based on tags
+4. **Monitor usage** with tag-based analytics
+
+#### Example Proxy Configuration with Tags
+
+```yaml
+# config.yaml
+model_list:
+  - model_name: gpt-4o
+    litellm_params:
+      model: gpt-4o
+      api_key: your-key
+
+# Tag-based routing rules
+tag_routing:
+  - tags: ["premium", "high-priority"]
+    models: ["gpt-4o", "claude-3-opus"]
+  - tags: ["standard"]
+    models: ["gpt-3.5-turbo", "claude-3-haiku"]
+```
+
+### Monitoring and Analytics
+
+Tags enable powerful analytics capabilities:
+
+```python
+# Example: Get spend reports by tags
+import requests
+
+response = requests.get(
+    "http://localhost:4000/global/spend/report",
+    headers={"Authorization": "Bearer sk-your-key"},
+    params={
+        "start_date": "2024-01-01",
+        "end_date": "2024-12-31",
+        "group_by": "tags"
+    }
+)
+
+spend_by_tags = response.json()
+```
+
+This documentation covers the essential patterns for using tags effectively with LangChain and LiteLLM, enabling better organization, tracking, and analytics of your LLM requests.

docs/my-website/docs/load_test_advanced.md

@@ -27,13 +27,13 @@ Tutorial on how to get to 1K+ RPS with LiteLLM Proxy on locust
 
 **Use this config for testing:**
 
-**Note:**  we're currently migrating to aiohttp which has 10x higher throughput. We recommend using the `aiohttp_openai/` provider for load testing.
+**Note:**  we're currently migrating to aiohttp which has 10x higher throughput. We recommend using the `openai/` provider for load testing.
 
 ```yaml
 model_list:
   - model_name: "fake-openai-endpoint"
     litellm_params:
-      model: aiohttp_openai/any
+      model: openai/any
       api_base: https://your-fake-openai-endpoint.com/chat/completions
       api_key: "test"
 ```
@@ -58,7 +58,7 @@ litellm provides a hosted `fake-openai-endpoint` you can load test against
 model_list:
   - model_name: fake-openai-endpoint
     litellm_params:
-      model: aiohttp_openai/fake
+      model: openai/fake
       api_key: fake-key
       api_base: https://exampleopenaiendpoint-production.up.railway.app/
 

docs/my-website/docs/load_test_rpm.md

@@ -53,8 +53,8 @@ model_list = [
     },
 ]
 
-router_1 = Router(model_list=model_list, num_retries=0, enable_pre_call_checks=True, routing_strategy="usage-based-routing-v2", redis_host=os.getenv("REDIS_HOST"), redis_port=os.getenv("REDIS_PORT"), redis_password=os.getenv("REDIS_PASSWORD"))
-router_2 = Router(model_list=model_list, num_retries=0, routing_strategy="usage-based-routing-v2", enable_pre_call_checks=True, redis_host=os.getenv("REDIS_HOST"), redis_port=os.getenv("REDIS_PORT"), redis_password=os.getenv("REDIS_PASSWORD"))
+router_1 = Router(model_list=model_list, num_retries=0, enable_pre_call_checks=True, routing_strategy="simple-shuffle", redis_host=os.getenv("REDIS_HOST"), redis_port=os.getenv("REDIS_PORT"), redis_password=os.getenv("REDIS_PASSWORD"))
+router_2 = Router(model_list=model_list, num_retries=0, routing_strategy="simple-shuffle", enable_pre_call_checks=True, redis_host=os.getenv("REDIS_HOST"), redis_port=os.getenv("REDIS_PORT"), redis_password=os.getenv("REDIS_PASSWORD"))
 
 
 
@@ -142,7 +142,7 @@ router_settings:
   redis_host: os.environ/REDIS_HOST ## 👈 IMPORTANT! Setup the proxy w/ redis
   redis_password: os.environ/REDIS_PASSWORD
   redis_port: os.environ/REDIS_PORT
-  routing_strategy: usage-based-routing-v2
+  routing_strategy: simple-shuffle # recommended for best performance
 ```
 
 ### 2. Start proxy 2 instances

docs/my-website/docs/mcp.md

@@ -2,7 +2,7 @@ import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
 import Image from '@theme/IdealImage';
 
-# /mcp - Model Context Protocol
+# MCP Overview
 
 LiteLLM Proxy provides an MCP Gateway that allows you to use a fixed endpoint for all MCP tools and control MCP access by Key, Team. 
 
@@ -23,6 +23,43 @@ LiteLLM Proxy provides an MCP Gateway that allows you to use a fixed endpoint fo
 
 ## Adding your MCP
 
+### Prerequisites
+
+To store MCP servers in the database, you need to enable database storage:
+
+**Environment Variable:**
+```bash
+export STORE_MODEL_IN_DB=True
+```
+
+**OR in config.yaml:**
+```yaml
+general_settings:
+  store_model_in_db: true
+```
+
+#### Fine-grained Database Storage Control
+
+By default, when `store_model_in_db` is `true`, all object types (models, MCPs, guardrails, vector stores, etc.) are stored in the database. If you want to store only specific object types, use the `supported_db_objects` setting.
+
+**Example: Store only MCP servers in the database**
+
+```yaml title="config.yaml" showLineNumbers
+general_settings:
+  store_model_in_db: true
+  supported_db_objects: ["mcp"]  # Only store MCP servers in DB
+
+model_list:
+  - model_name: gpt-4o
+    litellm_params:
+      model: openai/gpt-4o
+      api_key: sk-xxxxxxx
+```
+
+**See all available object types:** [Config Settings - supported_db_objects](./proxy/config_settings.md#general_settings---reference)
+
+If `supported_db_objects` is not set, all object types are loaded from the database (default behavior).
+
 <Tabs>
 <TabItem value="ui" label="LiteLLM UI">
 
@@ -40,7 +77,28 @@ LiteLLM supports the following MCP transports:
   style={{width: '80%', display: 'block', margin: '0'}}
 />
 
-### Adding a stdio MCP Server
+<br/>
+<br/>
+
+### Add HTTP MCP Server
+
+This video walks through adding and using an HTTP MCP server on LiteLLM UI and using it in Cursor IDE.
+
+<iframe width="840" height="500" src="https://www.loom.com/embed/e2aebce78e8d46beafeb4bacdde31f14" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>
+
+<br/>
+<br/>
+
+### Add SSE MCP Server
+
+This video walks through adding and using an SSE MCP server on LiteLLM UI and using it in Cursor IDE.
+
+<iframe width="840" height="500" src="https://www.loom.com/embed/07e04e27f5e74475b9cf8ef8247d2c3e" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>
+
+<br/>
+<br/>
+
+### Add STDIO MCP Server
 
 For stdio MCP servers, select "Standard Input/Output (stdio)" as the transport type and provide the stdio configuration in JSON format:
 
@@ -92,7 +150,7 @@ mcp_servers:
     transport: "http"
     description: "My custom MCP server"
     auth_type: "api_key"
-    spec_version: "2025-03-26"
+    auth_value: "abc123"
 ```
 
 **Configuration Options:**
@@ -107,8 +165,60 @@ mcp_servers:
 - **Args**: Array of arguments to pass to the command (optional for stdio)
 - **Env**: Environment variables to set for the stdio process (optional for stdio)
 - **Description**: Optional description for the server
-- **Auth Type**: Optional authentication type
-- **Spec Version**: Optional MCP specification version (defaults to `2025-03-26`)
+- **Auth Type**: Optional authentication type. Supported values:
+
+  | Value | Header sent |
+  |-------|-------------|
+  | `api_key` | `X-API-Key: <auth_value>` |
+  | `bearer_token` | `Authorization: Bearer <auth_value>` |
+  | `basic` | `Authorization: Basic <auth_value>` |
+  | `authorization` | `Authorization: <auth_value>` |
+
+- **Extra Headers**: Optional list of additional header names that should be forwarded from client to the MCP server
+- **Spec Version**: Optional MCP specification version (defaults to `2025-06-18`)
+
+Examples for each auth type:
+
+```yaml title="MCP auth examples (config.yaml)" showLineNumbers
+mcp_servers:
+  api_key_example:
+    url: "https://my-mcp-server.com/mcp"
+    auth_type: "api_key"
+    auth_value: "abc123"        # headers={"X-API-Key": "abc123"}
+
+  # NEW – OAuth 2.0 Client Credentials (v1.77.5)
+  oauth2_example:
+    url: "https://my-mcp-server.com/mcp"
+    auth_type: "oauth2"         # 👈 KEY CHANGE
+    authorization_url: "https://my-mcp-server.com/oauth/authorize" # optional for client-credentials
+    token_url: "https://my-mcp-server.com/oauth/token"             # required
+    client_id: os.environ/OAUTH_CLIENT_ID
+    client_secret: os.environ/OAUTH_CLIENT_SECRET
+    scopes: ["tool.read", "tool.write"] # optional
+
+  bearer_example:
+    url: "https://my-mcp-server.com/mcp"
+    auth_type: "bearer_token"
+    auth_value: "abc123"        # headers={"Authorization": "Bearer abc123"}
+
+  basic_example:
+    url: "https://my-mcp-server.com/mcp"
+    auth_type: "basic"
+    auth_value: "dXNlcjpwYXNz"  # headers={"Authorization": "Basic dXNlcjpwYXNz"}
+
+  custom_auth_example:
+    url: "https://my-mcp-server.com/mcp"
+    auth_type: "authorization"
+    auth_value: "Token example123"  # headers={"Authorization": "Token example123"}
+
+  # Example with extra headers forwarding
+  github_mcp:
+    url: "https://api.githubcopilot.com/mcp"
+    auth_type: "bearer_token"
+    auth_value: "ghp_example_token"
+    extra_headers: ["custom_key", "x-custom-header"]  # These headers will be forwarded from client
+```
+
 
 ### MCP Aliases
 
@@ -136,87 +246,121 @@ litellm_settings:
 </TabItem>
 </Tabs>
 
+## Converting OpenAPI Specs to MCP Servers
 
-## Using your MCP
+LiteLLM can automatically convert OpenAPI specifications into MCP servers, allowing you to expose any REST API as MCP tools. This is useful when you have existing APIs with OpenAPI/Swagger documentation and want to make them available as MCP tools.
 
-<Tabs>
-<TabItem value="openai" label="OpenAI API">
+### Benefits
 
-#### Connect via OpenAI Responses API
+- **Rapid Integration**: Convert existing APIs to MCP tools without writing custom MCP server code
+- **Automatic Tool Generation**: LiteLLM automatically generates MCP tools from your OpenAPI spec
+- **Unified Interface**: Use the same MCP interface for both native MCP servers and OpenAPI-based APIs
+- **Easy Testing**: Test and iterate on API integrations quickly
 
-Use the OpenAI Responses API to connect to your LiteLLM MCP server:
+### Configuration
 
-```bash title="cURL Example" showLineNumbers
-curl --location 'https://api.openai.com/v1/responses' \
---header 'Content-Type: application/json' \
---header "Authorization: Bearer $OPENAI_API_KEY" \
---data '{
-    "model": "gpt-4o",
-    "tools": [
-        {
-            "type": "mcp",
-            "server_label": "litellm",
-            "server_url": "litellm_proxy",
-            "require_approval": "never",
-            "headers": {
-                "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY"
-            }
-        }
-    ],
-    "input": "Run available tools",
-    "tool_choice": "required"
-}'
+Add your OpenAPI-based MCP server to your `config.yaml`:
+
+```yaml title="config.yaml - OpenAPI to MCP" showLineNumbers
+model_list:
+  - model_name: gpt-4o
+    litellm_params:
+      model: openai/gpt-4o
+      api_key: sk-xxxxxxx
+
+mcp_servers:
+  # OpenAPI Spec Example - Petstore API
+  petstore_mcp:
+    url: "https://petstore.swagger.io/v2"
+    spec_path: "/path/to/openapi.json"
+    auth_type: "none"
+  
+  # OpenAPI Spec with API Key Authentication
+  my_api_mcp:
+    url: "http://0.0.0.0:8090"
+    spec_path: "/path/to/openapi.json"
+    auth_type: "api_key"
+    auth_value: "your-api-key-here"
+  
+  # OpenAPI Spec with Bearer Token
+  secured_api_mcp:
+    url: "https://api.example.com"
+    spec_path: "/path/to/openapi.json" 
+    auth_type: "bearer_token"
+    auth_value: "your-bearer-token"
 ```
 
-</TabItem>
+### Configuration Parameters
 
-<TabItem value="litellm" label="LiteLLM Proxy">
+| Parameter | Required | Description |
+|-----------|----------|-------------|
+| `url` | Yes | The base URL of your API endpoint |
+| `spec_path` | Yes | Path or URL to your OpenAPI specification file (JSON or YAML) |
+| `auth_type` | No | Authentication type: `none`, `api_key`, `bearer_token`, `basic`, `authorization` |
+| `auth_value` | No | Authentication value (required if `auth_type` is set) |
+| `description` | No | Optional description for the MCP server |
+| `allowed_tools` | No | List of specific tools to allow (see [MCP Tool Filtering](#mcp-tool-filtering)) |
+| `disallowed_tools` | No | List of specific tools to block (see [MCP Tool Filtering](#mcp-tool-filtering)) |
 
-#### Connect via LiteLLM Proxy Responses API
+### Usage Example
 
-Use this when calling LiteLLM Proxy for LLM API requests to `/v1/responses` endpoint.
+Once configured, you can use the OpenAPI-based MCP server just like any other MCP server:
 
-```bash title="cURL Example" showLineNumbers
-curl --location '<your-litellm-proxy-base-url>/v1/responses' \
---header 'Content-Type: application/json' \
---header "Authorization: Bearer $LITELLM_API_KEY" \
---data '{
-    "model": "gpt-4o",
-    "tools": [
-        {
-            "type": "mcp",
-            "server_label": "litellm",
-            "server_url": "litellm_proxy",
-            "require_approval": "never",
+<Tabs>
+<TabItem value="fastmcp" label="Python FastMCP">
+
+```python title="Using OpenAPI-based MCP Server" showLineNumbers
+from fastmcp import Client
+import asyncio
+
+# Standard MCP configuration
+config = {
+    "mcpServers": {
+        "petstore": {
+            "url": "http://localhost:4000/petstore_mcp/mcp",
             "headers": {
-                "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY"
+                "x-litellm-api-key": "Bearer sk-1234"
             }
         }
-    ],
-    "input": "Run available tools",
-    "tool_choice": "required"
-}'
+    }
+}
+
+# Create a client that connects to the server
+client = Client(config)
+
+async def main():
+    async with client:
+        # List available tools generated from OpenAPI spec
+        tools = await client.list_tools()
+        print(f"Available tools: {[tool.name for tool in tools]}")
+
+        # Example: Get a pet by ID (from Petstore API)
+        response = await client.call_tool(
+            name="getpetbyid", 
+            arguments={"petId": "1"}
+        )
+        print(f"Response:\n{response}\n")
+
+        # Example: Find pets by status
+        response = await client.call_tool(
+            name="findpetsbystatus", 
+            arguments={"status": "available"}
+        )
+        print(f"Response:\n{response}\n")
+
+if __name__ == "__main__":
+    asyncio.run(main())
 ```
 
 </TabItem>
 
 <TabItem value="cursor" label="Cursor IDE">
 
-#### Connect via Cursor IDE
-
-Use tools directly from Cursor IDE with LiteLLM MCP:
-
-**Setup Instructions:**
-
-1. **Open Cursor Settings**: Use `⇧+⌘+J` (Mac) or `Ctrl+Shift+J` (Windows/Linux)
-2. **Navigate to MCP Tools**: Go to the "MCP Tools" tab and click "New MCP Server"
-3. **Add Configuration**: Copy and paste the JSON configuration below, then save with `Cmd+S` or `Ctrl+S`
-
-```json title="Basic Cursor MCP Configuration" showLineNumbers
+```json title="Cursor MCP Configuration for OpenAPI Server" showLineNumbers
 {
   "mcpServers": {
-    "LiteLLM": {
-      "url": "litellm_proxy",
+    "Petstore": {
+      "url": "http://localhost:4000/petstore_mcp/mcp",
       "headers": {
         "x-litellm-api-key": "Bearer $LITELLM_API_KEY"
       }
@@ -225,26 +369,250 @@ Use tools directly from Cursor IDE with LiteLLM MCP:
 }
 ```
 
+</TabItem>
+
+<TabItem value="openai" label="OpenAI Responses API">
+
+```bash title="Using OpenAPI MCP Server with OpenAI" showLineNumbers
+curl --location 'https://api.openai.com/v1/responses' \
+--header 'Content-Type: application/json' \
+--header "Authorization: Bearer $OPENAI_API_KEY" \
+--data '{
+    "model": "gpt-4o",
+    "tools": [
+        {
+            "type": "mcp",
+            "server_label": "petstore",
+            "server_url": "http://localhost:4000/petstore_mcp/mcp",
+            "require_approval": "never",
+            "headers": {
+                "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY"
+            }
+        }
+    ],
+    "input": "Find all available pets in the petstore",
+    "tool_choice": "required"
+}'
+```
+
 </TabItem>
 </Tabs>
 
-#### How it works when server_url="litellm_proxy"
+### How It Works
 
-When server_url="litellm_proxy", LiteLLM bridges non-MCP providers to your MCP tools.
+1. **Spec Loading**: LiteLLM loads your OpenAPI specification from the provided `spec_path`
+2. **Tool Generation**: Each API endpoint in the spec becomes an MCP tool
+3. **Parameter Mapping**: OpenAPI parameters are automatically mapped to MCP tool parameters
+4. **Request Handling**: When a tool is called, LiteLLM converts the MCP request to the appropriate HTTP request
+5. **Response Translation**: API responses are converted back to MCP format
 
-- Tool Discovery: LiteLLM fetches MCP tools and converts them to OpenAI-compatible definitions
-- LLM Call: Tools are sent to the LLM with your input; LLM selects which tools to call
-- Tool Execution: LiteLLM automatically parses arguments, routes calls to MCP servers, executes tools, and retrieves results
-- Response Integration: Tool results are sent back to LLM for final response generation
-- Output: Complete response combining LLM reasoning with tool execution results
+### OpenAPI Spec Requirements
 
-This enables MCP tool usage with any LiteLLM-supported provider, regardless of native MCP support.
+Your OpenAPI specification should follow standard OpenAPI/Swagger conventions:
+- **Supported versions**: OpenAPI 3.0.x, OpenAPI 3.1.x, Swagger 2.0
+- **Required fields**: `paths`, `info` sections should be properly defined
+- **Operation IDs**: Each operation should have a unique `operationId` (this becomes the tool name)
+- **Parameters**: Request parameters should be properly documented with types and descriptions
 
-#### Auto-execution for require_approval: "never"
+### Example OpenAPI Spec Structure
 
-Setting require_approval: "never" triggers automatic tool execution, returning the final response in a single API call without additional user interaction.
+```yaml title="sample-openapi.yaml" showLineNumbers
+openapi: 3.0.0
+info:
+  title: My API
+  version: 1.0.0
+paths:
+  /pets/{petId}:
+    get:
+      operationId: getPetById
+      summary: Get a pet by ID
+      parameters:
+        - name: petId
+          in: path
+          required: true
+          schema:
+            type: integer
+      responses:
+        '200':
+          description: Successful response
+          content:
+            application/json:
+              schema:
+                type: object
+```
 
+## Allow/Disallow MCP Tools
+  
+Control which tools are available from your MCP servers. You can either allow only specific tools or block dangerous ones.
 
+<Tabs>
+<TabItem value="allowed" label="Only Allow Specific Tools">
+
+Use `allowed_tools` to specify exactly which tools users can access. All other tools will be blocked.
+
+```yaml title="config.yaml" showLineNumbers
+mcp_servers:
+  github_mcp:
+    url: "https://api.githubcopilot.com/mcp"
+    auth_type: oauth2
+    authorization_url: https://github.com/login/oauth/authorize
+    token_url: https://github.com/login/oauth/access_token
+    client_id: os.environ/GITHUB_OAUTH_CLIENT_ID
+    client_secret: os.environ/GITHUB_OAUTH_CLIENT_SECRET
+    scopes: ["public_repo", "user:email"]
+    allowed_tools: ["list_tools"]
+    # only list_tools will be available
+```
+
+**Use this when:**
+- You want strict control over which tools are available
+- You're in a high-security environment
+- You're testing a new MCP server with limited tools
+
+</TabItem>
+<TabItem value="blocked" label="Block Specific Tools">
+
+Use `disallowed_tools` to block specific tools. All other tools will be available.
+
+```yaml title="config.yaml" showLineNumbers
+mcp_servers:
+  github_mcp:
+    url: "https://api.githubcopilot.com/mcp"
+    auth_type: oauth2
+    authorization_url: https://github.com/login/oauth/authorize
+    token_url: https://github.com/login/oauth/access_token
+    client_id: os.environ/GITHUB_OAUTH_CLIENT_ID
+    client_secret: os.environ/GITHUB_OAUTH_CLIENT_SECRET
+    scopes: ["public_repo", "user:email"]
+    disallowed_tools: ["repo_delete"]
+    # only repo_delete will be blocked
+```
+
+**Use this when:**
+- Most tools are safe, but you want to block a few dangerous ones
+- You want to prevent expensive API calls
+- You're gradually adding restrictions to an existing server
+
+</TabItem>
+</Tabs>
+
+### Important Notes
+
+- If you specify both `allowed_tools` and `disallowed_tools`, the allowed list takes priority
+- Tool names are case-sensitive
+
+---
+
+## Allow/Disallow MCP Tool Parameters
+
+Control which parameters are allowed for specific MCP tools using the `allowed_params` configuration. This provides fine-grained control over tool usage by restricting the parameters that can be passed to each tool.
+
+### Configuration
+
+`allowed_params` is a dictionary that maps tool names to lists of allowed parameter names. When configured, only the specified parameters will be accepted for that tool - any other parameters will be rejected with a 403 error.
+
+```yaml title="config.yaml with allowed_params" showLineNumbers
+mcp_servers:
+  deepwiki_mcp:
+    url: https://mcp.deepwiki.com/mcp
+    transport: "http"
+    auth_type: "none"
+    allowed_params:
+      # Tool name: list of allowed parameters
+      read_wiki_contents: ["status"]
+  
+  my_api_mcp:
+    url: "https://my-api-server.com"
+    auth_type: "api_key"
+    auth_value: "my-key"
+    allowed_params:
+      # Using unprefixed tool name
+      getpetbyid: ["status"]
+      # Using prefixed tool name (both formats work)
+      my_api_mcp-findpetsbystatus: ["status", "limit"]
+      # Another tool with multiple allowed params
+      create_issue: ["title", "body", "labels"]
+```
+
+### How It Works
+
+1. **Tool-specific filtering**: Each tool can have its own list of allowed parameters
+2. **Flexible naming**: Tool names can be specified with or without the server prefix (e.g., both `"getpetbyid"` and `"my_api_mcp-getpetbyid"` work)
+3. **Whitelist approach**: Only parameters in the allowed list are permitted
+4. **Unlisted tools**: If `allowed_params` is not set, all parameters are allowed
+5. **Error handling**: Requests with disallowed parameters receive a 403 error with details about which parameters are allowed
+
+### Example Request Behavior
+
+With the configuration above, here's how requests would be handled:
+
+**✅ Allowed Request:**
+```json
+{
+  "tool": "read_wiki_contents",
+  "arguments": {
+    "status": "active"
+  }
+}
+```
+
+**❌ Rejected Request:**
+```json
+{
+  "tool": "read_wiki_contents",
+  "arguments": {
+    "status": "active",
+    "limit": 10  // This parameter is not allowed
+  }
+}
+```
+
+**Error Response:**
+```json
+{
+  "error": "Parameters ['limit'] are not allowed for tool read_wiki_contents. Allowed parameters: ['status']. Contact proxy admin to allow these parameters."
+}
+```
+
+### Use Cases
+
+- **Security**: Prevent users from accessing sensitive parameters or dangerous operations
+- **Cost control**: Restrict expensive parameters (e.g., limiting result counts)
+- **Compliance**: Enforce parameter usage policies for regulatory requirements
+- **Staged rollouts**: Gradually enable parameters as tools are tested
+- **Multi-tenant isolation**: Different parameter access for different user groups
+
+### Combining with Tool Filtering
+
+`allowed_params` works alongside `allowed_tools` and `disallowed_tools` for complete control:
+
+```yaml title="Combined filtering example" showLineNumbers
+mcp_servers:
+  github_mcp:
+    url: "https://api.githubcopilot.com/mcp"
+    auth_type: oauth2
+    authorization_url: https://github.com/login/oauth/authorize
+    token_url: https://github.com/login/oauth/access_token
+    client_id: os.environ/GITHUB_OAUTH_CLIENT_ID
+    client_secret: os.environ/GITHUB_OAUTH_CLIENT_SECRET
+    scopes: ["public_repo", "user:email"]
+    # Only allow specific tools
+    allowed_tools: ["create_issue", "list_issues", "search_issues"]
+    # Block dangerous operations
+    disallowed_tools: ["delete_repo"]
+    # Restrict parameters per tool
+    allowed_params:
+      create_issue: ["title", "body", "labels"]
+      list_issues: ["state", "sort", "perPage"]
+      search_issues: ["query", "sort", "order", "perPage"]
+```
+
+This configuration ensures that:
+1. Only the three listed tools are available
+2. The `delete_repo` tool is explicitly blocked
+3. Each tool can only use its specified parameters
+
+---
 
 ## MCP Server Access Control
 
@@ -564,7 +932,6 @@ mcp_servers:
     url: https://mcp.deepwiki.com/mcp
     transport: "http"
     auth_type: "none"
-    spec_version: "2025-03-26"
     access_groups: ["dev_group"]
 ```
 
@@ -621,6 +988,224 @@ When creating API keys, you can assign them to specific access groups for permis
 />
 
 
+## Forwarding Custom Headers to MCP Servers
+
+LiteLLM supports forwarding additional custom headers from MCP clients to backend MCP servers using the `extra_headers` configuration parameter. This allows you to pass custom authentication tokens, API keys, or other headers that your MCP server requires.
+
+### Configuration
+
+
+<Tabs>
+<TabItem value="config" label="config.yaml">
+Configure `extra_headers` in your MCP server configuration to specify which header names should be forwarded:
+
+```yaml title="config.yaml with extra_headers" showLineNumbers
+mcp_servers:
+  github_mcp:
+    url: "https://api.githubcopilot.com/mcp"
+    auth_type: "bearer_token"
+    auth_value: "ghp_default_token"
+    extra_headers: ["custom_key", "x-custom-header", "Authorization"]
+    description: "GitHub MCP server with custom header forwarding"
+```
+</TabItem>
+<TabItem value="clientside" label="Dynamically on Client Side">
+
+Use this when giving users access to a [group of MCP servers](#grouping-mcps-access-groups).
+
+**Format:** `x-mcp-{server_alias}-{header_name}: value`
+
+This allows you to use different authentication for different MCP servers.
+
+
+**Examples:**
+- `x-mcp-github-authorization: Bearer ghp_xxxxxxxxx` - GitHub MCP server with Bearer token
+- `x-mcp-zapier-x-api-key: sk-xxxxxxxxx` - Zapier MCP server with API key
+- `x-mcp-deepwiki-authorization: Basic base64_encoded_creds` - DeepWiki MCP server with Basic auth
+
+```python title="Python Client with Server-Specific Auth" showLineNumbers
+from fastmcp import Client
+import asyncio
+
+# Standard MCP configuration with multiple servers
+config = {
+    "mcpServers": {
+        "mcp_group": {
+            "url": "http://localhost:4000/mcp",
+            "headers": {
+                "x-mcp-servers": "dev_group", # assume this gives access to github, zapier and deepwiki
+                "x-litellm-api-key": "Bearer sk-1234",
+                "x-mcp-github-authorization": "Bearer gho_token", 
+                "x-mcp-zapier-x-api-key": "sk-xxxxxxxxx",
+                "x-mcp-deepwiki-authorization": "Basic base64_encoded_creds",
+                "custom_key": "value"
+            }
+        }
+    }
+}
+
+# Create a client that connects to all servers
+client = Client(config)
+
+
+async def main():
+    async with client:
+        tools = await client.list_tools()
+        print(f"Available tools: {tools}")
+
+        # call mcp 
+        await client.call_tool(
+            name="github_mcp-search_issues",
+            arguments={'query': 'created:>2024-01-01', 'sort': 'created', 'order': 'desc', 'perPage': 30}
+        )
+
+if __name__ == "__main__":
+    asyncio.run(main())
+
+```
+
+
+
+**Benefits:**
+- **Server-specific authentication**: Each MCP server can use different auth methods
+- **Better security**: No need to share the same auth token across all servers
+- **Flexible header names**: Support for different auth header types (authorization, x-api-key, etc.)
+- **Clean separation**: Each server's auth is clearly identified
+
+
+
+</TabItem>
+</Tabs>
+
+
+### Client Usage
+
+When connecting from MCP clients, include the custom headers that match the `extra_headers` configuration:
+
+<Tabs>
+<TabItem value="fastmcp" label="Python FastMCP">
+
+```python title="FastMCP Client with Custom Headers" showLineNumbers
+from fastmcp import Client
+import asyncio
+
+# MCP client configuration with custom headers
+config = {
+    "mcpServers": {
+        "github": {
+            "url": "http://localhost:4000/github_mcp/mcp",
+            "headers": {
+                "x-litellm-api-key": "Bearer sk-1234",
+                "Authorization": "Bearer gho_token", 
+                "custom_key": "custom_value",
+                "x-custom-header": "additional_data"
+            }
+        }
+    }
+}
+
+# Create a client that connects to the server
+client = Client(config)
+
+async def main():
+    async with client:
+        # List available tools
+        tools = await client.list_tools()
+        print(f"Available tools: {tools}")
+        
+        # Call a tool if available
+        if tools:
+            result = await client.call_tool(tools[0].name, {})
+            print(f"Tool result: {result}")
+
+# Run the client
+asyncio.run(main())
+```
+
+</TabItem>
+
+<TabItem value="cursor" label="Cursor IDE">
+
+```json title="Cursor MCP Configuration with Custom Headers" showLineNumbers
+{
+  "mcpServers": {
+    "GitHub": {
+      "url": "http://localhost:4000/github_mcp/mcp",
+      "headers": {
+        "x-litellm-api-key": "Bearer $LITELLM_API_KEY",
+        "Authorization": "Bearer $GITHUB_TOKEN",
+        "custom_key": "custom_value",
+        "x-custom-header": "additional_data"
+      }
+    }
+  }
+}
+```
+
+</TabItem>
+
+<TabItem value="http" label="HTTP Client">
+
+```bash title="cURL with Custom Headers" showLineNumbers
+curl --location 'http://localhost:4000/github_mcp/mcp' \
+--header 'Content-Type: application/json' \
+--header 'x-litellm-api-key: Bearer sk-1234' \
+--header 'Authorization: Bearer gho_token' \
+--header 'custom_key: custom_value' \
+--header 'x-custom-header: additional_data' \
+--data '{
+    "jsonrpc": "2.0",
+    "id": 1,
+    "method": "tools/list"
+}'
+```
+
+</TabItem>
+</Tabs>
+
+### How It Works
+
+1. **Configuration**: Define `extra_headers` in your MCP server config with the header names you want to forward
+2. **Client Headers**: Include the corresponding headers in your MCP client requests
+3. **Header Forwarding**: LiteLLM automatically forwards matching headers to the backend MCP server
+4. **Authentication**: The backend MCP server receives both the configured auth headers and the custom headers
+
+### Use Cases
+
+- **Custom Authentication**: Forward custom API keys or tokens required by specific MCP servers
+- **Request Context**: Pass user identification, session data, or request tracking headers
+- **Third-party Integration**: Include headers required by external services that your MCP server integrates with
+- **Multi-tenant Systems**: Forward tenant-specific headers for proper request routing
+
+### Security Considerations
+
+- Only headers listed in `extra_headers` are forwarded to maintain security
+- Sensitive headers should be passed through environment variables when possible
+- Consider using server-specific auth headers for better security isolation
+
+---
+
+## MCP Oauth
+
+LiteLLM v 1.77.6 added support for OAuth 2.0 Client Credentials for MCP servers.
+
+
+This configuration is currently available on the config.yaml, with UI support coming soon.
+
+```yaml
+mcp_servers:
+  github_mcp:
+    url: "https://api.githubcopilot.com/mcp"
+    auth_type: oauth2
+    authorization_url: https://github.com/login/oauth/authorize
+    token_url: https://github.com/login/oauth/access_token
+    client_id: os.environ/GITHUB_OAUTH_CLIENT_ID
+    client_secret: os.environ/GITHUB_OAUTH_CLIENT_SECRET
+    scopes: ["public_repo", "user:email"]
+```
+
+[**See Claude Code Tutorial**](./tutorials/claude_responses_api#connecting-mcp-servers)
+
 ## Using your MCP with client side credentials
 
 Use this if you want to pass a client side authentication token to LiteLLM to then pass to your MCP to auth to your MCP.
@@ -630,13 +1215,6 @@ Use this if you want to pass a client side authentication token to LiteLLM to th
 
 You can specify MCP auth tokens using server-specific headers in the format `x-mcp-{server_alias}-{header_name}`. This allows you to use different authentication for different MCP servers.
 
-**Format:** `x-mcp-{server_alias}-{header_name}: value`
-
-**Examples:**
-- `x-mcp-github-authorization: Bearer ghp_xxxxxxxxx` - GitHub MCP server with Bearer token
-- `x-mcp-zapier-x-api-key: sk-xxxxxxxxx` - Zapier MCP server with API key
-- `x-mcp-deepwiki-authorization: Basic base64_encoded_creds` - DeepWiki MCP server with Basic auth
-
 **Benefits:**
 - **Server-specific authentication**: Each MCP server can use different auth methods
 - **Better security**: No need to share the same auth token across all servers
@@ -1016,136 +1594,6 @@ curl --location '<your-litellm-proxy-base-url>/v1/responses' \
 }'
 ```
 
-
-
-## MCP Cost Tracking
-
-LiteLLM provides two ways to track costs for MCP tool calls:
-
-| Method | When to Use | What It Does |
-|--------|-------------|--------------|
-| **Config-based Cost Tracking** | Simple cost tracking with fixed costs per tool/server | Automatically tracks costs based on configuration |
-| **Custom Post-MCP Hook** | Dynamic cost tracking with custom logic | Allows custom cost calculations and response modifications |
-
-### Config-based Cost Tracking
-
-Configure fixed costs for MCP servers directly in your config.yaml:
-
-```yaml title="config.yaml" showLineNumbers
-model_list:
-  - model_name: gpt-4o
-    litellm_params:
-      model: openai/gpt-4o
-      api_key: sk-xxxxxxx
-
-mcp_servers:
-  zapier_server:
-    url: "https://actions.zapier.com/mcp/sk-xxxxx/sse"
-    mcp_info:
-      mcp_server_cost_info:
-        # Default cost for all tools in this server
-        default_cost_per_query: 0.01
-        # Custom cost for specific tools
-        tool_name_to_cost_per_query:
-          send_email: 0.05
-          create_document: 0.03
-          
-  expensive_api_server:
-    url: "https://api.expensive-service.com/mcp"
-    mcp_info:
-      mcp_server_cost_info:
-        default_cost_per_query: 1.50
-```
-
-### Custom Post-MCP Hook
-
-Use this when you need dynamic cost calculation or want to modify the MCP response before it's returned to the user.
-
-#### 1. Create a custom MCP hook file
-
-```python title="custom_mcp_hook.py" showLineNumbers
-from typing import Optional
-from litellm.integrations.custom_logger import CustomLogger
-from litellm.types.mcp import MCPPostCallResponseObject
-
-
-class CustomMCPCostTracker(CustomLogger):
-    """
-    Custom handler for MCP cost tracking and response modification
-    """
-    
-    async def async_post_mcp_tool_call_hook(
-        self, 
-        kwargs, 
-        response_obj: MCPPostCallResponseObject, 
-        start_time, 
-        end_time
-    ) -> Optional[MCPPostCallResponseObject]:
-        """
-        Called after each MCP tool call. 
-        Modify costs and response before returning to user.
-        """
-        
-        # Extract tool information from kwargs
-        tool_name = kwargs.get("name", "")
-        server_name = kwargs.get("server_name", "")
-        
-        # Calculate custom cost based on your logic
-        custom_cost = 42.00
-        
-        # Set the response cost
-        response_obj.hidden_params.response_cost = custom_cost
-        
-  
-      
-        return response_obj
-    
-
-# Create instance for LiteLLM to use
-custom_mcp_cost_tracker = CustomMCPCostTracker()
-```
-
-#### 2. Configure in config.yaml
-
-```yaml title="config.yaml" showLineNumbers
-model_list:
-  - model_name: gpt-4o
-    litellm_params:
-      model: openai/gpt-4o
-      api_key: sk-xxxxxxx
-
-# Add your custom MCP hook
-callbacks:
-  - custom_mcp_hook.custom_mcp_cost_tracker
-
-mcp_servers:
-  zapier_server:
-    url: "https://actions.zapier.com/mcp/sk-xxxxx/sse"
-```
-
-#### 3. Start the proxy
-
-```shell
-$ litellm --config /path/to/config.yaml 
-```
-
-When MCP tools are called, your custom hook will:
-1. Calculate costs based on your custom logic
-2. Modify the response if needed
-3. Track costs in LiteLLM's logging system
-
-## MCP Permission Management
-
-LiteLLM supports managing permissions for MCP Servers by Keys, Teams, Organizations (entities) on LiteLLM. When a MCP client attempts to list tools, LiteLLM will only return the tools the entity has permissions to access.
-
-When Creating a Key, Team, or Organization, you can select the allowed MCP Servers that the entity has access to.
-
-<Image 
-  img={require('../img/mcp_key.png')}
-  style={{width: '80%', display: 'block', margin: '0'}}
-/>
-
-
 ## LiteLLM Proxy - Walk through MCP Gateway
 LiteLLM exposes an MCP Gateway for admins to add all their MCP servers to LiteLLM. The key benefits of using LiteLLM Proxy with MCP are:
 

docs/my-website/docs/mcp_control.md

@@ -0,0 +1,45 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import Image from '@theme/IdealImage';
+
+# MCP Permission Management
+
+Control which MCP servers and tools can be accessed by specific keys, teams, or organizations in LiteLLM. When a client attempts to list or call tools, LiteLLM enforces access controls based on configured permissions.
+
+## Overview
+
+LiteLLM provides fine-grained permission management for MCP servers, allowing you to:
+
+- **Restrict MCP access by entity**: Control which keys, teams, or organizations can access specific MCP servers
+- **Tool-level filtering**: Automatically filter available tools based on entity permissions
+- **Centralized control**: Manage all MCP permissions from the LiteLLM Admin UI or API
+
+This ensures that only authorized entities can discover and use MCP tools, providing an additional security layer for your MCP infrastructure.
+
+:::info Related Documentation
+- [MCP Overview](./mcp.md) - Learn about MCP in LiteLLM
+- [MCP Cost Tracking](./mcp_cost.md) - Track costs for MCP tool calls
+- [MCP Guardrails](./mcp_guardrail.md) - Apply security guardrails to MCP calls
+- [Using MCP](./mcp_usage.md) - How to use MCP with LiteLLM
+:::
+
+## How It Works
+
+LiteLLM supports managing permissions for MCP Servers by Keys, Teams, Organizations (entities) on LiteLLM. When a MCP client attempts to list tools, LiteLLM will only return the tools the entity has permissions to access.
+
+When Creating a Key, Team, or Organization, you can select the allowed MCP Servers that the entity has access to.
+
+<Image 
+  img={require('../img/mcp_key.png')}
+  style={{width: '80%', display: 'block', margin: '0'}}
+/>
+
+
+## Set Allowed Tools for a Key, Team, or Organization
+
+Control which tools different teams can access from the same MCP server. For example, give your Engineering team access to `list_repositories`, `create_issue`, and `search_code`, while Sales only gets `search_code` and `close_issue`.
+
+
+This video shows how to set allowed tools for a Key, Team, or Organization.
+
+<iframe width="840" height="500" src="https://www.loom.com/embed/7464d444c3324078892367272fe50745" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>

docs/my-website/docs/mcp_cost.md

@@ -0,0 +1,121 @@
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import Image from '@theme/IdealImage';
+
+# MCP Cost Tracking
+
+LiteLLM provides two ways to track costs for MCP tool calls:
+
+| Method | When to Use | What It Does |
+|--------|-------------|--------------|
+| **Config-based Cost Tracking** | Simple cost tracking with fixed costs per tool/server | Automatically tracks costs based on configuration |
+| **Custom Post-MCP Hook** | Dynamic cost tracking with custom logic | Allows custom cost calculations and response modifications |
+
+### Config-based Cost Tracking
+
+Configure fixed costs for MCP servers directly in your config.yaml:
+
+```yaml title="config.yaml" showLineNumbers
+model_list:
+  - model_name: gpt-4o
+    litellm_params:
+      model: openai/gpt-4o
+      api_key: sk-xxxxxxx
+
+mcp_servers:
+  zapier_server:
+    url: "https://actions.zapier.com/mcp/sk-xxxxx/sse"
+    mcp_info:
+      mcp_server_cost_info:
+        # Default cost for all tools in this server
+        default_cost_per_query: 0.01
+        # Custom cost for specific tools
+        tool_name_to_cost_per_query:
+          send_email: 0.05
+          create_document: 0.03
+          
+  expensive_api_server:
+    url: "https://api.expensive-service.com/mcp"
+    mcp_info:
+      mcp_server_cost_info:
+        default_cost_per_query: 1.50
+```
+
+### Custom Post-MCP Hook
+
+Use this when you need dynamic cost calculation or want to modify the MCP response before it's returned to the user.
+
+#### 1. Create a custom MCP hook file
+
+```python title="custom_mcp_hook.py" showLineNumbers
+from typing import Optional
+from litellm.integrations.custom_logger import CustomLogger
+from litellm.types.mcp import MCPPostCallResponseObject
+
+
+class CustomMCPCostTracker(CustomLogger):
+    """
+    Custom handler for MCP cost tracking and response modification
+    """
+    
+    async def async_post_mcp_tool_call_hook(
+        self, 
+        kwargs, 
+        response_obj: MCPPostCallResponseObject, 
+        start_time, 
+        end_time
+    ) -> Optional[MCPPostCallResponseObject]:
+        """
+        Called after each MCP tool call. 
+        Modify costs and response before returning to user.
+        """
+        
+        # Extract tool information from kwargs
+        tool_name = kwargs.get("name", "")
+        server_name = kwargs.get("server_name", "")
+        
+        # Calculate custom cost based on your logic
+        custom_cost = 42.00
+        
+        # Set the response cost
+        response_obj.hidden_params.response_cost = custom_cost
+        
+  
+      
+        return response_obj
+    
+
+# Create instance for LiteLLM to use
+custom_mcp_cost_tracker = CustomMCPCostTracker()
+```
+
+#### 2. Configure in config.yaml
+
+```yaml title="config.yaml" showLineNumbers
+model_list:
+  - model_name: gpt-4o
+    litellm_params:
+      model: openai/gpt-4o
+      api_key: sk-xxxxxxx
+
+# Add your custom MCP hook
+callbacks:
+  - custom_mcp_hook.custom_mcp_cost_tracker
+
+mcp_servers:
+  zapier_server:
+    url: "https://actions.zapier.com/mcp/sk-xxxxx/sse"
+```
+
+#### 3. Start the proxy
+
+```shell
+$ litellm --config /path/to/config.yaml 
+```
+
+When MCP tools are called, your custom hook will:
+1. Calculate costs based on your custom logic
+2. Modify the response if needed
+3. Track costs in LiteLLM's logging system
+

docs/my-website/docs/mcp_guardrail.md

@@ -0,0 +1,88 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import Image from '@theme/IdealImage';
+
+# MCP Guardrails
+
+LiteLLM supports applying guardrails to MCP tool calls to ensure security and compliance. You can configure guardrails to run before or during MCP calls to validate inputs and block or mask sensitive information.
+
+### Supported MCP Guardrail Modes
+
+MCP guardrails support the following modes:
+
+- `pre_mcp_call`: Run **before** MCP call, on **input**. Use this mode when you want to apply validation/masking/blocking for MCP requests
+- `during_mcp_call`: Run **during** MCP call execution. Use this mode for real-time monitoring and intervention
+
+### Configuration Examples
+
+Configure guardrails to run before MCP tool calls to validate and sanitize inputs:
+
+```yaml title="config.yaml" showLineNumbers
+guardrails:
+  - guardrail_name: "mcp-input-validation"
+    litellm_params:
+      guardrail: presidio  # or other supported guardrails
+      mode: "pre_mcp_call" # or during_mcp_call
+      pii_entities_config:
+        CREDIT_CARD: "BLOCK"  # Will block requests containing credit card numbers
+        EMAIL_ADDRESS: "MASK"  # Will mask email addresses
+        PHONE_NUMBER: "MASK"   # Will mask phone numbers
+      default_on: true
+```
+
+
+### Usage Examples
+
+#### Testing Pre-MCP Call Guardrails
+
+Test your MCP guardrails with a request that includes sensitive information:
+
+```bash title="Test MCP Guardrail" showLineNumbers
+curl http://localhost:4000/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer sk-1234" \
+  -d '{
+    "model": "gpt-3.5-turbo",
+    "messages": [
+      {"role": "user", "content": "My credit card is 4111-1111-1111-1111 and my email is john@example.com"}
+    ],
+    "guardrails": ["mcp-input-validation"]
+  }'
+```
+
+The request will be processed as follows:
+1. Credit card number will be blocked (request rejected)
+2. Email address will be masked (e.g., replaced with `<EMAIL_ADDRESS>`)
+
+#### Using with MCP Tools
+
+When using MCP tools, guardrails will be applied to the tool inputs:
+
+```python title="Python Example with MCP Guardrails" showLineNumbers
+import openai
+
+client = openai.OpenAI(
+    api_key="your-api-key",
+    base_url="http://localhost:4000"
+)
+
+# This request will trigger MCP guardrails
+response = client.chat.completions.create(
+    model="gpt-3.5-turbo",
+    messages=[
+        {"role": "user", "content": "Send an email to 555-123-4567 with my SSN 123-45-6789"}
+    ],
+    tools=[{"type": "mcp", "server_label": "litellm", "server_url": "litellm_proxy"}],
+    guardrails=["mcp-input-validation"]
+)
+```
+
+### Supported Guardrail Providers
+
+MCP guardrails work with all LiteLLM-supported guardrail providers:
+
+- **Presidio**: PII detection and masking
+- **Bedrock**: AWS Bedrock guardrails
+- **Lakera**: Content moderation
+- **Aporia**: Custom guardrails
+- **Custom**: Your own guardrail implementations

docs/my-website/docs/mcp_usage.md

@@ -0,0 +1,209 @@
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import Image from '@theme/IdealImage';
+
+# Using your MCP
+
+This document covers how to use LiteLLM as an MCP Gateway. You can see how to use it with Responses API, Cursor IDE, and OpenAI SDK.
+
+### Use on LiteLLM UI 
+
+Follow this walkthrough to use your MCP on LiteLLM UI
+
+<iframe width="840" height="500" src="https://www.loom.com/embed/57e0763267254bc79dbe6658d0b8758c" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>
+
+### Use with Responses API
+
+Replace `http://localhost:4000` with your LiteLLM Proxy base URL.
+
+Demo Video Using Responses API with LiteLLM Proxy: [Demo video here](https://www.loom.com/share/34587e618c5c47c0b0d67b4e4d02718f?sid=2caf3d45-ead4-4490-bcc1-8d6dd6041c02)
+
+
+<Tabs>
+<TabItem value="curl" label="cURL">
+
+```bash title="cURL Example" showLineNumbers
+curl --location 'http://localhost:4000/v1/responses' \
+--header 'Content-Type: application/json' \
+--header "Authorization: Bearer sk-1234" \
+--data '{
+    "model": "gpt-5",
+    "input": [
+    {
+      "role": "user",
+      "content": "give me TLDR of what BerriAI/litellm repo is about",
+      "type": "message"
+    }
+  ],
+    "tools": [
+        {
+            "type": "mcp",
+            "server_label": "litellm",
+            "server_url": "litellm_proxy",
+            "require_approval": "never"
+        }
+    ],
+    "stream": true,
+    "tool_choice": "required"
+}'
+```
+
+</TabItem>
+<TabItem value="python" label="Python SDK">
+
+```python title="Python SDK Example" showLineNumbers
+"""
+Use LiteLLM Proxy MCP Gateway to call MCP tools.
+
+When using LiteLLM Proxy, you can use the same MCP tools across all your LLM providers.
+"""
+import openai
+
+client = openai.OpenAI(
+    api_key="sk-1234", # paste your litellm proxy api key here
+    base_url="http://localhost:4000" # paste your litellm proxy base url here
+)
+print("Making API request to Responses API with MCP tools")
+
+response = client.responses.create(
+    model="gpt-5",
+    input=[
+        {
+            "role": "user",
+            "content": "give me TLDR of what BerriAI/litellm repo is about",
+            "type": "message"
+        }
+    ],
+    tools=[
+        {
+            "type": "mcp",
+            "server_label": "litellm",
+            "server_url": "litellm_proxy",
+            "require_approval": "never"
+        }
+    ],
+    stream=True,
+    tool_choice="required"
+)
+
+for chunk in response:
+    print("response chunk: ", chunk)
+```
+
+</TabItem>
+</Tabs>
+
+#### Specifying MCP Tools
+
+You can specify which MCP tools are available by using the `allowed_tools` parameter. This allows you to restrict access to specific tools within an MCP server.
+
+To get the list of allowed tools when using LiteLLM MCP Gateway, you can naigate to the LiteLLM UI on MCP Servers > MCP Tools > Click the Tool > Copy Tool Name.
+
+<Tabs>
+<TabItem value="curl" label="cURL">
+
+```bash title="cURL Example with allowed_tools" showLineNumbers
+curl --location 'http://localhost:4000/v1/responses' \
+--header 'Content-Type: application/json' \
+--header "Authorization: Bearer sk-1234" \
+--data '{
+    "model": "gpt-5",
+    "input": [
+    {
+      "role": "user",
+      "content": "give me TLDR of what BerriAI/litellm repo is about",
+      "type": "message"
+    }
+  ],
+    "tools": [
+        {
+            "type": "mcp",
+            "server_label": "litellm",
+            "server_url": "litellm_proxy/mcp",
+            "require_approval": "never",
+            "allowed_tools": ["GitMCP-fetch_litellm_documentation"]
+        }
+    ],
+    "stream": true,
+    "tool_choice": "required"
+}'
+```
+
+</TabItem>
+<TabItem value="python" label="Python SDK">
+
+```python title="Python SDK Example with allowed_tools" showLineNumbers
+import openai
+
+client = openai.OpenAI(
+    api_key="sk-1234",
+    base_url="http://localhost:4000"
+)
+
+response = client.responses.create(
+    model="gpt-5",
+    input=[
+        {
+            "role": "user",
+            "content": "give me TLDR of what BerriAI/litellm repo is about",
+            "type": "message"
+        }
+    ],
+    tools=[
+        {
+            "type": "mcp",
+            "server_label": "litellm",
+            "server_url": "litellm_proxy/mcp",
+            "require_approval": "never",
+            "allowed_tools": ["GitMCP-fetch_litellm_documentation"]
+        }
+    ],
+    stream=True,
+    tool_choice="required"
+)
+
+print(response)
+```
+
+</TabItem>
+</Tabs>
+
+### Use with Cursor IDE
+
+Use tools directly from Cursor IDE with LiteLLM MCP:
+
+**Setup Instructions:**
+
+1. **Open Cursor Settings**: Use `⇧+⌘+J` (Mac) or `Ctrl+Shift+J` (Windows/Linux)
+2. **Navigate to MCP Tools**: Go to the "MCP Tools" tab and click "New MCP Server"
+3. **Add Configuration**: Copy and paste the JSON configuration below, then save with `Cmd+S` or `Ctrl+S`
+
+```json title="Basic Cursor MCP Configuration" showLineNumbers
+{
+  "mcpServers": {
+    "LiteLLM": {
+      "url": "litellm_proxy",
+      "headers": {
+        "x-litellm-api-key": "Bearer $LITELLM_API_KEY"
+      }
+    }
+  }
+}
+```
+
+#### How it works when server_url="litellm_proxy"
+
+When server_url="litellm_proxy", LiteLLM bridges non-MCP providers to your MCP tools.
+
+- Tool Discovery: LiteLLM fetches MCP tools and converts them to OpenAI-compatible definitions
+- LLM Call: Tools are sent to the LLM with your input; LLM selects which tools to call
+- Tool Execution: LiteLLM automatically parses arguments, routes calls to MCP servers, executes tools, and retrieves results
+- Response Integration: Tool results are sent back to LLM for final response generation
+- Output: Complete response combining LLM reasoning with tool execution results
+
+This enables MCP tool usage with any LiteLLM-supported provider, regardless of native MCP support.
+
+#### Auto-execution for require_approval: "never"
+
+Setting require_approval: "never" triggers automatic tool execution, returning the final response in a single API call without additional user interaction.

docs/my-website/docs/moderation.md

@@ -130,6 +130,8 @@ Here's the exact json output and type you can expect from all moderation calls:
 
 ## **Supported Providers**
 
+#### ⚡️See all supported models and providers at [models.litellm.ai](https://models.litellm.ai/)
+
 | Provider    |
 |-------------|
 | OpenAI      |  

docs/my-website/docs/observability/braintrust.md

@@ -15,6 +15,7 @@ import os
 
 # set env
 os.environ["BRAINTRUST_API_KEY"] = ""
+os.environ["BRAINTRUST_API_BASE"] = "https://api.braintrustdata.com/v1"
 os.environ['OPENAI_API_KEY']=""
 
 # set braintrust as a callback, litellm will send the data to braintrust
@@ -35,6 +36,7 @@ response = litellm.completion(
 
 ```env
 BRAINTRUST_API_KEY=""
+BRAINTRUST_API_BASE="https://api.braintrustdata.com/v1"
 ```
 
 2. Add braintrust to callbacks
@@ -69,6 +71,10 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
 
 It is recommended that you include the `project_id` or `project_name` to ensure your traces are being written out to the correct Braintrust project.
 
+### Custom Span Names
+
+You can customize the span name in Braintrust logging by passing `span_name` in the metadata. By default, the span name is set to "Chat Completion".
+
 <Tabs>
 <TabItem value="sdk" label="SDK">
 
@@ -82,7 +88,9 @@ response = litellm.completion(
     "project_id": "1234",
     # passing project_name will try to find a project with that name, or create one if it doesn't exist
     # if both project_id and project_name are passed, project_id will be used
-    # "project_name": "my-special-project"
+    # "project_name": "my-special-project",
+    # custom span name for this operation (default: "Chat Completion")
+    "span_name": "User Greeting Handler"
   }
 )
 ```
@@ -97,6 +105,7 @@ response = litellm.completion(
   ],
   metadata={
     "project_id": "1234",
+    "span_name": "Custom Operation",
     "item1": "an item",
     "item2": "another item"
   }
@@ -119,7 +128,8 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
         { "role": "user", "content": "What time is it now? Use your tool"}
     ],
     "metadata": {
-        "project_id": "my-special-project"
+        "project_id": "my-special-project",
+        "span_name": "Tool Usage Request"
     }
 }'
 ```
@@ -144,7 +154,8 @@ response = client.chat.completions.create(
     ],
     extra_body={ # pass in any provider-specific param, if not supported by openai, https://docs.litellm.ai/docs/completion/input#provider-specific-params
         "metadata": { # 👈 use for logging additional params (e.g. to braintrust)
-            "project_id": "my-special-project"
+            "project_id": "my-special-project",
+            "span_name": "Poetry Generation"
         }
     }
 )
@@ -157,6 +168,8 @@ For more examples, [**Click Here**](../proxy/user_keys.md#chatcompletions)
 </TabItem>
 </Tabs>
 
+You can use `BRAINTRUST_API_BASE` to point to your self-hosted Braintrust data plane. Read more about this [here](https://www.braintrust.dev/docs/guides/self-hosting).
+
 ## Full API Spec
 
 Here's everything you can pass in metadata for a braintrust request
@@ -164,3 +177,7 @@ Here's everything you can pass in metadata for a braintrust request
 `braintrust_*` - If you are adding metadata from _proxy request headers_, any metadata field starting with `braintrust_` will be passed as metadata to the logging request. If you are using the SDK, just pass your metadata like normal (e.g., `metadata={"project_name": "my-test-project", "item1": "an item", "item2": "another item"}`)
 
 `project_id` - Set the project id for a braintrust call. Default is `litellm`.
+
+`project_name` - Set the project name for a braintrust call. Will try to find a project with that name, or create one if it doesn't exist. If both `project_id` and `project_name` are passed, `project_id` will be used.
+
+`span_name` - Set a custom span name for the operation. Default is `"Chat Completion"`. Use this to provide more descriptive names for different types of operations in your application (e.g., "User Query", "Document Summary", "Code Generation").

docs/my-website/docs/observability/callbacks.md

@@ -4,9 +4,16 @@
 
 liteLLM provides `input_callbacks`, `success_callbacks` and `failure_callbacks`, making it easy for you to send data to a particular provider depending on the status of your responses.
 
-liteLLM supports:
+:::tip
+**New to LiteLLM Callbacks?**
+
+- For proxy/server logging and observability, see the [Proxy Logging Guide](https://docs.litellm.ai/docs/proxy/logging).
+- To write your own callback logic, see the [Custom Callbacks Guide](https://docs.litellm.ai/docs/observability/custom_callback).
+:::
+
+
+### Supported Callback Integrations
 
-- [Custom Callback Functions](https://docs.litellm.ai/docs/observability/custom_callback)
 - [Lunary](https://lunary.ai/docs)
 - [Langfuse](https://langfuse.com/docs)
 - [LangSmith](https://www.langchain.com/langsmith)
@@ -16,9 +23,20 @@ liteLLM supports:
 - [Sentry](https://docs.sentry.io/platforms/python/)
 - [PostHog](https://posthog.com/docs/libraries/python)
 - [Slack](https://slack.dev/bolt-python/concepts)
+- [Arize](https://docs.arize.com/)
+- [PromptLayer](https://docs.promptlayer.com/)
 
 This is **not** an extensive list. Please check the dropdown for all logging integrations.
 
+### Related Cookbooks
+Try out our cookbooks for code snippets and interactive demos:
+
+- [Langfuse Callback Example (Colab)](https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/logging_observability/LiteLLM_Langfuse.ipynb)
+- [Lunary Callback Example (Colab)](https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/logging_observability/LiteLLM_Lunary.ipynb)
+- [Arize Callback Example (Colab)](https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/logging_observability/LiteLLM_Arize.ipynb)
+- [Proxy + Langfuse Callback Example (Colab)](https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/logging_observability/LiteLLM_Proxy_Langfuse.ipynb)
+- [PromptLayer Callback Example (Colab)](https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/LiteLLM_PromptLayer.ipynb)
+
 ### Quick Start
 
 ```python

docs/my-website/docs/observability/cloudzero.md

@@ -0,0 +1,209 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# CloudZero Integration
+
+LiteLLM provides an integration with CloudZero's AnyCost API, allowing you to export your LLM usage data to CloudZero for cost tracking analysis.
+
+## Overview
+
+| Property | Details |
+|----------|---------|
+| Description | Export LiteLLM usage data to CloudZero AnyCost API for cost tracking and analysis |
+| callback name | `cloudzero`|
+| Supported Operations | • Automatic hourly data export<br/>• Manual data export<br/>• Dry run testing<br/>• Cost and token usage tracking |
+| Data Format | CloudZero Billing Format (CBF) with proper resource tagging |
+| Export Frequency | Hourly (configurable via `CLOUDZERO_EXPORT_INTERVAL_MINUTES`) |
+
+## Environment Variables
+
+| Variable | Required | Description | Example |
+|----------|----------|-------------|---------|
+| `CLOUDZERO_API_KEY` | Yes | Your CloudZero API key | `cz_api_xxxxxxxxxx` |
+| `CLOUDZERO_CONNECTION_ID` | Yes | CloudZero connection ID for data submission | `conn_xxxxxxxxxx` |
+| `CLOUDZERO_TIMEZONE` | No | Timezone for date handling (default: UTC) | `America/New_York` |
+| `CLOUDZERO_EXPORT_INTERVAL_MINUTES` | No | Export frequency in minutes (default: 60) | `60` |
+
+## Setup
+
+### End to End Video Walkthrough
+This video walks through the entire process of setting up LiteLLM with CloudZero integration and viewing LiteLLM exported usage data in CloudZero.
+
+<iframe width="840" height="500" src="https://www.loom.com/embed/59b57593183f4cc3b1c05a2dd3277f92" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>
+
+### Step 1: Configure Environment Variables
+
+Set your CloudZero credentials in your environment:
+
+```bash
+export CLOUDZERO_API_KEY="cz_api_xxxxxxxxxx"
+export CLOUDZERO_CONNECTION_ID="conn_xxxxxxxxxx"
+export CLOUDZERO_TIMEZONE="UTC"  # Optional, defaults to UTC
+```
+
+### Step 2: Enable CloudZero Integration
+
+Add the CloudZero callback to your LiteLLM configuration YAML file:
+
+
+```yaml
+model_list:
+  - model_name: gpt-4o
+    litellm_params:
+      model: openai/gpt-4o
+      api_key: sk-xxxxxxx
+
+litellm_settings:
+  callbacks: ["cloudzero"]  # Enable CloudZero integration
+```
+
+### Step 3: Start LiteLLM Proxy
+
+Start your LiteLLM proxy with the configuration:
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+## Testing Your Setup
+
+### Dry Run Export
+
+Call the dry run endpoint to test your CloudZero configuration without sending data to CloudZero. This endpoint will not send any data to CloudZero, but will return the data that would be exported.
+
+```bash
+curl -X POST "http://localhost:4000/cloudzero/dry-run" \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer sk-1234" \
+  -d '{
+    "limit": 10
+  }' | jq
+```
+
+**Expected Response:**
+```json
+{
+  "message": "CloudZero dry run export completed successfully.",
+  "status": "success",
+  "dry_run_data": {
+    "usage_data": [...],
+    "cbf_data": [...],
+    "summary": {
+      "total_cost": 0.05,
+      "total_tokens": 1250,
+      "total_records": 10
+    }
+  }
+}
+```
+
+### Manual Export
+
+Call the export endpoint to send data immediately to CloudZero. We suggest setting a small `limit` to test the export. This will only export the last 10 records to CloudZero. Note: Cloudzero can take up to 15 minutes to process the exported data.
+
+```bash
+curl -X POST "http://localhost:4000/cloudzero/export" \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer sk-1234" \
+  -d '{
+    "limit": 10
+  }' | jq
+```
+
+**Expected Response:**
+```json
+{
+  "message": "CloudZero export completed successfully",
+  "status": "success"
+}
+```
+
+## Data Export Details
+
+### Automatic Export Schedule
+
+- **Frequency**: Every 60 minutes (configurable via `CLOUDZERO_EXPORT_INTERVAL_MINUTES`)
+- **Data Processing**: LiteLLM automatically processes and exports usage data hourly
+- **CloudZero Processing**: CloudZero typically takes 10-15 minutes to process data from LiteLLM
+
+### Data Format
+
+LiteLLM exports data in CloudZero Billing Format (CBF) with the following structure:
+
+```json
+{
+  "time/usage_start": "2024-01-15T14:00:00Z",
+  "cost/cost": 0.002,
+  "usage/amount": 150,
+  "usage/units": "tokens",
+  "resource/id": "czrn:litellm:openai:cross-region:team-123:llm-usage:gpt-4o",
+  "resource/service": "litellm",
+  "resource/account": "team-123",
+  "resource/region": "cross-region",
+  "resource/usage_family": "llm-usage",
+  "resource/tag:provider": "openai",
+  "resource/tag:model": "gpt-4o",
+  "resource/tag:prompt_tokens": "100",
+  "resource/tag:completion_tokens": "50"
+}
+```
+
+### Resource Tagging
+
+LiteLLM automatically creates comprehensive resource tags for cost attribution:
+
+- **Provider Tags**: `openai`, `anthropic`, `azure`, etc.
+- **Model Tags**: Specific model names like `gpt-4o`, `claude-3-sonnet`
+- **Team/User Tags**: Team IDs and user IDs for cost allocation
+- **Token Breakdown**: Separate tracking of prompt and completion tokens
+- **Usage Metrics**: Total tokens consumed per request
+
+## Advanced Configuration
+
+### Custom Export Frequency
+
+Change the export frequency (not recommended to go below 60 minutes):
+
+```bash
+export CLOUDZERO_EXPORT_INTERVAL_MINUTES=120  # Export every 2 hours
+```
+
+### Custom Time Range Export
+
+Export data for a specific time range:
+
+```bash
+curl -X POST "http://localhost:4000/cloudzero/export" \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer sk-1234" \
+  -d '{
+    "start_time_utc": "2024-01-15T00:00:00Z",
+    "end_time_utc": "2024-01-15T23:59:59Z",
+    "operation": "replace_hourly"
+  }' | jq
+```
+
+## Troubleshooting
+
+### Common Issues
+
+1. **Missing Credentials Error**
+   ```
+   CloudZero configuration missing. Please set CLOUDZERO_API_KEY and CLOUDZERO_CONNECTION_ID environment variables.
+   ```
+   **Solution**: Ensure both environment variables are set with valid values.
+
+2. **Connection Issues**
+   - Verify your CloudZero API key is valid
+   - Check that the connection ID exists in your CloudZero account
+   - Ensure your proxy has internet access to reach CloudZero's API
+
+3. **No Data in CloudZero**
+   - CloudZero can take 10-15 minutes to process data
+   - Check that your LiteLLM proxy is generating usage data
+   - Use the dry-run endpoint to verify data is being formatted correctly
+
+## Related Links
+
+- [CloudZero Documentation](https://docs.cloudzero.com/)
+- [CloudZero AnyCost API](https://docs.cloudzero.com/reference/anycost-api)

docs/my-website/docs/observability/custom_callback.md

@@ -4,7 +4,6 @@
 **For PROXY** [Go Here](../proxy/logging.md#custom-callback-class-async)
 ::: 
 
-
 ## Callback Class
 You can create a custom callback class to precisely log events as they occur in litellm. 
 
@@ -57,6 +56,34 @@ def async completion():
 asyncio.run(completion())
 ```
 
+## Common Hooks
+
+- `async_log_success_event` - Log successful API calls
+- `async_log_failure_event` - Log failed API calls  
+- `log_pre_api_call` - Log before API call
+- `log_post_api_call` - Log after API call
+
+**Proxy-only hooks** (only work with LiteLLM Proxy):
+- `async_post_call_success_hook` - Access user data + modify responses
+- `async_pre_call_hook` - Modify requests before sending
+
+### Example: Modifying the Response in async_post_call_success_hook
+
+You can use `async_post_call_success_hook` to add custom headers or metadata to the response before it is returned to the client. For example:
+
+```python
+async def async_post_call_success_hook(data, user_api_key_dict, response):
+    # Add a custom header to the response
+    additional_headers = getattr(response, "_hidden_params", {}).get("additional_headers", {}) or {}
+    additional_headers["x-litellm-custom-header"] = "my-value"
+    if not hasattr(response, "_hidden_params"):
+        response._hidden_params = {}
+    response._hidden_params["additional_headers"] = additional_headers
+    return response
+```
+
+This allows you to inject custom metadata or headers into the response for downstream consumers. You can use this pattern to pass information to clients, proxies, or observability tools.
+
 ## Callback Functions
 If you just want to log on a specific event (e.g. on input) - you can use callback functions. 
 
@@ -174,260 +201,87 @@ async def test_chat_openai():
 asyncio.run(test_chat_openai())
 ```
 
-:::info
+## What's Available in kwargs?
 
-We're actively trying to expand this to other event types. [Tell us if you need this!](https://github.com/BerriAI/litellm/issues/1007)
-:::
-
-## What's in kwargs? 
-
-Notice we pass in a kwargs argument to custom callback. 
-```python
-def custom_callback(
-    kwargs,                 # kwargs to completion
-    completion_response,    # response from completion
-    start_time, end_time    # start/end time
-):
-    # Your custom code here
-    print("LITELLM: in custom callback function")
-    print("kwargs", kwargs)
-    print("completion_response", completion_response)
-    print("start_time", start_time)
-    print("end_time", end_time)
-```
-
-This is a dictionary containing all the model-call details (the params we receive, the values we send to the http endpoint, the response we receive, stacktrace in case of errors, etc.). 
-
-This is all logged in the [model_call_details via our Logger](https://github.com/BerriAI/litellm/blob/fc757dc1b47d2eb9d0ea47d6ad224955b705059d/litellm/utils.py#L246).
-
-Here's exactly what you can expect in the kwargs dictionary:
-```shell
-### DEFAULT PARAMS ### 
-"model": self.model,
-"messages": self.messages,
-"optional_params": self.optional_params, # model-specific params passed in
-"litellm_params": self.litellm_params, # litellm-specific params passed in (e.g. metadata passed to completion call)
-"start_time": self.start_time, # datetime object of when call was started
-
-### PRE-API CALL PARAMS ### (check via kwargs["log_event_type"]="pre_api_call")
-"input" = input # the exact prompt sent to the LLM API
-"api_key" = api_key # the api key used for that LLM API 
-"additional_args" = additional_args # any additional details for that API call (e.g. contains optional params sent)
-
-### POST-API CALL PARAMS ### (check via kwargs["log_event_type"]="post_api_call")
-"original_response" = original_response # the original http response received (saved via response.text)
-
-### ON-SUCCESS PARAMS ### (check via kwargs["log_event_type"]="successful_api_call")
-"complete_streaming_response" = complete_streaming_response # the complete streamed response (only set if `completion(..stream=True)`)
-"end_time" = end_time # datetime object of when call was completed
-
-### ON-FAILURE PARAMS ### (check via kwargs["log_event_type"]="failed_api_call")
-"exception" = exception # the Exception raised
-"traceback_exception" = traceback_exception # the traceback generated via `traceback.format_exc()`
-"end_time" = end_time # datetime object of when call was completed
-```
-
-
-### Cache hits
-
-Cache hits are logged in success events as `kwarg["cache_hit"]`. 
-
-Here's an example of accessing it: 
-
-  ```python
-  import litellm
-from litellm.integrations.custom_logger import CustomLogger
-from litellm import completion, acompletion, Cache
-
-class MyCustomHandler(CustomLogger):
-    async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): 
-        print(f"On Success")
-        print(f"Value of Cache hit: {kwargs['cache_hit']"})
-
-async def test_async_completion_azure_caching():
-    customHandler_caching = MyCustomHandler()
-    litellm.cache = Cache(type="redis", host=os.environ['REDIS_HOST'], port=os.environ['REDIS_PORT'], password=os.environ['REDIS_PASSWORD'])
-    litellm.callbacks = [customHandler_caching]
-    unique_time = time.time()
-    response1 = await litellm.acompletion(model="azure/chatgpt-v-2",
-                            messages=[{
-                                "role": "user",
-                                "content": f"Hi 👋 - i'm async azure {unique_time}"
-                            }],
-                            caching=True)
-    await asyncio.sleep(1)
-    print(f"customHandler_caching.states pre-cache hit: {customHandler_caching.states}")
-    response2 = await litellm.acompletion(model="azure/chatgpt-v-2",
-                            messages=[{
-                                "role": "user",
-                                "content": f"Hi 👋 - i'm async azure {unique_time}"
-                            }],
-                            caching=True)
-    await asyncio.sleep(1) # success callbacks are done in parallel
-    print(f"customHandler_caching.states post-cache hit: {customHandler_caching.states}")
-    assert len(customHandler_caching.errors) == 0
-    assert len(customHandler_caching.states) == 4 # pre, post, success, success
-  ```
-
-### Get complete streaming response
-
-LiteLLM will pass you the complete streaming response in the final streaming chunk as part of the kwargs for your custom callback function.
+The kwargs dictionary contains all the details about your API call:
 
 ```python
-# litellm.set_verbose = False
-        def custom_callback(
-            kwargs,                 # kwargs to completion
-            completion_response,    # response from completion
-            start_time, end_time    # start/end time
-        ):
-            # print(f"streaming response: {completion_response}")
-            if "complete_streaming_response" in kwargs: 
-                print(f"Complete Streaming Response: {kwargs['complete_streaming_response']}")
-        
-        # Assign the custom callback function
-        litellm.success_callback = [custom_callback]
-
-        response = completion(model="claude-instant-1", messages=messages, stream=True)
-        for idx, chunk in enumerate(response): 
-            pass
-```
-
-
-### Log additional metadata
-
-LiteLLM accepts a metadata dictionary in the completion call. You can pass additional metadata into your completion call via `completion(..., metadata={"key": "value"})`. 
-
-Since this is a [litellm-specific param](https://github.com/BerriAI/litellm/blob/b6a015404eed8a0fa701e98f4581604629300ee3/litellm/main.py#L235), it's accessible via kwargs["litellm_params"]
-
-```python
-from litellm import completion
-import os, litellm
-
-## set ENV variables
-os.environ["OPENAI_API_KEY"] = "your-api-key"
-
-messages = [{ "content": "Hello, how are you?","role": "user"}]
-
-def custom_callback(
-    kwargs,                 # kwargs to completion
-    completion_response,    # response from completion
-    start_time, end_time    # start/end time
-):
-    print(kwargs["litellm_params"]["metadata"])
+def custom_callback(kwargs, completion_response, start_time, end_time):
+    # Access common data
+    model = kwargs.get("model")
+    messages = kwargs.get("messages", [])
+    cost = kwargs.get("response_cost", 0)
+    cache_hit = kwargs.get("cache_hit", False)
     
-
-# Assign the custom callback function
-litellm.success_callback = [custom_callback]
-
-response = litellm.completion(model="gpt-3.5-turbo", messages=messages, metadata={"hello": "world"})
+    # Access metadata you passed in
+    metadata = kwargs.get("litellm_params", {}).get("metadata", {})
 ```
 
-## Examples
+**Key fields in kwargs:**
+- `model` - The model name
+- `messages` - Input messages  
+- `response_cost` - Calculated cost
+- `cache_hit` - Whether response was cached
+- `litellm_params.metadata` - Your custom metadata
 
-### Custom Callback to track costs for Streaming + Non-Streaming
-By default, the response cost is accessible in the logging object via `kwargs["response_cost"]` on success (sync + async)
+## Practical Examples
+
+### Track API Costs
 ```python
+def track_cost_callback(kwargs, completion_response, start_time, end_time):
+    cost = kwargs["response_cost"] # litellm calculates this for you
+    print(f"Request cost: ${cost}")
 
-# Step 1. Write your custom callback function
-def track_cost_callback(
-    kwargs,                 # kwargs to completion
-    completion_response,    # response from completion
-    start_time, end_time    # start/end time
-):
-    try:
-        response_cost = kwargs["response_cost"] # litellm calculates response cost for you
-        print("regular response_cost", response_cost)
-    except:
-        pass
-
-# Step 2. Assign the custom callback function
 litellm.success_callback = [track_cost_callback]
 
-# Step 3. Make litellm.completion call
-response = completion(
-    model="gpt-3.5-turbo",
-    messages=[
-        {
-            "role": "user",
-            "content": "Hi 👋 - i'm openai"
-        }
-    ]
-)
-
-print(response)
+response = completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hello"}])
 ```
 
-### Custom Callback to log transformed Input to LLMs
+### Log Inputs to LLMs
 ```python
-def get_transformed_inputs(
-    kwargs,
-):
+def get_transformed_inputs(kwargs):
     params_to_model = kwargs["additional_args"]["complete_input_dict"]
     print("params to model", params_to_model)
 
 litellm.input_callback = [get_transformed_inputs]
 
-def test_chat_openai():
-    try:
-        response = completion(model="claude-2",
-                              messages=[{
-                                  "role": "user",
-                                  "content": "Hi 👋 - i'm openai"
-                              }])
-
-        print(response)
-
-    except Exception as e:
-        print(e)
-        pass
+response = completion(model="claude-2", messages=[{"role": "user", "content": "Hello"}])
 ```
 
-#### Output
-```shell
-params to model {'model': 'claude-2', 'prompt': "\n\nHuman: Hi 👋 - i'm openai\n\nAssistant: ", 'max_tokens_to_sample': 256}
+### Send to External Service
+```python
+import requests
+
+def send_to_analytics(kwargs, completion_response, start_time, end_time):
+    data = {
+        "model": kwargs.get("model"),
+        "cost": kwargs.get("response_cost", 0),
+        "duration": (end_time - start_time).total_seconds()
+    }
+    requests.post("https://your-analytics.com/api", json=data)
+
+litellm.success_callback = [send_to_analytics]
 ```
 
-### Custom Callback to write to Mixpanel
+## Common Issues
+
+### Callback Not Called
+Make sure you:
+1. Register callbacks correctly: `litellm.callbacks = [MyHandler()]`
+2. Use the right hook names (check spelling)
+3. Don't use proxy-only hooks in library mode
+
+### Performance Issues  
+- Use async hooks for I/O operations
+- Don't block in callback functions
+- Handle exceptions properly:
 
 ```python
-import mixpanel
-import litellm
-from litellm import completion
-
-def custom_callback(
-    kwargs,                 # kwargs to completion
-    completion_response,    # response from completion
-    start_time, end_time    # start/end time
-):
-    # Your custom code here
-    mixpanel.track("LLM Response", {"llm_response": completion_response})
-
-
-# Assign the custom callback function
-litellm.success_callback = [custom_callback]
-
-response = completion(
-    model="gpt-3.5-turbo",
-    messages=[
-        {
-            "role": "user",
-            "content": "Hi 👋 - i'm openai"
-        }
-    ]
-)
-
-print(response)
-
+class SafeHandler(CustomLogger):
+    async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
+        try:
+            await external_service(response_obj)
+        except Exception as e:
+            print(f"Callback error: {e}")  # Log but don't break the flow
 ```
 
-
-
-
-
-
-
-
-
-
-
-

docs/my-website/docs/observability/datadog.md

@@ -9,8 +9,14 @@ LiteLLM Supports logging to the following Datdog Integrations:
 - `datadog_llm_observability` [Datadog LLM Observability](https://www.datadoghq.com/product/llm-observability/)
 - `ddtrace-run` [Datadog Tracing](#datadog-tracing)
 
-<Tabs>
-<TabItem value="datadog" label="Datadog Logs">
+## Datadog Logs
+
+| Feature | Details |
+|---------|---------|
+| **What is logged** | [StandardLoggingPayload](../proxy/logging_spec) |
+| **Events** | Success + Failure |
+| **Product Link** | [Datadog Logs](https://docs.datadoghq.com/logs/) |
+
 
 We will use the `--config` to set `litellm.callbacks = ["datadog"]` this will log all successful LLM calls to DataDog
 
@@ -26,8 +32,16 @@ litellm_settings:
   service_callback: ["datadog"] # logs redis, postgres failures on datadog
 ```
 
-</TabItem>
-<TabItem value="datadog_llm_observability" label="Datadog LLM Observability">
+
+## Datadog LLM Observability
+
+**Overview**
+
+| Feature | Details |
+|---------|---------|
+| **What is logged** | [StandardLoggingPayload](../proxy/logging_spec) |
+| **Events** | Success + Failure |
+| **Product Link** | [Datadog LLM Observability](https://www.datadoghq.com/product/llm-observability/) |
 
 ```yaml
 model_list:
@@ -38,8 +52,7 @@ litellm_settings:
   callbacks: ["datadog_llm_observability"] # logs llm success logs on datadog
 ```
 
-</TabItem>
-</Tabs>
+
 
 **Step 2**: Set Required env variables for datadog
 
@@ -80,7 +93,53 @@ Expected output on Datadog
 
 <Image img={require('../../img/dd_small1.png')} />
 
-#### Datadog Tracing
+### Redacting Messages and Responses
+
+This section covers how to redact sensitive data from messages and responses in the logged payload on Datadog LLM Observability.
+
+
+When redaction is enabled, the actual message content and response text will be excluded from Datadog logs while preserving metadata like token counts, latency, and model information.
+
+**Step 1**: Configure redaction in your `config.yaml`
+
+```yaml showLineNumbers title="config.yaml"
+model_list:
+ - model_name: gpt-3.5-turbo
+    litellm_params:
+      model: gpt-3.5-turbo
+litellm_settings:
+  callbacks: ["datadog_llm_observability"] # logs llm success logs on datadog
+
+  # Params to apply only for "datadog_llm_observability" callback
+  datadog_llm_observability_params:
+    turn_off_message_logging: true # redacts input messages and output responses
+```
+
+**Step 2**: Send a chat completion request
+
+```shell
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+    --header 'Content-Type: application/json' \
+    --data '{
+    "model": "gpt-3.5-turbo",
+    "messages": [
+        {
+        "role": "user",
+        "content": "what llm are you"
+        }
+    ]
+}'
+```
+
+**Step 3**: Verify redaction in Datadog LLM Observability
+
+On the Datadog LLM Observability page, you should see that both input messages and output responses are redacted, while metadata (token counts, timing, model info) remains visible.
+
+<Image img={require('../../img/dd_llm_obs.png')} />
+
+
+
+### Datadog Tracing
 
 Use `ddtrace-run` to enable [Datadog Tracing](https://ddtrace.readthedocs.io/en/stable/installation_quickstart.html) on litellm proxy
 
@@ -104,7 +163,7 @@ docker run \
     --config /app/config.yaml --detailed_debug
 ```
 
-### Set DD variables (`DD_SERVICE` etc)
+## Set DD variables (`DD_SERVICE` etc)
 
 LiteLLM supports customizing the following Datadog environment variables
 

docs/my-website/docs/observability/helicone_integration.md

@@ -1,3 +1,6 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
 # Helicone - OSS LLM Observability Platform
 
 :::tip
@@ -9,9 +12,68 @@ https://github.com/BerriAI/litellm
 
 [Helicone](https://helicone.ai/) is an open source observability platform that proxies your LLM requests and provides key insights into your usage, spend, latency and more.
 
-## Using Helicone with LiteLLM
+## Quick Start
 
-LiteLLM provides `success_callbacks` and `failure_callbacks`, allowing you to easily log data to Helicone based on the status of your responses.
+<Tabs>
+<TabItem value="sdk" label="Python SDK">
+
+Use just 1 line of code to instantly log your responses **across all providers** with Helicone:
+
+```python
+import os
+from litellm import completion
+
+## Set env variables
+os.environ["HELICONE_API_KEY"] = "your-helicone-key"
+os.environ["OPENAI_API_KEY"] = "your-openai-key"
+
+# Set callbacks
+litellm.success_callback = ["helicone"]
+
+# OpenAI call
+response = completion(
+    model="gpt-4o",
+    messages=[{"role": "user", "content": "Hi 👋 - I'm OpenAI"}],
+)
+
+print(response)
+```
+
+</TabItem>
+<TabItem value="proxy" label="LiteLLM Proxy">
+
+Add Helicone to your LiteLLM proxy configuration:
+
+```yaml title="config.yaml"
+model_list:
+  - model_name: gpt-4
+    litellm_params:
+      model: gpt-4
+      api_key: os.environ/OPENAI_API_KEY
+
+# Add Helicone callback
+litellm_settings:
+  success_callback: ["helicone"]
+  
+# Set Helicone API key
+environment_variables:
+  HELICONE_API_KEY: "your-helicone-key"
+```
+
+Start the proxy:
+```bash
+litellm --config config.yaml
+```
+
+</TabItem>
+</Tabs>
+
+## Integration Methods
+
+There are two main approaches to integrate Helicone with LiteLLM:
+
+1. **Callbacks**: Log to Helicone while using any provider
+2. **Proxy Mode**: Use Helicone as a proxy for advanced features
 
 ### Supported LLM Providers
 
@@ -26,27 +88,16 @@ Helicone can log requests across [various LLM providers](https://docs.helicone.a
 - Replicate
 - And more
 
-### Integration Methods
+## Method 1: Using Callbacks
 
-There are two main approaches to integrate Helicone with LiteLLM:
+Log requests to Helicone while using any LLM provider directly.
 
-1. Using callbacks
-2. Using Helicone as a proxy
-
-Let's explore each method in detail.
-
-### Approach 1: Use Callbacks
-
-Use just 1 line of code to instantly log your responses **across all providers** with Helicone:
-
-```python
-litellm.success_callback = ["helicone"]
-```
-
-Complete Code
+<Tabs>
+<TabItem value="sdk" label="Python SDK">
 
 ```python
 import os
+import litellm
 from litellm import completion
 
 ## Set env variables
@@ -66,28 +117,78 @@ response = completion(
 print(response)
 ```
 
-### Approach 2: Use Helicone as a proxy
+</TabItem>
+<TabItem value="proxy" label="LiteLLM Proxy">
+
+```yaml title="config.yaml"
+model_list:
+  - model_name: gpt-4
+    litellm_params:
+      model: gpt-4
+      api_key: os.environ/OPENAI_API_KEY
+  - model_name: claude-3
+    litellm_params:
+      model: anthropic/claude-3-sonnet-20240229
+      api_key: os.environ/ANTHROPIC_API_KEY
+
+# Add Helicone logging
+litellm_settings:
+  success_callback: ["helicone"]
+  
+# Environment variables
+environment_variables:
+  HELICONE_API_KEY: "your-helicone-key"
+  OPENAI_API_KEY: "your-openai-key"
+  ANTHROPIC_API_KEY: "your-anthropic-key"
+```
+
+Start the proxy:
+```bash
+litellm --config config.yaml
+```
+
+Make requests to your proxy:
+```python
+import openai
+
+client = openai.OpenAI(
+    api_key="anything",  # proxy doesn't require real API key
+    base_url="http://localhost:4000"
+)
+
+response = client.chat.completions.create(
+    model="gpt-4",  # This gets logged to Helicone
+    messages=[{"role": "user", "content": "Hello!"}]
+)
+```
+
+</TabItem>
+</Tabs>
+
+## Method 2: Using Helicone as a Proxy
 
 Helicone's proxy provides [advanced functionality](https://docs.helicone.ai/getting-started/proxy-vs-async) like caching, rate limiting, LLM security through [PromptArmor](https://promptarmor.com/) and more.
 
-To use Helicone as a proxy for your LLM requests:
+<Tabs>
+<TabItem value="sdk" label="Python SDK">
 
-1. Set Helicone as your base URL via: litellm.api_base
-2. Pass in Helicone request headers via: litellm.metadata
-
-Complete Code:
+Set Helicone as your base URL and pass authentication headers:
 
 ```python
 import os
 import litellm
 from litellm import completion
 
+# Configure LiteLLM to use Helicone proxy
 litellm.api_base = "https://oai.hconeai.com/v1"
 litellm.headers = {
-    "Helicone-Auth": f"Bearer {os.getenv('HELICONE_API_KEY')}",  # Authenticate to send requests to Helicone API
+    "Helicone-Auth": f"Bearer {os.getenv('HELICONE_API_KEY')}",
 }
 
-response = litellm.completion(
+# Set your OpenAI API key
+os.environ["OPENAI_API_KEY"] = "your-openai-key"
+
+response = completion(
     model="gpt-3.5-turbo",
     messages=[{"role": "user", "content": "How does a court case get to the Supreme Court?"}]
 )
@@ -136,36 +237,119 @@ litellm.metadata = {
 }
 ```
 
-### Session Tracking and Tracing
+</TabItem>
+</Tabs>
+
+## Session Tracking and Tracing
 
 Track multi-step and agentic LLM interactions using session IDs and paths:
 
-```python
-litellm.metadata = {
-    "Helicone-Auth": f"Bearer {os.getenv('HELICONE_API_KEY')}",  # Authenticate to send requests to Helicone API
-    "Helicone-Session-Id": "session-abc-123",  # The session ID you want to track
-    "Helicone-Session-Path": "parent-trace/child-trace",  # The path of the session
-}
-```
-
-- `Helicone-Session-Id`: Use this to specify the unique identifier for the session you want to track. This allows you to group related requests together.
-- `Helicone-Session-Path`: This header defines the path of the session, allowing you to represent parent and child traces. For example, "parent/child" represents a child trace of a parent trace.
-
-By using these two headers, you can effectively group and visualize multi-step LLM interactions, gaining insights into complex AI workflows.
-
-### Retry and Fallback Mechanisms
-
-Set up retry mechanisms and fallback options:
+<Tabs>
+<TabItem value="sdk" label="Python SDK">
 
 ```python
+import litellm
+
+litellm.api_base = "https://oai.hconeai.com/v1"
 litellm.metadata = {
-    "Helicone-Auth": f"Bearer {os.getenv('HELICONE_API_KEY')}",  # Authenticate to send requests to Helicone API
-    "Helicone-Retry-Enabled": "true",  # Enable retry mechanism
-    "helicone-retry-num": "3",  # Set number of retries
-    "helicone-retry-factor": "2",  # Set exponential backoff factor
-    "Helicone-Fallbacks": '["gpt-3.5-turbo", "gpt-4"]',  # Set fallback models
+    "Helicone-Auth": f"Bearer {os.getenv('HELICONE_API_KEY')}",
+    "Helicone-Session-Id": "session-abc-123",
+    "Helicone-Session-Path": "parent-trace/child-trace",
 }
+
+response = litellm.completion(
+    model="gpt-3.5-turbo",
+    messages=[{"role": "user", "content": "Start a conversation"}]
+)
 ```
 
+</TabItem>
+<TabItem value="proxy" label="LiteLLM Proxy">
+
+```python
+import openai
+
+client = openai.OpenAI(
+    api_key="anything",
+    base_url="http://localhost:4000"
+)
+
+# First request in session
+response1 = client.chat.completions.create(
+    model="gpt-4",
+    messages=[{"role": "user", "content": "Hello"}],
+    extra_headers={
+        "Helicone-Session-Id": "session-abc-123",
+        "Helicone-Session-Path": "conversation/greeting"
+    }
+)
+
+# Follow-up request in same session
+response2 = client.chat.completions.create(
+    model="gpt-4",
+    messages=[{"role": "user", "content": "Tell me more"}],
+    extra_headers={
+        "Helicone-Session-Id": "session-abc-123",
+        "Helicone-Session-Path": "conversation/follow-up"
+    }
+)
+```
+
+</TabItem>
+</Tabs>
+
+- `Helicone-Session-Id`: Unique identifier for the session to group related requests
+- `Helicone-Session-Path`: Hierarchical path to represent parent/child traces (e.g., "parent/child")
+
+## Retry and Fallback Mechanisms
+
+<Tabs>
+<TabItem value="sdk" label="Python SDK">
+
+```python
+import litellm
+
+litellm.api_base = "https://oai.hconeai.com/v1"
+litellm.metadata = {
+    "Helicone-Auth": f"Bearer {os.getenv('HELICONE_API_KEY')}",
+    "Helicone-Retry-Enabled": "true",
+    "helicone-retry-num": "3",
+    "helicone-retry-factor": "2",  # Exponential backoff
+    "Helicone-Fallbacks": '["gpt-3.5-turbo", "gpt-4"]',
+}
+
+response = litellm.completion(
+    model="gpt-4",
+    messages=[{"role": "user", "content": "Hello"}]
+)
+```
+
+</TabItem>
+<TabItem value="proxy" label="LiteLLM Proxy">
+
+```yaml title="config.yaml"
+model_list:
+  - model_name: gpt-4
+    litellm_params:
+      model: gpt-4
+      api_key: os.environ/OPENAI_API_KEY
+      api_base: "https://oai.hconeai.com/v1"
+
+default_litellm_params:
+  headers:
+    Helicone-Auth: "Bearer ${HELICONE_API_KEY}"
+    Helicone-Retry-Enabled: "true"
+    helicone-retry-num: "3"
+    helicone-retry-factor: "2"
+    Helicone-Fallbacks: '["gpt-3.5-turbo", "gpt-4"]'
+
+environment_variables:
+  HELICONE_API_KEY: "your-helicone-key"
+  OPENAI_API_KEY: "your-openai-key"
+```
+
+</TabItem>
+</Tabs>
+
 > **Supported Headers** - For a full list of supported Helicone headers and their descriptions, please refer to the [Helicone documentation](https://docs.helicone.ai/getting-started/quick-start).
 > By utilizing these headers and metadata options, you can gain deeper insights into your LLM usage, optimize performance, and better manage your AI workflows with Helicone and LiteLLM.

docs/my-website/docs/observability/langfuse_otel_integration.md

@@ -35,14 +35,14 @@ The Langfuse OpenTelemetry integration allows you to send LiteLLM traces and obs
 |----------|----------|-------------|---------|
 | `LANGFUSE_PUBLIC_KEY` | Yes | Your Langfuse public key | `pk-lf-...` |
 | `LANGFUSE_SECRET_KEY` | Yes | Your Langfuse secret key | `sk-lf-...` |
-| `LANGFUSE_HOST` | No | Langfuse host URL | `https://us.cloud.langfuse.com` (default) |
+| `LANGFUSE_OTEL_HOST` | No | OTEL endpoint host | `https://otel.my-langfuse.com` |
 
 ### Endpoint Resolution
 
-The integration automatically constructs the OTEL endpoint from the `LANGFUSE_HOST`:
+The integration automatically constructs the OTEL endpoint from `LANGFUSE_OTEL_HOST`
 - **Default (US)**: `https://us.cloud.langfuse.com/api/public/otel`
 - **EU Region**: `https://cloud.langfuse.com/api/public/otel`
-- **Self-hosted**: `{LANGFUSE_HOST}/api/public/otel`
+- **Self-hosted**: `{LANGFUSE_OTEL_HOST}/api/public/otel`
 
 ## Usage
 
@@ -77,11 +77,11 @@ os.environ["LANGFUSE_PUBLIC_KEY"] = "pk-lf-..."
 os.environ["LANGFUSE_SECRET_KEY"] = "sk-lf-..."
 
 # Use EU region
-os.environ["LANGFUSE_HOST"] = "https://cloud.langfuse.com"  # EU region
-# os.environ["LANGFUSE_HOST"] = "https://us.cloud.langfuse.com"  # US region (default)
+os.environ["LANGFUSE_OTEL_HOST"] = "https://cloud.langfuse.com"  # EU region
+# os.environ["LANGFUSE_OTEL_HOST"] = "https://otel.my-langfuse.company.com"  # custom OTEL endpoint
 
 # Or use self-hosted instance
-# os.environ["LANGFUSE_HOST"] = "https://my-langfuse.company.com"
+# os.environ["LANGFUSE_OTEL_HOST"] = "https://my-langfuse.company.com"
 
 litellm.callbacks = ["langfuse_otel"]
 ```
@@ -98,14 +98,16 @@ import litellm
 # Get keys for your project from the project settings page: https://cloud.langfuse.com
 os.environ["LANGFUSE_PUBLIC_KEY"] = "pk-lf-..." 
 os.environ["LANGFUSE_SECRET_KEY"] = "sk-lf-..." 
-os.environ["LANGFUSE_HOST"] = "https://cloud.langfuse.com" # EU region
-# os.environ["LANGFUSE_HOST"] = "https://us.cloud.langfuse.com" # US region
+os.environ["LANGFUSE_OTEL_HOST"] = "https://cloud.langfuse.com" # EU region
+# os.environ["LANGFUSE_OTEL_HOST"] = "https://us.cloud.langfuse.com" # US region
+# os.environ["LANGFUSE_OTEL_HOST"] = "https://otel.my-langfuse.company.com" # custom OTEL endpoint
 
 LANGFUSE_AUTH = base64.b64encode(
     f"{os.environ.get('LANGFUSE_PUBLIC_KEY')}:{os.environ.get('LANGFUSE_SECRET_KEY')}".encode()
 ).decode()
 
-os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = os.environ.get("LANGFUSE_HOST") + "/api/public/otel"
+host = os.environ.get("LANGFUSE_OTEL_HOST")
+os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = host + "/api/public/otel"
 os.environ["OTEL_EXPORTER_OTLP_HEADERS"] = f"Authorization=Basic {LANGFUSE_AUTH}"
 
 litellm.callbacks = ["langfuse_otel"]
@@ -120,7 +122,8 @@ Add the integration to your proxy configuration:
 ```bash
 export LANGFUSE_PUBLIC_KEY="pk-lf-..."
 export LANGFUSE_SECRET_KEY="sk-lf-..."
-export LANGFUSE_HOST="https://us.cloud.langfuse.com"  # Default US region
+export LANGFUSE_OTEL_HOST="https://us.cloud.langfuse.com"  # Default US region
+# export LANGFUSE_OTEL_HOST="https://otel.my-langfuse.company.com"  # custom OTEL endpoint
 ```
 
 2. Setup config.yaml

docs/my-website/docs/observability/mlflow.md

@@ -17,7 +17,7 @@ MLflow’s integration with LiteLLM supports advanced observability compatible w
 Install MLflow:
 
 ```shell
-pip install mlflow
+pip install "litellm[mlflow]"
 ```
 
 To enable MLflow auto tracing for LiteLLM:
@@ -160,6 +160,102 @@ class CustomAgent:
 
 This approach generates a unified trace, combining your custom Python code with LiteLLM calls.
 
+## LiteLLM Proxy Server 
+
+### Dependencies
+
+For using `mlflow` on LiteLLM Proxy Server, you need to install the `mlflow` package on your docker container.
+
+```shell
+pip install "mlflow>=3.1.4"
+```
+
+### Configuration
+
+Configure MLflow in your LiteLLM proxy configuration file:
+
+```yaml
+model_list:
+  - model_name: openai/*
+    litellm_params:
+      model: openai/*
+
+litellm_settings:
+  success_callback: ["mlflow"]
+  failure_callback: ["mlflow"]
+```
+
+### Environment Variables
+
+For MLflow with Databricks service, set these required environment variables:
+
+```shell
+DATABRICKS_TOKEN="dapixxxxx"
+DATABRICKS_HOST="https://dbc-xxxx.cloud.databricks.com"
+MLFLOW_TRACKING_URI="databricks"
+MLFLOW_REGISTRY_URI="databricks-uc"
+MLFLOW_EXPERIMENT_ID="xxxx"
+```
+
+### Adding Tags for Better Tracing
+
+You can add custom tags to your requests for improved trace organization and filtering in MLflow. Tags help you categorize and search your traces by job ID, task name, or any custom metadata.
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+<Tabs>
+<TabItem value="curl" label="curl">
+
+```shell
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+    --header 'Content-Type: application/json' \
+    --header 'Authorization: Bearer sk-1234' \
+    --data '{
+    "model": "gemini-2.5-flash",
+    "messages": [
+        {
+        "role": "user",
+        "content": "what llm are you"
+        }
+    ],
+    "litellm_metadata": {
+        "tags": ["jobID:214590dsff09fds", "taskName:run_page_classification"]
+    }
+}'
+```
+
+</TabItem>
+<TabItem value="openai-python" label="OpenAI Python SDK">
+
+```python
+from openai import OpenAI
+
+# Initialize the OpenAI client pointing to your LiteLLM proxy
+client = OpenAI(
+    api_key="sk-1234",  # Your LiteLLM proxy API key
+    base_url="http://0.0.0.0:4000"  # Your LiteLLM proxy URL
+)
+
+# Make a request with tags in metadata
+response = client.chat.completions.create(
+    model="gemini-2.5-flash",
+    messages=[
+        {
+            "role": "user", 
+            "content": "what llm are you"
+        }
+    ],
+    extra_body={
+        "litellm_metadata": {
+            "tags": ["jobID:214590dsff09fds", "taskName:run_page_classification"]
+        }
+    }
+)
+```
+
+</TabItem>
+</Tabs>
 
 ## Support
 

docs/my-website/docs/observability/opik_integration.md

@@ -140,6 +140,7 @@ These can be passed inside metadata with the `opik` key.
 - `project_name` - Name of the Opik project to send data to.
 - `current_span_data` - The current span data to be used for tracing.
 - `tags` - Tags to be used for tracing.
+- `thread_id` - The thread id to group together multiple related traces.
 
 ### Usage
 
@@ -159,8 +160,10 @@ response = litellm.completion(
     messages=messages,
     metadata = {
         "opik": {
+            "project_name": "your-opik-project-name",
             "current_span_data": get_current_span_data(),
             "tags": ["streaming-test"],
+            "thread_id": "your-thread-id"
         },
     }
 )
@@ -174,7 +177,7 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
 -H 'Content-Type: application/json' \
 -H 'Authorization: Bearer sk-1234' \
 -d '{
-  "model": "gpt-3.5-turbo-testing",
+  "model": "gpt-3.5-turbo",
   "messages": [
     {
       "role": "user",
@@ -183,8 +186,10 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
   ],
   "metadata": {
     "opik": {
+      "project_name": "your-opik-project-name",
       "current_span_data": "...",
       "tags": ["streaming-test"],
+      "thread_id": "your-thread-id"
     },
   }
 }'
@@ -195,12 +200,25 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
 
 
 
+You can also pass the fields as part of the request header with a `opik_*` prefix:
 
-
-
-
-
-
+```shell
+curl --location --request POST 'http://0.0.0.0:4000/chat/completions' \
+    --header 'Content-Type: application/json' \
+    --header 'Authorization: Bearer sk-1234' \
+    --header 'opik_project_name: your-opik-project-name' \
+    --header 'opik_thread_id: your-thread-id' \
+    --header 'opik_tags: ["streaming-test"]' \
+    --data '{
+    "model": "gpt-3.5-turbo",
+    "messages": [
+        {
+        "role": "user",
+        "content": "What's the weather like in Boston today?"
+        }
+    ]
+}'
+```
 
 
 

docs/my-website/docs/observability/posthog_integration.md

@@ -0,0 +1,261 @@
+# PostHog - Tracking LLM Usage Analytics
+
+## What is PostHog?
+
+PostHog is an open-source product analytics platform that helps you track and analyze how users interact with your product. For LLM applications, PostHog provides specialized AI features to track model usage, performance, and user interactions with your AI features.
+
+## Usage with LiteLLM Proxy (LLM Gateway)
+
+**Step 1**: Create a `config.yaml` file and set `litellm_settings`: `success_callback`
+
+```yaml
+model_list:
+  - model_name: gpt-3.5-turbo
+    litellm_params:
+      model: gpt-3.5-turbo
+
+litellm_settings:
+  success_callback: ["posthog"]
+  failure_callback: ["posthog"]
+```
+
+**Step 2**: Set required environment variables
+
+```shell
+export POSTHOG_API_KEY="your-posthog-api-key"
+# Optional, defaults to https://app.posthog.com
+export POSTHOG_API_URL="https://app.posthog.com" # optional
+```
+
+**Step 3**: Start the proxy, make a test request
+
+Start proxy
+
+```shell
+litellm --config config.yaml --debug
+```
+
+Test Request
+
+```shell
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+    --header 'Content-Type: application/json' \
+    --data '{
+    "model": "gpt-3.5-turbo",
+    "messages": [
+        {
+        "role": "user",
+        "content": "what llm are you"
+        }
+    ],
+    "metadata": {
+        "user_id": "user-123",
+        "custom_field": "custom_value"
+    }
+}'
+```
+
+### Team-Based Logging
+
+Configure different PostHog credentials per team using the team callback settings:
+
+```bash
+curl -X POST 'http://localhost:4000/team/{team_id}/callback' \
+  -H 'Authorization: Bearer sk-1234' \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "callback_name": "posthog",
+    "callback_type": "success",
+    "callback_vars": {
+      "posthog_api_key": "ph_team_specific_key",
+      "posthog_api_url": "https://custom.posthog.com"
+    }
+  }'
+```
+
+Now all requests from that team will be logged to their specific PostHog project.
+
+## Usage with LiteLLM Python SDK
+
+### Quick Start
+
+Use just 2 lines of code, to instantly log your responses **across all providers** with PostHog:
+
+```python
+litellm.success_callback = ["posthog"]
+litellm.failure_callback = ["posthog"] # logs errors to posthog
+```
+```python
+import litellm
+import os
+
+# from PostHog
+os.environ["POSTHOG_API_KEY"] = ""
+# Optional, defaults to https://app.posthog.com
+os.environ["POSTHOG_API_URL"] = "" # optional
+
+# LLM API Keys
+os.environ['OPENAI_API_KEY']=""
+
+# set posthog as a callback, litellm will send the data to posthog
+litellm.success_callback = ["posthog"]
+
+# openai call
+response = litellm.completion(
+    model="gpt-3.5-turbo",
+    messages=[
+        {"role": "user", "content": "Hi - i'm openai"}
+    ],
+    metadata = {
+        "user_id": "user-123", # set posthog user ID
+    }
+)
+```
+
+### Advanced
+
+#### Set User ID and Custom Metadata
+
+Pass `user_id` in `metadata` to associate events with specific users in PostHog:
+
+**With LiteLLM Python SDK:**
+
+```python
+import litellm
+
+litellm.success_callback = ["posthog"]
+
+response = litellm.completion(
+    model="gpt-3.5-turbo",
+    messages=[
+        {"role": "user", "content": "Hello world"}
+    ],
+    metadata={
+        "user_id": "user-123",  # Add user ID for PostHog tracking
+        "custom_field": "custom_value"  # Add custom metadata
+    }
+)
+```
+
+**With LiteLLM Proxy using OpenAI Python SDK:**
+
+```python
+import openai
+
+client = openai.OpenAI(
+    api_key="sk-1234",  # Your LiteLLM Proxy API key
+    base_url="http://0.0.0.0:4000"  # Your LiteLLM Proxy URL
+)
+
+response = client.chat.completions.create(
+    model="gpt-3.5-turbo",
+    messages=[
+        {"role": "user", "content": "Hello world"}
+    ],
+    extra_body={
+        "metadata": {
+            "user_id": "user-123",  # Add user ID for PostHog tracking
+            "project_name": "my-project",  # Add custom metadata
+            "environment": "production"
+        }
+    }
+)
+```
+
+#### Per-Request Credentials
+
+You can override PostHog credentials on a per-request basis:
+
+```python
+import litellm
+
+litellm.success_callback = ["posthog"]
+
+# Use custom PostHog credentials for this specific request
+response = litellm.completion(
+    model="gpt-3.5-turbo",
+    messages=[
+        {"role": "user", "content": "Hello world"}
+    ],
+    posthog_api_key="ph_custom_project_key",
+    posthog_api_url="https://custom.posthog.com"
+)
+```
+
+This is useful when you need to:
+- Log different teams/projects to separate PostHog instances
+- Use different PostHog projects for staging vs production
+- Route logs based on customer or tenant
+
+#### Disable Logging for Specific Calls
+
+Use the `no-log` flag to prevent logging for specific calls:
+
+```python
+import litellm
+
+litellm.success_callback = ["posthog"]
+
+response = litellm.completion(
+    model="gpt-3.5-turbo",
+    messages=[
+        {"role": "user", "content": "This won't be logged"}
+    ],
+    metadata={"no-log": True}
+)
+```
+
+## What's Logged to PostHog?
+
+When LiteLLM logs to PostHog, it captures detailed information about your LLM usage:
+
+### For Completion Calls
+- **Model Information**: Provider, model name, model parameters
+- **Usage Metrics**: Input tokens, output tokens, total cost
+- **Performance**: Latency, completion time
+- **Content**: Input messages, model responses (respects privacy settings)
+- **Metadata**: Custom fields, user ID, trace information
+
+### For Embedding Calls
+- **Model Information**: Provider, model name
+- **Usage Metrics**: Input tokens, total cost
+- **Performance**: Latency
+- **Content**: Input text (respects privacy settings)
+- **Metadata**: Custom fields, user ID, trace information
+
+### For Errors
+- **Error Details**: Error type, error message, stack trace
+- **Context**: Model, provider, input that caused the error
+- **Timing**: When the error occurred, request duration
+
+## Environment Variables
+
+| Variable | Required | Description |
+|----------|----------|-------------|
+| `POSTHOG_API_KEY` | Yes | Your PostHog project API key |
+| `POSTHOG_API_URL` | No | PostHog API URL (defaults to https://app.posthog.com) |
+
+## Troubleshooting
+
+### 1. Missing API Key
+```
+Error: POSTHOG_API_KEY is not set
+```
+
+Set your PostHog API key:
+```python
+import os
+os.environ["POSTHOG_API_KEY"] = "your-api-key"
+```
+
+### 2. Custom PostHog Instance
+If you're using a self-hosted PostHog instance:
+```python
+import os
+os.environ["POSTHOG_API_URL"] = "https://your-posthog-instance.com"
+```
+
+### 3. Events Not Appearing
+- Check that your API key is correct
+- Verify network connectivity to PostHog
+- Events may take a few minutes to appear in PostHog dashboard

docs/my-website/docs/pass_through/azure_passthrough.md

@@ -0,0 +1,89 @@
+# Azure Passthrough
+
+Pass-through endpoints for `/azure`
+
+## Overview
+
+| Feature | Supported | Notes |
+|-------|-------|-------|
+| Cost Tracking | ❌ | Not supported |
+| Logging | ✅ | Works across all integrations |
+| Streaming | ✅ | Fully supported |
+
+### When to use this?
+
+- For most use cases, you should use the [native LiteLLM Azure OpenAI Integration](../providers/azure/azure) (`/chat/completions`, `/embeddings`, `/completions`, `/images`, etc.)
+- Use this passthrough to call newer or less common Azure OpenAI endpoints that LiteLLM doesn't fully support yet, such as `/assistants`, `/threads`, `/vector_stores`
+
+Simply replace your Azure endpoint (e.g. `https://<your-resource-name>.openai.azure.com`) with `LITELLM_PROXY_BASE_URL/azure`
+
+## Usage Examples
+
+### Assistants API
+
+#### Create Azure OpenAI Client
+
+Make sure you do the following:
+- Point `azure_endpoint` to your `LITELLM_PROXY_BASE_URL/azure`
+- Use your `LITELLM_API_KEY` as the `api_key`
+
+```python
+import openai
+
+client = openai.AzureOpenAI(
+    azure_endpoint="http://0.0.0.0:4000/azure",  # <your-proxy-url>/azure
+    api_key="sk-anything",  # <your-proxy-api-key>
+    api_version="2024-05-01-preview"  # required Azure API version
+)
+```
+
+#### Create an Assistant
+
+```python
+assistant = client.beta.assistants.create(
+    name="Math Tutor",
+    instructions="You are a math tutor. Help solve equations.",
+    model="gpt-4o",
+)
+```
+
+#### Create a Thread
+```python
+thread = client.beta.threads.create()
+```
+
+#### Add a Message to the Thread
+```python
+message = client.beta.threads.messages.create(
+    thread_id=thread.id,
+    role="user",
+    content="Solve 3x + 11 = 14",
+)
+```
+
+#### Run the Assistant
+```python
+run = client.beta.threads.runs.create(
+    thread_id=thread.id,
+    assistant_id=assistant.id,
+)
+
+# Check run status
+run_status = client.beta.threads.runs.retrieve(
+    thread_id=thread.id,
+    run_id=run.id
+)
+```
+
+#### Retrieve Messages
+```python
+messages = client.beta.threads.messages.list(
+    thread_id=thread.id
+)
+```
+
+#### Delete the Assistant
+
+```python
+client.beta.assistants.delete(assistant.id)
+```

docs/my-website/docs/pass_through/google_ai_studio.md

@@ -230,6 +230,13 @@ curl -X POST "https://generativelanguage.googleapis.com/v1beta/models/gemini-1.5
 ```
 
 
+## **Example 4: Video Generation with Veo**
+
+Generate videos using Google's Veo model through LiteLLM pass-through routes.
+
+[**→ Complete Veo Video Generation Guide**](../proxy/veo_video_generation.md)
+
+
 ## Advanced 
 
 Pre-requisites

docs/my-website/docs/pass_through/intro.md

@@ -11,3 +11,43 @@ These endpoints are useful for 2 scenarios:
 ## How is your request handled? 
 
 The request is passed through to the provider's endpoint. The response is then passed back to the client. **No translation is done.**
+
+### Request Forwarding Process
+
+1. **Request Reception**: LiteLLM receives your request at `/provider/endpoint`
+2. **Authentication**: Your LiteLLM API key is validated and mapped to the provider's API key
+3. **Request Transformation**: Request is reformatted for the target provider's API
+4. **Forwarding**: Request is sent to the actual provider endpoint
+5. **Response Handling**: Provider response is returned directly to you
+
+### Authentication Flow
+
+```mermaid
+graph LR
+    A[Client Request] --> B[LiteLLM Proxy]
+    B --> C[Validate LiteLLM API Key]
+    C --> D[Map to Provider API Key]
+    D --> E[Forward to Provider]
+    E --> F[Return Response]
+```
+
+**Key Points:**
+- Use your **LiteLLM API key** in requests, not the provider's key
+- LiteLLM handles the provider authentication internally
+- Same authentication works across all passthrough endpoints
+
+### Error Handling
+
+**Provider Errors**: Forwarded directly to you with original error codes and messages
+
+**LiteLLM Errors**: 
+- `401`: Invalid LiteLLM API key
+- `404`: Provider or endpoint not supported
+- `500`: Internal routing/forwarding errors
+
+### Benefits
+
+- **Unified Authentication**: One API key for all providers
+- **Centralized Logging**: All requests logged through LiteLLM
+- **Cost Tracking**: Usage tracked across all endpoints
+- **Access Control**: Same permissions apply to passthrough endpoints

docs/my-website/docs/pass_through/vertex_ai.md

@@ -15,10 +15,11 @@ Pass-through endpoints for Vertex AI - call provider-specific endpoint, in nativ
 
 ## Supported Endpoints
 
-LiteLLM supports 2 vertex ai passthrough routes:
+LiteLLM supports 3 vertex ai passthrough routes:
 
 1. `/vertex_ai` → routes to `https://{vertex_location}-aiplatform.googleapis.com/`
 2. `/vertex_ai/discovery` → routes to [`https://discoveryengine.googleapis.com`](https://discoveryengine.googleapis.com/)
+3. `/vertex_ai/live` → upgrades to the Vertex AI Live API WebSocket (`google.cloud.aiplatform.v1.LlmBidiService/BidiGenerateContent`)
 
 ## How to use
 
@@ -170,6 +171,50 @@ generateContent();
 </Tabs>
 
 
+## Vertex AI Live API WebSocket
+
+LiteLLM can now proxy the Vertex AI Live API to help you experiment with streaming audio/text from Gemini Live models without exposing Google credentials to clients.
+
+- Configure default Vertex credentials via `default_vertex_config` or environment variables (see examples above).
+- Connect to `wss://<PROXY_URL>/vertex_ai/live`. LiteLLM will exchange your saved credentials for a short-lived access token and forward messages bidirectionally.
+- Optional query params `vertex_project`, `vertex_location`, and `model` let you override defaults for multi-project setups or global-only models.
+
+```python title="client.py"
+import asyncio
+import json
+
+from websockets.asyncio.client import connect
+
+
+async def main() -> None:
+    headers = {
+        "x-litellm-api-key": "Bearer sk-your-litellm-key",
+        "Content-Type": "application/json",
+    }
+    async with connect(
+        "ws://localhost:4000/vertex_ai/live",
+        additional_headers=headers,
+    ) as ws:
+        await ws.send(
+            json.dumps(
+                {
+                    "setup": {
+                        "model": "projects/your-project/locations/us-central1/publishers/google/models/gemini-2.0-flash-live-preview-04-09",
+                        "generation_config": {"response_modalities": ["TEXT"]},
+                    }
+                }
+            )
+        )
+
+        async for message in ws:
+            print("server:", message)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+
 ## Quick Start
 
 Let's call the Vertex AI [`/generateContent` endpoint](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/inference)
@@ -415,4 +460,4 @@ generateContent();
 ```
 
 </TabItem>
-</Tabs>
+</Tabs>

docs/my-website/docs/pass_through/vertex_ai_live_websocket.md

@@ -0,0 +1,284 @@
+# Vertex AI Live API WebSocket Passthrough
+
+LiteLLM now supports WebSocket passthrough for the Vertex AI Live API, enabling real-time bidirectional communication with Gemini models.
+
+## Overview
+
+The Vertex AI Live API WebSocket passthrough allows you to:
+- Connect to Vertex AI Live API through LiteLLM proxy
+- Use existing Vertex AI authentication methods
+- Pass through all WebSocket messages bidirectionally
+- Support text, audio, video, and multimodal interactions
+- Track costs automatically for all usage types
+
+## Configuration
+
+### Environment Variables
+
+Set the following environment variables for Vertex AI authentication:
+
+```bash
+# Required
+DEFAULT_VERTEXAI_PROJECT=your-project-id
+DEFAULT_VERTEXAI_LOCATION=us-central1
+
+# Optional - use one of these for authentication
+DEFAULT_GOOGLE_APPLICATION_CREDENTIALS=/path/to/service-account.json
+# OR run: gcloud auth application-default login
+```
+
+### Configuration File
+
+Alternatively, configure in your `config.yaml`:
+
+```yaml
+litellm_settings:
+  default_vertex_config:
+    vertex_project: "your-project-id"
+    vertex_location: "us-central1"
+    vertex_credentials: "os.environ/GOOGLE_APPLICATION_CREDENTIALS"
+```
+
+## Usage
+
+### WebSocket Endpoints
+
+- `ws://your-proxy-host/v1/vertex-ai/live`
+- `ws://your-proxy-host/vertex-ai/live`
+
+### Query Parameters
+
+- `project_id` (optional): Google Cloud project ID (can be set in config)
+- `location` (optional): Vertex AI location (can be set in config, default: us-central1)
+
+### Example Connection
+
+```javascript
+// If project_id and location are set in config, you can connect without query params
+const ws = new WebSocket('ws://localhost:4000/v1/vertex-ai/live');
+
+// Or specify them explicitly
+const ws = new WebSocket('ws://localhost:4000/v1/vertex-ai/live?project_id=your-project-id&location=us-central1');
+```
+
+## Cost Tracking
+
+The WebSocket passthrough automatically tracks costs for all usage types based on the [Vertex AI pricing](https://cloud.google.com/vertex-ai/generative-ai/pricing#model-optimizer-pricing):
+
+### Supported Cost Tracking
+
+- **Text**: Character-based or token-based pricing depending on model
+- **Audio**: Per-second pricing for audio input/output
+- **Video**: Per-second pricing for video input
+- **Images**: Per-image pricing for image input
+
+### Cost Calculation
+
+Costs are calculated using the same methods as other Vertex AI models in LiteLLM:
+- Uses `cost_per_character` for Gemini models
+- Uses `cost_per_token` for partner models (Claude, Llama, etc.)
+- Includes audio, video, and image costs when applicable
+
+### Cost Logging
+
+Costs are automatically logged to:
+- LiteLLM proxy logs
+- Database (if configured)
+- Spend tracking system
+- Admin dashboard
+
+Example log output:
+```
+Vertex AI Live WebSocket session cost: $0.001234 (input: $0.000800, output: $0.000434) tokens: 150, characters: 1200, duration: 45.2s
+```
+
+## API Reference
+
+### Setup Message
+
+Send this message first to initialize the session:
+
+```json
+{
+  "setup": {
+    "model": "projects/your-project-id/locations/us-central1/publishers/google/models/gemini-2.0-flash-live-preview-04-09",
+    "generation_config": {
+      "response_modalities": ["TEXT"]
+    }
+  }
+}
+```
+
+### Text Input
+
+```json
+{
+  "client_content": {
+    "turns": [
+      {
+        "role": "user",
+        "parts": [{"text": "Hello! How are you?"}]
+      }
+    ],
+    "turn_complete": true
+  }
+}
+```
+
+### Audio Input
+
+```json
+{
+  "realtime_input": {
+    "media_chunks": [
+      {
+        "data": "base64-encoded-audio-data",
+        "mime_type": "audio/pcm"
+      }
+    ]
+  }
+}
+```
+
+## Supported Features
+
+### Response Modalities
+
+- **TEXT**: Text responses
+- **AUDIO**: Audio responses with voice synthesis
+
+### Tools
+
+- **Function Calling**: Define and use custom functions
+- **Code Execution**: Execute Python code
+- **Google Search**: Search the web
+- **Voice Activity Detection**: Detect when user is speaking
+
+### Advanced Features
+
+- **Audio Transcription**: Transcribe input and output audio
+- **Proactive Audio**: Model responds only when relevant
+- **Affective Dialog**: Understand emotional expressions
+
+## Examples
+
+### Python Client
+
+```python
+import asyncio
+import json
+import websockets
+
+async def chat_with_gemini():
+    uri = "ws://localhost:4000/v1/vertex-ai/live?project_id=your-project-id"
+    
+    async with websockets.connect(uri) as websocket:
+        # Setup
+        setup = {
+            "setup": {
+                "model": "projects/your-project-id/locations/us-central1/publishers/google/models/gemini-2.0-flash-live-preview-04-09",
+                "generation_config": {"response_modalities": ["TEXT"]}
+            }
+        }
+        await websocket.send(json.dumps(setup))
+        
+        # Wait for setup response
+        response = await websocket.recv()
+        print(f"Setup: {response}")
+        
+        # Send message
+        message = {
+            "client_content": {
+                "turns": [{"role": "user", "parts": [{"text": "Hello!"}]}],
+                "turn_complete": True
+            }
+        }
+        await websocket.send(json.dumps(message))
+        
+        # Receive response
+        async for response in websocket:
+            print(f"Response: {response}")
+            # Check if turn is complete
+            data = json.loads(response)
+            if data.get("serverContent", {}).get("turnComplete"):
+                break
+
+asyncio.run(chat_with_gemini())
+```
+
+### JavaScript Client
+
+```javascript
+const ws = new WebSocket('ws://localhost:4000/v1/vertex-ai/live?project_id=your-project-id');
+
+ws.onopen = function() {
+    // Send setup
+    const setup = {
+        setup: {
+            model: "projects/your-project-id/locations/us-central1/publishers/google/models/gemini-2.0-flash-live-preview-04-09",
+            generation_config: { response_modalities: ["TEXT"] }
+        }
+    };
+    ws.send(JSON.stringify(setup));
+};
+
+ws.onmessage = function(event) {
+    const data = JSON.parse(event.data);
+    console.log('Received:', data);
+    
+    // Check if setup is complete
+    if (data.setupComplete) {
+        // Send a message
+        const message = {
+            client_content: {
+                turns: [{ role: "user", parts: [{ text: "Hello!" }] }],
+                turn_complete: true
+            }
+        };
+        ws.send(JSON.stringify(message));
+    }
+};
+```
+
+## Error Handling
+
+The WebSocket connection may close with these codes:
+
+- `4001`: Vertex AI credentials not configured
+- `4002`: Project ID not provided
+- `1011`: Internal server error
+
+## Authentication
+
+The WebSocket passthrough uses the same authentication as other LiteLLM endpoints:
+
+1. **API Key**: Pass `Authorization: Bearer your-api-key` header
+2. **Vertex AI Credentials**: Set environment variables or config file
+
+## Limitations
+
+- Requires valid Google Cloud project with Vertex AI API enabled
+- WebSocket connections are not persistent across server restarts
+- Rate limits apply based on your Google Cloud quotas
+
+## Troubleshooting
+
+### Common Issues
+
+1. **Authentication Error**: Ensure Vertex AI credentials are properly configured
+2. **Project Not Found**: Verify the project ID exists and has Vertex AI enabled
+3. **Connection Refused**: Check that the LiteLLM proxy server is running
+
+### Debug Mode
+
+Enable debug logging to see detailed connection information:
+
+```bash
+export LITELLM_LOG=DEBUG
+```
+
+## Related Documentation
+
+- [Vertex AI Live API Reference](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/multimodal-live)
+- [LiteLLM Proxy Configuration](../proxy/)
+- [Vertex AI Passthrough Endpoints](./vertex_ai.md)

docs/my-website/docs/projects/Railtracks.md

@@ -0,0 +1,7 @@
+# Railtracks
+
+`Railtracks` is an open-source agentic framework that helps developers build resilient agentic systems offering local and remote monitoring tools.
+
+- [Github](https://github.com/RailtownAI/railtracks)
+- [Docs](https://railtownai.github.io/railtracks/)
+- [Railtracks](https://railtracks.org/)

docs/my-website/docs/providers/aiml.md

@@ -1,5 +1,23 @@
 # AI/ML API  
+https://aimlapi.com/
 
+## Overview
+
+| Property | Details |
+|-------|-------|
+| Description | AI/ML API provides access to state-of-the-art AI models including flux-pro/v1.1 for high-quality image generation. |
+| Provider Route on LiteLLM | `aiml/` |
+| Link to Provider Doc | [AI/ML API ↗](https://docs.aimlapi.com/) |
+| Supported Operations | [`/chat/completions`], [`/images/generations`](#image-generation) |
+
+LiteLLM supports AI/ML API Image Generation calls.
+
+## API Base, Key
+```python
+# env variable
+os.environ['AIML_API_KEY'] = "your-api-key"
+os.environ['AIML_API_BASE'] = "https://api.aimlapi.com"  # [optional] 
+```
 Getting started with the AI/ML API is simple. Follow these steps to set up your integration:
 
 ### 1. Get Your API Key  
@@ -24,7 +42,7 @@ You can choose from LLama, Qwen, Flux, and 200+ other open and closed-source mod
 import litellm
 
 response = litellm.completion(
-    model="openai/meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo", # The model name must include prefix "openai" + the model name from ai/ml api
+    model="aiml/meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo", # The model name must include prefix "openai" + the model name from ai/ml api
     api_key="", # your aiml api-key 
     api_base="https://api.aimlapi.com/v2",
     messages=[
@@ -42,7 +60,7 @@ response = litellm.completion(
 import litellm
 
 response = litellm.completion(
-    model="openai/Qwen/Qwen2-72B-Instruct",  # The model name must include prefix "openai" + the model name from ai/ml api
+    model="aiml/Qwen/Qwen2-72B-Instruct",  # The model name must include prefix "openai" + the model name from ai/ml api
     api_key="",  # your aiml api-key 
     api_base="https://api.aimlapi.com/v2",
     messages=[
@@ -67,7 +85,7 @@ import litellm
 
 async def main():
     response = await litellm.acompletion(
-        model="openai/anthropic/claude-3-5-haiku",  # The model name must include prefix "openai" + the model name from ai/ml api
+        model="aiml/anthropic/claude-3-5-haiku",  # The model name must include prefix "openai" + the model name from ai/ml api
         api_key="",  # your aiml api-key
         api_base="https://api.aimlapi.com/v2",
         messages=[
@@ -97,7 +115,7 @@ async def main():
     try:
         print("test acompletion + streaming")
         response = await litellm.acompletion(
-            model="openai/nvidia/Llama-3.1-Nemotron-70B-Instruct-HF", # The model name must include prefix "openai" + the model name from ai/ml api
+            model="aiml/nvidia/Llama-3.1-Nemotron-70B-Instruct-HF", # The model name must include prefix "openai" + the model name from ai/ml api
             api_key="", # your aiml api-key
             api_base="https://api.aimlapi.com/v2",
             messages=[{"content": "Hey, how's it going?", "role": "user"}],
@@ -125,7 +143,7 @@ import litellm
 
 async def main():
     response = await litellm.aembedding(
-        model="openai/text-embedding-3-small", # The model name must include prefix "openai" + the model name from ai/ml api
+        model="aiml/text-embedding-3-small", # The model name must include prefix "openai" + the model name from ai/ml api
         api_key="",  # your aiml api-key
         api_base="https://api.aimlapi.com/v1", # 👈 the URL has changed from v2 to v1
         input="Your text string",
@@ -147,7 +165,7 @@ import litellm
 
 async def main():
     response = await litellm.aimage_generation(
-        model="openai/dall-e-3",  # The model name must include prefix "openai" + the model name from ai/ml api
+        model="aiml/dall-e-3",  # The model name must include prefix "openai" + the model name from ai/ml api
         api_key="",  # your aiml api-key
         api_base="https://api.aimlapi.com/v1", # 👈 the URL has changed from v2 to v1
         prompt="A cute baby sea otter",