Merge branch 'main' into newrelic

.circleci/config.yml

@@ -3496,8 +3496,13 @@ jobs:
           command: |
             npx playwright test e2e_ui_tests/ --reporter=html --output=test-results
           no_output_timeout: 120m
-      - store_test_results:
+      - store_artifacts:
           path: test-results
+          destination: playwright-results
+
+      - store_artifacts:
+          path: playwright-report
+          destination: playwright-report
 
   test_nonroot_image:
     machine:

Dockerfile

@@ -1,8 +1,8 @@
 # Base image for building
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/python:latest-dev
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base
 
 # Runtime image
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/python:latest-dev
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base
 # Builder stage
 FROM $LITELLM_BUILD_IMAGE AS builder
 
@@ -12,11 +12,9 @@ WORKDIR /app
 USER root
 
 # Install build dependencies
-RUN apk add --no-cache gcc python3-dev openssl openssl-dev
+RUN apk add --no-cache bash gcc py3-pip python3 python3-dev openssl openssl-dev
 
-
-RUN pip install --upgrade pip>=24.3.1 && \
-    pip install build
+RUN python -m pip install build
 
 # Copy the current directory contents into the container at /app
 COPY . .
@@ -48,10 +46,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
 USER root
 
 # Install runtime dependencies
-RUN apk add --no-cache openssl tzdata nodejs npm
-
-# Upgrade pip to fix CVE-2025-8869
-RUN pip install --upgrade pip>=24.3.1
+RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip
 
 WORKDIR /app
 # Copy the current directory contents into the container at /app

README.md

@@ -348,7 +348,7 @@ curl 'http://0.0.0.0:4000/key/generate' \
 | [Fireworks AI (`fireworks_ai`)](https://docs.litellm.ai/docs/providers/fireworks_ai) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [FriendliAI (`friendliai`)](https://docs.litellm.ai/docs/providers/friendliai) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Galadriel (`galadriel`)](https://docs.litellm.ai/docs/providers/galadriel) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
-| [GitHub Copilot (`github_copilot`)](https://docs.litellm.ai/docs/providers/github_copilot) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [GitHub Copilot (`github_copilot`)](https://docs.litellm.ai/docs/providers/github_copilot) | ✅ | ✅ | ✅ | ✅ |  |  |  |  |  |  |
 | [GitHub Models (`github`)](https://docs.litellm.ai/docs/providers/github) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Google - PaLM](https://docs.litellm.ai/docs/providers/palm) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Google - Vertex AI (`vertex_ai`)](https://docs.litellm.ai/docs/providers/vertex) | ✅ | ✅ | ✅ | ✅ | ✅ |  |  |  |  |  |

cookbook/mock_guardrail_server/mock_bedrock_guardrail_server.py

@@ -0,0 +1,573 @@
+#!/usr/bin/env python3
+"""
+Mock Bedrock Guardrail API Server
+
+This is a FastAPI server that mimics the AWS Bedrock Guardrail API for testing purposes.
+It follows the same API spec as the real Bedrock guardrail endpoint.
+
+Usage:
+    python mock_bedrock_guardrail_server.py
+
+The server will start on http://localhost:8080
+"""
+
+import os
+import re
+from typing import Any, Dict, List, Literal, Optional
+
+from fastapi import Depends, FastAPI, Header, HTTPException, status
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel, Field
+
+# ============================================================================
+# Request/Response Models (matching Bedrock API spec)
+# ============================================================================
+
+
+class BedrockTextContent(BaseModel):
+    text: str
+
+
+class BedrockContentItem(BaseModel):
+    text: BedrockTextContent
+
+
+class BedrockRequest(BaseModel):
+    source: Literal["INPUT", "OUTPUT"]
+    content: List[BedrockContentItem] = Field(default_factory=list)
+
+
+class BedrockGuardrailOutput(BaseModel):
+    text: Optional[str] = None
+
+
+class TopicPolicyItem(BaseModel):
+    name: str
+    type: str
+    action: Literal["BLOCKED", "NONE"]
+
+
+class TopicPolicy(BaseModel):
+    topics: List[TopicPolicyItem] = Field(default_factory=list)
+
+
+class ContentFilterItem(BaseModel):
+    type: str
+    confidence: str
+    action: Literal["BLOCKED", "NONE"]
+
+
+class ContentPolicy(BaseModel):
+    filters: List[ContentFilterItem] = Field(default_factory=list)
+
+
+class CustomWord(BaseModel):
+    match: str
+    action: Literal["BLOCKED", "NONE"]
+
+
+class WordPolicy(BaseModel):
+    customWords: List[CustomWord] = Field(default_factory=list)
+    managedWordLists: List[Dict[str, Any]] = Field(default_factory=list)
+
+
+class PiiEntity(BaseModel):
+    type: str
+    match: str
+    action: Literal["BLOCKED", "ANONYMIZED", "NONE"]
+
+
+class RegexMatch(BaseModel):
+    name: str
+    match: str
+    regex: str
+    action: Literal["BLOCKED", "ANONYMIZED", "NONE"]
+
+
+class SensitiveInformationPolicy(BaseModel):
+    piiEntities: List[PiiEntity] = Field(default_factory=list)
+    regexes: List[RegexMatch] = Field(default_factory=list)
+
+
+class ContextualGroundingFilter(BaseModel):
+    type: str
+    threshold: float
+    score: float
+    action: Literal["BLOCKED", "NONE"]
+
+
+class ContextualGroundingPolicy(BaseModel):
+    filters: List[ContextualGroundingFilter] = Field(default_factory=list)
+
+
+class Assessment(BaseModel):
+    topicPolicy: Optional[TopicPolicy] = None
+    contentPolicy: Optional[ContentPolicy] = None
+    wordPolicy: Optional[WordPolicy] = None
+    sensitiveInformationPolicy: Optional[SensitiveInformationPolicy] = None
+    contextualGroundingPolicy: Optional[ContextualGroundingPolicy] = None
+
+
+class BedrockGuardrailResponse(BaseModel):
+    usage: Dict[str, int] = Field(
+        default_factory=lambda: {"topicPolicyUnits": 1, "contentPolicyUnits": 1}
+    )
+    action: Literal["NONE", "GUARDRAIL_INTERVENED"] = "NONE"
+    outputs: List[BedrockGuardrailOutput] = Field(default_factory=list)
+    assessments: List[Assessment] = Field(default_factory=list)
+
+
+# ============================================================================
+# Mock Guardrail Configuration
+# ============================================================================
+
+
+class GuardrailConfig(BaseModel):
+    """Configuration for mock guardrail behavior"""
+
+    blocked_words: List[str] = Field(
+        default_factory=lambda: ["offensive", "inappropriate", "badword"]
+    )
+    blocked_topics: List[str] = Field(default_factory=lambda: ["violence", "illegal"])
+    pii_patterns: Dict[str, str] = Field(
+        default_factory=lambda: {
+            "EMAIL": r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b",
+            "PHONE": r"\b\d{3}[-.]?\d{3}[-.]?\d{4}\b",
+            "SSN": r"\b\d{3}-\d{2}-\d{4}\b",
+            "CREDIT_CARD": r"\b\d{4}[\s-]?\d{4}[\s-]?\d{4}[\s-]?\d{4}\b",
+        }
+    )
+    anonymize_pii: bool = True  # If True, ANONYMIZE PII; if False, BLOCK it
+    bearer_token: str = "mock-bedrock-token-12345"
+
+
+# Global config
+GUARDRAIL_CONFIG = GuardrailConfig()
+
+# ============================================================================
+# FastAPI App Setup
+# ============================================================================
+
+app = FastAPI(
+    title="Mock Bedrock Guardrail API",
+    description="Mock server mimicking AWS Bedrock Guardrail API",
+    version="1.0.0",
+)
+
+
+# ============================================================================
+# Authentication
+# ============================================================================
+
+
+async def verify_bearer_token(authorization: Optional[str] = Header(None)) -> str:
+    """
+    Verify the Bearer token from the Authorization header.
+
+    Args:
+        authorization: The Authorization header value
+
+    Returns:
+        The token if valid
+
+    Raises:
+        HTTPException: If token is missing or invalid
+    """
+    if authorization is None:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Missing Authorization header",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+
+    # Check if it's a Bearer token
+    parts = authorization.split()
+    print(f"parts: {parts}")
+    if len(parts) != 2 or parts[0].lower() != "bearer":
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Invalid Authorization header format. Expected: Bearer <token>",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+
+    token = parts[1]
+
+    # Verify token
+    if token != GUARDRAIL_CONFIG.bearer_token:
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail="Invalid bearer token",
+        )
+
+    return token
+
+
+# ============================================================================
+# Guardrail Logic
+# ============================================================================
+
+
+def check_blocked_words(text: str) -> Optional[WordPolicy]:
+    """Check if text contains blocked words"""
+    found_words = []
+    text_lower = text.lower()
+
+    for word in GUARDRAIL_CONFIG.blocked_words:
+        if word.lower() in text_lower:
+            found_words.append(CustomWord(match=word, action="BLOCKED"))
+
+    if found_words:
+        return WordPolicy(customWords=found_words)
+    return None
+
+
+def check_blocked_topics(text: str) -> Optional[TopicPolicy]:
+    """Check if text contains blocked topics"""
+    found_topics = []
+    text_lower = text.lower()
+
+    for topic in GUARDRAIL_CONFIG.blocked_topics:
+        if topic.lower() in text_lower:
+            found_topics.append(
+                TopicPolicyItem(name=topic, type=topic.upper(), action="BLOCKED")
+            )
+
+    if found_topics:
+        return TopicPolicy(topics=found_topics)
+    return None
+
+
+def check_pii(text: str) -> tuple[Optional[SensitiveInformationPolicy], str]:
+    """
+    Check for PII in text and return policy + anonymized text
+
+    Returns:
+        Tuple of (SensitiveInformationPolicy or None, anonymized_text)
+    """
+    pii_entities = []
+    anonymized_text = text
+    action = "ANONYMIZED" if GUARDRAIL_CONFIG.anonymize_pii else "BLOCKED"
+
+    for pii_type, pattern in GUARDRAIL_CONFIG.pii_patterns.items():
+        try:
+            # Compile the regex pattern with a timeout to prevent ReDoS attacks
+            compiled_pattern = re.compile(pattern)
+            matches = compiled_pattern.finditer(text)
+            for match in matches:
+                matched_text = match.group()
+                pii_entities.append(
+                    PiiEntity(type=pii_type, match=matched_text, action=action)
+                )
+
+                # Anonymize the text if configured
+                if GUARDRAIL_CONFIG.anonymize_pii:
+                    anonymized_text = anonymized_text.replace(
+                        matched_text, f"[{pii_type}_REDACTED]"
+                    )
+        except re.error:
+            # Invalid regex pattern - skip it and log a warning
+            print(f"Warning: Invalid regex pattern for PII type {pii_type}: {pattern}")
+            continue
+
+    if pii_entities:
+        return SensitiveInformationPolicy(piiEntities=pii_entities), anonymized_text
+
+    return None, text
+
+
+def process_guardrail_request(
+    request: BedrockRequest,
+) -> tuple[BedrockGuardrailResponse, List[str]]:
+    """
+    Process a guardrail request and return the response.
+
+    Returns:
+        Tuple of (response, list of output texts)
+    """
+    all_text_content = []
+    output_texts = []
+
+    # Extract all text from content items
+    for content_item in request.content:
+        if content_item.text and content_item.text.text:
+            all_text_content.append(content_item.text.text)
+
+    # Combine all text for analysis
+    combined_text = " ".join(all_text_content)
+
+    # Initialize response
+    response = BedrockGuardrailResponse()
+    assessment = Assessment()
+    has_intervention = False
+
+    # Check for blocked words
+    word_policy = check_blocked_words(combined_text)
+    if word_policy:
+        assessment.wordPolicy = word_policy
+        has_intervention = True
+
+    # Check for blocked topics
+    topic_policy = check_blocked_topics(combined_text)
+    if topic_policy:
+        assessment.topicPolicy = topic_policy
+        has_intervention = True
+
+    # Check for PII
+    for text in all_text_content:
+        pii_policy, anonymized_text = check_pii(text)
+        if pii_policy:
+            assessment.sensitiveInformationPolicy = pii_policy
+            if GUARDRAIL_CONFIG.anonymize_pii:
+                # If anonymizing, we don't block, we modify the text
+                output_texts.append(anonymized_text)
+                has_intervention = True
+            else:
+                # If not anonymizing PII, we block it
+                output_texts.append(text)
+                has_intervention = True
+        else:
+            output_texts.append(text)
+
+    # Build response
+    if has_intervention:
+        response.action = "GUARDRAIL_INTERVENED"
+        # Only add assessment if there were interventions
+        response.assessments = [assessment]
+
+    # Add outputs (modified or original text)
+    response.outputs = [BedrockGuardrailOutput(text=txt) for txt in output_texts]
+
+    return response, output_texts
+
+
+# ============================================================================
+# API Endpoints
+# ============================================================================
+
+
+@app.get("/")
+async def root():
+    """Health check endpoint"""
+    return {
+        "service": "Mock Bedrock Guardrail API",
+        "status": "running",
+        "endpoint_format": "/guardrail/{guardrailIdentifier}/version/{guardrailVersion}/apply",
+    }
+
+
+@app.get("/health")
+async def health():
+    """Health check endpoint"""
+    return {"status": "healthy"}
+
+
+@app.post(
+    "/guardrail/{guardrailIdentifier}/version/{guardrailVersion}/apply",
+    response_model=BedrockGuardrailResponse,
+)
+async def apply_guardrail(
+    guardrailIdentifier: str,
+    guardrailVersion: str,
+    request: BedrockRequest,
+    token: str = Depends(verify_bearer_token),
+) -> BedrockGuardrailResponse:
+    """
+    Apply guardrail to input or output content.
+
+    This endpoint mimics the AWS Bedrock ApplyGuardrail API.
+
+    Args:
+        guardrailIdentifier: The guardrail ID
+        guardrailVersion: The guardrail version
+        request: The guardrail request containing content to analyze
+        token: Bearer token (verified by dependency)
+
+    Returns:
+        BedrockGuardrailResponse with analysis results
+    """
+    # Process the request
+    response, output_texts = process_guardrail_request(request)
+
+    # Log the request (optional, for debugging)
+    print(f"Guardrail applied: {guardrailIdentifier} v{guardrailVersion}")
+    print(f"Source: {request.source}")
+    print(f"Action: {response.action}")
+
+    return response
+
+
+"""
+LiteLLM exposes a basic guardrail API with the text extracted from the request and sent to the guardrail API, as well as the received request body for any further processing. 
+
+This works across all LiteLLM endpoints (completion, anthropic /v1/messages, responses api, image generation, embedding, etc.)
+
+This makes it easy to support your own guardrail API without having to make a PR to LiteLLM.
+
+LiteLLM supports passing any provider specific params from LiteLLM config.yaml to the guardrail API.
+
+Example:
+
+```yaml
+guardrails:
+  - guardrail_name: "bedrock-content-guard"
+    litellm_params:
+      guardrail: generic_guardrail_api
+      mode: "pre_call"
+      api_key: os.environ/GUARDRAIL_API_KEY
+      api_base: os.environ/GUARDRAIL_API_BASE
+      additional_provider_specific_params:
+        api_version: os.environ/GUARDRAIL_API_VERSION # additional provider specific params
+```
+
+This is a beta API. Please help us improve it. 
+"""
+
+
+class LitellmBasicGuardrailRequest(BaseModel):
+    texts: List[str]
+    images: Optional[List[str]] = None
+    tools: Optional[List[dict]] = None
+    request_data: Dict[str, Any] = Field(default_factory=dict)
+    additional_provider_specific_params: Dict[str, Any] = Field(default_factory=dict)
+    input_type: Literal["request", "response"]
+    litellm_call_id: Optional[str] = None
+    litellm_trace_id: Optional[str] = None
+
+
+class LitellmBasicGuardrailResponse(BaseModel):
+    action: Literal[
+        "BLOCKED", "NONE", "GUARDRAIL_INTERVENED"
+    ]  # BLOCKED = litellm will raise an error, NONE = litellm will continue, GUARDRAIL_INTERVENED = litellm will continue, but the text was modified by the guardrail
+    blocked_reason: Optional[str] = None  # only if action is BLOCKED, otherwise None
+    texts: Optional[List[str]] = None
+    images: Optional[List[str]] = None
+
+
+@app.post(
+    "/beta/litellm_basic_guardrail_api",
+    response_model=LitellmBasicGuardrailResponse,
+)
+async def beta_litellm_basic_guardrail_api(
+    request: LitellmBasicGuardrailRequest,
+) -> LitellmBasicGuardrailResponse:
+    """
+    Apply guardrail to input or output content.
+
+    This endpoint mimics the AWS Bedrock ApplyGuardrail API.
+
+    Args:
+        request: The guardrail request containing content to analyze
+        token: Bearer token (verified by dependency)
+
+    Returns:
+        LitellmBasicGuardrailResponse with analysis results
+    """
+    print(f"request: {request}")
+    if any("ishaan" in text.lower() for text in request.texts):
+        return LitellmBasicGuardrailResponse(
+            action="BLOCKED", blocked_reason="Ishaan is not allowed"
+        )
+    elif any("pii_value" in text for text in request.texts):
+        return LitellmBasicGuardrailResponse(
+            action="GUARDRAIL_INTERVENED",
+            texts=[
+                text.replace("pii_value", "pii_value_redacted")
+                for text in request.texts
+            ],
+        )
+    return LitellmBasicGuardrailResponse(action="NONE")
+
+
+@app.post("/config/update")
+async def update_config(
+    config: GuardrailConfig, token: str = Depends(verify_bearer_token)
+):
+    """
+    Update the guardrail configuration.
+
+    This is a testing endpoint to modify the mock guardrail behavior.
+
+    Args:
+        config: New guardrail configuration
+        token: Bearer token (verified by dependency)
+
+    Returns:
+        Updated configuration
+    """
+    global GUARDRAIL_CONFIG
+    GUARDRAIL_CONFIG = config
+    return {"status": "updated", "config": GUARDRAIL_CONFIG}
+
+
+@app.get("/config")
+async def get_config(token: str = Depends(verify_bearer_token)):
+    """
+    Get the current guardrail configuration.
+
+    Args:
+        token: Bearer token (verified by dependency)
+
+    Returns:
+        Current configuration
+    """
+    return GUARDRAIL_CONFIG
+
+
+# ============================================================================
+# Error Handlers
+# ============================================================================
+
+
+@app.exception_handler(HTTPException)
+async def http_exception_handler(request, exc: HTTPException):
+    """Custom error handler for HTTP exceptions"""
+    return JSONResponse(
+        status_code=exc.status_code,
+        content={"error": exc.detail},
+        headers=exc.headers,
+    )
+
+
+# ============================================================================
+# Main
+# ============================================================================
+
+if __name__ == "__main__":
+    import uvicorn
+
+    # Get configuration from environment
+    host = os.getenv("MOCK_BEDROCK_HOST", "0.0.0.0")
+    port = int(os.getenv("MOCK_BEDROCK_PORT", "8080"))
+    bearer_token = os.getenv("MOCK_BEDROCK_TOKEN", "mock-bedrock-token-12345")
+
+    # Update config with environment token
+    GUARDRAIL_CONFIG.bearer_token = bearer_token
+
+    print("=" * 80)
+    print("Mock Bedrock Guardrail API Server")
+    print("=" * 80)
+    print(f"Server starting on: http://{host}:{port}")
+    print(f"Bearer Token: {bearer_token}")
+    print(f"Endpoint: POST /guardrail/{{id}}/version/{{version}}/apply")
+    print("=" * 80)
+    print("\nExample curl command:")
+    print(
+        f"""
+curl -X POST "http://{host}:{port}/guardrail/test-guardrail/version/1/apply" \\
+  -H "Authorization: Bearer {bearer_token}" \\
+  -H "Content-Type: application/json" \\
+  -d '{{
+    "source": "INPUT",
+    "content": [
+      {{
+        "text": {{
+          "text": "Hello, my email is test@example.com"
+        }}
+      }}
+    ]
+  }}'
+    """
+    )
+    print("=" * 80)
+
+    uvicorn.run(app, host=host, port=port)

deploy/charts/litellm-helm/Chart.yaml

@@ -18,7 +18,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.4.8
+version: 0.4.9
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
@@ -33,5 +33,5 @@ dependencies:
     condition: db.deployStandalone
   - name: redis
     version: ">=18.0.0"
-    repository: oci://registry-1.docker.io/bitnamicharts 
+    repository: oci://registry-1.docker.io/bitnamicharts
     condition: redis.enabled

deploy/charts/litellm-helm/README.md

@@ -10,46 +10,48 @@
 - Helm 3.8.0+
 
 If `db.deployStandalone` is used:
+
 - PV provisioner support in the underlying infrastructure
 
 If `db.useStackgresOperator` is used (not yet implemented):
-- The Stackgres Operator must already be installed in the Kubernetes Cluster.  This chart will **not** install the operator if it is missing.
+
+- The Stackgres Operator must already be installed in the Kubernetes Cluster. This chart will **not** install the operator if it is missing.
 
 ## Parameters
 
 ### LiteLLM Proxy Deployment Settings
 
-| Name                                                       | Description                                                                                                                                                                           | Value |
-| ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- |
-| `replicaCount`                                             | The number of LiteLLM Proxy pods to be deployed                                                                                                                                       | `1`  |
-| `masterkeySecretName`                                      | The name of the Kubernetes Secret that contains the Master API Key for LiteLLM.  If not specified, use the generated secret name.                                                                                                         | N/A  |
-| `masterkeySecretKey`                                      | The key within the Kubernetes Secret that contains the Master API Key for LiteLLM.  If not specified, use `masterkey` as the key.                                                                                                         | N/A  |
-| `masterkey`                                                | The Master API Key for LiteLLM.  If not specified, a random key in the `sk-...` format is generated.                                                                                   | N/A  |
-| `environmentSecrets`                                       | An optional array of Secret object names.  The keys and values in these secrets will be presented to the LiteLLM proxy pod as environment variables.  See below for an example Secret object.  | `[]`  |
-| `environmentConfigMaps`                                       | An optional array of ConfigMap object names.  The keys and values in these configmaps will be presented to the LiteLLM proxy pod as environment variables.  See below for an example Secret object.  | `[]`  |
-| `image.repository`                                         | LiteLLM Proxy image repository                                                                                                                                                        | `ghcr.io/berriai/litellm`  |
-| `image.pullPolicy`                                         | LiteLLM Proxy image pull policy                                                                                                                                                       | `IfNotPresent`  |
-| `image.tag`                                                | Overrides the image tag whose default the latest version of LiteLLM at the time this chart was published.                                                                             | `""`  |
-| `imagePullSecrets`                                         | Registry credentials for the LiteLLM and initContainer images.                                                                                                                        | `[]`  |
-| `serviceAccount.create`                                    | Whether or not to create a Kubernetes Service Account for this deployment.  The default is `false` because LiteLLM has no need to access the Kubernetes API.                          | `false`  |
-| `service.type`                                             | Kubernetes Service type (e.g. `LoadBalancer`, `ClusterIP`, etc.)                                                                                                                      | `ClusterIP`  |
-| `service.port`                                             | TCP port that the Kubernetes Service will listen on.  Also the TCP port within the Pod that the proxy will listen on.                                                                 | `4000`  |
-| `service.loadBalancerClass`                                | Optional LoadBalancer implementation class (only used when `service.type` is `LoadBalancer`)                                                                                                          | `""`  |
-| `ingress.*`                                                | See [values.yaml](./values.yaml) for example settings                                                                                                                                 | N/A  |
-| `proxyConfigMap.create`     | When `true`, render a ConfigMap from `.Values.proxy_config` and mount it.                                                                                                                                      | `true` |
-| `proxyConfigMap.name`       | When `create=false`, name of the existing ConfigMap to mount.                                                                                                                                                  | `""`  |
-| `proxyConfigMap.key`        | Key in the ConfigMap that contains the proxy config file.                                                                                                                                                      | `"config.yaml"` |
-| `proxy_config.*`            | See [values.yaml](./values.yaml) for default settings. Rendered into the ConfigMap’s `config.yaml` only when `proxyConfigMap.create=true`. See [example_config_yaml](../../../litellm/proxy/example_config_yaml/) for configuration examples. | `N/A` |
-| `extraContainers[]`         | An array of additional containers to be deployed as sidecars alongside the LiteLLM Proxy.
-| `pdb.enabled`                   | Enable a PodDisruptionBudget for the LiteLLM proxy Deployment                                                                 | `false` |
-| `pdb.minAvailable`             | Minimum number/percentage of pods that must be available during **voluntary** disruptions (choose **one** of minAvailable/maxUnavailable) | `null`  |
-| `pdb.maxUnavailable`           | Maximum number/percentage of pods that can be unavailable during **voluntary** disruptions (choose **one** of minAvailable/maxUnavailable) | `null`  |
-| `pdb.annotations`              | Extra metadata annotations to add to the PDB                                                                                   | `{}`    |
-| `pdb.labels`                   | Extra metadata labels to add to the PDB                                                                                        | `{}`    |
+| Name                        | Description                                                                                                                                                                                                                                   | Value                     |
+| --------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------- |
+| `replicaCount`              | The number of LiteLLM Proxy pods to be deployed                                                                                                                                                                                               | `1`                       |
+| `masterkeySecretName`       | The name of the Kubernetes Secret that contains the Master API Key for LiteLLM. If not specified, use the generated secret name.                                                                                                              | N/A                       |
+| `masterkeySecretKey`        | The key within the Kubernetes Secret that contains the Master API Key for LiteLLM. If not specified, use `masterkey` as the key.                                                                                                              | N/A                       |
+| `masterkey`                 | The Master API Key for LiteLLM. If not specified, a random key in the `sk-...` format is generated.                                                                                                                                           | N/A                       |
+| `environmentSecrets`        | An optional array of Secret object names. The keys and values in these secrets will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object.                                                   | `[]`                      |
+| `environmentConfigMaps`     | An optional array of ConfigMap object names. The keys and values in these configmaps will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object.                                             | `[]`                      |
+| `image.repository`          | LiteLLM Proxy image repository                                                                                                                                                                                                                | `ghcr.io/berriai/litellm` |
+| `image.pullPolicy`          | LiteLLM Proxy image pull policy                                                                                                                                                                                                               | `IfNotPresent`            |
+| `image.tag`                 | Overrides the image tag whose default the latest version of LiteLLM at the time this chart was published.                                                                                                                                     | `""`                      |
+| `imagePullSecrets`          | Registry credentials for the LiteLLM and initContainer images.                                                                                                                                                                                | `[]`                      |
+| `serviceAccount.create`     | Whether or not to create a Kubernetes Service Account for this deployment. The default is `false` because LiteLLM has no need to access the Kubernetes API.                                                                                   | `false`                   |
+| `service.type`              | Kubernetes Service type (e.g. `LoadBalancer`, `ClusterIP`, etc.)                                                                                                                                                                              | `ClusterIP`               |
+| `service.port`              | TCP port that the Kubernetes Service will listen on. Also the TCP port within the Pod that the proxy will listen on.                                                                                                                          | `4000`                    |
+| `service.loadBalancerClass` | Optional LoadBalancer implementation class (only used when `service.type` is `LoadBalancer`)                                                                                                                                                  | `""`                      |
+| `ingress.labels`            | Additional labels for the Ingress resource                                                                                                                                                                                                    | `{}`                      |
+| `ingress.*`                 | See [values.yaml](./values.yaml) for example settings                                                                                                                                                                                         | N/A                       |
+| `proxyConfigMap.create`     | When `true`, render a ConfigMap from `.Values.proxy_config` and mount it.                                                                                                                                                                     | `true`                    |
+| `proxyConfigMap.name`       | When `create=false`, name of the existing ConfigMap to mount.                                                                                                                                                                                 | `""`                      |
+| `proxyConfigMap.key`        | Key in the ConfigMap that contains the proxy config file.                                                                                                                                                                                     | `"config.yaml"`           |
+| `proxy_config.*`            | See [values.yaml](./values.yaml) for default settings. Rendered into the ConfigMap’s `config.yaml` only when `proxyConfigMap.create=true`. See [example_config_yaml](../../../litellm/proxy/example_config_yaml/) for configuration examples. | `N/A`                     |
+| `extraContainers[]`         | An array of additional containers to be deployed as sidecars alongside the LiteLLM Proxy.                                                                                                                                                     |
+| `pdb.enabled`               | Enable a PodDisruptionBudget for the LiteLLM proxy Deployment                                                                                                                                                                                 | `false`                   |
+| `pdb.minAvailable`          | Minimum number/percentage of pods that must be available during **voluntary** disruptions (choose **one** of minAvailable/maxUnavailable)                                                                                                     | `null`                    |
+| `pdb.maxUnavailable`        | Maximum number/percentage of pods that can be unavailable during **voluntary** disruptions (choose **one** of minAvailable/maxUnavailable)                                                                                                    | `null`                    |
+| `pdb.annotations`           | Extra metadata annotations to add to the PDB                                                                                                                                                                                                  | `{}`                      |
+| `pdb.labels`                | Extra metadata labels to add to the PDB                                                                                                                                                                                                       | `{}`                      |
 
 #### Example `proxy_config` ConfigMap from values (default):
 
-
 ```
 proxyConfigMap:
   create: true
@@ -67,7 +69,6 @@ proxy_config:
 
 #### Example using existing `proxyConfigMap` instead of creating it:
 
-
 ```
 proxyConfigMap:
   create: false
@@ -77,8 +78,7 @@ proxyConfigMap:
 # proxy_config is ignored in this mode
 ```
 
-#### Example `environmentSecrets` Secret 
-
+#### Example `environmentSecrets` Secret
 
 ```
 apiVersion: v1
@@ -91,21 +91,23 @@ type: Opaque
 ```
 
 ### Database Settings
-| Name                                                       | Description                                                                                                                                                                           | Value |
-| ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- |
-| `db.useExisting`                                           | Use an existing Postgres database.  A Kubernetes Secret object must exist that contains credentials for connecting to the database.  An example secret object definition is provided below.  | `false`  |
-| `db.endpoint`                                              | If `db.useExisting` is `true`, this is the IP, Hostname or Service Name of the Postgres server to connect to.                                                                         | `localhost`  |
-| `db.database`                                              | If `db.useExisting` is `true`, the name of the existing database to connect to.                                                                                                       | `litellm`  |
-| `db.url`                                              | If `db.useExisting` is `true`, the connection url of the existing database to connect to can be overwritten with this value.                                                                                                       | `postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_HOST)/$(DATABASE_NAME)`  |
-| `db.secret.name`                                           | If `db.useExisting` is `true`, the name of the Kubernetes Secret that contains credentials.                                                                                           | `postgres`  |
-| `db.secret.usernameKey`                                    | If `db.useExisting` is `true`, the name of the key within the Kubernetes Secret that holds the username for authenticating with the Postgres instance.                                | `username`  |
-| `db.secret.passwordKey`                                    | If `db.useExisting` is `true`, the name of the key within the Kubernetes Secret that holds the password associates with the above user.                                               | `password`  |
-| `db.useStackgresOperator`                                  | Not yet implemented.                                                                                                                                                                  | `false`  |
-| `db.deployStandalone`                                      | Deploy a standalone, single instance deployment of Postgres, using the Bitnami postgresql chart.  This is useful for getting started but doesn't provide HA or (by default) data backups.  | `true`  |
-| `postgresql.*`                                             | If `db.deployStandalone` is `true`, configuration passed to the Bitnami postgresql chart.   See the [Bitnami Documentation](https://github.com/bitnami/charts/tree/main/bitnami/postgresql) for full configuration details.  See [values.yaml](./values.yaml) for the default configuration.  | See [values.yaml](./values.yaml) |
-| `postgresql.auth.*`                                        | If `db.deployStandalone` is `true`, care should be taken to ensure the default `password` and `postgres-password` values are **NOT** used.                                            | `NoTaGrEaTpAsSwOrD`  |
+
+| Name                      | Description                                                                                                                                                                                                                                                                               | Value                                                                                      |
+| ------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------ |
+| `db.useExisting`          | Use an existing Postgres database. A Kubernetes Secret object must exist that contains credentials for connecting to the database. An example secret object definition is provided below.                                                                                                 | `false`                                                                                    |
+| `db.endpoint`             | If `db.useExisting` is `true`, this is the IP, Hostname or Service Name of the Postgres server to connect to.                                                                                                                                                                             | `localhost`                                                                                |
+| `db.database`             | If `db.useExisting` is `true`, the name of the existing database to connect to.                                                                                                                                                                                                           | `litellm`                                                                                  |
+| `db.url`                  | If `db.useExisting` is `true`, the connection url of the existing database to connect to can be overwritten with this value.                                                                                                                                                              | `postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_HOST)/$(DATABASE_NAME)` |
+| `db.secret.name`          | If `db.useExisting` is `true`, the name of the Kubernetes Secret that contains credentials.                                                                                                                                                                                               | `postgres`                                                                                 |
+| `db.secret.usernameKey`   | If `db.useExisting` is `true`, the name of the key within the Kubernetes Secret that holds the username for authenticating with the Postgres instance.                                                                                                                                    | `username`                                                                                 |
+| `db.secret.passwordKey`   | If `db.useExisting` is `true`, the name of the key within the Kubernetes Secret that holds the password associates with the above user.                                                                                                                                                   | `password`                                                                                 |
+| `db.useStackgresOperator` | Not yet implemented.                                                                                                                                                                                                                                                                      | `false`                                                                                    |
+| `db.deployStandalone`     | Deploy a standalone, single instance deployment of Postgres, using the Bitnami postgresql chart. This is useful for getting started but doesn't provide HA or (by default) data backups.                                                                                                  | `true`                                                                                     |
+| `postgresql.*`            | If `db.deployStandalone` is `true`, configuration passed to the Bitnami postgresql chart. See the [Bitnami Documentation](https://github.com/bitnami/charts/tree/main/bitnami/postgresql) for full configuration details. See [values.yaml](./values.yaml) for the default configuration. | See [values.yaml](./values.yaml)                                                           |
+| `postgresql.auth.*`       | If `db.deployStandalone` is `true`, care should be taken to ensure the default `password` and `postgres-password` values are **NOT** used.                                                                                                                                                | `NoTaGrEaTpAsSwOrD`                                                                        |
 
 #### Example Postgres `db.useExisting` Secret
+
 ```yaml
 apiVersion: v1
 kind: Secret
@@ -143,7 +145,7 @@ metadata:
   name: litellm-env-secret
 type: Opaque
 data:
-  SOME_PASSWORD: cDZbUGVXeU5e0ZW    # base64 encoded
+  SOME_PASSWORD: cDZbUGVXeU5e0ZW # base64 encoded
   ANOTHER_PASSWORD: AAZbUGVXeU5e0ZB # base64 encoded
 ```
 
@@ -153,23 +155,23 @@ Source: [GitHub Gist from troyharvey](https://gist.github.com/troyharvey/4506472
 
 The migration job supports both ArgoCD and Helm hooks to ensure database migrations run at the appropriate time during deployments.
 
-| Name                                                       | Description                                                                                                                                                                           | Value |
-| ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- |
-| `migrationJob.enabled`                                     | Enable or disable the schema migration Job                                                                                                                                           | `true`  |
-| `migrationJob.backoffLimit`                                | Backoff limit for Job restarts                                                                                                                                                       | `4`  |
-| `migrationJob.ttlSecondsAfterFinished`                     | TTL for completed migration jobs                                                                                                                                                      | `120`  |
-| `migrationJob.annotations`                                 | Additional annotations for the migration job pod                                                                                                                                      | `{}`  |
-| `migrationJob.extraContainers`                             | Additional containers to run alongside the migration job                                                                                                                              | `[]`  |
-| `migrationJob.hooks.argocd.enabled`                       | Enable ArgoCD hooks for the migration job (uses PreSync hook with BeforeHookCreation delete policy)                                                                                  | `true`  |
-| `migrationJob.hooks.helm.enabled`                         | Enable Helm hooks for the migration job (uses pre-install,pre-upgrade hooks with before-hook-creation delete policy)                                                                 | `false`  |
-| `migrationJob.hooks.helm.weight`                          | Helm hook execution order (lower weights executed first). Optional - defaults to "1" if not specified.                                                                               | N/A  |
-
+| Name                                   | Description                                                                                                          | Value   |
+| -------------------------------------- | -------------------------------------------------------------------------------------------------------------------- | ------- |
+| `migrationJob.enabled`                 | Enable or disable the schema migration Job                                                                           | `true`  |
+| `migrationJob.backoffLimit`            | Backoff limit for Job restarts                                                                                       | `4`     |
+| `migrationJob.ttlSecondsAfterFinished` | TTL for completed migration jobs                                                                                     | `120`   |
+| `migrationJob.annotations`             | Additional annotations for the migration job pod                                                                     | `{}`    |
+| `migrationJob.extraContainers`         | Additional containers to run alongside the migration job                                                             | `[]`    |
+| `migrationJob.hooks.argocd.enabled`    | Enable ArgoCD hooks for the migration job (uses PreSync hook with BeforeHookCreation delete policy)                  | `true`  |
+| `migrationJob.hooks.helm.enabled`      | Enable Helm hooks for the migration job (uses pre-install,pre-upgrade hooks with before-hook-creation delete policy) | `false` |
+| `migrationJob.hooks.helm.weight`       | Helm hook execution order (lower weights executed first). Optional - defaults to "1" if not specified.               | N/A     |
 
 ## Accessing the Admin UI
+
 When browsing to the URL published per the settings in `ingress.*`, you will
-be prompted for **Admin Configuration**.  The **Proxy Endpoint** is the internal
+be prompted for **Admin Configuration**. The **Proxy Endpoint** is the internal
 (from the `litellm` pod's perspective) URL published by the `<RELEASE>-litellm`
-Kubernetes Service.  If the deployment uses the default settings for this
+Kubernetes Service. If the deployment uses the default settings for this
 service, the **Proxy Endpoint** should be set to `http://<RELEASE>-litellm:4000`.
 
 The **Proxy Key** is the value specified for `masterkey` or, if a `masterkey`
@@ -181,7 +183,8 @@ kubectl -n litellm get secret <RELEASE>-litellm-masterkey -o jsonpath="{.data.ma
 ```
 
 ## Admin UI Limitations
-At the time of writing, the Admin UI is unable to add models.  This is because
+
+At the time of writing, the Admin UI is unable to add models. This is because
 it would need to update the `config.yaml` file which is a exposed ConfigMap, and
-therefore, read-only.  This is a limitation of this helm chart, not the Admin UI
+therefore, read-only. This is a limitation of this helm chart, not the Admin UI
 itself.

deploy/charts/litellm-helm/templates/ingress.yaml

@@ -18,6 +18,9 @@ metadata:
   name: {{ $fullName }}
   labels:
     {{- include "litellm.labels" . | nindent 4 }}
+    {{- with .Values.ingress.labels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
   {{- with .Values.ingress.annotations }}
   annotations:
     {{- toYaml . | nindent 4 }}

deploy/charts/litellm-helm/tests/ingress_tests.yaml

@@ -0,0 +1,45 @@
+suite: Ingress Configuration Tests
+templates:
+  - ingress.yaml
+tests:
+  - it: should not create Ingress by default
+    asserts:
+      - hasDocuments:
+          count: 0
+
+  - it: should create Ingress when enabled
+    set:
+      ingress.enabled: true
+    asserts:
+      - hasDocuments:
+          count: 1
+      - isKind:
+          of: Ingress
+
+  - it: should add custom labels
+    set:
+      ingress.enabled: true
+      ingress.labels:
+        custom-label: "true"
+        another-label: "value"
+    asserts:
+      - isKind:
+          of: Ingress
+      - equal:
+          path: metadata.labels.custom-label
+          value: "true"
+      - equal:
+          path: metadata.labels.another-label
+          value: "value"
+
+  - it: should add annotations
+    set:
+      ingress.enabled: true
+      ingress.annotations:
+        kubernetes.io/ingress.class: "nginx"
+    asserts:
+      - isKind:
+          of: Ingress
+      - equal:
+          path: metadata.annotations["kubernetes.io/ingress.class"]
+          value: "nginx"

deploy/charts/litellm-helm/values.yaml

@@ -35,7 +35,8 @@ podAnnotations: {}
 podLabels: {}
 
 terminationGracePeriodSeconds: 90
-topologySpreadConstraints: []
+topologySpreadConstraints:
+  []
   # - maxSkew: 1
   #   topologyKey: kubernetes.io/hostname
   #   whenUnsatisfiable: DoNotSchedule
@@ -46,7 +47,8 @@ topologySpreadConstraints: []
 # At the time of writing, the litellm docker image requires write access to the
 #  filesystem on startup so that prisma can install some dependencies.
 podSecurityContext: {}
-securityContext: {}
+securityContext:
+  {}
   # capabilities:
   #   drop:
   #     - ALL
@@ -57,13 +59,15 @@ securityContext: {}
 # A list of Kubernetes Secret objects that will be exported to the LiteLLM proxy
 #  pod as environment variables.  These secrets can then be referenced in the
 #  configuration file (or "litellm" ConfigMap) with `os.environ/<Env Var Name>`
-environmentSecrets: []
+environmentSecrets:
+  []
   # - litellm-env-secret
 
 # A list of Kubernetes ConfigMap objects that will be exported to the LiteLLM proxy
 #  pod as environment variables.  The ConfigMap kv-pairs can then be referenced in the
 #  configuration file (or "litellm" ConfigMap) with `os.environ/<Env Var Name>`
-environmentConfigMaps: []
+environmentConfigMaps:
+  []
   # - litellm-env-configmap
 
 service:
@@ -82,7 +86,9 @@ separateHealthPort: 8081
 ingress:
   enabled: false
   className: "nginx"
-  annotations: {}
+  labels: {}
+  annotations:
+    {}
     # kubernetes.io/ingress.class: nginx
     # kubernetes.io/tls-acme: "true"
   hosts:
@@ -129,7 +135,8 @@ proxy_config:
   general_settings:
     master_key: os.environ/PROXY_MASTER_KEY
 
-resources: {}
+resources:
+  {}
   # We usually recommend not to specify default resources and to leave this as a conscious
   # choice for the user. This also increases chances charts run on environments with little
   # resources, such as Minikube. If you do want to specify resources, uncomment the following
@@ -231,7 +238,7 @@ migrationJob:
   #    cpu: 100m
   #    memory: 100Mi
   extraContainers: []
-  
+
   # Hook configuration
   hooks:
     argocd:
@@ -240,30 +247,30 @@ migrationJob:
       enabled: false
 
 # Additional environment variables to be added to the deployment as a map of key-value pairs
-envVars: {
-    # USE_DDTRACE: "true"
-}
+envVars: {}
 
+# USE_DDTRACE: "true"
 # Additional environment variables to be added to the deployment as a list of k8s env vars
-extraEnvVars: {
-    # - name: EXTRA_ENV_VAR
-    #   value: EXTRA_ENV_VAR_VALUE
-}
+extraEnvVars: {}
 
+# - name: EXTRA_ENV_VAR
+#   value: EXTRA_ENV_VAR_VALUE
 # Pod Disruption Budget
 pdb:
   enabled: false
   # Set exactly one of the following. If both are set, minAvailable takes precedence.
-  minAvailable: null     # e.g. "50%" or 1
-  maxUnavailable: null   # e.g. 1 or "20%"
+  minAvailable: null # e.g. "50%" or 1
+  maxUnavailable: null # e.g. 1 or "20%"
   annotations: {}
   labels: {}
 
 serviceMonitor:
   enabled: false
-  labels: {}
+  labels:
+    {}
     # test: test
-  annotations: {}
+  annotations:
+    {}
     # kubernetes.io/test: test
   interval: 15s
   scrapeTimeout: 10s
@@ -273,4 +280,4 @@ serviceMonitor:
   #   action: replace
   namespaceSelector:
     matchNames: []
-    # - test-namespace
+    # - test-namespace

docker/Dockerfile.database

@@ -1,8 +1,8 @@
 # Base image for building
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/python:latest-dev
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base
 
 # Runtime image
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/python:latest-dev
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base
 # Builder stage
 FROM $LITELLM_BUILD_IMAGE AS builder
 
@@ -13,13 +13,15 @@ USER root
 
 # Install build dependencies
 RUN apk add --no-cache \
-    build-base \
+    bash \
+    gcc \
+    py3-pip \
+    python3 \
     python3-dev \
+    openssl \
     openssl-dev
 
-
-RUN pip install --upgrade pip && \
-    pip install build
+RUN python -m pip install build
 
 # Copy the current directory contents into the container at /app
 COPY . .
@@ -46,7 +48,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
 USER root
 
 # Install runtime dependencies
-RUN apk add --no-cache openssl
+RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip
 
 WORKDIR /app
 # Copy the current directory contents into the container at /app

docker/Dockerfile.non_root

@@ -1,6 +1,6 @@
 # Base images
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/python:latest-dev
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/python:latest-dev
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base
 
 # -----------------
 # Builder Stage
@@ -10,7 +10,18 @@ WORKDIR /app
 
 # Install build dependencies including Node.js for UI build
 USER root
-RUN apk add --no-cache build-base bash nodejs npm \
+RUN apk add --no-cache \
+    python3 \
+    py3-pip \
+    clang \
+    llvm \
+    lld \
+    gcc \
+    linux-headers \
+    build-base \
+    bash \
+    nodejs \
+    npm \
   && pip install --no-cache-dir --upgrade pip build
 
 # Copy project files
@@ -62,7 +73,7 @@ WORKDIR /app
 # Install runtime dependencies
 USER root
 RUN apk upgrade --no-cache && \
-  apk add --no-cache bash libstdc++ ca-certificates openssl supervisor
+  apk add --no-cache python3 py3-pip bash openssl tzdata nodejs npm supervisor
 
 # Copy only necessary artifacts from builder stage for runtime
 COPY . .

docs/my-website/docs/a2a.md

@@ -0,0 +1,204 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import Image from '@theme/IdealImage';
+
+# /a2a - Agent Gateway (A2A Protocol)
+
+| Feature | Supported | 
+|---------|-----------|
+| Logging | ✅ |
+| Load Balancing | ✅ |
+| Streaming | ✅ |
+
+:::tip
+
+LiteLLM follows the [A2A (Agent-to-Agent) Protocol](https://github.com/google/A2A) for invoking agents.
+
+:::
+
+## Adding your Agent
+
+You can add A2A-compatible agents through the LiteLLM Admin UI.
+
+1. Navigate to the **Agents** tab
+2. Click **Add Agent**
+3. Enter the agent name (e.g., `ij-local`) and the URL of your A2A agent
+
+<Image 
+  img={require('../img/add_agent_1.png')}
+  style={{width: '80%', display: 'block', margin: '0'}}
+/>
+
+The URL should be the invocation URL for your A2A agent (e.g., `http://localhost:10001`).
+
+## Invoking your Agents
+
+Use the [A2A Python SDK](https://pypi.org/project/a2a/) to invoke agents through LiteLLM:
+
+- `base_url`: Your LiteLLM proxy URL + `/a2a/{agent_name}`
+- `headers`: Include your LiteLLM Virtual Key for authentication
+
+```python showLineNumbers title="invoke_a2a_agent.py"
+from uuid import uuid4
+import httpx
+import asyncio
+from a2a.client import A2ACardResolver, A2AClient
+from a2a.types import MessageSendParams, SendMessageRequest
+
+# === CONFIGURE THESE ===
+LITELLM_BASE_URL = "http://localhost:4000"  # Your LiteLLM proxy URL
+LITELLM_VIRTUAL_KEY = "sk-1234"             # Your LiteLLM Virtual Key
+LITELLM_AGENT_NAME = "ij-local"             # Agent name registered in LiteLLM
+# =======================
+
+async def main():
+    base_url = f"{LITELLM_BASE_URL}/a2a/{LITELLM_AGENT_NAME}"
+    headers = {"Authorization": f"Bearer {LITELLM_VIRTUAL_KEY}"}
+    
+    async with httpx.AsyncClient(headers=headers) as httpx_client:
+        # Resolve agent card and create client
+        resolver = A2ACardResolver(httpx_client=httpx_client, base_url=base_url)
+        agent_card = await resolver.get_agent_card()
+        client = A2AClient(httpx_client=httpx_client, agent_card=agent_card)
+
+        # Send a message
+        request = SendMessageRequest(
+            id=str(uuid4()),
+            params=MessageSendParams(
+                message={
+                    "role": "user",
+                    "parts": [{"kind": "text", "text": "Hello, what can you do?"}],
+                    "messageId": uuid4().hex,
+                }
+            ),
+        )
+        response = await client.send_message(request)
+        print(response.model_dump(mode="json", exclude_none=True))
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+### Streaming Responses
+
+For streaming responses, use `send_message_streaming`:
+
+```python showLineNumbers title="invoke_a2a_agent_streaming.py"
+from uuid import uuid4
+import httpx
+import asyncio
+from a2a.client import A2ACardResolver, A2AClient
+from a2a.types import MessageSendParams, SendStreamingMessageRequest
+
+# === CONFIGURE THESE ===
+LITELLM_BASE_URL = "http://localhost:4000"  # Your LiteLLM proxy URL
+LITELLM_VIRTUAL_KEY = "sk-1234"             # Your LiteLLM Virtual Key
+LITELLM_AGENT_NAME = "ij-local"             # Agent name registered in LiteLLM
+# =======================
+
+async def main():
+    base_url = f"{LITELLM_BASE_URL}/a2a/{LITELLM_AGENT_NAME}"
+    headers = {"Authorization": f"Bearer {LITELLM_VIRTUAL_KEY}"}
+    
+    async with httpx.AsyncClient(headers=headers) as httpx_client:
+        # Resolve agent card and create client
+        resolver = A2ACardResolver(httpx_client=httpx_client, base_url=base_url)
+        agent_card = await resolver.get_agent_card()
+        client = A2AClient(httpx_client=httpx_client, agent_card=agent_card)
+
+        # Send a streaming message
+        request = SendStreamingMessageRequest(
+            id=str(uuid4()),
+            params=MessageSendParams(
+                message={
+                    "role": "user",
+                    "parts": [{"kind": "text", "text": "Hello, what can you do?"}],
+                    "messageId": uuid4().hex,
+                }
+            ),
+        )
+        
+        # Stream the response
+        async for chunk in client.send_message_streaming(request):
+            print(chunk.model_dump(mode="json", exclude_none=True))
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+## Tracking Agent Logs
+
+After invoking an agent, you can view the request logs in the LiteLLM **Logs** tab.
+
+The logs show:
+- **Request/Response content** sent to and received from the agent
+- **User, Key, Team** information for tracking who made the request
+- **Latency and cost** metrics
+
+<Image 
+  img={require('../img/agent2.png')}
+  style={{width: '100%', display: 'block', margin: '2rem auto'}}
+/>
+
+## API Reference
+
+### Endpoint
+
+```
+POST /a2a/{agent_name}/message/send
+```
+
+### Authentication
+
+Include your LiteLLM Virtual Key in the `Authorization` header:
+
+```
+Authorization: Bearer sk-your-litellm-key
+```
+
+### Request Format
+
+LiteLLM follows the [A2A JSON-RPC 2.0 specification](https://github.com/google/A2A):
+
+```json title="Request Body"
+{
+  "jsonrpc": "2.0",
+  "id": "unique-request-id",
+  "method": "message/send",
+  "params": {
+    "message": {
+      "role": "user",
+      "parts": [{"kind": "text", "text": "Your message here"}],
+      "messageId": "unique-message-id"
+    }
+  }
+}
+```
+
+### Response Format
+
+```json title="Response"
+{
+  "jsonrpc": "2.0",
+  "id": "unique-request-id",
+  "result": {
+    "kind": "task",
+    "id": "task-id",
+    "contextId": "context-id",
+    "status": {"state": "completed", "timestamp": "2025-01-01T00:00:00Z"},
+    "artifacts": [
+      {
+        "artifactId": "artifact-id",
+        "name": "response",
+        "parts": [{"kind": "text", "text": "Agent response here"}]
+      }
+    ]
+  }
+}
+```
+
+## Agent Registry
+
+Want to create a central registry so your team can discover what agents are available within your company?
+
+Use the [AI Hub](./proxy/ai_hub) to make agents public and discoverable across your organization. This allows developers to browse available agents without needing to rebuild them.

docs/my-website/docs/adding_provider/generic_guardrail_api.md

@@ -0,0 +1,269 @@
+# [BETA] Generic Guardrail API - Integrate Without a PR
+
+## The Problem
+
+As a guardrail provider, integrating with LiteLLM traditionally requires:
+- Making a PR to the LiteLLM repository
+- Waiting for review and merge
+- Maintaining provider-specific code in LiteLLM's codebase
+- Updating the integration for changes to your API
+
+## The Solution
+
+The **Generic Guardrail API** lets you integrate with LiteLLM **instantly** by implementing a simple API endpoint. No PR required.
+
+### Key Benefits
+
+1. **No PR Needed** - Deploy and integrate immediately
+2. **Universal Support** - Works across ALL LiteLLM endpoints (chat, embeddings, image generation, etc.)
+3. **Simple Contract** - One endpoint, three response types
+4. **Multi-Modal Support** - Handle both text and images in requests/responses
+5. **Custom Parameters** - Pass provider-specific params via config
+6. **Full Control** - You own and maintain your guardrail API
+
+## Supported Endpoints
+
+The Generic Guardrail API works with the following LiteLLM endpoints:
+
+- `/v1/chat/completions` - OpenAI Chat Completions
+- `/v1/completions` - OpenAI Text Completions
+- `/v1/responses` - OpenAI Responses API
+- `/v1/images/generations` - OpenAI Image Generation
+- `/v1/audio/transcriptions` - OpenAI Audio Transcriptions
+- `/v1/audio/speech` - OpenAI Text-to-Speech
+- `/v1/messages` - Anthropic Messages
+- `/v1/rerank` - Cohere Rerank
+- Pass-through endpoints
+
+## How It Works
+
+1. LiteLLM extracts text and images from any request (chat messages, embeddings, image prompts, etc.)
+2. Sends extracted content + metadata to your API endpoint
+3. Your API responds with: `BLOCKED`, `NONE`, or `GUARDRAIL_INTERVENED`
+4. LiteLLM enforces the decision and applies any modifications
+
+## API Contract
+
+### Endpoint
+
+Implement `POST /beta/litellm_basic_guardrail_api`
+
+### Request Format
+
+```json
+{
+  "texts": ["extracted text from the request"],  // array of text strings
+  "images": ["base64_encoded_image_data"],  // optional array of images
+  "tools": [  // optional array of tools (OpenAI ChatCompletionToolParam format)
+    {
+      "type": "function",
+      "function": {
+        "name": "get_weather",
+        "description": "Get the current weather",
+        "parameters": {
+          "type": "object",
+          "properties": {
+            "location": {"type": "string"}
+          }
+        }
+      }
+    }
+  ],
+  "request_data": {
+    "user_api_key_hash": "hash of the litellm virtual key used",
+    "user_api_key_alias": "alias of the litellm virtual key used",
+    "user_api_key_user_id": "user id associated with the litellm virtual key used",
+    "user_api_key_user_email": "user email associated with the litellm virtual key used",
+    "user_api_key_team_id": "team id associated with the litellm virtual key used",
+    "user_api_key_team_alias": "team alias associated with the litellm virtual key used",
+    "user_api_key_end_user_id": "end user id associated with the litellm virtual key used",
+    "user_api_key_org_id": "org id associated with the litellm virtual key used"
+  },
+  "input_type": "request",  // "request" or "response"
+  "litellm_call_id": "unique_call_id",  // the call id of the individual LLM call
+  "litellm_trace_id": "trace_id",  // the trace id of the LLM call - useful if there are multiple LLM calls for the same conversation
+  "additional_provider_specific_params": {
+    // your custom params from config
+  }
+}
+```
+
+### Response Format
+
+```json
+{
+  "action": "BLOCKED" | "NONE" | "GUARDRAIL_INTERVENED",
+  "blocked_reason": "why content was blocked",  // required if action=BLOCKED
+  "texts": ["modified text"],  // optional array of modified text strings
+  "images": ["modified_base64_image"]  // optional array of modified images
+}
+```
+
+**Actions:**
+- `BLOCKED` - LiteLLM raises error and blocks request
+- `NONE` - Request proceeds unchanged  
+- `GUARDRAIL_INTERVENED` - Request proceeds with modified texts/images (provide `texts` and/or `images` fields)
+
+## Parameters
+
+### `tools` Parameter
+
+The `tools` parameter provides information about available function/tool definitions in the request.
+
+**Format:** OpenAI `ChatCompletionToolParam` format (see [OpenAI API reference](https://platform.openai.com/docs/api-reference/chat/create#chat-create-tools))
+
+**Example:**
+```json
+{
+  "type": "function",
+  "function": {
+    "name": "get_weather",
+    "description": "Get the current weather in a location",
+    "parameters": {
+      "type": "object",
+      "properties": {
+        "location": {
+          "type": "string",
+          "description": "City and state, e.g. San Francisco, CA"
+        },
+        "unit": {
+          "type": "string",
+          "enum": ["celsius", "fahrenheit"]
+        }
+      },
+      "required": ["location"]
+    }
+  }
+}
+```
+
+**Limitations:**
+- **Input only:** Tools are only passed for `input_type="request"` (pre-call guardrails). Output/response guardrails do not currently receive tool information.
+- **Supported endpoints:** The `tools` parameter is supported on: `/v1/chat/completions`, `/v1/responses`, and `/v1/messages`. Other endpoints do not have tool support.
+
+**Use cases:**
+- Enforce tool permission policies (e.g., only allow certain users/teams to access specific tools)
+- Validate tool schemas before sending to LLM
+- Log tool usage for audit purposes
+- Block sensitive tools based on user context
+
+## LiteLLM Configuration
+
+Add to `config.yaml`:
+
+```yaml
+litellm_settings:
+  guardrails:
+    - guardrail_name: "my-guardrail"
+      litellm_params:
+        guardrail: generic_guardrail_api
+        mode: pre_call  # or post_call, during_call
+        api_base: https://your-guardrail-api.com
+        api_key: os.environ/YOUR_GUARDRAIL_API_KEY  # optional
+        additional_provider_specific_params:
+          # your custom parameters
+          threshold: 0.8
+          language: "en"
+```
+
+## Usage
+
+Users apply your guardrail by name:
+
+```python
+response = client.chat.completions.create(
+    model="gpt-4",
+    messages=[{"role": "user", "content": "hello"}],
+    guardrails=["my-guardrail"]
+)
+```
+
+Or with dynamic parameters:
+
+```python
+response = client.chat.completions.create(
+    model="gpt-4",
+    messages=[{"role": "user", "content": "hello"}],
+    guardrails=[{
+        "my-guardrail": {
+            "extra_body": {
+                "custom_threshold": 0.9
+            }
+        }
+    }]
+)
+```
+
+## Implementation Example
+
+See [mock_bedrock_guardrail_server.py](https://github.com/BerriAI/litellm/blob/main/cookbook/mock_guardrail_server/mock_bedrock_guardrail_server.py) for a complete reference implementation.
+
+**Minimal FastAPI example:**
+
+```python
+from fastapi import FastAPI
+from pydantic import BaseModel
+from typing import List, Optional, Dict, Any
+
+app = FastAPI()
+
+class GuardrailRequest(BaseModel):
+    texts: List[str]
+    images: Optional[List[str]] = None
+    tools: Optional[List[Dict[str, Any]]] = None  # OpenAI ChatCompletionToolParam format
+    request_data: Dict[str, Any]
+    input_type: str  # "request" or "response"
+    litellm_call_id: Optional[str] = None
+    litellm_trace_id: Optional[str] = None
+    additional_provider_specific_params: Dict[str, Any]
+
+class GuardrailResponse(BaseModel):
+    action: str  # BLOCKED, NONE, or GUARDRAIL_INTERVENED
+    blocked_reason: Optional[str] = None
+    texts: Optional[List[str]] = None
+    images: Optional[List[str]] = None
+
+@app.post("/beta/litellm_basic_guardrail_api")
+async def apply_guardrail(request: GuardrailRequest):
+    # Your guardrail logic here
+    
+    # Example: Check text content
+    for text in request.texts:
+        if "badword" in text.lower():
+            return GuardrailResponse(
+                action="BLOCKED",
+                blocked_reason="Content contains prohibited terms"
+            )
+    
+    # Example: Check tools (if present in request)
+    if request.tools:
+        for tool in request.tools:
+            if tool.get("type") == "function":
+                function_name = tool.get("function", {}).get("name", "")
+                # Block sensitive tools
+                if function_name in ["delete_data", "access_admin_panel"]:
+                    return GuardrailResponse(
+                        action="BLOCKED",
+                        blocked_reason=f"Tool '{function_name}' is not allowed"
+                    )
+    
+    return GuardrailResponse(action="NONE")
+```
+
+## When to Use This
+
+✅ **Use Generic Guardrail API when:**
+- You want instant integration without waiting for PRs
+- You maintain your own guardrail service
+- You need full control over updates and features
+- You want to support all LiteLLM endpoints automatically
+
+❌ **Make a PR when:**
+- You want deeper integration with LiteLLM internals
+- Your guardrail requires complex LiteLLM-specific logic
+- You want to be featured as a built-in provider
+
+## Questions?
+
+This is a **beta API**. We're actively improving it based on feedback. Open an issue or PR if you need additional capabilities.
+

docs/my-website/docs/audio_transcription.md

@@ -13,7 +13,7 @@ import TabItem from '@theme/TabItem';
 | Fallbacks | ✅ | Works between supported models |
 | Loadbalancing | ✅ | Works between supported models |
 | Guardrails | ✅ | Applies to output transcribed text (non-streaming only) |
-| Supported Providers | `openai`, `azure`, `vertex_ai`, `gemini`, `deepgram`, `groq`, `fireworks_ai` | |
+| Supported Providers | `openai`, `azure`, `vertex_ai`, `gemini`, `deepgram`, `groq`, `fireworks_ai`, `ovhcloud` | |
 
 ## Quick Start
 
@@ -126,6 +126,7 @@ transcript = client.audio.transcriptions.create(
 - [Fireworks AI](./providers/fireworks_ai.md#audio-transcription)
 - [Groq](./providers/groq.md#speech-to-text---whisper)
 - [Deepgram](./providers/deepgram.md)
+- [OVHcloud AI Endpoints](./providers/ovhcloud.md)
 
 ---
 

docs/my-website/docs/completion/image_generation_chat.md

@@ -224,8 +224,8 @@ asyncio.run(generate_image())
 
 | Provider | Model | 
 |----------|--------|
-| Google AI Studio | `gemini/gemini-2.0-flash-preview-image-generation`, `gemini/gemini-2.5-flash-image-preview` |
-| Vertex AI | `vertex_ai/gemini-2.0-flash-preview-image-generation`, `vertex_ai/gemini-2.5-flash-image-preview` |
+| Google AI Studio | `gemini/gemini-2.0-flash-preview-image-generation`, `gemini/gemini-2.5-flash-image-preview`, `gemini/gemini-3-pro-image-preview` |
+| Vertex AI | `vertex_ai/gemini-2.0-flash-preview-image-generation`, `vertex_ai/gemini-2.5-flash-image-preview`, `vertex_ai/gemini-3-pro-image-preview` |
 
 ## Spec
 

docs/my-website/docs/completion/knowledgebase.md

@@ -20,6 +20,8 @@ LiteLLM integrates with vector stores, allowing your models to access your organ
 - [OpenAI Vector Stores](https://platform.openai.com/docs/api-reference/vector-stores/search)
 - [Azure Vector Stores](https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/file-search?tabs=python#vector-stores) (Cannot be directly queried. Only available for calling in Assistants messages. We will be adding Azure AI Search Vector Store API support soon.)
 - [Vertex AI RAG API](https://cloud.google.com/vertex-ai/generative-ai/docs/rag-overview)
+- [Gemini File Search](https://ai.google.dev/gemini-api/docs/file-search)
+- [RAGFlow Datasets](/docs/providers/ragflow_vector_store.md) (Dataset management only, search not supported)
 
 ## Quick Start
 

docs/my-website/docs/contribute_integration/custom_webhook_api.md

@@ -0,0 +1,106 @@
+# Contribute Custom Webhook API
+
+If your API just needs a Webhook event from LiteLLM, here's how to add a 'native' integration for it on LiteLLM: 
+
+1. Clone the repo and open the `generic_api_compatible_callbacks.json`
+
+```bash
+git clone https://github.com/BerriAI/litellm.git
+cd litellm
+open .
+```
+
+2. Add your API to the `generic_api_compatible_callbacks.json`
+
+Example:
+
+```json
+{
+    "rubrik": {
+        "event_types": ["llm_api_success"],
+        "endpoint": "{{environment_variables.RUBRIK_WEBHOOK_URL}}",
+        "headers": {
+            "Content-Type": "application/json",
+            "Authorization": "Bearer {{environment_variables.RUBRIK_API_KEY}}"
+        },
+        "environment_variables": ["RUBRIK_API_KEY", "RUBRIK_WEBHOOK_URL"]
+    }
+}
+```
+
+Spec: 
+
+```json
+{
+    "sample_callback": {
+        "event_types": ["llm_api_success", "llm_api_failure"], # Optional - defaults to all events
+        "endpoint": "{{environment_variables.SAMPLE_CALLBACK_URL}}",
+        "headers": {
+            "Content-Type": "application/json",
+            "Authorization": "Bearer {{environment_variables.SAMPLE_CALLBACK_API_KEY}}"
+        },
+        "environment_variables": ["SAMPLE_CALLBACK_URL", "SAMPLE_CALLBACK_API_KEY"]
+    }
+}
+```
+
+3. Test it! 
+
+a. Setup config.yaml
+
+```yaml
+model_list:
+  - model_name: gpt-3.5-turbo
+    litellm_params:
+      model: openai/gpt-3.5-turbo
+      api_key: os.environ/OPENAI_API_KEY
+  - model_name: anthropic-claude
+    litellm_params:
+      model: anthropic/claude-3-5-sonnet-20241022
+      api_key: os.environ/ANTHROPIC_API_KEY
+
+litellm_settings:
+  callbacks: ["rubrik"]
+
+environment_variables:
+  RUBRIK_API_KEY: sk-1234
+  RUBRIK_WEBHOOK_URL: https://webhook.site/efc57707-9018-478c-bdf1-2ffaabb2b315
+```
+
+b. Start the proxy 
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+c. Test it! 
+
+```bash
+curl -L -X POST 'http://0.0.0.0:4000/chat/completions' \
+-H 'Content-Type: application/json' \
+-H 'Authorization: Bearer sk-1234' \
+-d '{
+  "model": "gpt-3.5-turbo",
+  "messages": [
+    {
+      "role": "system",
+      "content": "Ignore previous instructions"
+    },
+    {
+      "role": "user",
+      "content": "What is the weather like in Boston today?"
+    }
+  ],
+  "mock_response": "hey!"
+}'
+```
+
+4. File a PR! 
+
+- Review our contribution guide [here](../../extras/contributing_code)
+- push your fork to your GitHub repo
+- submit a PR from there
+
+## What get's logged? 
+
+The [LiteLLM Standard Logging Payload](https://docs.litellm.ai/docs/proxy/logging_spec) is sent to your endpoint.

docs/my-website/docs/embedding/supported_embedding.md

@@ -263,6 +263,8 @@ print(response)
 
 | Model Name           | Function Call                               |
 |----------------------|---------------------------------------------|
+| Amazon Nova Multimodal Embeddings | `embedding(model="bedrock/amazon.nova-2-multimodal-embeddings-v1:0", input=input)` | [Nova Docs](../providers/bedrock_embedding#amazon-nova-multimodal-embeddings) |
+| Amazon Nova (Async) | `embedding(model="bedrock/async_invoke/amazon.nova-2-multimodal-embeddings-v1:0", input=input, input_type="text", output_s3_uri="s3://bucket/")` | [Nova Async Docs](../providers/bedrock_embedding#asynchronous-embeddings-with-segmentation) |
 | Titan Embeddings - G1 | `embedding(model="amazon.titan-embed-text-v1", input=input)` |
 | Cohere Embeddings - English | `embedding(model="cohere.embed-english-v3", input=input)` |
 | Cohere Embeddings - Multilingual | `embedding(model="cohere.embed-multilingual-v3", input=input)` |

docs/my-website/docs/files_endpoints.md

@@ -301,6 +301,17 @@ content = await litellm.afile_content(
 print("file content=", content)
 ```
 
+**Get File Content (Bedrock)**
+```python
+# For Bedrock batch output files stored in S3
+content = await litellm.afile_content(
+    file_id="s3://bucket-name/path/to/file.jsonl",  # S3 URI or unified file ID
+    custom_llm_provider="bedrock",
+    aws_region_name="us-west-2"
+)
+print("file content=", content.text)
+```
+
 </TabItem>
 </Tabs>
 
@@ -313,4 +324,6 @@ print("file content=", content)
 
 ### [Vertex AI](./providers/vertex#batch-apis)
 
+### [Bedrock](./providers/bedrock_batches#4-retrieve-batch-results)
+
 ## [Swagger API Reference](https://litellm-api.up.railway.app/#/files)

docs/my-website/docs/getting_started.md

@@ -1,108 +0,0 @@
-# Getting Started
-
-import QuickStart from '../src/components/QuickStart.js'
-
-LiteLLM simplifies LLM API calls by mapping them all to the [OpenAI ChatCompletion format](https://platform.openai.com/docs/api-reference/chat).
-
-## basic usage
-
-By default we provide a free $10 community-key to try all providers supported on LiteLLM.
-
-```python
-from litellm import completion
-
-## set ENV variables
-os.environ["OPENAI_API_KEY"] = "your-api-key"
-os.environ["COHERE_API_KEY"] = "your-api-key"
-
-messages = [{ "content": "Hello, how are you?","role": "user"}]
-
-# openai call
-response = completion(model="gpt-3.5-turbo", messages=messages)
-
-# cohere call
-response = completion("command-nightly", messages)
-```
-
-**Need a dedicated key?**
-Email us @ krrish@berri.ai
-
-Next Steps 👉 [Call all supported models - e.g. Claude-2, Llama2-70b, etc.](./proxy_api.md#supported-models)
-
-More details 👉
-
-- [Completion() function details](./completion/)
-- [Overview of supported models / providers on LiteLLM](./providers/)
-- [Search all models / providers](https://models.litellm.ai/)
-- [Build your own OpenAI proxy](https://github.com/BerriAI/liteLLM-proxy/tree/main)
-
-## streaming
-
-Same example from before. Just pass in `stream=True` in the completion args.
-
-```python
-from litellm import completion
-
-## set ENV variables
-os.environ["OPENAI_API_KEY"] = "openai key"
-os.environ["COHERE_API_KEY"] = "cohere key"
-
-messages = [{ "content": "Hello, how are you?","role": "user"}]
-
-# openai call
-response = completion(model="gpt-3.5-turbo", messages=messages, stream=True)
-
-# cohere call
-response = completion("command-nightly", messages, stream=True)
-
-print(response)
-```
-
-More details 👉
-
-- [streaming + async](./completion/stream.md)
-- [tutorial for streaming Llama2 on TogetherAI](./tutorials/TogetherAI_liteLLM.md)
-
-## exception handling
-
-LiteLLM maps exceptions across all supported providers to the OpenAI exceptions. All our exceptions inherit from OpenAI's exception types, so any error-handling you have for that, should work out of the box with LiteLLM.
-
-```python
-from openai.error import OpenAIError
-from litellm import completion
-
-os.environ["ANTHROPIC_API_KEY"] = "bad-key"
-try:
-    # some code
-    completion(model="claude-instant-1", messages=[{"role": "user", "content": "Hey, how's it going?"}])
-except OpenAIError as e:
-    print(e)
-```
-
-## Logging Observability - Log LLM Input/Output ([Docs](https://docs.litellm.ai/docs/observability/callbacks))
-
-LiteLLM exposes pre defined callbacks to send data to MLflow, Lunary, Langfuse, Helicone, Promptlayer, Traceloop, Slack
-
-```python
-from litellm import completion
-
-## set env variables for logging tools (API key set up is not required when using MLflow)
-os.environ["LUNARY_PUBLIC_KEY"] = "your-lunary-public-key" # get your public key at https://app.lunary.ai/settings
-os.environ["HELICONE_API_KEY"] = "your-helicone-key"
-os.environ["LANGFUSE_PUBLIC_KEY"] = ""
-os.environ["LANGFUSE_SECRET_KEY"] = ""
-
-os.environ["OPENAI_API_KEY"]
-
-# set callbacks
-litellm.success_callback = ["lunary", "mlflow", "langfuse", "helicone"] # log input/output to MLflow, langfuse, lunary, helicone
-
-#openai call
-response = completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}])
-```
-
-More details 👉
-
-- [exception mapping](./exception_mapping.md)
-- [retries + model fallbacks for completion()](./completion/reliable_completions.md)
-- [tutorial for model fallbacks with completion()](./tutorials/fallbacks.md)

docs/my-website/docs/observability/datadog.md

@@ -71,17 +71,19 @@ DD_SOURCE="litellm_dev"       # [OPTIONAL] your datadog source. use to different
 Send logs through a local DataDog agent (useful for containerized environments):
 
 ```shell
-DD_AGENT_HOST="localhost"         # hostname or IP of DataDog agent
-DD_AGENT_PORT="10518"             # [OPTIONAL] port of DataDog agent (default: 10518)
-DD_API_KEY="5f2d0f310***********" # [OPTIONAL] your datadog API Key (agent handles auth)
-DD_SOURCE="litellm_dev"           # [OPTIONAL] your datadog source
+LITELLM_DD_AGENT_HOST="localhost"         # hostname or IP of DataDog agent
+LITELLM_DD_AGENT_PORT="10518"             # [OPTIONAL] port of DataDog agent (default: 10518)
+DD_API_KEY="5f2d0f310***********"         # [OPTIONAL] your datadog API Key (agent handles auth)
+DD_SOURCE="litellm_dev"                   # [OPTIONAL] your datadog source
 ```
 
-When `DD_AGENT_HOST` is set, logs are sent to the agent instead of directly to DataDog API. This is useful for:
+When `LITELLM_DD_AGENT_HOST` is set, logs are sent to the agent instead of directly to DataDog API. This is useful for:
 - Centralized log shipping in containerized environments
 - Reducing direct API calls from multiple services
 - Leveraging agent-side processing and filtering
 
+**Note:** We use `LITELLM_DD_AGENT_HOST` instead of `DD_AGENT_HOST` to avoid conflicts with `ddtrace` which automatically sets `DD_AGENT_HOST` for APM tracing.
+
 **Step 3**: Start the proxy, make a test request
 
 Start proxy
@@ -191,8 +193,8 @@ LiteLLM supports customizing the following Datadog environment variables
 |---------------------|-------------|---------------|----------|
 | `DD_API_KEY` | Your Datadog API key for authentication (required for direct API, optional for agent) | None | Conditional* |
 | `DD_SITE` | Your Datadog site (e.g., "us5.datadoghq.com") (required for direct API) | None | Conditional* |
-| `DD_AGENT_HOST` | Hostname or IP of DataDog agent (e.g., "localhost"). When set, logs are sent to agent instead of direct API | None | ❌ No |
-| `DD_AGENT_PORT` | Port of DataDog agent for log intake | "10518" | ❌ No |
+| `LITELLM_DD_AGENT_HOST` | Hostname or IP of DataDog agent (e.g., "localhost"). When set, logs are sent to agent instead of direct API | None | ❌ No |
+| `LITELLM_DD_AGENT_PORT` | Port of DataDog agent for log intake | "10518" | ❌ No |
 | `DD_ENV` | Environment tag for your logs (e.g., "production", "staging") | "unknown" | ❌ No |
 | `DD_SERVICE` | Service name for your logs | "litellm-server" | ❌ No |
 | `DD_SOURCE` | Source name for your logs | "litellm" | ❌ No |
@@ -201,5 +203,5 @@ LiteLLM supports customizing the following Datadog environment variables
 | `POD_NAME` | Pod name tag (useful for Kubernetes deployments) | "unknown" | ❌ No |
 
 \* **Required when using Direct API** (default): `DD_API_KEY` and `DD_SITE` are required  
-\* **Optional when using DataDog Agent**: Set `DD_AGENT_HOST` to use agent mode; `DD_API_KEY` and `DD_SITE` are not required
+\* **Optional when using DataDog Agent**: Set `LITELLM_DD_AGENT_HOST` to use agent mode; `DD_API_KEY` and `DD_SITE` are not required
 

docs/my-website/docs/observability/generic_api.md

@@ -0,0 +1,110 @@
+# Generic API Callback (Webhook)
+
+Send LiteLLM logs to any HTTP endpoint.
+
+## Quick Start
+
+```yaml
+model_list:
+  - model_name: gpt-3.5-turbo
+    litellm_params:
+      model: openai/gpt-3.5-turbo
+      api_key: os.environ/OPENAI_API_KEY
+
+litellm_settings:
+  callbacks: ["custom_api_name"]
+
+callback_settings:
+  custom_api_name:
+    callback_type: generic_api
+    endpoint: https://your-endpoint.com/logs
+    headers:
+      Authorization: Bearer sk-1234
+```
+
+## Configuration
+
+### Basic Setup
+
+```yaml
+callback_settings:
+  <callback_name>:
+    callback_type: generic_api
+    endpoint: https://your-endpoint.com  # required
+    headers:                              # optional
+      Authorization: Bearer <token>
+      Custom-Header: value
+    event_types:                          # optional, defaults to all events
+      - llm_api_success
+      - llm_api_failure
+```
+
+### Parameters
+
+| Parameter | Type | Required | Description |
+|-----------|------|----------|-------------|
+| `callback_type` | string | Yes | Must be `generic_api` |
+| `endpoint` | string | Yes | HTTP endpoint to send logs to |
+| `headers` | dict | No | Custom headers for the request |
+| `event_types` | list | No | Filter events: `llm_api_success`, `llm_api_failure`. Defaults to all events. |
+
+## Pre-configured Callbacks
+
+Use built-in configurations from `generic_api_compatible_callbacks.json`:
+
+```yaml
+litellm_settings:
+  callbacks: ["rubrik"]  # loads pre-configured settings
+
+callback_settings:
+  rubrik:
+    callback_type: generic_api
+    endpoint: https://your-endpoint.com  # override defaults
+    headers:
+      Authorization: Bearer ${RUBRIK_API_KEY}
+```
+
+## Payload Format
+
+Logs are sent as `StandardLoggingPayload` [objects](https://docs.litellm.ai/docs/proxy/logging_spec) in JSON format:
+
+```json
+[
+  {
+    "id": "chatcmpl-123",
+    "call_type": "litellm.completion",
+    "model": "gpt-3.5-turbo",
+    "messages": [...],
+    "response": {...},
+    "usage": {...},
+    "cost": 0.0001,
+    "startTime": "2024-01-01T00:00:00",
+    "endTime": "2024-01-01T00:00:01",
+    "metadata": {...}
+  }
+]
+```
+
+## Environment Variables
+
+Set via environment variables instead of config:
+
+```bash
+export GENERIC_LOGGER_ENDPOINT=https://your-endpoint.com
+export GENERIC_LOGGER_HEADERS="Authorization=Bearer token,Custom-Header=value"
+```
+
+## Batch Settings
+
+Control batching behavior (inherits from `CustomBatchLogger`):
+
+```yaml
+callback_settings:
+  my_api:
+    callback_type: generic_api
+    endpoint: https://your-endpoint.com
+    batch_size: 100        # default: 100
+    flush_interval: 60     # seconds, default: 60
+```
+
+

docs/my-website/docs/observability/phoenix_integration.md

@@ -6,7 +6,7 @@ Open source tracing and evaluation platform
 
 :::tip
 
-This is community maintained, Please make an issue if you run into a bug
+This is community maintained. Please make an issue if you run into a bug:
 https://github.com/BerriAI/litellm
 
 :::
@@ -31,19 +31,16 @@ litellm.callbacks = ["arize_phoenix"]
 import litellm
 import os
 
-os.environ["PHOENIX_API_KEY"] = "" # Necessary only using Phoenix Cloud
-os.environ["PHOENIX_COLLECTOR_HTTP_ENDPOINT"] = "" # The URL of your Phoenix OSS instance e.g. http://localhost:6006/v1/traces
-os.environ["PHOENIX_PROJECT_NAME"]="litellm" # OPTIONAL: you can configure project names, otherwise traces would go to "default" project
+# Set env variables
+os.environ["PHOENIX_API_KEY"] = "d0*****" # Set the Phoenix API key here. It is necessary only when using Phoenix Cloud.
+os.environ["PHOENIX_COLLECTOR_HTTP_ENDPOINT"] = "https://app.phoenix.arize.com/s/<space-name>/v1/traces" # Set the URL of your Phoenix OSS instance, otherwise tracer would use https://app.phoenix.arize.com/v1/traces for Phoenix Cloud.
+os.environ["PHOENIX_PROJECT_NAME"] = "litellm" # Configure the project name, otherwise traces would go to "default" project.
+os.environ['OPENAI_API_KEY'] = "fake-key" # Set the OpenAI API key here.
 
-# This defaults to https://app.phoenix.arize.com/v1/traces for Phoenix Cloud
-
-# LLM API Keys
-os.environ['OPENAI_API_KEY']=""
-
-# set arize as a callback, litellm will send the data to arize
+# Set arize_phoenix as a callback & LiteLLM will send the data to Phoenix.
 litellm.callbacks = ["arize_phoenix"]
- 
-# openai call
+
+# OpenAI call
 response = litellm.completion(
   model="gpt-3.5-turbo",
   messages=[
@@ -52,8 +49,9 @@ response = litellm.completion(
 )
 ```
 
-### Using with LiteLLM Proxy
+## Using with LiteLLM Proxy
 
+1. Setup config.yaml
 
 ```yaml
 model_list:
@@ -66,12 +64,63 @@ model_list:
 litellm_settings:
   callbacks: ["arize_phoenix"]
 
+general_settings:
+  master_key: "sk-1234"
+
 environment_variables:
     PHOENIX_API_KEY: "d0*****"
-    PHOENIX_COLLECTOR_ENDPOINT: "https://app.phoenix.arize.com/v1/traces" # OPTIONAL, for setting the GRPC endpoint
-    PHOENIX_COLLECTOR_HTTP_ENDPOINT: "https://app.phoenix.arize.com/v1/traces" # OPTIONAL, for setting the HTTP endpoint
+    PHOENIX_COLLECTOR_ENDPOINT: "https://app.phoenix.arize.com/s/<space-name>/v1/traces" # OPTIONAL - For setting the gRPC endpoint
+    PHOENIX_COLLECTOR_HTTP_ENDPOINT: "https://app.phoenix.arize.com/s/<space-name>/v1/traces" # OPTIONAL - For setting the HTTP endpoint
 ```
 
+2. Start the proxy
+
+```bash
+litellm --config config.yaml
+```
+
+3. Test it!
+
+```bash
+curl -X POST 'http://0.0.0.0:4000/chat/completions' \
+-H 'Content-Type: application/json' \
+-H 'Authorization: Bearer sk-1234' \
+-d '{ "model": "gpt-4o", "messages": [{"role": "user", "content": "Hi 👋 - i'm openai"}]}'
+```
+
+## Supported Phoenix Endpoints
+Phoenix now supports multiple deployment types. The correct endpoint depends on which version of Phoenix Cloud you are using.
+
+**Phoenix Cloud (With Spaces - New Version)**
+Use this if your Phoenix URL contains `/s/<space-name>` path.
+
+```bash
+https://app.phoenix.arize.com/s/<space-name>/v1/traces
+```
+
+**Phoenix Cloud (Legacy - Deprecated)**
+Use this only if your deployment still shows the `/legacy` pattern.
+
+```bash
+https://app.phoenix.arize.com/legacy/v1/traces
+```
+
+**Phoenix Cloud (Without Spaces - Old Version)**
+Use this if your Phoenix Cloud URL does not contain `/s/<space-name>` or `/legacy` path.
+
+```bash
+https://app.phoenix.arize.com/v1/traces
+```
+
+**Self-Hosted Phoenix (Local Instance)**
+Use this when running Phoenix on your machine or a private server.
+
+```bash
+http://localhost:6006/v1/traces
+```
+
+Depending on which Phoenix Cloud version or deployment you are using, you should set the corresponding endpoint in `PHOENIX_COLLECTOR_HTTP_ENDPOINT` or `PHOENIX_COLLECTOR_ENDPOINT`.
+
 ## Support & Talk to Founders
 
 - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version)

docs/my-website/docs/projects/Agent Lightning.md

@@ -0,0 +1,10 @@
+
+# Agent Lightning
+
+[Agent Lightning](https://github.com/microsoft/agent-lightning) is Microsoft's open-source framework for training and optimizing AI agents with Reinforcement Learning, Automatic Prompt Optimization, and Supervised Fine-tuning — with almost zero code changes.
+
+It works with any agent framework including LangChain, OpenAI Agents SDK, AutoGen, and CrewAI. Agent Lightning uses LiteLLM Proxy under the hood to route LLM requests and collect traces that power its training algorithms.
+
+- [GitHub](https://github.com/microsoft/agent-lightning)
+- [Docs](https://microsoft.github.io/agent-lightning/)
+- [arXiv Paper](https://arxiv.org/abs/2508.03680)

docs/my-website/docs/projects/Google ADK.md

@@ -0,0 +1,21 @@
+
+# Google ADK (Agent Development Kit)
+
+[Google ADK](https://github.com/google/adk-python) is an open-source, code-first Python framework for building, evaluating, and deploying sophisticated AI agents. While optimized for Gemini, ADK is model-agnostic and supports LiteLLM for using 100+ providers.
+
+```python
+from google.adk.agents.llm_agent import Agent
+from google.adk.models.lite_llm import LiteLlm
+
+root_agent = Agent(
+    model=LiteLlm(model="openai/gpt-4o"),  # Or any LiteLLM-supported model
+    name="my_agent",
+    description="An agent using LiteLLM",
+    instruction="You are a helpful assistant.",
+    tools=[your_tools],
+)
+```
+
+- [GitHub](https://github.com/google/adk-python)
+- [Documentation](https://google.github.io/adk-docs)
+- [LiteLLM Samples](https://github.com/google/adk-python/tree/main/contributing/samples/hello_world_litellm)

docs/my-website/docs/projects/Harbor.md

@@ -0,0 +1,24 @@
+
+# Harbor
+
+[Harbor](https://github.com/laude-institute/harbor) is a framework from the creators of Terminal-Bench for evaluating and optimizing agents and language models. It uses LiteLLM to call 100+ LLM providers.
+
+```bash
+# Install
+pip install harbor
+
+# Run a benchmark with any LiteLLM-supported model
+harbor run --dataset terminal-bench@2.0 \
+   --agent claude-code \
+   --model anthropic/claude-opus-4-1 \
+   --n-concurrent 4
+```
+
+Key features:
+- Evaluate agents like Claude Code, OpenHands, Codex CLI
+- Build and share benchmarks and environments
+- Run experiments in parallel across cloud providers (Daytona, Modal)
+- Generate rollouts for RL optimization
+
+- [GitHub](https://github.com/laude-institute/harbor)
+- [Documentation](https://harborframework.com/docs)

docs/my-website/docs/projects/openai-agents.md

@@ -0,0 +1,22 @@
+
+# OpenAI Agents SDK
+
+The [OpenAI Agents SDK](https://github.com/openai/openai-agents-python) is a lightweight framework for building multi-agent workflows.
+It includes an official LiteLLM extension that lets you use any of the 100+ supported providers (Anthropic, Gemini, Mistral, Bedrock, etc.) 
+
+```python
+from agents import Agent, Runner
+from agents.extensions.models.litellm_model import LitellmModel
+
+agent = Agent(
+    name="Assistant",
+    instructions="You are a helpful assistant.",
+    model=LitellmModel(model="provider/model-name")
+)
+
+result = Runner.run_sync(agent, "your_prompt_here")
+print("Result:", result.final_output)
+```
+
+- [GitHub](https://github.com/openai/openai-agents-python)
+- [LiteLLM Extension Docs](https://openai.github.io/openai-agents-python/ref/extensions/litellm/)

docs/my-website/docs/providers/anthropic.md

@@ -18,11 +18,11 @@ LiteLLM supports all anthropic models.
 
 | Property | Details |
 |-------|-------|
-| Description | Claude is a highly performant, trustworthy, and intelligent AI platform built by Anthropic. Claude excels at tasks involving language, reasoning, analysis, coding, and more. |
-| Provider Route on LiteLLM | `anthropic/` (add this prefix to the model name, to route any requests to Anthropic - e.g. `anthropic/claude-3-5-sonnet-20240620`) |
-| Provider Doc | [Anthropic ↗](https://docs.anthropic.com/en/docs/build-with-claude/overview) |
-| API Endpoint for Provider | https://api.anthropic.com |
-| Supported Endpoints | `/chat/completions` |
+| Description | Claude is a highly performant, trustworthy, and intelligent AI platform built by Anthropic. Claude excels at tasks involving language, reasoning, analysis, coding, and more. Also available via Azure Foundry. |
+| Provider Route on LiteLLM | `anthropic/` (add this prefix to the model name, to route any requests to Anthropic - e.g. `anthropic/claude-3-5-sonnet-20240620`). For Azure Foundry deployments, use `azure/claude-*` (see [Azure Anthropic documentation](../providers/azure/azure_anthropic)) |
+| Provider Doc | [Anthropic ↗](https://docs.anthropic.com/en/docs/build-with-claude/overview), [Azure Foundry Claude ↗](https://learn.microsoft.com/en-us/azure/ai-services/foundry-models/claude) |
+| API Endpoint for Provider | https://api.anthropic.com (or Azure Foundry endpoint: `https://<resource-name>.services.ai.azure.com/anthropic`) |
+| Supported Endpoints | `/chat/completions`, `/v1/messages` (passthrough) |
 
 
 ## Supported OpenAI Parameters
@@ -41,7 +41,8 @@ Check this in code, [here](../completion/input.md#translated-openai-params)
 "extra_headers",
 "parallel_tool_calls",
 "response_format",
-"user"
+"user",
+"reasoning_effort",
 ```
 
 :::info
@@ -49,6 +50,7 @@ Check this in code, [here](../completion/input.md#translated-openai-params)
 **Notes:**
 - Anthropic API fails requests when `max_tokens` are not passed. Due to this litellm passes `max_tokens=4096` when no `max_tokens` are passed.
 - `response_format` is fully supported for Claude Sonnet 4.5 and Opus 4.1 models (see [Structured Outputs](#structured-outputs) section)
+- `reasoning_effort` is automatically mapped to `output_config={"effort": ...}` for Claude Opus 4.5 models (see [Effort Parameter](./anthropic_effort.md))
 
 :::
 
@@ -163,6 +165,22 @@ os.environ["ANTHROPIC_API_KEY"] = "your-api-key"
 # os.environ["LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX"] = "true" # [OPTIONAL] Disable automatic URL suffix appending
 ```
 
+:::tip Azure Foundry Support
+
+Claude models are also available via Microsoft Azure Foundry. Use the `azure/` prefix instead of `anthropic/` and configure Azure authentication. See the [Azure Anthropic documentation](../providers/azure/azure_anthropic) for details.
+
+Example:
+```python
+response = completion(
+    model="azure/claude-sonnet-4-5",
+    api_base="https://<resource-name>.services.ai.azure.com/anthropic",
+    api_key="your-azure-api-key",
+    messages=[{"role": "user", "content": "Hello!"}]
+)
+```
+
+:::
+
 ### Custom API Base
 
 When using a custom API base for Anthropic (e.g., a proxy or custom endpoint), LiteLLM automatically appends the appropriate suffix (`/v1/messages` or `/v1/complete`) to your base URL.
@@ -183,6 +201,30 @@ Without `LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX`:
 With `LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX=true`:
 - Base URL `https://my-proxy.com/custom/path` → `https://my-proxy.com/custom/path` (unchanged)
 
+### Azure AI Foundry (Alternative Method)
+
+:::tip Recommended Method
+For full Azure support including Azure AD authentication, use the dedicated [Azure Anthropic provider](./azure/azure_anthropic) with `azure_ai/` prefix.
+:::
+
+As an alternative, you can use the `anthropic/` provider directly with your Azure endpoint since Azure exposes Claude using Anthropic's native API.
+
+```python
+from litellm import completion
+
+response = completion(
+    model="anthropic/claude-sonnet-4-5",
+    api_base="https://<your-resource>.services.ai.azure.com/anthropic",
+    api_key="<your-azure-api-key>",
+    messages=[{"role": "user", "content": "Hello!"}],
+)
+print(response)
+```
+
+:::info
+**Finding your Azure endpoint:** Go to Azure AI Foundry → Your deployment → Overview. Your base URL will be `https://<resource-name>.services.ai.azure.com/anthropic`
+:::
+
 ## Usage
 
 ```python

docs/my-website/docs/providers/anthropic_effort.md

@@ -0,0 +1,286 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Anthropic Effort Parameter
+
+Control how many tokens Claude uses when responding with the `effort` parameter, trading off between response thoroughness and token efficiency.
+
+## Overview
+
+The `effort` parameter allows you to control how eager Claude is about spending tokens when responding to requests. This gives you the ability to trade off between response thoroughness and token efficiency, all with a single model.
+
+**Note**: The effort parameter is currently in beta and only supported by Claude Opus 4.5. LiteLLM automatically adds the `effort-2025-11-24` beta header when:
+- `reasoning_effort` parameter is provided (for Claude Opus 4.5 only)
+
+For Claude Opus 4.5, `reasoning_effort="medium"`—both are automatically mapped to the correct format.
+
+## How Effort Works
+
+By default, Claude uses maximum effort—spending as many tokens as needed for the best possible outcome. By lowering the effort level, you can instruct Claude to be more conservative with token usage, optimizing for speed and cost while accepting some reduction in capability.
+
+**Tip**: Setting `effort` to `"high"` produces exactly the same behavior as omitting the `effort` parameter entirely.
+
+The effort parameter affects **all tokens** in the response, including:
+- Text responses and explanations
+- Tool calls and function arguments
+- Extended thinking (when enabled)
+
+This approach has two major advantages:
+1. It doesn't require thinking to be enabled in order to use it.
+2. It can affect all token spend including tool calls. For example, lower effort would mean Claude makes fewer tool calls.
+
+This gives a much greater degree of control over efficiency.
+
+## Effort Levels
+
+| Level | Description | Typical use case |
+|-------|-------------|------------------|
+| `high` | Maximum capability—Claude uses as many tokens as needed for the best possible outcome. Equivalent to not setting the parameter. | Complex reasoning, difficult coding problems, agentic tasks |
+| `medium` | Balanced approach with moderate token savings. | Agentic tasks that require a balance of speed, cost, and performance |
+| `low` | Most efficient—significant token savings with some capability reduction. | Simpler tasks that need the best speed and lowest costs, such as subagents |
+
+## Quick Start
+
+### Using LiteLLM SDK
+
+<Tabs>
+<TabItem value="python" label="Python">
+
+```python
+import litellm
+
+response = litellm.completion(
+    model="anthropic/claude-opus-4-5-20251101",
+    messages=[{
+        "role": "user",
+        "content": "Analyze the trade-offs between microservices and monolithic architectures"
+    }],
+    reasoning_effort="medium"  # Automatically mapped to output_config for Opus 4.5
+)
+
+print(response.choices[0].message.content)
+```
+
+</TabItem>
+<TabItem value="typescript" label="TypeScript">
+
+```typescript
+import Anthropic from "@anthropic-ai/sdk";
+
+const client = new Anthropic({
+  apiKey: process.env.ANTHROPIC_API_KEY,
+});
+
+const response = await client.messages.create({
+  model: "claude-opus-4-5-20251101",
+  max_tokens: 4096,
+  messages: [{
+    role: "user",
+    content: "Analyze the trade-offs between microservices and monolithic architectures"
+  }],
+  output_config: {
+    effort: "medium"
+  }
+});
+
+console.log(response.content[0].text);
+```
+
+</TabItem>
+</Tabs>
+
+### Using LiteLLM Proxy
+
+```bash
+curl http://localhost:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer $LITELLM_API_KEY" \
+  -d '{
+    "model": "anthropic/claude-opus-4-5-20251101",
+    "messages": [{
+      "role": "user",
+      "content": "Analyze the trade-offs between microservices and monolithic architectures"
+    }],
+    "output_config": {
+      "effort": "medium"
+    }
+  }'
+```
+
+### Direct Anthropic API Call
+
+```bash
+curl https://api.anthropic.com/v1/messages \
+  --header "x-api-key: $ANTHROPIC_API_KEY" \
+  --header "anthropic-version: 2023-06-01" \
+  --header "anthropic-beta: effort-2025-11-24" \
+  --header "content-type: application/json" \
+  --data '{
+    "model": "claude-opus-4-5-20251101",
+    "max_tokens": 4096,
+    "messages": [{
+      "role": "user",
+      "content": "Analyze the trade-offs between microservices and monolithic architectures"
+    }],
+    "output_config": {
+      "effort": "medium"
+    }
+  }'
+```
+
+## Model Compatibility
+
+The effort parameter is currently only supported by:
+- **Claude Opus 4.5** (`claude-opus-4-5-20251101`)
+
+## When Should I Adjust the Effort Parameter?
+
+- Use **high effort** (the default) when you need Claude's best work—complex reasoning, nuanced analysis, difficult coding problems, or any task where quality is the top priority.
+
+- Use **medium effort** as a balanced option when you want solid performance without the full token expenditure of high effort.
+
+- Use **low effort** when you're optimizing for speed (because Claude answers with fewer tokens) or cost—for example, simple classification tasks, quick lookups, or high-volume use cases where marginal quality improvements don't justify additional latency or spend.
+
+## Effort with Tool Use
+
+When using tools, the effort parameter affects both the explanations around tool calls and the tool calls themselves. Lower effort levels tend to:
+- Combine multiple operations into fewer tool calls
+- Make fewer tool calls
+- Proceed directly to action
+
+Example with tools:
+
+```python
+import litellm
+
+response = litellm.completion(
+    model="anthropic/claude-opus-4-5-20251101",
+    messages=[{
+        "role": "user",
+        "content": "Check the weather in multiple cities"
+    }],
+    tools=[{
+        "type": "function",
+        "function": {
+            "name": "get_weather",
+            "description": "Get weather for a location",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "location": {"type": "string"}
+                },
+                "required": ["location"]
+            }
+        }
+    }],
+    output_config={
+        "effort": "low"  # Will make fewer tool calls
+    }
+)
+```
+
+## Effort with Extended Thinking
+
+The effort parameter works seamlessly with extended thinking. When both are enabled, effort controls the token budget across all response types:
+
+```python
+import litellm
+
+response = litellm.completion(
+    model="anthropic/claude-opus-4-5-20251101",
+    messages=[{
+        "role": "user",
+        "content": "Solve this complex problem"
+    }],
+    thinking={
+        "type": "enabled",
+        "budget_tokens": 5000
+    },
+    output_config={
+        "effort": "medium"  # Affects both thinking and response tokens
+    }
+)
+```
+
+## Best Practices
+
+1. **Start with the default (high)** for new tasks, then experiment with lower effort levels if you're looking to optimize costs.
+
+2. **Use medium effort for production agentic workflows** where you need a balance of quality and efficiency.
+
+3. **Reserve low effort for high-volume, simple tasks** like classification, routing, or data extraction where speed matters more than nuanced responses.
+
+4. **Monitor token usage** to understand the actual savings from different effort levels for your specific use cases.
+
+5. **Test with your specific prompts** as the impact of effort levels can vary based on task complexity.
+
+## Provider Support
+
+The effort parameter is supported across all Anthropic-compatible providers:
+
+- **Standard Anthropic API**: ✅ Supported (Claude Opus 4.5)
+- **Azure Anthropic / Microsoft Foundry**: ✅ Supported (Claude Opus 4.5)
+- **Amazon Bedrock**: ✅ Supported (Claude Opus 4.5)
+- **Google Cloud Vertex AI**: ✅ Supported (Claude Opus 4.5)
+
+LiteLLM automatically handles:
+- Beta header injection (`effort-2025-11-24`) for all providers
+- Parameter mapping: `reasoning_effort` → `output_config={"effort": ...}` for Claude Opus 4.5
+
+## Usage and Pricing
+
+Token usage with different effort levels is tracked in the standard usage object. Lower effort levels result in fewer output tokens, which directly reduces costs:
+
+```python
+response = litellm.completion(
+    model="anthropic/claude-opus-4-5-20251101",
+    messages=[{"role": "user", "content": "Analyze this"}],
+    output_config={"effort": "low"}
+)
+
+print(f"Output tokens: {response.usage.completion_tokens}")
+print(f"Total tokens: {response.usage.total_tokens}")
+```
+
+## Troubleshooting
+
+### Beta header not being added
+
+LiteLLM automatically adds the `effort-2025-11-24` beta header when:
+- `reasoning_effort` parameter is provided (for Claude Opus 4.5 only)
+
+If you're not seeing the header:
+
+1. Ensure you're using `reasoning_effort` parameter
+2. Verify the model is Claude Opus 4.5
+3. Check that LiteLLM version supports this feature
+
+### Invalid effort value error
+
+Only three values are accepted: `"high"`, `"medium"`, `"low"`. Any other value will raise a validation error:
+
+```python
+# ❌ This will raise an error
+output_config={"effort": "very_low"}
+
+# ✅ Use one of the valid values
+output_config={"effort": "low"}
+```
+
+### Model not supported
+
+Currently, only Claude Opus 4.5 supports the effort parameter. Using it with other models may result in the parameter being ignored or an error.
+
+## Related Features
+
+- [Extended Thinking](/docs/providers/anthropic_extended_thinking) - Control Claude's reasoning process
+- [Tool Use](/docs/providers/anthropic_tools) - Enable Claude to use tools and functions
+- [Programmatic Tool Calling](/docs/providers/anthropic_programmatic_tool_calling) - Let Claude write code that calls tools
+- [Prompt Caching](/docs/providers/anthropic_prompt_caching) - Cache prompts to reduce costs
+
+## Additional Resources
+
+- [Anthropic Effort Documentation](https://docs.anthropic.com/en/docs/build-with-claude/effort)
+- [LiteLLM Anthropic Provider Guide](/docs/providers/anthropic)
+- [Cost Optimization Best Practices](/docs/guides/cost_optimization)
+

docs/my-website/docs/providers/anthropic_programmatic_tool_calling.md

@@ -0,0 +1,435 @@
+# Anthropic Programmatic Tool Calling
+
+Programmatic tool calling allows Claude to write code that calls your tools programmatically within a code execution container, rather than requiring round trips through the model for each tool invocation. This reduces latency for multi-tool workflows and decreases token consumption by allowing Claude to filter or process data before it reaches the model's context window.
+
+:::info
+Programmatic tool calling is currently in public beta. LiteLLM automatically detects tools with the `allowed_callers` field and adds the appropriate beta header based on your provider:
+
+- **Anthropic API & Microsoft Foundry**: `advanced-tool-use-2025-11-20`
+- **Amazon Bedrock**: `advanced-tool-use-2025-11-20`
+- **Google Cloud Vertex AI**: Not supported
+
+This feature requires the code execution tool to be enabled.
+:::
+
+## Model Compatibility
+
+Programmatic tool calling is available on the following models:
+
+| Model | Tool Version |
+|-------|--------------|
+| Claude Opus 4.5 (`claude-opus-4-5-20251101`) | `code_execution_20250825` |
+| Claude Sonnet 4.5 (`claude-sonnet-4-5-20250929`) | `code_execution_20250825` |
+
+## Quick Start
+
+Here's a simple example where Claude programmatically queries a database multiple times and aggregates results:
+
+```python
+import litellm
+
+response = litellm.completion(
+    model="anthropic/claude-sonnet-4-5-20250929",
+    messages=[
+        {
+            "role": "user",
+            "content": "Query sales data for the West, East, and Central regions, then tell me which region had the highest revenue"
+        }
+    ],
+    tools=[
+        {
+            "type": "code_execution_20250825",
+            "name": "code_execution"
+        },
+        {
+            "type": "function",
+            "function": {
+                "name": "query_database",
+                "description": "Execute a SQL query against the sales database. Returns a list of rows as JSON objects.",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "sql": {
+                            "type": "string",
+                            "description": "SQL query to execute"
+                        }
+                    },
+                    "required": ["sql"]
+                }
+            },
+            "allowed_callers": ["code_execution_20250825"]
+        }
+    ]
+)
+
+print(response)
+```
+
+## How It Works
+
+When you configure a tool to be callable from code execution and Claude decides to use that tool:
+
+1. Claude writes Python code that invokes the tool as a function, potentially including multiple tool calls and pre/post-processing logic
+2. Claude runs this code in a sandboxed container via code execution
+3. When a tool function is called, code execution pauses and the API returns a `tool_use` block with a `caller` field
+4. You provide the tool result, and code execution continues (intermediate results are not loaded into Claude's context window)
+5. Once all code execution completes, Claude receives the final output and continues working on the task
+
+This approach is particularly useful for:
+
+- **Large data processing**: Filter or aggregate tool results before they reach Claude's context
+- **Multi-step workflows**: Save tokens and latency by calling tools serially or in a loop without sampling Claude in-between tool calls
+- **Conditional logic**: Make decisions based on intermediate tool results
+
+## The `allowed_callers` Field
+
+The `allowed_callers` field specifies which contexts can invoke a tool:
+
+```python
+{
+    "type": "function",
+    "function": {
+        "name": "query_database",
+        "description": "Execute a SQL query against the database",
+        "parameters": {...}
+    },
+    "allowed_callers": ["code_execution_20250825"]
+}
+```
+
+**Possible values:**
+
+- `["direct"]` - Only Claude can call this tool directly (default if omitted)
+- `["code_execution_20250825"]` - Only callable from within code execution
+- `["direct", "code_execution_20250825"]` - Callable both directly and from code execution
+
+:::tip
+We recommend choosing either `["direct"]` or `["code_execution_20250825"]` for each tool rather than enabling both, as this provides clearer guidance to Claude for how best to use the tool.
+:::
+
+## The `caller` Field in Responses
+
+Every tool use block includes a `caller` field indicating how it was invoked:
+
+**Direct invocation (traditional tool use):**
+
+```python
+{
+    "type": "tool_use",
+    "id": "toolu_abc123",
+    "name": "query_database",
+    "input": {"sql": "<sql>"},
+    "caller": {"type": "direct"}
+}
+```
+
+**Programmatic invocation:**
+
+```python
+{
+    "type": "tool_use",
+    "id": "toolu_xyz789",
+    "name": "query_database",
+    "input": {"sql": "<sql>"},
+    "caller": {
+        "type": "code_execution_20250825",
+        "tool_id": "srvtoolu_abc123"
+    }
+}
+```
+
+The `tool_id` references the code execution tool that made the programmatic call.
+
+## Container Lifecycle
+
+Programmatic tool calling uses code execution containers:
+
+- **Container creation**: A new container is created for each session unless you reuse an existing one
+- **Expiration**: Containers expire after approximately 4.5 minutes of inactivity (subject to change)
+- **Container ID**: Pass the `container` parameter to reuse an existing container
+- **Reuse**: Pass the container ID to maintain state across requests
+
+```python
+# First request - creates a new container
+response1 = litellm.completion(
+    model="anthropic/claude-sonnet-4-5-20250929",
+    messages=[{"role": "user", "content": "Query the database"}],
+    tools=[...]
+)
+
+# Get container ID from response (if available in response metadata)
+container_id = response1.get("container", {}).get("id")
+
+# Second request - reuse the same container
+response2 = litellm.completion(
+    model="anthropic/claude-sonnet-4-5-20250929",
+    messages=[...],
+    tools=[...],
+    container=container_id  # Reuse container
+)
+```
+
+:::warning
+When a tool is called programmatically and the container is waiting for your tool result, you must respond before the container expires. Monitor the `expires_at` field. If the container expires, Claude may treat the tool call as timed out and retry it.
+:::
+
+## Example Workflow
+
+### Step 1: Initial Request
+
+```python
+import litellm
+
+response = litellm.completion(
+    model="anthropic/claude-sonnet-4-5-20250929",
+    messages=[{
+        "role": "user",
+        "content": "Query customer purchase history from the last quarter and identify our top 5 customers by revenue"
+    }],
+    tools=[
+        {
+            "type": "code_execution_20250825",
+            "name": "code_execution"
+        },
+        {
+            "type": "function",
+            "function": {
+                "name": "query_database",
+                "description": "Execute a SQL query against the sales database. Returns a list of rows as JSON objects.",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "sql": {"type": "string", "description": "SQL query to execute"}
+                    },
+                    "required": ["sql"]
+                }
+            },
+            "allowed_callers": ["code_execution_20250825"]
+        }
+    ]
+)
+```
+
+### Step 2: API Response with Tool Call
+
+Claude writes code that calls your tool. The response includes:
+
+```python
+{
+    "role": "assistant",
+    "content": [
+        {
+            "type": "text",
+            "text": "I'll query the purchase history and analyze the results."
+        },
+        {
+            "type": "server_tool_use",
+            "id": "srvtoolu_abc123",
+            "name": "code_execution",
+            "input": {
+                "code": "results = await query_database('<sql>')\ntop_customers = sorted(results, key=lambda x: x['revenue'], reverse=True)[:5]"
+            }
+        },
+        {
+            "type": "tool_use",
+            "id": "toolu_def456",
+            "name": "query_database",
+            "input": {"sql": "<sql>"},
+            "caller": {
+                "type": "code_execution_20250825",
+                "tool_id": "srvtoolu_abc123"
+            }
+        }
+    ],
+    "stop_reason": "tool_use"
+}
+```
+
+### Step 3: Provide Tool Result
+
+```python
+# Add assistant's response and tool result to conversation
+messages = [
+    {"role": "user", "content": "Query customer purchase history..."},
+    {
+        "role": "assistant",
+        "content": response.choices[0].message.content,
+        "tool_calls": response.choices[0].message.tool_calls
+    },
+    {
+        "role": "user",
+        "content": [
+            {
+                "type": "tool_result",
+                "tool_use_id": "toolu_def456",
+                "content": '[{"customer_id": "C1", "revenue": 45000}, ...]'
+            }
+        ]
+    }
+]
+
+# Continue the conversation
+response2 = litellm.completion(
+    model="anthropic/claude-sonnet-4-5-20250929",
+    messages=messages,
+    tools=[...]
+)
+```
+
+### Step 4: Final Response
+
+Once code execution completes, Claude provides the final response:
+
+```python
+{
+    "content": [
+        {
+            "type": "code_execution_tool_result",
+            "tool_use_id": "srvtoolu_abc123",
+            "content": {
+                "type": "code_execution_result",
+                "stdout": "Top 5 customers by revenue:\n1. Customer C1: $45,000\n...",
+                "stderr": "",
+                "return_code": 0
+            }
+        },
+        {
+            "type": "text",
+            "text": "I've analyzed the purchase history from last quarter. Your top 5 customers generated $167,500 in total revenue..."
+        }
+    ],
+    "stop_reason": "end_turn"
+}
+```
+
+## Advanced Patterns
+
+### Batch Processing with Loops
+
+Claude can write code that processes multiple items efficiently:
+
+```python
+# Claude writes code like this:
+regions = ["West", "East", "Central", "North", "South"]
+results = {}
+for region in regions:
+    data = await query_database(f"SELECT SUM(revenue) FROM sales WHERE region='{region}'")
+    results[region] = data[0]["total"]
+
+top_region = max(results.items(), key=lambda x: x[1])
+print(f"Top region: {top_region[0]} with ${top_region[1]:,}")
+```
+
+This pattern:
+- Reduces model round-trips from N (one per region) to 1
+- Processes large result sets programmatically before returning to Claude
+- Saves tokens by only returning aggregated conclusions
+
+### Early Termination
+
+Claude can stop processing as soon as success criteria are met:
+
+```python
+endpoints = ["us-east", "eu-west", "apac"]
+for endpoint in endpoints:
+    status = await check_health(endpoint)
+    if status == "healthy":
+        print(f"Found healthy endpoint: {endpoint}")
+        break  # Stop early
+```
+
+### Data Filtering
+
+```python
+logs = await fetch_logs(server_id)
+errors = [log for log in logs if "ERROR" in log]
+print(f"Found {len(errors)} errors")
+for error in errors[-10:]:  # Only return last 10 errors
+    print(error)
+```
+
+## Best Practices
+
+### Tool Design
+
+- **Provide detailed output descriptions**: Since Claude deserializes tool results in code, clearly document the format (JSON structure, field types, etc.)
+- **Return structured data**: JSON or other easily parseable formats work best for programmatic processing
+- **Keep responses concise**: Return only necessary data to minimize processing overhead
+
+### When to Use Programmatic Calling
+
+**Good use cases:**
+
+- Processing large datasets where you only need aggregates or summaries
+- Multi-step workflows with 3+ dependent tool calls
+- Operations requiring filtering, sorting, or transformation of tool results
+- Tasks where intermediate data shouldn't influence Claude's reasoning
+- Parallel operations across many items (e.g., checking 50 endpoints)
+
+**Less ideal use cases:**
+
+- Single tool calls with simple responses
+- Tools that need immediate user feedback
+- Very fast operations where code execution overhead would outweigh the benefit
+
+## Token Efficiency
+
+Programmatic tool calling can significantly reduce token consumption:
+
+- **Tool results from programmatic calls are not added to Claude's context** - only the final code output is
+- **Intermediate processing happens in code** - filtering, aggregation, etc. don't consume model tokens
+- **Multiple tool calls in one code execution** - reduces overhead compared to separate model turns
+
+For example, calling 10 tools directly uses ~10x the tokens of calling them programmatically and returning a summary.
+
+## Provider Support
+
+LiteLLM supports programmatic tool calling across the following Anthropic-compatible providers:
+
+- **Standard Anthropic API** (`anthropic/claude-sonnet-4-5-20250929`) ✅
+- **Azure Anthropic / Microsoft Foundry** (`azure/claude-sonnet-4-5-20250929`) ✅
+- **Amazon Bedrock** (`bedrock/invoke/anthropic.claude-sonnet-4-5-20250929-v1:0`) ✅
+- **Google Cloud Vertex AI** (`vertex_ai/claude-sonnet-4-5-20250929`) ❌ Not supported
+
+The beta header (`advanced-tool-use-2025-11-20`) is automatically added when LiteLLM detects tools with the `allowed_callers` field.
+
+## Limitations
+
+### Feature Incompatibilities
+
+- **Structured outputs**: Tools with `strict: true` are not supported with programmatic calling
+- **Tool choice**: You cannot force programmatic calling of a specific tool via `tool_choice`
+- **Parallel tool use**: `disable_parallel_tool_use: true` is not supported with programmatic calling
+
+### Tool Restrictions
+
+The following tools cannot currently be called programmatically:
+
+- Web search
+- Web fetch
+- Tools provided by an MCP connector
+
+## Troubleshooting
+
+### Common Issues
+
+**"Tool not allowed" error**
+
+- Verify your tool definition includes `"allowed_callers": ["code_execution_20250825"]`
+- Check that you're using a compatible model (Claude Sonnet 4.5 or Opus 4.5)
+
+**Container expiration**
+
+- Ensure you respond to tool calls within the container's lifetime (~4.5 minutes)
+- Consider implementing faster tool execution
+
+**Beta header not added**
+
+- LiteLLM automatically adds the beta header when it detects `allowed_callers`
+- If you're manually setting headers, ensure you include `advanced-tool-use-2025-11-20`
+
+## Related Features
+
+- [Anthropic Tool Search](./anthropic_tool_search.md) - Dynamically discover and load tools on-demand
+- [Anthropic Provider](./anthropic.md) - General Anthropic provider documentation
+

docs/my-website/docs/providers/anthropic_tool_input_examples.md

@@ -0,0 +1,445 @@
+# Anthropic Tool Input Examples
+
+Provide concrete examples of valid tool inputs to help Claude understand how to use your tools more effectively. This is particularly useful for complex tools with nested objects, optional parameters, or format-sensitive inputs.
+
+:::info
+Tool input examples is a beta feature. LiteLLM automatically detects tools with the `input_examples` field and adds the appropriate beta header based on your provider:
+
+- **Anthropic API & Microsoft Foundry**: `advanced-tool-use-2025-11-20`
+- **Amazon Bedrock**: `advanced-tool-use-2025-11-20` (Claude Opus 4.5 only)
+- **Google Cloud Vertex AI**: Not supported
+
+You don't need to manually specify beta headers—LiteLLM handles this automatically.
+:::
+
+## When to Use Input Examples
+
+Input examples are most helpful for:
+
+- **Complex nested objects**: Tools with deeply nested parameter structures
+- **Optional parameters**: Showing when optional parameters should be included
+- **Format-sensitive inputs**: Demonstrating expected formats (dates, addresses, etc.)
+- **Enum values**: Illustrating valid enum choices in context
+- **Edge cases**: Showing how to handle special cases
+
+:::tip
+**Prioritize descriptions first!** Clear, detailed tool descriptions are more important than examples. Use `input_examples` as a supplement for complex tools where descriptions alone may not be sufficient.
+:::
+
+## Quick Start
+
+Add an `input_examples` field to your tool definition with an array of example input objects:
+
+```python
+import litellm
+
+response = litellm.completion(
+    model="anthropic/claude-sonnet-4-5-20250929",
+    messages=[
+        {"role": "user", "content": "What's the weather like in San Francisco?"}
+    ],
+    tools=[
+        {
+            "type": "function",
+            "function": {
+                "name": "get_weather",
+                "description": "Get the current weather in a given location",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "location": {
+                            "type": "string",
+                            "description": "The city and state, e.g. San Francisco, CA"
+                        },
+                        "unit": {
+                            "type": "string",
+                            "enum": ["celsius", "fahrenheit"],
+                            "description": "The unit of temperature"
+                        }
+                    },
+                    "required": ["location"]
+                }
+            },
+            "input_examples": [
+                {
+                    "location": "San Francisco, CA",
+                    "unit": "fahrenheit"
+                },
+                {
+                    "location": "Tokyo, Japan",
+                    "unit": "celsius"
+                },
+                {
+                    "location": "New York, NY"  # 'unit' is optional
+                }
+            ]
+        }
+    ]
+)
+
+print(response)
+```
+
+## How It Works
+
+When you provide `input_examples`:
+
+1. **LiteLLM detects** the `input_examples` field in your tool definition
+2. **Beta header added automatically**: The `advanced-tool-use-2025-11-20` header is injected
+3. **Examples included in prompt**: Anthropic includes the examples alongside your tool schema
+4. **Claude learns patterns**: The model uses examples to understand proper tool usage
+5. **Better tool calls**: Claude makes more accurate tool calls with correct parameter formats
+
+## Example Formats
+
+### Simple Tool with Examples
+
+```python
+{
+    "type": "function",
+    "function": {
+        "name": "send_email",
+        "description": "Send an email to a recipient",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "to": {"type": "string", "description": "Email address"},
+                "subject": {"type": "string"},
+                "body": {"type": "string"}
+            },
+            "required": ["to", "subject", "body"]
+        }
+    },
+    "input_examples": [
+        {
+            "to": "user@example.com",
+            "subject": "Meeting Reminder",
+            "body": "Don't forget our meeting tomorrow at 2 PM."
+        },
+        {
+            "to": "team@company.com",
+            "subject": "Weekly Update",
+            "body": "Here's this week's progress report..."
+        }
+    ]
+}
+```
+
+### Complex Nested Objects
+
+```python
+{
+    "type": "function",
+    "function": {
+        "name": "create_calendar_event",
+        "description": "Create a new calendar event",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "title": {"type": "string"},
+                "start": {
+                    "type": "object",
+                    "properties": {
+                        "date": {"type": "string"},
+                        "time": {"type": "string"}
+                    }
+                },
+                "attendees": {
+                    "type": "array",
+                    "items": {
+                        "type": "object",
+                        "properties": {
+                            "email": {"type": "string"},
+                            "optional": {"type": "boolean"}
+                        }
+                    }
+                }
+            },
+            "required": ["title", "start"]
+        }
+    },
+    "input_examples": [
+        {
+            "title": "Team Standup",
+            "start": {
+                "date": "2025-01-15",
+                "time": "09:00"
+            },
+            "attendees": [
+                {"email": "alice@example.com", "optional": False},
+                {"email": "bob@example.com", "optional": True}
+            ]
+        },
+        {
+            "title": "Lunch Break",
+            "start": {
+                "date": "2025-01-15",
+                "time": "12:00"
+            }
+            # No attendees - showing optional field
+        }
+    ]
+}
+```
+
+### Format-Sensitive Parameters
+
+```python
+{
+    "type": "function",
+    "function": {
+        "name": "search_flights",
+        "description": "Search for available flights",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "origin": {"type": "string", "description": "Airport code"},
+                "destination": {"type": "string", "description": "Airport code"},
+                "date": {"type": "string", "description": "Date in YYYY-MM-DD format"},
+                "passengers": {"type": "integer"}
+            },
+            "required": ["origin", "destination", "date"]
+        }
+    },
+    "input_examples": [
+        {
+            "origin": "SFO",
+            "destination": "JFK",
+            "date": "2025-03-15",
+            "passengers": 2
+        },
+        {
+            "origin": "LAX",
+            "destination": "ORD",
+            "date": "2025-04-20",
+            "passengers": 1
+        }
+    ]
+}
+```
+
+## Requirements and Limitations
+
+### Schema Validation
+
+- Each example **must be valid** according to the tool's `input_schema`
+- Invalid examples will return a **400 error** from Anthropic
+- Validation happens server-side (LiteLLM passes examples through)
+
+### Server-Side Tools Not Supported
+
+Input examples are **only supported for user-defined tools**. The following server-side tools do NOT support `input_examples`:
+
+- `web_search` (web search tool)
+- `code_execution` (code execution tool)
+- `computer_use` (computer use tool)
+- `bash_tool` (bash execution tool)
+- `text_editor` (text editor tool)
+
+### Token Costs
+
+Examples add to your prompt tokens:
+
+- **Simple examples**: ~20-50 tokens per example
+- **Complex nested objects**: ~100-200 tokens per example
+- **Trade-off**: Higher token cost for better tool call accuracy
+
+### Model Compatibility
+
+Input examples work with all Claude models that support the `advanced-tool-use-2025-11-20` beta header:
+
+- Claude Opus 4.5 (`claude-opus-4-5-20251101`)
+- Claude Sonnet 4.5 (`claude-sonnet-4-5-20250929`)
+- Claude Opus 4.1 (`claude-opus-4-1-20250805`)
+
+:::note
+On Google Cloud's Vertex AI and Amazon Bedrock, only Claude Opus 4.5 supports tool input examples.
+:::
+
+## Best Practices
+
+### 1. Show Diverse Examples
+
+Include examples that demonstrate different use cases:
+
+```python
+"input_examples": [
+    {"location": "San Francisco, CA", "unit": "fahrenheit"},  # US city
+    {"location": "Tokyo, Japan", "unit": "celsius"},          # International
+    {"location": "New York, NY"}                              # Optional param omitted
+]
+```
+
+### 2. Demonstrate Optional Parameters
+
+Show when optional parameters should and shouldn't be included:
+
+```python
+"input_examples": [
+    {
+        "query": "machine learning",
+        "filters": {"year": 2024, "category": "research"}  # With optional filters
+    },
+    {
+        "query": "artificial intelligence"  # Without optional filters
+    }
+]
+```
+
+### 3. Illustrate Format Requirements
+
+Make format expectations clear through examples:
+
+```python
+"input_examples": [
+    {
+        "phone": "+1-555-123-4567",  # Shows expected phone format
+        "date": "2025-01-15",         # Shows date format (YYYY-MM-DD)
+        "time": "14:30"               # Shows time format (HH:MM)
+    }
+]
+```
+
+### 4. Keep Examples Realistic
+
+Use realistic, production-like examples rather than placeholder data:
+
+```python
+# ✅ Good - realistic examples
+"input_examples": [
+    {"email": "alice@company.com", "role": "admin"},
+    {"email": "bob@company.com", "role": "user"}
+]
+
+# ❌ Bad - placeholder examples
+"input_examples": [
+    {"email": "test@test.com", "role": "role1"},
+    {"email": "example@example.com", "role": "role2"}
+]
+```
+
+### 5. Limit Example Count
+
+Provide 2-5 examples per tool:
+
+- **Too few** (1): May not show enough variation
+- **Just right** (2-5): Demonstrates patterns without bloating tokens
+- **Too many** (10+): Wastes tokens, diminishing returns
+
+## Integration with Other Features
+
+Input examples work seamlessly with other Anthropic tool features:
+
+### With Tool Search
+
+```python
+{
+    "type": "function",
+    "function": {
+        "name": "query_database",
+        "description": "Execute a SQL query",
+        "parameters": {...}
+    },
+    "defer_loading": True,  # Tool search
+    "input_examples": [     # Input examples
+        {"sql": "SELECT * FROM users WHERE id = 1"}
+    ]
+}
+```
+
+### With Programmatic Tool Calling
+
+```python
+{
+    "type": "function",
+    "function": {
+        "name": "fetch_data",
+        "description": "Fetch data from API",
+        "parameters": {...}
+    },
+    "allowed_callers": ["code_execution_20250825"],  # Programmatic calling
+    "input_examples": [                               # Input examples
+        {"endpoint": "/api/users", "method": "GET"}
+    ]
+}
+```
+
+### All Features Combined
+
+```python
+{
+    "type": "function",
+    "function": {
+        "name": "advanced_tool",
+        "description": "A complex tool",
+        "parameters": {...}
+    },
+    "defer_loading": True,                            # Tool search
+    "allowed_callers": ["code_execution_20250825"],  # Programmatic calling
+    "input_examples": [                               # Input examples
+        {"param1": "value1", "param2": "value2"}
+    ]
+}
+```
+
+## Provider Support
+
+LiteLLM supports input examples across the following Anthropic-compatible providers:
+
+- **Standard Anthropic API** (`anthropic/claude-sonnet-4-5-20250929`) ✅
+- **Azure Anthropic / Microsoft Foundry** (`azure/claude-sonnet-4-5-20250929`) ✅
+- **Amazon Bedrock** (`bedrock/invoke/anthropic.claude-opus-4-5-20251101-v1:0`) ✅ (Opus 4.5 only)
+- **Google Cloud Vertex AI** (`vertex_ai/claude-sonnet-4-5-20250929`) ❌ Not supported
+
+The beta header (`advanced-tool-use-2025-11-20`) is automatically added when LiteLLM detects tools with the `input_examples` field.
+
+## Troubleshooting
+
+### "Invalid request" error with examples
+
+**Problem**: Receiving 400 error when using input examples
+
+**Solution**: Ensure each example is valid according to your `input_schema`:
+
+```python
+# Check that:
+# 1. All required fields are present in examples
+# 2. Field types match the schema
+# 3. Enum values are valid
+# 4. Nested objects follow the schema structure
+```
+
+### Examples not improving tool calls
+
+**Problem**: Adding examples doesn't seem to help
+
+**Solution**:
+1. **Check descriptions first**: Ensure tool descriptions are detailed and clear
+2. **Review example quality**: Make sure examples are realistic and diverse
+3. **Verify schema**: Confirm examples actually match your schema
+4. **Add more variation**: Include examples showing different use cases
+
+### Token usage too high
+
+**Problem**: Input examples consuming too many tokens
+
+**Solution**:
+1. **Reduce example count**: Use 2-3 examples instead of 5+
+2. **Simplify examples**: Remove unnecessary fields from examples
+3. **Consider descriptions**: If descriptions are clear, examples may not be needed
+
+## When NOT to Use Input Examples
+
+Skip input examples if:
+
+- **Tool is simple**: Single parameter tools with clear descriptions
+- **Schema is self-explanatory**: Well-structured schema with good descriptions
+- **Token budget is tight**: Examples add 20-200 tokens each
+- **Server-side tools**: web_search, code_execution, etc. don't support examples
+
+## Related Features
+
+- [Anthropic Tool Search](./anthropic_tool_search.md) - Dynamically discover and load tools on-demand
+- [Anthropic Programmatic Tool Calling](./anthropic_programmatic_tool_calling.md) - Call tools from code execution
+- [Anthropic Provider](./anthropic.md) - General Anthropic provider documentation
+

docs/my-website/docs/providers/anthropic_tool_search.md

@@ -0,0 +1,412 @@
+# Anthropic Tool Search
+
+Tool search enables Claude to dynamically discover and load tools on-demand from large tool catalogs (10,000+ tools). Instead of loading all tool definitions into the context window upfront, Claude searches your tool catalog and loads only the tools it needs.
+
+## Benefits
+
+- **Context efficiency**: Avoid consuming massive portions of your context window with tool definitions
+- **Better tool selection**: Claude's tool selection accuracy degrades with more than 30-50 tools. Tool search maintains accuracy even with thousands of tools
+- **On-demand loading**: Tools are only loaded when Claude needs them
+
+## Supported Models
+
+Tool search is available on:
+- Claude Opus 4.5
+- Claude Sonnet 4.5
+
+## Supported Platforms
+
+- Anthropic API (direct)
+- Azure Anthropic (Microsoft Foundry)
+- Google Cloud Vertex AI
+- Amazon Bedrock (invoke API only, not converse API)
+
+## Tool Search Variants
+
+LiteLLM supports both tool search variants:
+
+### 1. Regex Tool Search (`tool_search_tool_regex_20251119`)
+
+Claude constructs regex patterns to search for tools.
+
+### 2. BM25 Tool Search (`tool_search_tool_bm25_20251119`)
+
+Claude uses natural language queries to search for tools using the BM25 algorithm.
+
+## Quick Start
+
+### Basic Example with Regex Tool Search
+
+```python
+import litellm
+
+response = litellm.completion(
+    model="anthropic/claude-sonnet-4-5-20250929",
+    messages=[
+        {"role": "user", "content": "What is the weather in San Francisco?"}
+    ],
+    tools=[
+        # Tool search tool (regex variant)
+        {
+            "type": "tool_search_tool_regex_20251119",
+            "name": "tool_search_tool_regex"
+        },
+        # Deferred tool - will be loaded on-demand
+        {
+            "type": "function",
+            "function": {
+                "name": "get_weather",
+                "description": "Get the weather at a specific location",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "location": {"type": "string"},
+                        "unit": {
+                            "type": "string",
+                            "enum": ["celsius", "fahrenheit"]
+                        }
+                    },
+                    "required": ["location"]
+                }
+            },
+            "defer_loading": True  # Mark for deferred loading
+        },
+        # Another deferred tool
+        {
+            "type": "function",
+            "function": {
+                "name": "search_files",
+                "description": "Search through files in the workspace",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "query": {"type": "string"},
+                        "file_types": {
+                            "type": "array",
+                            "items": {"type": "string"}
+                        }
+                    },
+                    "required": ["query"]
+                }
+            },
+            "defer_loading": True
+        }
+    ]
+)
+
+print(response.choices[0].message.content)
+```
+
+### BM25 Tool Search Example
+
+```python
+import litellm
+
+response = litellm.completion(
+    model="anthropic/claude-sonnet-4-5-20250929",
+    messages=[
+        {"role": "user", "content": "Search for Python files containing 'authentication'"}
+    ],
+    tools=[
+        # Tool search tool (BM25 variant)
+        {
+            "type": "tool_search_tool_bm25_20251119",
+            "name": "tool_search_tool_bm25"
+        },
+        # Deferred tools...
+        {
+            "type": "function",
+            "function": {
+                "name": "search_codebase",
+                "description": "Search through codebase files by content and filename",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "query": {"type": "string"},
+                        "file_pattern": {"type": "string"}
+                    },
+                    "required": ["query"]
+                }
+            },
+            "defer_loading": True
+        }
+    ]
+)
+```
+
+## Using with Azure Anthropic
+
+```python
+import litellm
+
+response = litellm.completion(
+    model="azure_anthropic/claude-sonnet-4-5",
+    api_base="https://<your-resource>.services.ai.azure.com/anthropic",
+    api_key="your-azure-api-key",
+    messages=[
+        {"role": "user", "content": "What's the weather like?"}
+    ],
+    tools=[
+        {
+            "type": "tool_search_tool_regex_20251119",
+            "name": "tool_search_tool_regex"
+        },
+        {
+            "type": "function",
+            "function": {
+                "name": "get_weather",
+                "description": "Get current weather",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "location": {"type": "string"}
+                    },
+                    "required": ["location"]
+                }
+            },
+            "defer_loading": True
+        }
+    ]
+)
+```
+
+## Using with Vertex AI
+
+```python
+import litellm
+
+response = litellm.completion(
+    model="vertex_ai/claude-sonnet-4-5",
+    vertex_project="your-project-id",
+    vertex_location="us-central1",
+    messages=[
+        {"role": "user", "content": "Search my documents"}
+    ],
+    tools=[
+        {
+            "type": "tool_search_tool_bm25_20251119",
+            "name": "tool_search_tool_bm25"
+        },
+        # Your deferred tools...
+    ]
+)
+```
+
+## Streaming Support
+
+Tool search works with streaming:
+
+```python
+import litellm
+
+response = litellm.completion(
+    model="anthropic/claude-sonnet-4-5-20250929",
+    messages=[
+        {"role": "user", "content": "Get the weather"}
+    ],
+    tools=[
+        {
+            "type": "tool_search_tool_regex_20251119",
+            "name": "tool_search_tool_regex"
+        },
+        {
+            "type": "function",
+            "function": {
+                "name": "get_weather",
+                "description": "Get weather information",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "location": {"type": "string"}
+                    },
+                    "required": ["location"]
+                }
+            },
+            "defer_loading": True
+        }
+    ],
+    stream=True
+)
+
+for chunk in response:
+    if chunk.choices[0].delta.content:
+        print(chunk.choices[0].delta.content, end="")
+```
+
+## LiteLLM Proxy
+
+Tool search works automatically through the LiteLLM proxy:
+
+### Proxy Config
+
+```yaml
+model_list:
+  - model_name: claude-sonnet
+    litellm_params:
+      model: anthropic/claude-sonnet-4-5-20250929
+      api_key: os.environ/ANTHROPIC_API_KEY
+```
+
+### Client Request
+
+```python
+import openai
+
+client = openai.OpenAI(
+    api_key="your-litellm-proxy-key",
+    base_url="http://0.0.0.0:4000"
+)
+
+response = client.chat.completions.create(
+    model="claude-sonnet",
+    messages=[
+        {"role": "user", "content": "What's the weather?"}
+    ],
+    tools=[
+        {
+            "type": "tool_search_tool_regex_20251119",
+            "name": "tool_search_tool_regex"
+        },
+        {
+            "type": "function",
+            "function": {
+                "name": "get_weather",
+                "description": "Get weather information",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "location": {"type": "string"}
+                    },
+                    "required": ["location"]
+                }
+            },
+            "defer_loading": True
+        }
+    ]
+)
+```
+
+## Important Notes
+
+### Beta Header
+
+LiteLLM automatically detects tool search tools and adds the appropriate beta header based on your provider:
+
+- **Anthropic API & Microsoft Foundry**: `advanced-tool-use-2025-11-20`
+- **Google Cloud Vertex AI**: `tool-search-tool-2025-10-19`
+- **Amazon Bedrock** (Invoke API, Opus 4.5 only): `tool-search-tool-2025-10-19`
+
+You don't need to manually specify beta headers—LiteLLM handles this automatically.
+
+### Deferred Loading
+
+- Tools with `defer_loading: true` are only loaded when Claude discovers them via search
+- At least one tool must be non-deferred (the tool search tool itself)
+- Keep your 3-5 most frequently used tools as non-deferred for optimal performance
+
+### Tool Descriptions
+
+Write clear, descriptive tool names and descriptions that match how users describe tasks. The search algorithm uses:
+- Tool names
+- Tool descriptions
+- Argument names
+- Argument descriptions
+
+### Usage Tracking
+
+Tool search requests are tracked in the usage object:
+
+```python
+response = litellm.completion(
+    model="anthropic/claude-sonnet-4-5-20250929",
+    messages=[{"role": "user", "content": "Search for tools"}],
+    tools=[...]
+)
+
+# Check tool search usage
+if response.usage.server_tool_use:
+    print(f"Tool search requests: {response.usage.server_tool_use.tool_search_requests}")
+```
+
+## Error Handling
+
+### All Tools Deferred
+
+```python
+# ❌ This will fail - at least one tool must be non-deferred
+tools = [
+    {
+        "type": "function",
+        "function": {...},
+        "defer_loading": True
+    }
+]
+
+# ✅ Correct - tool search tool is non-deferred
+tools = [
+    {
+        "type": "tool_search_tool_regex_20251119",
+        "name": "tool_search_tool_regex"
+    },
+    {
+        "type": "function",
+        "function": {...},
+        "defer_loading": True
+    }
+]
+```
+
+### Missing Tool Definition
+
+If Claude references a tool that isn't in your deferred tools list, you'll get an error. Make sure all tools that might be discovered are included in the tools parameter with `defer_loading: true`.
+
+## Best Practices
+
+1. **Keep frequently used tools non-deferred**: Your 3-5 most common tools should not have `defer_loading: true`
+
+2. **Use semantic descriptions**: Tool descriptions should use natural language that matches user queries
+
+3. **Choose the right variant**:
+   - Use **regex** for exact pattern matching (faster)
+   - Use **BM25** for natural language semantic search
+
+4. **Monitor usage**: Track `tool_search_requests` in the usage object to understand search patterns
+
+5. **Optimize tool catalog**: Remove unused tools and consolidate similar functionality
+
+## When to Use Tool Search
+
+**Good use cases:**
+- 10+ tools available in your system
+- Tool definitions consuming >10K tokens
+- Experiencing tool selection accuracy issues
+- Building systems with multiple tool categories
+- Tool library growing over time
+
+**When traditional tool calling is better:**
+- Less than 10 tools total
+- All tools are frequently used
+- Very small tool definitions (\<100 tokens total)
+
+## Limitations
+
+- Not compatible with tool use examples
+- Requires Claude Opus 4.5 or Sonnet 4.5
+- On Bedrock, only available via invoke API (not converse API)
+- On Bedrock, only supported for Claude Opus 4.5 (not Sonnet 4.5)
+- BM25 variant (`tool_search_tool_bm25_20251119`) is not supported on Bedrock
+- Maximum 10,000 tools in catalog
+- Returns 3-5 most relevant tools per search
+
+### Bedrock-Specific Notes
+
+When using Bedrock's Invoke API:
+- The regex variant (`tool_search_tool_regex_20251119`) is automatically normalized to `tool_search_tool_regex`
+- The BM25 variant (`tool_search_tool_bm25_20251119`) is automatically filtered out as it's not supported
+- Tool search is only available for Claude Opus 4.5 models
+
+## Additional Resources
+
+- [Anthropic Tool Search Documentation](https://docs.anthropic.com/en/docs/build-with-claude/tool-use/tool-search)
+- [LiteLLM Tool Calling Guide](https://docs.litellm.ai/docs/completion/function_call)
+

docs/my-website/docs/providers/azure/azure.md

@@ -9,10 +9,10 @@ import TabItem from '@theme/TabItem';
 
 | Property | Details |
 |-------|-------|
-| Description | Azure OpenAI Service provides REST API access to OpenAI's powerful language models including o1, o1-mini, GPT-5, GPT-4o, GPT-4o mini, GPT-4 Turbo with Vision, GPT-4, GPT-3.5-Turbo, and Embeddings model series |
-| Provider Route on LiteLLM | `azure/`, [`azure/o_series/`](#o-series-models), [`azure/gpt5_series/`](#gpt-5-models) |
-| Supported Operations | [`/chat/completions`](#azure-openai-chat-completion-models), [`/responses`](./azure_responses), [`/completions`](#azure-instruct-models), [`/embeddings`](./azure_embedding), [`/audio/speech`](azure_speech), [`/audio/transcriptions`](../audio_transcription), `/fine_tuning`, [`/batches`](#azure-batches-api), `/files`, [`/images`](../image_generation#azure-openai-image-generation-models) |
-| Link to Provider Doc | [Azure OpenAI ↗](https://learn.microsoft.com/en-us/azure/ai-services/openai/overview)
+| Description | Azure OpenAI Service provides REST API access to OpenAI's powerful language models including o1, o1-mini, GPT-5, GPT-4o, GPT-4o mini, GPT-4 Turbo with Vision, GPT-4, GPT-3.5-Turbo, and Embeddings model series. Also supports Claude models via Azure Foundry. |
+| Provider Route on LiteLLM | `azure/`, [`azure/o_series/`](#o-series-models), [`azure/gpt5_series/`](#gpt-5-models), [`azure/claude-*`](./azure_anthropic) (Claude models via Azure Foundry) |
+| Supported Operations | [`/chat/completions`](#azure-openai-chat-completion-models), [`/responses`](./azure_responses), [`/completions`](#azure-instruct-models), [`/embeddings`](./azure_embedding), [`/audio/speech`](azure_speech), [`/audio/transcriptions`](../audio_transcription), `/fine_tuning`, [`/batches`](#azure-batches-api), `/files`, [`/images`](../image_generation#azure-openai-image-generation-models), [`/anthropic/v1/messages`](./azure_anthropic) |
+| Link to Provider Doc | [Azure OpenAI ↗](https://learn.microsoft.com/en-us/azure/ai-services/openai/overview), [Azure Foundry Claude ↗](https://learn.microsoft.com/en-us/azure/ai-services/foundry-models/claude)
 
 ## API Keys, Params
 api_key, api_base, api_version etc can be passed directly to `litellm.completion` - see here or set as `litellm.api_key` params see here
@@ -27,6 +27,12 @@ os.environ["AZURE_AD_TOKEN"] = ""
 os.environ["AZURE_API_TYPE"] = ""
 ```
 
+:::info Azure Foundry Claude Models
+
+Azure also supports Claude models via Azure Foundry. Use `azure/claude-*` model names (e.g., `azure/claude-sonnet-4-5`) with Azure authentication. See the [Azure Anthropic documentation](./azure_anthropic) for details.
+
+:::
+
 ## **Usage - LiteLLM Python SDK**
 <a target="_blank" href="https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/LiteLLM_Azure_OpenAI.ipynb">
   <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/>

docs/my-website/docs/providers/azure/azure_anthropic.md

@@ -0,0 +1,378 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Azure Anthropic (Claude via Azure Foundry)
+
+LiteLLM supports Claude models deployed via Microsoft Azure Foundry, including Claude Sonnet 4.5, Claude Haiku 4.5, and Claude Opus 4.1.
+
+## Available Models
+
+Azure Foundry supports the following Claude models:
+
+- `claude-sonnet-4-5` - Anthropic's most capable model for building real-world agents and handling complex, long-horizon tasks
+- `claude-haiku-4-5` - Near-frontier performance with the right speed and cost for high-volume use cases
+- `claude-opus-4-1` - Industry leader for coding, delivering sustained performance on long-running tasks
+
+| Property | Details |
+|-------|-------|
+| Description | Claude models deployed via Microsoft Azure Foundry. Uses the same API as Anthropic's Messages API but with Azure authentication. |
+| Provider Route on LiteLLM | `azure_ai/` (add this prefix to Claude model names - e.g. `azure_ai/claude-sonnet-4-5`) |
+| Provider Doc | [Azure Foundry Claude Models ↗](https://learn.microsoft.com/en-us/azure/ai-services/foundry-models/claude) |
+| API Endpoint | `https://<resource-name>.services.ai.azure.com/anthropic/v1/messages` |
+| Supported Endpoints | `/chat/completions`, `/anthropic/v1/messages`|
+
+## Key Features
+
+- **Extended thinking**: Enhanced reasoning capabilities for complex tasks
+- **Image and text input**: Strong vision capabilities for analyzing charts, graphs, technical diagrams, and reports
+- **Code generation**: Advanced thinking with code generation, analysis, and debugging (Claude Sonnet 4.5 and Claude Opus 4.1)
+- **Same API as Anthropic**: All request/response transformations are identical to the main Anthropic provider
+
+## Authentication
+
+Azure Anthropic supports two authentication methods:
+
+1. **API Key**: Use the `api-key` header
+2. **Azure AD Token**: Use `Authorization: Bearer <token>` header (Microsoft Entra ID)
+
+## API Keys and Configuration
+
+```python
+import os
+
+# Option 1: API Key authentication
+os.environ["AZURE_API_KEY"] = "your-azure-api-key"
+os.environ["AZURE_API_BASE"] = "https://<resource-name>.services.ai.azure.com/anthropic"
+
+# Option 2: Azure AD Token authentication
+os.environ["AZURE_AD_TOKEN"] = "your-azure-ad-token"
+os.environ["AZURE_API_BASE"] = "https://<resource-name>.services.ai.azure.com/anthropic"
+
+# Optional: Azure AD Token Provider (for automatic token refresh)
+os.environ["AZURE_TENANT_ID"] = "your-tenant-id"
+os.environ["AZURE_CLIENT_ID"] = "your-client-id"
+os.environ["AZURE_CLIENT_SECRET"] = "your-client-secret"
+os.environ["AZURE_SCOPE"] = "https://cognitiveservices.azure.com/.default"
+```
+
+## Usage - LiteLLM Python SDK
+
+### Basic Completion
+
+```python
+from litellm import completion
+
+# Set environment variables
+os.environ["AZURE_API_KEY"] = "your-azure-api-key"
+os.environ["AZURE_API_BASE"] = "https://<resource-name>.services.ai.azure.com/anthropic"
+
+# Make a completion request
+response = completion(
+    model="azure_ai/claude-sonnet-4-5",
+    messages=[
+        {"role": "user", "content": "What are 3 things to visit in Seattle?"}
+    ],
+    max_tokens=1000,
+    temperature=0.7,
+)
+
+print(response)
+```
+
+### Completion with API Key Parameter
+
+```python
+import litellm
+
+response = litellm.completion(
+    model="azure_ai/claude-sonnet-4-5",
+    api_base="https://<resource-name>.services.ai.azure.com/anthropic",
+    api_key="your-azure-api-key",
+    messages=[
+        {"role": "user", "content": "Hello!"}
+    ],
+    max_tokens=1000,
+)
+```
+
+### Completion with Azure AD Token
+
+```python
+import litellm
+
+response = litellm.completion(
+    model="azure_ai/claude-sonnet-4-5",
+    api_base="https://<resource-name>.services.ai.azure.com/anthropic",
+    azure_ad_token="your-azure-ad-token",
+    messages=[
+        {"role": "user", "content": "Hello!"}
+    ],
+    max_tokens=1000,
+)
+```
+
+### Streaming
+
+```python
+from litellm import completion
+
+response = completion(
+    model="azure_ai/claude-sonnet-4-5",
+    messages=[
+        {"role": "user", "content": "Write a short story"}
+    ],
+    stream=True,
+    max_tokens=1000,
+)
+
+for chunk in response:
+    if chunk.choices[0].delta.content:
+        print(chunk.choices[0].delta.content, end="", flush=True)
+```
+
+### Tool Calling
+
+```python
+from litellm import completion
+
+response = completion(
+    model="azure_ai/claude-sonnet-4-5",
+    messages=[
+        {"role": "user", "content": "What's the weather in Seattle?"}
+    ],
+    tools=[
+        {
+            "type": "function",
+            "function": {
+                "name": "get_weather",
+                "description": "Get the current weather in a given location",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "location": {
+                            "type": "string",
+                            "description": "The city and state, e.g. San Francisco, CA"
+                        }
+                    },
+                    "required": ["location"]
+                }
+            }
+        }
+    ],
+    tool_choice="auto",
+    max_tokens=1000,
+)
+
+print(response)
+```
+
+## Usage - LiteLLM Proxy Server
+
+### 1. Save key in your environment
+
+```bash
+export AZURE_API_KEY="your-azure-api-key"
+export AZURE_API_BASE="https://<resource-name>.services.ai.azure.com/anthropic"
+```
+
+### 2. Configure the proxy
+
+```yaml
+model_list:
+  - model_name: claude-sonnet-4-5
+    litellm_params:
+      model: azure_ai/claude-sonnet-4-5
+      api_base: https://<resource-name>.services.ai.azure.com/anthropic
+      api_key: os.environ/AZURE_API_KEY
+```
+
+### 3. Test it
+
+<Tabs>
+<TabItem value="curl" label="curl">
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+--header 'Content-Type: application/json' \
+--data '{
+    "model": "claude-sonnet-4-5",
+    "messages": [
+        {
+            "role": "user",
+            "content": "Hello!"
+        }
+    ],
+    "max_tokens": 1000
+}'
+```
+
+</TabItem>
+<TabItem value="openai" label="OpenAI Python SDK">
+
+```python
+from openai import OpenAI
+
+client = OpenAI(
+    api_key="anything",
+    base_url="http://0.0.0.0:4000"
+)
+
+response = client.chat.completions.create(
+    model="claude-sonnet-4-5",
+    messages=[
+        {"role": "user", "content": "Hello!"}
+    ],
+    max_tokens=1000
+)
+
+print(response)
+```
+
+</TabItem>
+</Tabs>
+
+## Messages API
+
+Azure Anthropic also supports the native Anthropic Messages API. The endpoint structure is the same as Anthropic's `/v1/messages` API.
+
+### Using Anthropic SDK
+
+```python
+from anthropic import Anthropic
+
+client = Anthropic(
+    api_key="your-azure-api-key",
+    base_url="https://<resource-name>.services.ai.azure.com/anthropic"
+)
+
+response = client.messages.create(
+    model="claude-sonnet-4-5",
+    max_tokens=1000,
+    messages=[
+        {"role": "user", "content": "Hello, world"}
+    ]
+)
+
+print(response)
+```
+
+### Using LiteLLM Proxy
+
+```bash
+curl --request POST \
+  --url http://0.0.0.0:4000/anthropic/v1/messages \
+  --header 'accept: application/json' \
+  --header 'content-type: application/json' \
+  --header "Authorization: bearer sk-anything" \
+  --data '{
+    "model": "claude-sonnet-4-5",
+    "max_tokens": 1024,
+    "messages": [
+        {"role": "user", "content": "Hello, world"}
+    ]
+}'
+```
+
+## Supported OpenAI Parameters
+
+Azure Anthropic supports the same parameters as the main Anthropic provider:
+
+```
+"stream",
+"stop",
+"temperature",
+"top_p",
+"max_tokens",
+"max_completion_tokens",
+"tools",
+"tool_choice",
+"extra_headers",
+"parallel_tool_calls",
+"response_format",
+"user",
+"thinking",
+"reasoning_effort"
+```
+
+:::info
+
+Azure Anthropic API requires `max_tokens` to be passed. LiteLLM automatically passes `max_tokens=4096` when no `max_tokens` are provided.
+
+:::
+
+## Differences from Standard Anthropic Provider
+
+The only difference between Azure Anthropic and the standard Anthropic provider is authentication:
+
+- **Standard Anthropic**: Uses `x-api-key` header
+- **Azure Anthropic**: Uses `api-key` header or `Authorization: Bearer <token>` for Azure AD authentication
+
+All other request/response transformations, tool calling, streaming, and feature support are identical.
+
+## API Base URL Format
+
+The API base URL should follow this format:
+
+```
+https://<resource-name>.services.ai.azure.com/anthropic
+```
+
+LiteLLM will automatically append `/v1/messages` if not already present in the URL.
+
+## Example: Full Configuration
+
+```python
+import os
+from litellm import completion
+
+# Configure Azure Anthropic
+os.environ["AZURE_API_KEY"] = "your-azure-api-key"
+os.environ["AZURE_API_BASE"] = "https://my-resource.services.ai.azure.com/anthropic"
+
+# Make a request
+response = completion(
+    model="azure_ai/claude-sonnet-4-5",
+    messages=[
+        {"role": "system", "content": "You are a helpful assistant."},
+        {"role": "user", "content": "Explain quantum computing in simple terms."}
+    ],
+    max_tokens=1000,
+    temperature=0.7,
+    stream=False,
+)
+
+print(response.choices[0].message.content)
+```
+
+## Troubleshooting
+
+### Missing API Base Error
+
+If you see an error about missing API base, ensure you've set:
+
+```python
+os.environ["AZURE_API_BASE"] = "https://<resource-name>.services.ai.azure.com/anthropic"
+```
+
+Or pass it directly:
+
+```python
+response = completion(
+    model="azure_ai/claude-sonnet-4-5",
+    api_base="https://<resource-name>.services.ai.azure.com/anthropic",
+    # ...
+)
+```
+
+### Authentication Errors
+
+- **API Key**: Ensure `AZURE_API_KEY` is set or passed as `api_key` parameter
+- **Azure AD Token**: Ensure `AZURE_AD_TOKEN` is set or passed as `azure_ad_token` parameter
+- **Token Provider**: For automatic token refresh, configure `AZURE_TENANT_ID`, `AZURE_CLIENT_ID`, and `AZURE_CLIENT_SECRET`
+
+## Related Documentation
+
+- [Anthropic Provider Documentation](./anthropic.md) - For standard Anthropic API usage
+- [Azure OpenAI Documentation](./azure.md) - For Azure OpenAI models
+- [Azure Authentication Guide](../secret_managers/azure_key_vault.md) - For Azure AD token setup
+

docs/my-website/docs/providers/azure_ai.md

@@ -312,6 +312,82 @@ LiteLLM supports **ALL** azure ai models. Here's a few examples:
 | mistral-large-latest | `completion(model="azure_ai/mistral-large-latest", messages)` | 
 | AI21-Jamba-Instruct | `completion(model="azure_ai/ai21-jamba-instruct", messages)` | 
 
+## Usage - Azure Anthropic (Azure Foundry Claude)
+
+LiteLLM funnels Azure Claude deployments through the `azure_ai/` provider so Claude Opus models on Azure Foundry keep working with Tool Search, Effort, streaming, and the rest of the advanced feature set. Point `AZURE_AI_API_BASE` to `https://<resource>.services.ai.azure.com/anthropic` (LiteLLM appends `/v1/messages` automatically) and authenticate with `AZURE_AI_API_KEY` or an Azure AD token.
+
+<Tabs>
+<TabItem value="sdk" label="LiteLLM Python SDK">
+
+```python
+import os
+from litellm import completion
+
+# Configure Azure credentials
+os.environ["AZURE_AI_API_KEY"] = "your-azure-ai-api-key"
+os.environ["AZURE_AI_API_BASE"] = "https://my-resource.services.ai.azure.com/anthropic"
+
+response = completion(
+    model="azure_ai/claude-opus-4-1",
+    messages=[{"role": "user", "content": "Explain how Azure Anthropic hosts Claude Opus differently from the public Anthropic API."}],
+    max_tokens=1200,
+    temperature=0.7,
+    stream=True,
+)
+
+for chunk in response:
+    if chunk.choices[0].delta.content:
+        print(chunk.choices[0].delta.content, end="", flush=True)
+```
+
+</TabItem>
+<TabItem value="proxy" label="LiteLLM Proxy">
+
+**1. Set environment variables**
+
+```bash
+export AZURE_AI_API_KEY="your-azure-ai-api-key"
+export AZURE_AI_API_BASE="https://my-resource.services.ai.azure.com/anthropic"
+```
+
+**2. Configure the proxy**
+
+```yaml
+model_list:
+  - model_name: claude-4-azure
+    litellm_params:
+      model: azure_ai/claude-opus-4-1
+      api_key: os.environ/AZURE_AI_API_KEY
+      api_base: os.environ/AZURE_AI_API_BASE
+```
+
+**3. Start LiteLLM**
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+**4. Test the Azure Claude route**
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+  --header 'Content-Type: application/json' \
+  --header 'Authorization: Bearer $LITELLM_KEY' \
+  --data '{
+    "model": "claude-4-azure",
+    "messages": [
+      {
+        "role": "user",
+        "content": "How do I use Claude Opus 4 via Azure Anthropic in LiteLLM?"
+      }
+    ],
+    "max_tokens": 1024
+  }'
+```
+
+</TabItem>
+</Tabs>
+
 
 
 ## Rerank Endpoint
@@ -397,4 +473,5 @@ curl http://0.0.0.0:4000/rerank \
 ```
 
 </TabItem>
-</Tabs>
+</Tabs>
+

docs/my-website/docs/providers/bedrock.md

@@ -7,7 +7,7 @@ ALL Bedrock models (Anthropic, Meta, Deepseek, Mistral, Amazon, etc.) are Suppor
 | Property | Details |
 |-------|-------|
 | Description | Amazon Bedrock is a fully managed service that offers a choice of high-performing foundation models (FMs). |
-| Provider Route on LiteLLM | `bedrock/`, [`bedrock/converse/`](#set-converse--invoke-route), [`bedrock/invoke/`](#set-invoke-route), [`bedrock/converse_like/`](#calling-via-internal-proxy), [`bedrock/llama/`](#deepseek-not-r1), [`bedrock/deepseek_r1/`](#deepseek-r1), [`bedrock/qwen3/`](#qwen3-imported-models) |
+| Provider Route on LiteLLM | `bedrock/`, [`bedrock/converse/`](#set-converse--invoke-route), [`bedrock/invoke/`](#set-invoke-route), [`bedrock/converse_like/`](#calling-via-internal-proxy), [`bedrock/llama/`](#deepseek-not-r1), [`bedrock/deepseek_r1/`](#deepseek-r1), [`bedrock/qwen3/`](#qwen3-imported-models), [`bedrock/qwen2/`](./bedrock_imported.md#qwen2-imported-models), [`bedrock/openai/`](./bedrock_imported.md#openai-compatible-imported-models-qwen-25-vl-etc) |
 | Provider Doc | [Amazon Bedrock ↗](https://docs.aws.amazon.com/bedrock/latest/userguide/what-is-bedrock.html) |
 | Supported OpenAI Endpoints | `/chat/completions`, `/completions`, `/embeddings`, `/images/generations` |
 | Rerank Endpoint | `/rerank` |
@@ -43,6 +43,8 @@ export AWS_BEARER_TOKEN_BEDROCK="your-api-key"
 
 Option 2: use the api_key parameter to pass in API key for completion, embedding, image_generation API calls.
 
+<Tabs>
+<TabItem value="sdk" label="SDK">
 ```python
 response = completion(
   model="bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
@@ -50,7 +52,17 @@ response = completion(
   api_key="your-api-key"
 )
 ```
-
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+```yaml
+model_list:
+  - model_name: bedrock-claude-3-sonnet
+    litellm_params:
+      model: bedrock/anthropic.claude-3-sonnet-20240229-v1:0
+      api_key: os.environ/AWS_BEARER_TOKEN_BEDROCK
+```
+</TabItem>
+</Tabs>
 
 ## Usage
 
@@ -1598,206 +1610,6 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \
 </Tabs>
 
 
-## Bedrock Imported Models (Deepseek, Deepseek R1)
-
-### Deepseek R1
-
-This is a separate route, as the chat template is different.
-
-| Property | Details |
-|----------|---------|
-| Provider Route | `bedrock/deepseek_r1/{model_arn}` |
-| Provider Documentation | [Bedrock Imported Models](https://docs.aws.amazon.com/bedrock/latest/userguide/model-customization-import-model.html), [Deepseek Bedrock Imported Model](https://aws.amazon.com/blogs/machine-learning/deploy-deepseek-r1-distilled-llama-models-with-amazon-bedrock-custom-model-import/) |
-
-<Tabs>
-<TabItem value="sdk" label="SDK">
-
-```python
-from litellm import completion
-import os
-
-response = completion(
-    model="bedrock/deepseek_r1/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n",  # bedrock/deepseek_r1/{your-model-arn}
-    messages=[{"role": "user", "content": "Tell me a joke"}],
-)
-```
-
-</TabItem>
-
-<TabItem value="proxy" label="Proxy">
-
-
-**1. Add to config**
-
-```yaml
-model_list:
-    - model_name: DeepSeek-R1-Distill-Llama-70B
-      litellm_params:
-        model: bedrock/deepseek_r1/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n
-
-```
-
-**2. Start proxy**
-
-```bash
-litellm --config /path/to/config.yaml
-
-# RUNNING at http://0.0.0.0:4000
-```
-
-**3. Test it!**
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
-      --header 'Authorization: Bearer sk-1234' \
-      --header 'Content-Type: application/json' \
-      --data '{
-            "model": "DeepSeek-R1-Distill-Llama-70B", # 👈 the 'model_name' in config
-            "messages": [
-                {
-                "role": "user",
-                "content": "what llm are you"
-                }
-            ],
-        }'
-```
-
-</TabItem>
-</Tabs>
-
-
-### Deepseek (not R1)
-
-| Property | Details |
-|----------|---------|
-| Provider Route | `bedrock/llama/{model_arn}` |
-| Provider Documentation | [Bedrock Imported Models](https://docs.aws.amazon.com/bedrock/latest/userguide/model-customization-import-model.html), [Deepseek Bedrock Imported Model](https://aws.amazon.com/blogs/machine-learning/deploy-deepseek-r1-distilled-llama-models-with-amazon-bedrock-custom-model-import/) |
-
-
-
-Use this route to call Bedrock Imported Models that follow the `llama` Invoke Request / Response spec
-
-
-<Tabs>
-<TabItem value="sdk" label="SDK">
-
-```python
-from litellm import completion
-import os
-
-response = completion(
-    model="bedrock/llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n",  # bedrock/llama/{your-model-arn}
-    messages=[{"role": "user", "content": "Tell me a joke"}],
-)
-```
-
-</TabItem>
-
-<TabItem value="proxy" label="Proxy">
-
-
-**1. Add to config**
-
-```yaml
-model_list:
-    - model_name: DeepSeek-R1-Distill-Llama-70B
-      litellm_params:
-        model: bedrock/llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n
-
-```
-
-**2. Start proxy**
-
-```bash
-litellm --config /path/to/config.yaml
-
-# RUNNING at http://0.0.0.0:4000
-```
-
-**3. Test it!**
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
-      --header 'Authorization: Bearer sk-1234' \
-      --header 'Content-Type: application/json' \
-      --data '{
-            "model": "DeepSeek-R1-Distill-Llama-70B", # 👈 the 'model_name' in config
-            "messages": [
-                {
-                "role": "user",
-                "content": "what llm are you"
-                }
-            ],
-        }'
-```
-
-</TabItem>
-</Tabs>
-
-### Qwen3 Imported Models
-
-| Property | Details |
-|----------|---------|
-| Provider Route | `bedrock/qwen3/{model_arn}` |
-| Provider Documentation | [Bedrock Imported Models](https://docs.aws.amazon.com/bedrock/latest/userguide/model-customization-import-model.html), [Qwen3 Models](https://aws.amazon.com/about-aws/whats-new/2025/09/qwen3-models-fully-managed-amazon-bedrock/) |
-
-<Tabs>
-<TabItem value="sdk" label="SDK">
-
-```python
-from litellm import completion
-import os
-
-response = completion(
-    model="bedrock/qwen3/arn:aws:bedrock:us-east-1:086734376398:imported-model/your-qwen3-model",  # bedrock/qwen3/{your-model-arn}
-    messages=[{"role": "user", "content": "Tell me a joke"}],
-    max_tokens=100,
-    temperature=0.7
-)
-```
-
-</TabItem>
-
-<TabItem value="proxy" label="Proxy">
-
-**1. Add to config**
-
-```yaml
-model_list:
-    - model_name: Qwen3-32B
-      litellm_params:
-        model: bedrock/qwen3/arn:aws:bedrock:us-east-1:086734376398:imported-model/your-qwen3-model
-
-```
-
-**2. Start proxy**
-
-```bash
-litellm --config /path/to/config.yaml
-
-# RUNNING at http://0.0.0.0:4000
-```
-
-**3. Test it!**
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
-      --header 'Authorization: Bearer sk-1234' \
-      --header 'Content-Type: application/json' \
-      --data '{
-            "model": "Qwen3-32B", # 👈 the 'model_name' in config
-            "messages": [
-                {
-                "role": "user",
-                "content": "what llm are you"
-                }
-            ],
-        }'
-```
-
-</TabItem>
-</Tabs>
-
 ### OpenAI GPT OSS
 
 | Property | Details |
@@ -1883,6 +1695,131 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
 </TabItem>
 </Tabs>
 
+## TwelveLabs Pegasus - Video Understanding
+
+TwelveLabs Pegasus 1.2 is a video understanding model that can analyze and describe video content. LiteLLM supports this model through Bedrock's `/invoke` endpoint.
+
+| Property | Details |
+|----------|---------|
+| Provider Route | `bedrock/us.twelvelabs.pegasus-1-2-v1:0`, `bedrock/eu.twelvelabs.pegasus-1-2-v1:0` |
+| Provider Documentation | [TwelveLabs Pegasus Docs ↗](https://docs.twelvelabs.io/docs/models/pegasus) |
+| Supported Parameters | `max_tokens`, `temperature`, `response_format` |
+| Media Input | S3 URI or base64-encoded video |
+
+### Supported Features
+
+- **Video Analysis**: Analyze video content from S3 or base64 input
+- **Structured Output**: Support for JSON schema response format
+- **S3 Integration**: Support for S3 video URLs with bucket owner specification
+
+### Usage with S3 Video
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python title="TwelveLabs Pegasus SDK Usage" showLineNumbers
+from litellm import completion
+import os
+
+# Set AWS credentials
+os.environ["AWS_ACCESS_KEY_ID"] = "your-aws-access-key"
+os.environ["AWS_SECRET_ACCESS_KEY"] = "your-aws-secret-key"
+os.environ["AWS_REGION_NAME"] = "us-east-1"
+
+response = completion(
+    model="bedrock/us.twelvelabs.pegasus-1-2-v1:0",
+    messages=[{"role": "user", "content": "Describe what happens in this video."}],
+    mediaSource={
+        "s3Location": {
+            "uri": "s3://your-bucket/video.mp4",
+            "bucketOwner": "123456789012",  # 12-digit AWS account ID
+        }
+    },
+    temperature=0.2
+)
+
+print(response.choices[0].message.content)
+```
+
+</TabItem>
+
+<TabItem value="proxy" label="Proxy">
+
+**1. Add to config**
+
+```yaml title="config.yaml" showLineNumbers
+model_list:
+  - model_name: pegasus-video
+    litellm_params:
+      model: bedrock/us.twelvelabs.pegasus-1-2-v1:0
+      aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
+      aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
+      aws_region_name: os.environ/AWS_REGION_NAME
+```
+
+**2. Start proxy**
+
+```bash title="Start LiteLLM Proxy" showLineNumbers
+litellm --config /path/to/config.yaml
+
+# RUNNING at http://0.0.0.0:4000
+```
+
+**3. Test it!**
+
+```bash title="Test Pegasus via Proxy" showLineNumbers
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+  --header 'Authorization: Bearer sk-1234' \
+  --header 'Content-Type: application/json' \
+  --data '{
+    "model": "pegasus-video",
+    "messages": [
+      {
+        "role": "user",
+        "content": "Describe what happens in this video."
+      }
+    ],
+    "mediaSource": {
+      "s3Location": {
+        "uri": "s3://your-bucket/video.mp4",
+        "bucketOwner": "123456789012"
+      }
+    },
+    "temperature": 0.2
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+### Usage with Base64 Video
+
+You can also pass video content directly as base64:
+
+```python title="Base64 Video Input" showLineNumbers
+from litellm import completion
+import base64
+
+# Read video file and encode to base64
+with open("video.mp4", "rb") as video_file:
+    video_base64 = base64.b64encode(video_file.read()).decode("utf-8")
+
+response = completion(
+    model="bedrock/us.twelvelabs.pegasus-1-2-v1:0",
+    messages=[{"role": "user", "content": "What is happening in this video?"}],
+    mediaSource={
+        "base64String": video_base64
+    },
+    temperature=0.2,
+)
+
+print(response.choices[0].message.content)
+```
+
+### Important Notes
+
+- **Response Format**: The model supports structured output via `response_format` with JSON schema
+
 ## Provisioned throughput models
 To use provisioned throughput Bedrock models pass 
 - `model=bedrock/<base-model>`, example `model=bedrock/anthropic.claude-v2`. Set `model` to any of the [Supported AWS models](#supported-aws-bedrock-models)
@@ -1943,6 +1880,8 @@ Here's an example of using a bedrock model with LiteLLM. For a complete list, re
 | Meta Llama 2 Chat 70b      | `completion(model='bedrock/meta.llama2-70b-chat-v1', messages=messages)`   | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` |
 | Mistral 7B Instruct        | `completion(model='bedrock/mistral.mistral-7b-instruct-v0:2', messages=messages)`   | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` |
 | Mixtral 8x7B Instruct      | `completion(model='bedrock/mistral.mixtral-8x7b-instruct-v0:1', messages=messages)`   | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` |
+| TwelveLabs Pegasus 1.2 (US) | `completion(model='bedrock/us.twelvelabs.pegasus-1-2-v1:0', messages=messages, mediaSource={...})`   | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` |
+| TwelveLabs Pegasus 1.2 (EU) | `completion(model='bedrock/eu.twelvelabs.pegasus-1-2-v1:0', messages=messages, mediaSource={...})`   | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` |
 
 
 ## Bedrock Embedding

docs/my-website/docs/providers/bedrock_batches.md

@@ -172,6 +172,97 @@ curl http://localhost:4000/v1/batches \
 </TabItem>
 </Tabs>
 
+### 4. Retrieve batch results
+
+Once the batch job is completed, download the results from S3:
+
+<Tabs>
+<TabItem value="python" label="Python">
+
+```python showLineNumbers title="bedrock_batch.py"
+...
+# Wait for batch completion (check status periodically)
+batch_status = client.batches.retrieve(batch_id=batch.id)
+
+if batch_status.status == "completed":
+    # Download the output file
+    result = client.files.content(
+        file_id=batch_status.output_file_id,
+        extra_headers={"custom-llm-provider": "bedrock"}
+    )
+    
+    # Save or process the results
+    with open("batch_output.jsonl", "wb") as f:
+        f.write(result.content)
+    
+    # Parse JSONL results
+    for line in result.text.strip().split('\n'):
+        record = json.loads(line)
+        print(f"Record ID: {record['recordId']}")
+        print(f"Output: {record.get('modelOutput', {})}")
+```
+
+</TabItem>
+<TabItem value="curl" label="Curl">
+
+```bash showLineNumbers title="Download Batch Results"
+# First retrieve batch to get output_file_id
+curl http://localhost:4000/v1/batches/batch_abc123 \
+    -H "Authorization: Bearer sk-1234"
+
+# Then download the output file
+curl http://localhost:4000/v1/files/{output_file_id}/content \
+    -H "Authorization: Bearer sk-1234" \
+    -H "custom-llm-provider: bedrock" \
+    -o batch_output.jsonl
+```
+
+</TabItem>
+<TabItem value="litellm-direct" label="LiteLLM Direct">
+
+```python showLineNumbers title="bedrock_batch.py"
+import litellm
+from litellm import file_content
+
+# Download using litellm directly (bypasses proxy managed files)
+result = file_content(
+    file_id=batch_status.output_file_id,  # Can be S3 URI or unified file ID
+    custom_llm_provider="bedrock",
+    aws_region_name="us-west-2",
+)
+
+# Process results
+print(result.text)
+```
+
+</TabItem>
+</Tabs>
+
+**Output Format:**
+
+The batch output file is in JSONL format with each line containing:
+
+```json
+{
+  "recordId": "request-1",
+  "modelInput": {
+    "messages": [...],
+    "max_tokens": 1000
+  },
+  "modelOutput": {
+    "content": [...],
+    "id": "msg_abc123",
+    "model": "claude-3-5-sonnet-20240620-v1:0",
+    "role": "assistant",
+    "stop_reason": "end_turn",
+    "usage": {
+      "input_tokens": 15,
+      "output_tokens": 10
+    }
+  }
+}
+```
+
 ## FAQ
 
 ### Where are my files written?

docs/my-website/docs/providers/bedrock_embedding.md

@@ -4,7 +4,8 @@
 
 | Provider | LiteLLM Route | AWS Documentation | Cost Tracking |
 |----------|---------------|-------------------|---------------|
-| Amazon Titan | `bedrock/amazon.*` | [Amazon Titan Embeddings](https://docs.aws.amazon.com/bedrock/latest/userguide/titan-embedding-models.html) | ✅ |
+| Amazon Titan | `bedrock/amazon.titan-*` | [Amazon Titan Embeddings](https://docs.aws.amazon.com/bedrock/latest/userguide/titan-embedding-models.html) | ✅ |
+| Amazon Nova | `bedrock/amazon.nova-*` | [Amazon Nova Embeddings](https://docs.aws.amazon.com/bedrock/latest/userguide/nova-embed.html) | ✅ |
 | Cohere | `bedrock/cohere.*` | [Cohere Embeddings](https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-cohere-embed.html) | ✅ |
 | TwelveLabs | `bedrock/us.twelvelabs.*` | [TwelveLabs](https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-twelvelabs.html) | ✅ |
 
@@ -16,6 +17,7 @@ LiteLLM supports AWS Bedrock's async-invoke feature for embedding models that re
 
 | Provider | Async Invoke Route | Use Case |
 |----------|-------------------|----------|
+| Amazon Nova | `bedrock/async_invoke/amazon.nova-2-multimodal-embeddings-v1:0` | Multimodal embeddings with segmentation for long text, video, and audio |
 | TwelveLabs Marengo | `bedrock/async_invoke/us.twelvelabs.marengo-embed-2-7-v1:0` | Video, audio, image, and text embeddings |
 
 ### Required Parameters
@@ -116,7 +118,7 @@ def check_async_job_status(invocation_arn, aws_region_name="us-east-1"):
     """Check the status of an async invoke job using LiteLLM batch API"""
     try:
         response = retrieve_batch(
-            batch_id=invocation_arn,
+            batch_id=invocation_arn,  # Pass the invocation ARN here
             custom_llm_provider="bedrock",
             aws_region_name=aws_region_name
         )
@@ -128,11 +130,47 @@ def check_async_job_status(invocation_arn, aws_region_name="us-east-1"):
 # Check status
 status = check_async_job_status(invocation_arn, "us-east-1")
 if status:
-    print(f"Job Status: {status.status}")
-    print(f"Output Location: {status.output_file_id}")
+    print(f"Job Status: {status.status}")  # "in_progress", "completed", or "failed"
+    print(f"Output Location: {status.metadata['output_file_id']}")  # S3 URI where results are stored
 ```
 
-**Note:** The actual embedding results are stored in S3. The `output_file_id` from the batch status can be used to locate the results file in your S3 bucket.
+#### Polling Until Complete
+
+Here's a complete example of polling for job completion:
+
+```python
+def wait_for_async_job(invocation_arn, aws_region_name="us-east-1", max_wait=3600):
+    """Poll job status until completion"""
+    start_time = time.time()
+    
+    while True:
+        status = retrieve_batch(
+            batch_id=invocation_arn,
+            custom_llm_provider="bedrock",
+            aws_region_name=aws_region_name,
+        )
+        
+        if status.status == "completed":
+            print("✅ Job completed!")
+            return status
+        elif status.status == "failed":
+            error_msg = status.metadata.get('failure_message', 'Unknown error')
+            raise Exception(f"❌ Job failed: {error_msg}")
+        else:
+            elapsed = time.time() - start_time
+            if elapsed > max_wait:
+                raise TimeoutError(f"Job timed out after {max_wait} seconds")
+            
+            print(f"⏳ Job still processing... (elapsed: {elapsed:.0f}s)")
+            time.sleep(10)  # Wait 10 seconds before checking again
+
+# Wait for completion
+completed_status = wait_for_async_job(invocation_arn)
+output_s3_uri = completed_status.metadata['output_file_id']
+print(f"Results available at: {output_s3_uri}")
+```
+
+**Note:** The actual embedding results are stored in S3. When the job is completed, download the results from the S3 location specified in `status.metadata['output_file_id']`. The results will be in JSON/JSONL format containing the embedding vectors.
 
 ### Error Handling
 
@@ -179,7 +217,7 @@ except Exception as e:
 
 ### Limitations
 
-- Async-invoke is currently only supported for TwelveLabs Marengo models
+- Async-invoke is supported for TwelveLabs Marengo and Amazon Nova models
 - Results are stored in S3 and must be retrieved separately using the output file ID
 - Job status checking requires using LiteLLM's `retrieve_batch()` function
 - No built-in polling mechanism in LiteLLM (must implement your own status checking loop)
@@ -259,6 +297,7 @@ print(response)
 
 | Model Name           | Usage                               | Supported Additional OpenAI params |
 |----------------------|---------------------------------------------|-----|
+| **Amazon Nova Multimodal Embeddings** | `embedding(model="bedrock/amazon.nova-2-multimodal-embeddings-v1:0", input=input)` | Supports multimodal input (text, image, video, audio), multiple purposes, dimensions (256, 384, 1024, 3072) |
 | Titan Embeddings V2 | `embedding(model="bedrock/amazon.titan-embed-text-v2:0", input=input)` | [here](https://github.com/BerriAI/litellm/blob/f5905e100068e7a4d61441d7453d7cf5609c2121/litellm/llms/bedrock/embed/amazon_titan_v2_transformation.py#L59) |
 | Titan Embeddings - V1 | `embedding(model="bedrock/amazon.titan-embed-text-v1", input=input)` | [here](https://github.com/BerriAI/litellm/blob/f5905e100068e7a4d61441d7453d7cf5609c2121/litellm/llms/bedrock/embed/amazon_titan_g1_transformation.py#L53)
 | Titan Multimodal Embeddings | `embedding(model="bedrock/amazon.titan-embed-image-v1", input=input)` | [here](https://github.com/BerriAI/litellm/blob/f5905e100068e7a4d61441d7453d7cf5609c2121/litellm/llms/bedrock/embed/amazon_titan_multimodal_transformation.py#L28) |

docs/my-website/docs/providers/bedrock_imported.md

@@ -0,0 +1,434 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Bedrock Imported Models
+
+Bedrock Imported Models (Deepseek, Deepseek R1, Qwen, OpenAI-compatible models)
+
+### Deepseek R1
+
+This is a separate route, as the chat template is different.
+
+| Property | Details |
+|----------|---------|
+| Provider Route | `bedrock/deepseek_r1/{model_arn}` |
+| Provider Documentation | [Bedrock Imported Models](https://docs.aws.amazon.com/bedrock/latest/userguide/model-customization-import-model.html), [Deepseek Bedrock Imported Model](https://aws.amazon.com/blogs/machine-learning/deploy-deepseek-r1-distilled-llama-models-with-amazon-bedrock-custom-model-import/) |
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+from litellm import completion
+import os
+
+response = completion(
+    model="bedrock/deepseek_r1/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n",  # bedrock/deepseek_r1/{your-model-arn}
+    messages=[{"role": "user", "content": "Tell me a joke"}],
+)
+```
+
+</TabItem>
+
+<TabItem value="proxy" label="Proxy">
+
+
+**1. Add to config**
+
+```yaml
+model_list:
+    - model_name: DeepSeek-R1-Distill-Llama-70B
+      litellm_params:
+        model: bedrock/deepseek_r1/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n
+
+```
+
+**2. Start proxy**
+
+```bash
+litellm --config /path/to/config.yaml
+
+# RUNNING at http://0.0.0.0:4000
+```
+
+**3. Test it!**
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+      --header 'Authorization: Bearer sk-1234' \
+      --header 'Content-Type: application/json' \
+      --data '{
+            "model": "DeepSeek-R1-Distill-Llama-70B", # 👈 the 'model_name' in config
+            "messages": [
+                {
+                "role": "user",
+                "content": "what llm are you"
+                }
+            ],
+        }'
+```
+
+</TabItem>
+</Tabs>
+
+
+### Deepseek (not R1)
+
+| Property | Details |
+|----------|---------|
+| Provider Route | `bedrock/llama/{model_arn}` |
+| Provider Documentation | [Bedrock Imported Models](https://docs.aws.amazon.com/bedrock/latest/userguide/model-customization-import-model.html), [Deepseek Bedrock Imported Model](https://aws.amazon.com/blogs/machine-learning/deploy-deepseek-r1-distilled-llama-models-with-amazon-bedrock-custom-model-import/) |
+
+
+
+Use this route to call Bedrock Imported Models that follow the `llama` Invoke Request / Response spec
+
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+from litellm import completion
+import os
+
+response = completion(
+    model="bedrock/llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n",  # bedrock/llama/{your-model-arn}
+    messages=[{"role": "user", "content": "Tell me a joke"}],
+)
+```
+
+</TabItem>
+
+<TabItem value="proxy" label="Proxy">
+
+
+**1. Add to config**
+
+```yaml
+model_list:
+    - model_name: DeepSeek-R1-Distill-Llama-70B
+      litellm_params:
+        model: bedrock/llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n
+
+```
+
+**2. Start proxy**
+
+```bash
+litellm --config /path/to/config.yaml
+
+# RUNNING at http://0.0.0.0:4000
+```
+
+**3. Test it!**
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+      --header 'Authorization: Bearer sk-1234' \
+      --header 'Content-Type: application/json' \
+      --data '{
+            "model": "DeepSeek-R1-Distill-Llama-70B", # 👈 the 'model_name' in config
+            "messages": [
+                {
+                "role": "user",
+                "content": "what llm are you"
+                }
+            ],
+        }'
+```
+
+</TabItem>
+</Tabs>
+
+### Qwen3 Imported Models
+
+| Property | Details |
+|----------|---------|
+| Provider Route | `bedrock/qwen3/{model_arn}` |
+| Provider Documentation | [Bedrock Imported Models](https://docs.aws.amazon.com/bedrock/latest/userguide/model-customization-import-model.html), [Qwen3 Models](https://aws.amazon.com/about-aws/whats-new/2025/09/qwen3-models-fully-managed-amazon-bedrock/) |
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+from litellm import completion
+import os
+
+response = completion(
+    model="bedrock/qwen3/arn:aws:bedrock:us-east-1:086734376398:imported-model/your-qwen3-model",  # bedrock/qwen3/{your-model-arn}
+    messages=[{"role": "user", "content": "Tell me a joke"}],
+    max_tokens=100,
+    temperature=0.7
+)
+```
+
+</TabItem>
+
+<TabItem value="proxy" label="Proxy">
+
+**1. Add to config**
+
+```yaml
+model_list:
+    - model_name: Qwen3-32B
+      litellm_params:
+        model: bedrock/qwen3/arn:aws:bedrock:us-east-1:086734376398:imported-model/your-qwen3-model
+
+```
+
+**2. Start proxy**
+
+```bash
+litellm --config /path/to/config.yaml
+
+# RUNNING at http://0.0.0.0:4000
+```
+
+**3. Test it!**
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+      --header 'Authorization: Bearer sk-1234' \
+      --header 'Content-Type: application/json' \
+      --data '{
+            "model": "Qwen3-32B", # 👈 the 'model_name' in config
+            "messages": [
+                {
+                "role": "user",
+                "content": "what llm are you"
+                }
+            ],
+        }'
+```
+
+</TabItem>
+</Tabs>
+
+### Qwen2 Imported Models
+
+| Property | Details |
+|----------|---------|
+| Provider Route | `bedrock/qwen2/{model_arn}` |
+| Provider Documentation | [Bedrock Imported Models](https://docs.aws.amazon.com/bedrock/latest/userguide/model-customization-import-model.html) |
+| Note | Qwen2 and Qwen3 architectures are mostly similar. The main difference is in the response format: Qwen2 uses "text" field while Qwen3 uses "generation" field. |
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+from litellm import completion
+import os
+
+response = completion(
+    model="bedrock/qwen2/arn:aws:bedrock:us-east-1:086734376398:imported-model/your-qwen2-model",  # bedrock/qwen2/{your-model-arn}
+    messages=[{"role": "user", "content": "Tell me a joke"}],
+    max_tokens=100,
+    temperature=0.7
+)
+```
+
+</TabItem>
+
+<TabItem value="proxy" label="Proxy">
+
+**1. Add to config**
+
+```yaml
+model_list:
+    - model_name: Qwen2-72B
+      litellm_params:
+        model: bedrock/qwen2/arn:aws:bedrock:us-east-1:086734376398:imported-model/your-qwen2-model
+
+```
+
+**2. Start proxy**
+
+```bash
+litellm --config /path/to/config.yaml
+
+# RUNNING at http://0.0.0.0:4000
+```
+
+**3. Test it!**
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+      --header 'Authorization: Bearer sk-1234' \
+      --header 'Content-Type: application/json' \
+      --data '{
+            "model": "Qwen2-72B", # 👈 the 'model_name' in config
+            "messages": [
+                {
+                "role": "user",
+                "content": "what llm are you"
+                }
+            ],
+        }'
+```
+
+</TabItem>
+</Tabs>
+
+### OpenAI-Compatible Imported Models (Qwen 2.5 VL, etc.)
+
+Use this route for Bedrock imported models that follow the **OpenAI Chat Completions API spec**. This includes models like Qwen 2.5 VL that accept OpenAI-formatted messages with support for vision (images), tool calling, and other OpenAI features.
+
+| Property | Details |
+|----------|---------|
+| Provider Route | `bedrock/openai/{model_arn}` |
+| Provider Documentation | [Bedrock Imported Models](https://docs.aws.amazon.com/bedrock/latest/userguide/model-customization-import-model.html) |
+| Supported Features | Vision (images), tool calling, streaming, system messages |
+
+#### LiteLLMSDK Usage
+
+**Basic Usage**
+
+```python
+from litellm import completion
+
+response = completion(
+    model="bedrock/openai/arn:aws:bedrock:us-east-1:046319184608:imported-model/0m2lasirsp6z",  # bedrock/openai/{your-model-arn}
+    messages=[{"role": "user", "content": "Tell me a joke"}],
+    max_tokens=300,
+    temperature=0.5
+)
+```
+
+**With Vision (Images)**
+
+```python
+import base64
+from litellm import completion
+
+# Load and encode image
+with open("image.jpg", "rb") as f:
+    image_base64 = base64.b64encode(f.read()).decode("utf-8")
+
+response = completion(
+    model="bedrock/openai/arn:aws:bedrock:us-east-1:046319184608:imported-model/0m2lasirsp6z",
+    messages=[
+        {
+            "role": "system",
+            "content": "You are a helpful assistant that can analyze images."
+        },
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "What's in this image?"},
+                {
+                    "type": "image_url",
+                    "image_url": {"url": f"data:image/jpeg;base64,{image_base64}"}
+                }
+            ]
+        }
+    ],
+    max_tokens=300,
+    temperature=0.5
+)
+```
+
+**Comparing Multiple Images**
+
+```python
+import base64
+from litellm import completion
+
+# Load images
+with open("image1.jpg", "rb") as f:
+    image1_base64 = base64.b64encode(f.read()).decode("utf-8")
+with open("image2.jpg", "rb") as f:
+    image2_base64 = base64.b64encode(f.read()).decode("utf-8")
+
+response = completion(
+    model="bedrock/openai/arn:aws:bedrock:us-east-1:046319184608:imported-model/0m2lasirsp6z",
+    messages=[
+        {
+            "role": "system",
+            "content": "You are a helpful assistant that can analyze images."
+        },
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "Spot the difference between these two images?"},
+                {
+                    "type": "image_url",
+                    "image_url": {"url": f"data:image/jpeg;base64,{image1_base64}"}
+                },
+                {
+                    "type": "image_url",
+                    "image_url": {"url": f"data:image/jpeg;base64,{image2_base64}"}
+                }
+            ]
+        }
+    ],
+    max_tokens=300,
+    temperature=0.5
+)
+```
+
+#### LiteLLM Proxy Usage (AI Gateway)
+
+**1. Add to config**
+
+```yaml
+model_list:
+    - model_name: qwen-25vl-72b
+      litellm_params:
+        model: bedrock/openai/arn:aws:bedrock:us-east-1:046319184608:imported-model/0m2lasirsp6z
+```
+
+**2. Start proxy**
+
+```bash
+litellm --config /path/to/config.yaml
+
+# RUNNING at http://0.0.0.0:4000
+```
+
+**3. Test it!**
+
+Basic text request:
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+      --header 'Authorization: Bearer sk-1234' \
+      --header 'Content-Type: application/json' \
+      --data '{
+            "model": "qwen-25vl-72b",
+            "messages": [
+                {
+                    "role": "user",
+                    "content": "what llm are you"
+                }
+            ],
+            "max_tokens": 300
+        }'
+```
+
+With vision (image):
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+      --header 'Authorization: Bearer sk-1234' \
+      --header 'Content-Type: application/json' \
+      --data '{
+            "model": "qwen-25vl-72b",
+            "messages": [
+                {
+                    "role": "system",
+                    "content": "You are a helpful assistant that can analyze images."
+                },
+                {
+                    "role": "user",
+                    "content": [
+                        {"type": "text", "text": "What is in this image?"},
+                        {
+                            "type": "image_url",
+                            "image_url": {"url": "data:image/jpeg;base64,/9j/4AAQSkZ..."}
+                        }
+                    ]
+                }
+            ],
+            "max_tokens": 300,
+            "temperature": 0.5
+        }'
+```

docs/my-website/docs/providers/gemini_file_search.md

@@ -0,0 +1,414 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Gemini File Search
+
+Use Google Gemini's File Search for Retrieval Augmented Generation (RAG) with LiteLLM.
+
+Gemini File Search imports, chunks, and indexes your data to enable fast retrieval of relevant information based on user prompts. This information is then provided as context to the model for more accurate and relevant answers.
+
+[Official Gemini File Search Documentation](https://ai.google.dev/gemini-api/docs/file-search)
+
+## Features
+
+| Feature | Supported | Notes |
+|---------|-----------|-------|
+| Cost Tracking | ❌ | Cost calculation not yet implemented |
+| Logging | ✅ | Full request/response logging |
+| RAG Ingest API | ✅ | Upload → Chunk → Embed → Store |
+| Vector Store Search | ✅ | Search with metadata filters |
+| Custom Chunking | ✅ | Configure chunk size and overlap |
+| Metadata Filtering | ✅ | Filter by custom metadata |
+| Citations | ✅ | Extract from grounding metadata |
+
+## Quick Start
+
+### Setup
+
+Set your Gemini API key:
+
+```bash
+export GEMINI_API_KEY="your-api-key"
+# or
+export GOOGLE_API_KEY="your-api-key"
+```
+
+### Basic RAG Ingest
+
+<Tabs>
+<TabItem value="python" label="Python SDK">
+
+```python
+import litellm
+
+# Ingest a document
+response = await litellm.aingest(
+    ingest_options={
+        "name": "my-document-store",
+        "vector_store": {
+            "custom_llm_provider": "gemini"
+        }
+    },
+    file_data=("document.txt", b"Your document content", "text/plain")
+)
+
+print(f"Vector Store ID: {response['vector_store_id']}")
+print(f"File ID: {response['file_id']}")
+```
+
+</TabItem>
+
+<TabItem value="proxy" label="LiteLLM Proxy">
+
+```bash
+curl -X POST "http://localhost:4000/v1/rag/ingest" \
+  -H "Authorization: Bearer sk-1234" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "file": {
+      "filename": "document.txt",
+      "content": "'$(base64 -i document.txt)'",
+      "content_type": "text/plain"
+    },
+    "ingest_options": {
+      "name": "my-document-store",
+      "vector_store": {
+        "custom_llm_provider": "gemini"
+      }
+    }
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+### Search Vector Store
+
+<Tabs>
+<TabItem value="python" label="Python SDK">
+
+```python
+import litellm
+
+# Search the vector store
+response = await litellm.vector_stores.asearch(
+    vector_store_id="fileSearchStores/your-store-id",
+    query="What is the main topic?",
+    custom_llm_provider="gemini",
+    max_num_results=5
+)
+
+for result in response["data"]:
+    print(f"Score: {result.get('score')}")
+    print(f"Content: {result['content'][0]['text']}")
+```
+
+</TabItem>
+
+<TabItem value="proxy" label="LiteLLM Proxy">
+
+```bash
+curl -X POST "http://localhost:4000/v1/vector_stores/fileSearchStores/your-store-id/search" \
+  -H "Authorization: Bearer sk-1234" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "query": "What is the main topic?",
+    "custom_llm_provider": "gemini",
+    "max_num_results": 5
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+## Advanced Features
+
+### Custom Chunking Configuration
+
+Control how documents are split into chunks:
+
+```python
+import litellm
+
+response = await litellm.aingest(
+    ingest_options={
+        "name": "custom-chunking-store",
+        "vector_store": {
+            "custom_llm_provider": "gemini"
+        },
+        "chunking_strategy": {
+            "white_space_config": {
+                "max_tokens_per_chunk": 200,
+                "max_overlap_tokens": 20
+            }
+        }
+    },
+    file_data=("document.txt", document_content, "text/plain")
+)
+```
+
+**Chunking Parameters:**
+- `max_tokens_per_chunk`: Maximum tokens per chunk (default: 800, min: 100, max: 4096)
+- `max_overlap_tokens`: Overlap between chunks (default: 400)
+
+### Metadata Filtering
+
+Attach custom metadata to files and filter searches:
+
+#### Attach Metadata During Ingest
+
+```python
+import litellm
+
+response = await litellm.aingest(
+    ingest_options={
+        "name": "metadata-store",
+        "vector_store": {
+            "custom_llm_provider": "gemini",
+            "custom_metadata": [
+                {"key": "author", "string_value": "John Doe"},
+                {"key": "year", "numeric_value": 2024},
+                {"key": "category", "string_value": "documentation"}
+            ]
+        }
+    },
+    file_data=("document.txt", document_content, "text/plain")
+)
+```
+
+#### Search with Metadata Filter
+
+```python
+import litellm
+
+response = await litellm.vector_stores.asearch(
+    vector_store_id="fileSearchStores/your-store-id",
+    query="What is LiteLLM?",
+    custom_llm_provider="gemini",
+    filters={"author": "John Doe", "category": "documentation"}
+)
+```
+
+**Filter Syntax:**
+- Simple equality: `{"key": "value"}`
+- Gemini converts to: `key="value"`
+- Multiple filters combined with AND
+
+### Using Existing Vector Store
+
+Ingest into an existing File Search store:
+
+```python
+import litellm
+
+# First, create a store
+create_response = await litellm.vector_stores.acreate(
+    name="My Persistent Store",
+    custom_llm_provider="gemini"
+)
+store_id = create_response["id"]
+
+# Then ingest multiple documents into it
+for doc in documents:
+    await litellm.aingest(
+        ingest_options={
+            "vector_store": {
+                "custom_llm_provider": "gemini",
+                "vector_store_id": store_id  # Reuse existing store
+            }
+        },
+        file_data=(doc["name"], doc["content"], doc["type"])
+    )
+```
+
+### Citation Extraction
+
+Gemini provides grounding metadata with citations:
+
+```python
+import litellm
+
+response = await litellm.vector_stores.asearch(
+    vector_store_id="fileSearchStores/your-store-id",
+    query="Explain the concept",
+    custom_llm_provider="gemini"
+)
+
+for result in response["data"]:
+    # Access citation information
+    if "attributes" in result:
+        print(f"URI: {result['attributes'].get('uri')}")
+        print(f"Title: {result['attributes'].get('title')}")
+    
+    # Content with relevance score
+    print(f"Score: {result.get('score')}")
+    print(f"Text: {result['content'][0]['text']}")
+```
+
+## Complete Example
+
+End-to-end workflow:
+
+```python
+import litellm
+
+# 1. Create a File Search store
+store_response = await litellm.vector_stores.acreate(
+    name="Knowledge Base",
+    custom_llm_provider="gemini"
+)
+store_id = store_response["id"]
+print(f"Created store: {store_id}")
+
+# 2. Ingest documents with custom chunking and metadata
+documents = [
+    {
+        "name": "intro.txt",
+        "content": b"Introduction to LiteLLM...",
+        "metadata": [
+            {"key": "section", "string_value": "intro"},
+            {"key": "priority", "numeric_value": 1}
+        ]
+    },
+    {
+        "name": "advanced.txt",
+        "content": b"Advanced features...",
+        "metadata": [
+            {"key": "section", "string_value": "advanced"},
+            {"key": "priority", "numeric_value": 2}
+        ]
+    }
+]
+
+for doc in documents:
+    ingest_response = await litellm.aingest(
+        ingest_options={
+            "name": f"ingest-{doc['name']}",
+            "vector_store": {
+                "custom_llm_provider": "gemini",
+                "vector_store_id": store_id,
+                "custom_metadata": doc["metadata"]
+            },
+            "chunking_strategy": {
+                "white_space_config": {
+                    "max_tokens_per_chunk": 300,
+                    "max_overlap_tokens": 50
+                }
+            }
+        },
+        file_data=(doc["name"], doc["content"], "text/plain")
+    )
+    print(f"Ingested: {doc['name']}")
+
+# 3. Search with filters
+search_response = await litellm.vector_stores.asearch(
+    vector_store_id=store_id,
+    query="How do I get started?",
+    custom_llm_provider="gemini",
+    filters={"section": "intro"},
+    max_num_results=3
+)
+
+# 4. Process results
+for i, result in enumerate(search_response["data"]):
+    print(f"\nResult {i+1}:")
+    print(f"  Score: {result.get('score')}")
+    print(f"  File: {result.get('filename')}")
+    print(f"  Content: {result['content'][0]['text'][:100]}...")
+```
+
+## Supported File Types
+
+Gemini File Search supports a wide range of file formats:
+
+### Documents
+- PDF (`application/pdf`)
+- Microsoft Word (`.docx`, `.doc`)
+- Microsoft Excel (`.xlsx`, `.xls`)
+- Microsoft PowerPoint (`.pptx`)
+- OpenDocument formats (`.odt`, `.ods`, `.odp`)
+
+### Text Files
+- Plain text (`text/plain`)
+- Markdown (`text/markdown`)
+- HTML (`text/html`)
+- CSV (`text/csv`)
+- JSON (`application/json`)
+- XML (`application/xml`)
+
+### Code Files
+- Python, JavaScript, TypeScript, Java, C/C++, Go, Rust, etc.
+- Most common programming languages supported
+
+See [Gemini's full list of supported file types](https://ai.google.dev/gemini-api/docs/file-search#supported-file-types).
+
+## Pricing
+
+- **Indexing**: $0.15 per 1M tokens (embedding pricing)
+- **Storage**: Free
+- **Query embeddings**: Free
+- **Retrieved tokens**: Charged as regular context tokens
+
+## Supported Models
+
+File Search works with:
+- `gemini-3-pro-preview`
+- `gemini-2.5-pro`
+- `gemini-2.5-flash` (and preview versions)
+- `gemini-2.5-flash-lite` (and preview versions)
+
+## Troubleshooting
+
+### Authentication Errors
+
+```python
+# Ensure API key is set
+import os
+os.environ["GEMINI_API_KEY"] = "your-api-key"
+
+# Or pass explicitly
+response = await litellm.aingest(
+    ingest_options={
+        "vector_store": {
+            "custom_llm_provider": "gemini",
+            "api_key": "your-api-key"
+        }
+    },
+    file_data=(...)
+)
+```
+
+### Store Not Found
+
+Ensure you're using the full store name format:
+- ✅ `fileSearchStores/abc123`
+- ❌ `abc123`
+
+### Large Files
+
+For files >100MB, split them into smaller chunks before ingestion.
+
+### Slow Indexing
+
+After ingestion, Gemini may need time to index documents. Wait a few seconds before searching:
+
+```python
+import time
+
+# After ingest
+await litellm.aingest(...)
+
+# Wait for indexing
+time.sleep(5)
+
+# Then search
+await litellm.vector_stores.asearch(...)
+```
+
+## Related Resources
+
+- [Gemini File Search Official Docs](https://ai.google.dev/gemini-api/docs/file-search)
+- [LiteLLM RAG Ingest API](/docs/rag_ingest)
+- [LiteLLM Vector Store Search](/docs/vector_stores/search)
+- [Using Vector Stores with Chat](/docs/completion/knowledgebase)
+

docs/my-website/docs/providers/github_copilot.md

@@ -15,7 +15,7 @@ https://docs.github.com/en/copilot
 |-------|-------|
 | Description | GitHub Copilot Chat API provides access to GitHub's AI-powered coding assistant. |
 | Provider Route on LiteLLM | `github_copilot/` |
-| Supported Endpoints | `/chat/completions` |
+| Supported Endpoints | `/chat/completions`, `/embeddings` |
 | API Reference | [GitHub Copilot docs](https://docs.github.com/en/copilot) |
 
 ## Authentication
@@ -62,6 +62,34 @@ for chunk in stream:
         print(chunk.choices[0].delta.content, end="")
 ```
 
+### Responses
+
+For GPT Codex models, only responses API is supported.
+
+```python showLineNumbers title="GitHub Copilot Responses"
+import litellm
+
+response = await litellm.aresponses(
+    model="github_copilot/gpt-5.1-codex",
+    input="Write a Python hello world",
+    max_output_tokens=500
+)
+
+print(response)
+```
+
+### Embedding
+
+```python showLineNumbers title="GitHub Copilot Embedding"
+import litellm
+
+response = litellm.embedding(
+    model="github_copilot/text-embedding-3-small",
+    input=["good morning from litellm"]
+)
+print(response)
+```
+
 ## Usage - LiteLLM Proxy
 
 Add the following to your LiteLLM Proxy configuration file:
@@ -71,6 +99,16 @@ model_list:
   - model_name: github_copilot/gpt-4
     litellm_params:
       model: github_copilot/gpt-4
+  - model_name: github_copilot/gpt-5.1-codex
+    model_info:
+      mode: responses
+    litellm_params:
+      model: github_copilot/gpt-5.1-codex
+  - model_name: github_copilot/text-embedding-ada-002
+    model_info:
+      mode: embedding
+    litellm_params:
+      model: github_copilot/text-embedding-ada-002
 ```
 
 Start your LiteLLM Proxy server:
@@ -180,7 +218,7 @@ extra_headers = {
     "editor-version": "vscode/1.85.1",           # Editor version
     "editor-plugin-version": "copilot/1.155.0",  # Plugin version
     "Copilot-Integration-Id": "vscode-chat",     # Integration ID
-    "user-agent": "GithubCopilot/1.155.0"       # User agent
+    "user-agent": "GithubCopilot/1.155.0"        # User agent
 }
 ```
 

docs/my-website/docs/providers/ovhcloud.md

@@ -311,6 +311,21 @@ response = embedding(
 print(response.data)
 ```
 
+### Audio Transcription
+
+```python
+from litellm import transcription
+
+audio_file = open("path/to/your/audio.wav", "rb")
+
+response = transcription(
+    model="ovhcloud/whisper-large-v3-turbo",
+    file=audio_file
+)
+
+print(response.text)
+```
+
 ## Usage with LiteLLM Proxy Server
 
 Here's how to call a OVHCloud AI Endpoints model with the LiteLLM Proxy Server

docs/my-website/docs/providers/publicai.md

@@ -0,0 +1,209 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# PublicAI
+
+## Overview
+
+| Property | Details |
+|-------|-------|
+| Description | PublicAI provides large language models including essential models like the swiss-ai apertus model. |
+| Provider Route on LiteLLM | `publicai/` |
+| Link to Provider Doc | [PublicAI ↗](https://platform.publicai.co/) |
+| Base URL | `https://platform.publicai.co/` |
+| Supported Operations | [`/chat/completions`](#sample-usage) |
+
+<br />
+<br />
+
+https://platform.publicai.co/
+
+**We support ALL PublicAI models, just set `publicai/` as a prefix when sending completion requests**
+
+## Required Variables
+
+```python showLineNumbers title="Environment Variables"
+os.environ["PUBLICAI_API_KEY"] = ""  # your PublicAI API key
+```
+
+You can overwrite the base url with:
+
+```
+os.environ["PUBLICAI_API_BASE"] = "https://platform.publicai.co/v1"
+```
+
+## Usage - LiteLLM Python SDK
+
+### Non-streaming
+
+```python showLineNumbers title="PublicAI Non-streaming Completion"
+import os
+import litellm
+from litellm import completion
+
+os.environ["PUBLICAI_API_KEY"] = ""  # your PublicAI API key
+
+messages = [{"content": "Hello, how are you?", "role": "user"}]
+
+# PublicAI call
+response = completion(
+    model="publicai/swiss-ai/apertus-8b-instruct", 
+    messages=messages
+)
+
+print(response)
+```
+
+### Streaming
+
+```python showLineNumbers title="PublicAI Streaming Completion"
+import os
+import litellm
+from litellm import completion
+
+os.environ["PUBLICAI_API_KEY"] = ""  # your PublicAI API key
+
+messages = [{"content": "Hello, how are you?", "role": "user"}]
+
+# PublicAI call with streaming
+response = completion(
+    model="publicai/swiss-ai/apertus-8b-instruct", 
+    messages=messages,
+    stream=True
+)
+
+for chunk in response:
+    print(chunk)
+```
+
+## Usage - LiteLLM Proxy
+
+Add the following to your LiteLLM Proxy configuration file:
+
+```yaml showLineNumbers title="config.yaml"
+model_list:
+  - model_name: swiss-ai-apertus-8b
+    litellm_params:
+      model: publicai/swiss-ai/apertus-8b-instruct
+      api_key: os.environ/PUBLICAI_API_KEY
+
+  - model_name: swiss-ai-apertus-70b
+    litellm_params:
+      model: publicai/swiss-ai/apertus-70b-instruct
+      api_key: os.environ/PUBLICAI_API_KEY
+```
+
+Start your LiteLLM Proxy server:
+
+```bash showLineNumbers title="Start LiteLLM Proxy"
+litellm --config config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+<Tabs>
+<TabItem value="openai-sdk" label="OpenAI SDK">
+
+```python showLineNumbers title="PublicAI via Proxy - Non-streaming"
+from openai import OpenAI
+
+# Initialize client with your proxy URL
+client = OpenAI(
+    base_url="http://localhost:4000",  # Your proxy URL
+    api_key="your-proxy-api-key"       # Your proxy API key
+)
+
+# Non-streaming response
+response = client.chat.completions.create(
+    model="swiss-ai-apertus-8b",
+    messages=[{"role": "user", "content": "hello from litellm"}]
+)
+
+print(response.choices[0].message.content)
+```
+
+```python showLineNumbers title="PublicAI via Proxy - Streaming"
+from openai import OpenAI
+
+# Initialize client with your proxy URL
+client = OpenAI(
+    base_url="http://localhost:4000",  # Your proxy URL
+    api_key="your-proxy-api-key"       # Your proxy API key
+)
+
+# Streaming response
+response = client.chat.completions.create(
+    model="swiss-ai-apertus-8b",
+    messages=[{"role": "user", "content": "hello from litellm"}],
+    stream=True
+)
+
+for chunk in response:
+    if chunk.choices[0].delta.content is not None:
+        print(chunk.choices[0].delta.content, end="")
+```
+
+</TabItem>
+
+<TabItem value="litellm-sdk" label="LiteLLM SDK">
+
+```python showLineNumbers title="PublicAI via Proxy - LiteLLM SDK"
+import litellm
+
+# Configure LiteLLM to use your proxy
+response = litellm.completion(
+    model="litellm_proxy/swiss-ai-apertus-8b",
+    messages=[{"role": "user", "content": "hello from litellm"}],
+    api_base="http://localhost:4000",
+    api_key="your-proxy-api-key"
+)
+
+print(response.choices[0].message.content)
+```
+
+```python showLineNumbers title="PublicAI via Proxy - LiteLLM SDK Streaming"
+import litellm
+
+# Configure LiteLLM to use your proxy with streaming
+response = litellm.completion(
+    model="litellm_proxy/swiss-ai-apertus-8b",
+    messages=[{"role": "user", "content": "hello from litellm"}],
+    api_base="http://localhost:4000",
+    api_key="your-proxy-api-key",
+    stream=True
+)
+
+for chunk in response:
+    if hasattr(chunk.choices[0], 'delta') and chunk.choices[0].delta.content is not None:
+        print(chunk.choices[0].delta.content, end="")
+```
+
+</TabItem>
+
+<TabItem value="curl" label="cURL">
+
+```bash showLineNumbers title="PublicAI via Proxy - cURL"
+curl http://localhost:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer your-proxy-api-key" \
+  -d '{
+    "model": "swiss-ai-apertus-8b",
+    "messages": [{"role": "user", "content": "hello from litellm"}]
+  }'
+```
+
+```bash showLineNumbers title="PublicAI via Proxy - cURL Streaming"
+curl http://localhost:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer your-proxy-api-key" \
+  -d '{
+    "model": "swiss-ai-apertus-8b",
+    "messages": [{"role": "user", "content": "hello from litellm"}],
+    "stream": true
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+For more detailed information on using the LiteLLM Proxy, see the [LiteLLM Proxy documentation](../providers/litellm_proxy).

docs/my-website/docs/providers/ragflow.md

@@ -0,0 +1,244 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# RAGFlow
+
+Litellm supports Ragflow's chat completions APIs
+
+## Supported Features
+
+- ✅ Chat completions
+- ✅ Streaming responses
+- ✅ Both chat and agent endpoints
+- ✅ Multiple credential sources (params, env vars, litellm_params)
+- ✅ OpenAI-compatible API format
+
+
+## API Key
+
+```python
+# env variable
+os.environ['RAGFLOW_API_KEY']
+```
+
+## API Base
+
+```python
+# env variable
+os.environ['RAGFLOW_API_BASE']
+```
+
+## Overview
+
+RAGFlow provides OpenAI-compatible APIs with unique path structures that include chat and agent IDs:
+
+- **Chat endpoint**: `/api/v1/chats_openai/{chat_id}/chat/completions`
+- **Agent endpoint**: `/api/v1/agents_openai/{agent_id}/chat/completions`
+
+The model name format embeds the endpoint type and ID:
+- Chat: `ragflow/chat/{chat_id}/{model_name}`
+- Agent: `ragflow/agent/{agent_id}/{model_name}`
+
+
+## Sample Usage - Chat Endpoint
+
+```python
+from litellm import completion
+import os
+
+os.environ['RAGFLOW_API_KEY'] = "your-ragflow-api-key"
+os.environ['RAGFLOW_API_BASE'] = "http://localhost:9380"  # or your hosted URL
+
+response = completion(
+    model="ragflow/chat/my-chat-id/gpt-4o-mini",
+    messages=[{"role": "user", "content": "How does the deep doc understanding work?"}]
+)
+print(response)
+```
+
+## Sample Usage - Agent Endpoint
+
+```python
+from litellm import completion
+import os
+
+os.environ['RAGFLOW_API_KEY'] = "your-ragflow-api-key"
+os.environ['RAGFLOW_API_BASE'] = "http://localhost:9380"  # or your hosted URL
+
+response = completion(
+    model="ragflow/agent/my-agent-id/gpt-4o-mini",
+    messages=[{"role": "user", "content": "What are the key features?"}]
+)
+print(response)
+```
+
+## Sample Usage - With Parameters
+
+You can also pass `api_key` and `api_base` directly as parameters:
+
+```python
+from litellm import completion
+
+response = completion(
+    model="ragflow/chat/my-chat-id/gpt-4o-mini",
+    messages=[{"role": "user", "content": "Hello!"}],
+    api_key="your-ragflow-api-key",
+    api_base="http://localhost:9380"
+)
+print(response)
+```
+
+## Sample Usage - Streaming
+
+```python
+from litellm import completion
+import os
+
+os.environ['RAGFLOW_API_KEY'] = "your-ragflow-api-key"
+os.environ['RAGFLOW_API_BASE'] = "http://localhost:9380"
+
+response = completion(
+    model="ragflow/agent/my-agent-id/gpt-4o-mini",
+    messages=[{"role": "user", "content": "Explain RAGFlow"}],
+    stream=True
+)
+
+for chunk in response:
+    print(chunk)
+```
+
+## Model Name Format
+
+The model name must follow one of these formats:
+
+### Chat Endpoint
+```
+ragflow/chat/{chat_id}/{model_name}
+```
+
+Example: `ragflow/chat/my-chat-id/gpt-4o-mini`
+
+### Agent Endpoint
+```
+ragflow/agent/{agent_id}/{model_name}
+```
+
+Example: `ragflow/agent/my-agent-id/gpt-4o-mini`
+
+Where:
+- `{chat_id}` or `{agent_id}` is the ID of your chat or agent in RAGFlow
+- `{model_name}` is the actual model name (e.g., `gpt-4o-mini`, `gpt-4o`, etc.)
+
+## Configuration Sources
+
+LiteLLM supports multiple ways to provide credentials, checked in this order:
+
+1. **Function parameters**: `api_key="..."`, `api_base="..."`
+2. **litellm_params**: `litellm_params={"api_key": "...", "api_base": "..."}`
+3. **Environment variables**: `RAGFLOW_API_KEY`, `RAGFLOW_API_BASE`
+4. **Global litellm settings**: `litellm.api_key`, `litellm.api_base`
+
+## Usage - LiteLLM Proxy Server
+
+### 1. Save key in your environment
+
+```bash
+export RAGFLOW_API_KEY="your-ragflow-api-key"
+export RAGFLOW_API_BASE="http://localhost:9380"
+```
+
+### 2. Start the proxy
+
+<Tabs>
+<TabItem value="config" label="config.yaml">
+
+```yaml
+model_list:
+  - model_name: ragflow-chat-gpt4
+    litellm_params:
+      model: ragflow/chat/my-chat-id/gpt-4o-mini
+      api_key: os.environ/RAGFLOW_API_KEY
+      api_base: os.environ/RAGFLOW_API_BASE
+  - model_name: ragflow-agent-gpt4
+    litellm_params:
+      model: ragflow/agent/my-agent-id/gpt-4o-mini
+      api_key: os.environ/RAGFLOW_API_KEY
+      api_base: os.environ/RAGFLOW_API_BASE
+```
+
+</TabItem>
+<TabItem value="cli" label="CLI">
+
+```bash
+$ litellm --config /path/to/config.yaml
+
+# Server running on http://0.0.0.0:4000
+```
+
+</TabItem>
+</Tabs>
+
+### 3. Test it
+
+<Tabs>
+<TabItem value="Curl" label="Curl Request">
+
+```bash
+curl http://0.0.0.0:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer sk-1234" \
+  -d '{
+    "model": "ragflow-chat-gpt4",
+    "messages": [
+      {"role": "user", "content": "How does RAGFlow work?"}
+    ]
+  }'
+```
+
+</TabItem>
+<TabItem value="Python" label="Python SDK">
+
+```python
+from openai import OpenAI
+
+client = OpenAI(
+    api_key="sk-1234",  # Your LiteLLM proxy key
+    base_url="http://0.0.0.0:4000"
+)
+
+response = client.chat.completions.create(
+    model="ragflow-chat-gpt4",
+    messages=[
+        {"role": "user", "content": "How does RAGFlow work?"}
+    ]
+)
+print(response)
+```
+
+</TabItem>
+</Tabs>
+
+## API Base URL Handling
+
+The `api_base` parameter can be provided with or without `/v1` suffix. LiteLLM will automatically handle it:
+
+- `http://localhost:9380` → `http://localhost:9380/api/v1/chats_openai/{chat_id}/chat/completions`
+- `http://localhost:9380/v1` → `http://localhost:9380/api/v1/chats_openai/{chat_id}/chat/completions`
+- `http://localhost:9380/api/v1` → `http://localhost:9380/api/v1/chats_openai/{chat_id}/chat/completions`
+
+All three formats will work correctly.
+
+## Error Handling
+
+If you encounter errors:
+
+1. **Invalid model format**: Ensure your model name follows `ragflow/{chat|agent}/{id}/{model_name}` format
+2. **Missing api_base**: Provide `api_base` via parameter, environment variable, or litellm_params
+3. **Connection errors**: Verify your RAGFlow server is running and accessible at the provided `api_base`
+
+:::info
+
+For more information about passing provider-specific parameters, [go here](../completion/provider_specific_params.md)
+
+:::
+

docs/my-website/docs/providers/ragflow_vector_store.md

@@ -0,0 +1,349 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import Image from '@theme/IdealImage';
+
+# RAGFlow Vector Stores
+
+Litellm support creation and management of datasets for document processing and knowledge base management in Ragflow.
+
+| Property | Details |
+|----------|---------|
+| Description | RAGFlow datasets enable document processing, chunking, and knowledge base management for RAG applications. |
+| Provider Route on LiteLLM | `ragflow` in the litellm vector_store_registry |
+| Provider Doc | [RAGFlow API Documentation ↗](https://ragflow.io/docs) |
+| Supported Operations | Dataset Management (Create, List, Update, Delete) |
+| Search/Retrieval | ❌ Not supported (management only) |
+
+## Quick Start
+
+### LiteLLM Python SDK
+
+```python showLineNumbers title="Example using LiteLLM Python SDK"
+import os
+import litellm
+
+# Set RAGFlow credentials
+os.environ["RAGFLOW_API_KEY"] = "your-ragflow-api-key"
+os.environ["RAGFLOW_API_BASE"] = "http://localhost:9380"  # Optional, defaults to localhost:9380
+
+# Create a RAGFlow dataset
+response = litellm.vector_stores.create(
+    name="my-dataset",
+    custom_llm_provider="ragflow",
+    metadata={
+        "description": "My knowledge base dataset",
+        "embedding_model": "BAAI/bge-large-zh-v1.5@BAAI",
+        "chunk_method": "naive"
+    }
+)
+
+print(f"Created dataset ID: {response.id}")
+print(f"Dataset name: {response.name}")
+```
+
+### LiteLLM Proxy
+
+#### 1. Configure your vector_store_registry
+
+<Tabs>
+<TabItem value="config-yaml" label="config.yaml">
+
+```yaml
+model_list:
+  - model_name: gpt-4o-mini
+    litellm_params:
+      model: gpt-4o-mini
+      api_key: os.environ/OPENAI_API_KEY
+
+vector_store_registry:
+  - vector_store_name: "ragflow-knowledge-base"
+    litellm_params:
+      vector_store_id: "your-dataset-id"
+      custom_llm_provider: "ragflow"
+      api_key: os.environ/RAGFLOW_API_KEY
+      api_base: os.environ/RAGFLOW_API_BASE  # Optional
+      vector_store_description: "RAGFlow dataset for knowledge base"
+      vector_store_metadata:
+        source: "Company documentation"
+```
+
+</TabItem>
+
+<TabItem value="litellm-ui" label="LiteLLM UI">
+
+On the LiteLLM UI, Navigate to Experimental > Vector Stores > Create Vector Store. On this page you can create a vector store with a name, vector store id and credentials.
+
+<Image 
+  img={require('../../img/kb_2.png')}
+  style={{width: '50%'}}
+/>
+
+</TabItem>
+</Tabs>
+
+#### 2. Create a dataset via Proxy
+
+<Tabs>
+<TabItem value="curl" label="Curl">
+
+```bash
+curl http://localhost:4000/v1/vector_stores \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer $LITELLM_API_KEY" \
+  -d '{
+    "name": "my-ragflow-dataset",
+    "custom_llm_provider": "ragflow",
+    "metadata": {
+      "description": "Test dataset",
+      "chunk_method": "naive"
+    }
+  }'
+```
+
+</TabItem>
+
+<TabItem value="openai-sdk" label="OpenAI Python SDK">
+
+```python
+from openai import OpenAI
+
+# Initialize client with your LiteLLM proxy URL
+client = OpenAI(
+    base_url="http://localhost:4000",
+    api_key="your-litellm-api-key"
+)
+
+# Create a RAGFlow dataset
+response = client.vector_stores.create(
+    name="my-ragflow-dataset",
+    custom_llm_provider="ragflow",
+    metadata={
+        "description": "Test dataset",
+        "chunk_method": "naive"
+    }
+)
+
+print(f"Created dataset: {response.id}")
+```
+
+</TabItem>
+</Tabs>
+
+## Configuration
+
+### Environment Variables
+
+RAGFlow vector stores support configuration via environment variables:
+
+- `RAGFLOW_API_KEY` - Your RAGFlow API key (required)
+- `RAGFLOW_API_BASE` - RAGFlow API base URL (optional, defaults to `http://localhost:9380`)
+
+### Parameters
+
+You can also pass these via `litellm_params`:
+
+- `api_key` - RAGFlow API key (overrides `RAGFLOW_API_KEY` env var)
+- `api_base` - RAGFlow API base URL (overrides `RAGFLOW_API_BASE` env var)
+
+## Dataset Creation Options
+
+### Basic Dataset Creation
+
+```python
+response = litellm.vector_stores.create(
+    name="basic-dataset",
+    custom_llm_provider="ragflow"
+)
+```
+
+### Dataset with Chunk Method
+
+RAGFlow supports various chunk methods for different document types:
+
+<Tabs>
+<TabItem value="naive" label="Naive (General)">
+
+```python
+response = litellm.vector_stores.create(
+    name="general-dataset",
+    custom_llm_provider="ragflow",
+    metadata={
+        "chunk_method": "naive",
+        "parser_config": {
+            "chunk_token_num": 512,
+            "delimiter": "\n",
+            "html4excel": False,
+            "layout_recognize": "DeepDOC"
+        }
+    }
+)
+```
+
+</TabItem>
+
+<TabItem value="book" label="Book">
+
+```python
+response = litellm.vector_stores.create(
+    name="book-dataset",
+    custom_llm_provider="ragflow",
+    metadata={
+        "chunk_method": "book",
+        "parser_config": {
+            "raptor": {
+                "use_raptor": False
+            }
+        }
+    }
+)
+```
+
+</TabItem>
+
+<TabItem value="qa" label="Q&A">
+
+```python
+response = litellm.vector_stores.create(
+    name="qa-dataset",
+    custom_llm_provider="ragflow",
+    metadata={
+        "chunk_method": "qa",
+        "parser_config": {
+            "raptor": {
+                "use_raptor": False
+            }
+        }
+    }
+)
+```
+
+</TabItem>
+
+<TabItem value="paper" label="Paper">
+
+```python
+response = litellm.vector_stores.create(
+    name="paper-dataset",
+    custom_llm_provider="ragflow",
+    metadata={
+        "chunk_method": "paper",
+        "parser_config": {
+            "raptor": {
+                "use_raptor": False
+            }
+        }
+    }
+)
+```
+
+</TabItem>
+</Tabs>
+
+### Dataset with Ingestion Pipeline
+
+Instead of using a chunk method, you can use an ingestion pipeline:
+
+```python
+response = litellm.vector_stores.create(
+    name="pipeline-dataset",
+    custom_llm_provider="ragflow",
+    metadata={
+        "parse_type": 2,  # Number of parsers in your pipeline
+        "pipeline_id": "d0bebe30ae2211f0970942010a8e0005"  # 32-character hex ID
+    }
+)
+```
+
+**Note**: `chunk_method` and `pipeline_id` are mutually exclusive. Use one or the other.
+
+### Advanced Parser Configuration
+
+```python
+response = litellm.vector_stores.create(
+    name="advanced-dataset",
+    custom_llm_provider="ragflow",
+    metadata={
+        "chunk_method": "naive",
+        "description": "Advanced dataset with custom parser config",
+        "embedding_model": "BAAI/bge-large-zh-v1.5@BAAI",
+        "permission": "me",  # or "team"
+        "parser_config": {
+            "chunk_token_num": 1024,
+            "delimiter": "\n!?;。；！？",
+            "html4excel": True,
+            "layout_recognize": "DeepDOC",
+            "auto_keywords": 5,
+            "auto_questions": 3,
+            "task_page_size": 12,
+            "raptor": {
+                "use_raptor": True
+            },
+            "graphrag": {
+                "use_graphrag": False
+            }
+        }
+    }
+)
+```
+
+## Supported Chunk Methods
+
+RAGFlow supports the following chunk methods:
+
+- `naive` - General purpose (default)
+- `book` - For book documents
+- `email` - For email documents
+- `laws` - For legal documents
+- `manual` - Manual chunking
+- `one` - Single chunk
+- `paper` - For academic papers
+- `picture` - For image documents
+- `presentation` - For presentation documents
+- `qa` - Q&A format
+- `table` - For table documents
+- `tag` - Tag-based chunking
+
+## RAGFlow-Specific Parameters
+
+All RAGFlow-specific parameters should be passed via the `metadata` field:
+
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| `avatar` | string | Base64 encoding of the avatar (max 65535 chars) |
+| `description` | string | Brief description of the dataset (max 65535 chars) |
+| `embedding_model` | string | Embedding model name (e.g., "BAAI/bge-large-zh-v1.5@BAAI") |
+| `permission` | string | Access permission: "me" (default) or "team" |
+| `chunk_method` | string | Chunking method (see supported methods above) |
+| `parser_config` | object | Parser configuration (varies by chunk_method) |
+| `parse_type` | int | Number of parsers in pipeline (required with pipeline_id) |
+| `pipeline_id` | string | 32-character hex pipeline ID (required with parse_type) |
+
+## Error Handling
+
+RAGFlow returns error responses in the following format:
+
+```json
+{
+    "code": 101,
+    "message": "Dataset name 'my-dataset' already exists"
+}
+```
+
+LiteLLM automatically maps these to appropriate exceptions:
+
+- `code != 0` → Raises exception with the error message
+- Missing required fields → Raises `ValueError`
+- Mutually exclusive parameters → Raises `ValueError`
+
+## Limitations
+
+- **Search/Retrieval**: RAGFlow vector stores support dataset management only. Search operations are not supported and will raise `NotImplementedError`.
+- **List/Update/Delete**: These operations are not yet implemented through the standard vector store API. Use RAGFlow's native API endpoints directly.
+
+## Further Reading
+
+Vector Stores:
+- [Vector Store Creation](../vector_stores/create.md)
+- [Using Vector Stores with Completions](../completion/knowledgebase.md)
+- [Vector Store Registry](../completion/knowledgebase.md#vectorstoreregistry)
+

docs/my-website/docs/providers/vertex.md

@@ -2550,355 +2550,6 @@ print(response)
 </TabItem>
 </Tabs>
 
-
-## **Gemini TTS (Text-to-Speech) Audio Output**
-
-:::info
-
-LiteLLM supports Gemini TTS models on Vertex AI that can generate audio responses using the OpenAI-compatible `audio` parameter format.
-
-:::
-
-### Supported Models
-
-LiteLLM supports Gemini TTS models with audio capabilities on Vertex AI (e.g. `vertex_ai/gemini-2.5-flash-preview-tts` and `vertex_ai/gemini-2.5-pro-preview-tts`). For the complete list of available TTS models and voices, see the [official Gemini TTS documentation](https://ai.google.dev/gemini-api/docs/speech-generation).
-
-### Limitations
-
-:::warning
-
-**Important Limitations**:
-- Gemini TTS models only support the `pcm16` audio format
-- **Streaming support has not been added** to TTS models yet
-- The `modalities` parameter must be set to `['audio']` for TTS requests
-
-:::
-
-### Quick Start
-
-<Tabs>
-<TabItem value="sdk" label="SDK">
-
-```python
-from litellm import completion
-import json
-
-## GET CREDENTIALS
-file_path = 'path/to/vertex_ai_service_account.json'
-
-# Load the JSON file
-with open(file_path, 'r') as file:
-    vertex_credentials = json.load(file)
-
-# Convert to JSON string
-vertex_credentials_json = json.dumps(vertex_credentials)
-
-response = completion(
-    model="vertex_ai/gemini-2.5-flash-preview-tts",
-    messages=[{"role": "user", "content": "Say hello in a friendly voice"}],
-    modalities=["audio"],  # Required for TTS models
-    audio={
-        "voice": "Kore",
-        "format": "pcm16"  # Required: must be "pcm16"
-    },
-    vertex_credentials=vertex_credentials_json
-)
-
-print(response)
-```
-
-</TabItem>
-<TabItem value="proxy" label="PROXY">
-
-1. Setup config.yaml
-
-```yaml
-model_list:
-  - model_name: gemini-tts-flash
-    litellm_params:
-      model: vertex_ai/gemini-2.5-flash-preview-tts
-      vertex_project: "your-project-id"
-      vertex_location: "us-central1"
-      vertex_credentials: "/path/to/service_account.json"
-  - model_name: gemini-tts-pro
-    litellm_params:
-      model: vertex_ai/gemini-2.5-pro-preview-tts
-      vertex_project: "your-project-id"
-      vertex_location: "us-central1"
-      vertex_credentials: "/path/to/service_account.json"
-```
-
-2. Start proxy
-
-```bash
-litellm --config /path/to/config.yaml
-```
-
-3. Make TTS request
-
-```bash
-curl http://0.0.0.0:4000/v1/chat/completions \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer <YOUR-LITELLM-KEY>" \
-  -d '{
-    "model": "gemini-tts-flash",
-    "messages": [{"role": "user", "content": "Say hello in a friendly voice"}],
-    "modalities": ["audio"],
-    "audio": {
-      "voice": "Kore",
-      "format": "pcm16"
-    }
-  }'
-```
-
-</TabItem>
-</Tabs>
-
-### Advanced Usage
-
-You can combine TTS with other Gemini features:
-
-```python
-response = completion(
-    model="vertex_ai/gemini-2.5-pro-preview-tts",
-    messages=[
-        {"role": "system", "content": "You are a helpful assistant that speaks clearly."},
-        {"role": "user", "content": "Explain quantum computing in simple terms"}
-    ],
-    modalities=["audio"],
-    audio={
-        "voice": "Charon",
-        "format": "pcm16"
-    },
-    temperature=0.7,
-    max_tokens=150,
-    vertex_credentials=vertex_credentials_json
-)
-```
-
-For more information about Gemini's TTS capabilities and available voices, see the [official Gemini TTS documentation](https://ai.google.dev/gemini-api/docs/speech-generation).
-
-## **Text to Speech APIs**
-
-:::info
-
-LiteLLM supports calling [Vertex AI Text to Speech API](https://console.cloud.google.com/vertex-ai/generative/speech/text-to-speech) in the OpenAI text to speech API format
-
-:::
-
-
-
-### Usage - Basic
-
-<Tabs>
-<TabItem value="sdk" label="SDK">
-
-Vertex AI does not support passing a `model` param - so passing `model=vertex_ai/` is the only required param
-
-**Sync Usage**
-
-```python
-speech_file_path = Path(__file__).parent / "speech_vertex.mp3"
-response = litellm.speech(
-    model="vertex_ai/",
-    input="hello what llm guardrail do you have",
-)
-response.stream_to_file(speech_file_path)
-```
-
-**Async Usage**
-```python
-speech_file_path = Path(__file__).parent / "speech_vertex.mp3"
-response = litellm.aspeech(
-    model="vertex_ai/",
-    input="hello what llm guardrail do you have",
-)
-response.stream_to_file(speech_file_path)
-```
-
-</TabItem>
-<TabItem value="proxy" label="LiteLLM PROXY (Unified Endpoint)">
-
-1. Add model to config.yaml
-```yaml
-model_list:
-  - model_name: vertex-tts
-    litellm_params:
-      model: vertex_ai/ # Vertex AI does not support passing a `model` param - so passing `model=vertex_ai/` is the only required param
-      vertex_project: "adroit-crow-413218"
-      vertex_location: "us-central1"
-      vertex_credentials: adroit-crow-413218-a956eef1a2a8.json 
-
-litellm_settings:
-  drop_params: True
-```
-
-2. Start Proxy 
-
-```
-$ litellm --config /path/to/config.yaml
-```
-
-3. Make Request use OpenAI Python SDK
-
-
-```python
-import openai
-
-client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
-
-# see supported values for "voice" on vertex here: 
-# https://console.cloud.google.com/vertex-ai/generative/speech/text-to-speech
-response = client.audio.speech.create(
-    model = "vertex-tts",
-    input="the quick brown fox jumped over the lazy dogs",
-    voice={'languageCode': 'en-US', 'name': 'en-US-Studio-O'}
-)
-print("response from proxy", response)
-```
-
-</TabItem>
-</Tabs>
-
-
-### Usage - `ssml` as input
-
-Pass your `ssml` as input to the `input` param, if it contains `<speak>`, it will be automatically detected and passed as `ssml` to the Vertex AI API
-
-If you need to force your `input` to be passed as `ssml`, set `use_ssml=True`
-
-<Tabs>
-<TabItem value="sdk" label="SDK">
-
-Vertex AI does not support passing a `model` param - so passing `model=vertex_ai/` is the only required param
-
-
-```python
-speech_file_path = Path(__file__).parent / "speech_vertex.mp3"
-
-
-ssml = """
-<speak>
-    <p>Hello, world!</p>
-    <p>This is a test of the <break strength="medium" /> text-to-speech API.</p>
-</speak>
-"""
-
-response = litellm.speech(
-    input=ssml,
-    model="vertex_ai/test",
-    voice={
-        "languageCode": "en-UK",
-        "name": "en-UK-Studio-O",
-    },
-    audioConfig={
-        "audioEncoding": "LINEAR22",
-        "speakingRate": "10",
-    },
-)
-response.stream_to_file(speech_file_path)
-```
-
-</TabItem>
-
-<TabItem value="proxy" label="LiteLLM PROXY (Unified Endpoint)">
-
-```python
-import openai
-
-client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
-
-ssml = """
-<speak>
-    <p>Hello, world!</p>
-    <p>This is a test of the <break strength="medium" /> text-to-speech API.</p>
-</speak>
-"""
-
-# see supported values for "voice" on vertex here: 
-# https://console.cloud.google.com/vertex-ai/generative/speech/text-to-speech
-response = client.audio.speech.create(
-    model = "vertex-tts",
-    input=ssml,
-    voice={'languageCode': 'en-US', 'name': 'en-US-Studio-O'},
-)
-print("response from proxy", response)
-```
-
-</TabItem>
-</Tabs>
-
-
-### Forcing SSML Usage
-
-You can force the use of SSML by setting the `use_ssml` parameter to `True`. This is useful when you want to ensure that your input is treated as SSML, even if it doesn't contain the `<speak>` tags.
-
-Here are examples of how to force SSML usage:
-
-
-<Tabs>
-<TabItem value="sdk" label="SDK">
-
-Vertex AI does not support passing a `model` param - so passing `model=vertex_ai/` is the only required param
-
-
-```python
-speech_file_path = Path(__file__).parent / "speech_vertex.mp3"
-
-
-ssml = """
-<speak>
-    <p>Hello, world!</p>
-    <p>This is a test of the <break strength="medium" /> text-to-speech API.</p>
-</speak>
-"""
-
-response = litellm.speech(
-    input=ssml,
-    use_ssml=True,
-    model="vertex_ai/test",
-    voice={
-        "languageCode": "en-UK",
-        "name": "en-UK-Studio-O",
-    },
-    audioConfig={
-        "audioEncoding": "LINEAR22",
-        "speakingRate": "10",
-    },
-)
-response.stream_to_file(speech_file_path)
-```
-
-</TabItem>
-
-<TabItem value="proxy" label="LiteLLM PROXY (Unified Endpoint)">
-
-```python
-import openai
-
-client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
-
-ssml = """
-<speak>
-    <p>Hello, world!</p>
-    <p>This is a test of the <break strength="medium" /> text-to-speech API.</p>
-</speak>
-"""
-
-# see supported values for "voice" on vertex here: 
-# https://console.cloud.google.com/vertex-ai/generative/speech/text-to-speech
-response = client.audio.speech.create(
-    model = "vertex-tts",
-    input=ssml, # pass as None since OpenAI SDK requires this param
-    voice={'languageCode': 'en-US', 'name': 'en-US-Studio-O'},
-    extra_body={"use_ssml": True},
-)
-print("response from proxy", response)
-```
-
-</TabItem>
-</Tabs>
-
 ## **Fine Tuning APIs**
 
 

docs/my-website/docs/providers/vertex_image.md

@@ -1,18 +1,65 @@
 # Vertex AI Image Generation
 
-Vertex AI Image Generation uses Google's Imagen models to generate high-quality images from text descriptions.
+Vertex AI supports two types of image generation:
+
+1. **Gemini Image Generation Models** (Nano Banana 🍌) - Conversational image generation using `generateContent` API
+2. **Imagen Models** - Traditional image generation using `predict` API
 
 | Property | Details |
 |----------|---------|
-| Description | Vertex AI Image Generation uses Google's Imagen models to generate high-quality images from text descriptions. |
+| Description | Vertex AI Image Generation supports both Gemini image generation models |
 | Provider Route on LiteLLM | `vertex_ai/` |
 | Provider Doc | [Google Cloud Vertex AI Image Generation ↗](https://cloud.google.com/vertex-ai/docs/generative-ai/image/generate-images) |
+| Gemini Image Generation Docs | [Gemini Image Generation ↗](https://ai.google.dev/gemini-api/docs/image-generation) |
 
 ## Quick Start
 
-### LiteLLM Python SDK
+### Gemini Image Generation Models
 
-```python showLineNumbers title="Basic Image Generation"
+Gemini image generation models support conversational image creation with features like:
+- Text-to-Image generation
+- Image editing (text + image → image)
+- Multi-turn image refinement
+- High-fidelity text rendering
+- Up to 4K resolution (Gemini 3 Pro)
+
+```python showLineNumbers title="Gemini 2.5 Flash Image"
+import litellm
+
+# Generate a single image
+response = await litellm.aimage_generation(
+    prompt="A nano banana dish in a fancy restaurant with a Gemini theme",
+    model="vertex_ai/gemini-2.5-flash-image",
+    vertex_ai_project="your-project-id",
+    vertex_ai_location="us-central1",
+    n=1,
+    size="1024x1024",
+)
+
+print(response.data[0].b64_json)  # Gemini returns base64 images
+```
+
+```python showLineNumbers title="Gemini 3 Pro Image Preview (4K output)"
+import litellm
+
+# Generate high-resolution image
+response = await litellm.aimage_generation(
+    prompt="Da Vinci style anatomical sketch of a dissected Monarch butterfly",
+    model="vertex_ai/gemini-3-pro-image-preview",
+    vertex_ai_project="your-project-id",
+    vertex_ai_location="us-central1",
+    n=1,
+    size="1024x1024",
+    # Optional: specify image size for Gemini 3 Pro
+    # imageSize="4K",  # Options: "1K", "2K", "4K"
+)
+
+print(response.data[0].b64_json)
+```
+
+### Imagen Models
+
+```python showLineNumbers title="Imagen Image Generation"
 import litellm
 
 # Generate a single image
@@ -21,9 +68,11 @@ response = await litellm.aimage_generation(
     model="vertex_ai/imagen-4.0-generate-001",
     vertex_ai_project="your-project-id",
     vertex_ai_location="us-central1",
+    n=1,
+    size="1024x1024",
 )
 
-print(response.data[0].url)
+print(response.data[0].b64_json)  # Imagen also returns base64 images
 ```
 
 ### LiteLLM Proxy
@@ -70,6 +119,18 @@ print(response.data[0].url)
 
 ## Supported Models
 
+### Gemini Image Generation Models
+
+- `vertex_ai/gemini-2.5-flash-image` - Fast, efficient image generation (1024px resolution)
+- `vertex_ai/gemini-3-pro-image-preview` - Advanced model with 4K output, Google Search grounding, and thinking mode
+- `vertex_ai/gemini-2.0-flash-preview-image` - Preview model
+- `vertex_ai/gemini-2.5-flash-image-preview` - Preview model
+
+### Imagen Models
+
+- `vertex_ai/imagegeneration@006` - Legacy Imagen model
+- `vertex_ai/imagen-4.0-generate-001` - Latest Imagen model
+- `vertex_ai/imagen-3.0-generate-001` - Imagen 3.0 model
 
 :::tip
 
@@ -77,7 +138,5 @@ print(response.data[0].url)
 
 :::
 
-LiteLLM supports all Vertex AI Imagen models available through Google Cloud.
-
 For the complete and up-to-date list of supported models, visit: [https://models.litellm.ai/](https://models.litellm.ai/)
 

docs/my-website/docs/providers/vertex_speech.md

@@ -0,0 +1,423 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Vertex AI Text to Speech
+
+| Property | Details |
+|-------|-------|
+| Description | Google Cloud Text-to-Speech with Chirp3 HD voices and Gemini TTS |
+| Provider Route on LiteLLM | `vertex_ai/chirp` (Chirp), `vertex_ai/gemini-*-tts` (Gemini) |
+
+## Chirp3 HD Voices
+
+Google Cloud Text-to-Speech API with high-quality Chirp3 HD voices.
+
+### Quick Start
+
+#### LiteLLM Python SDK
+
+```python showLineNumbers title="Chirp3 Quick Start"
+from litellm import speech
+from pathlib import Path
+
+speech_file_path = Path(__file__).parent / "speech.mp3"
+response = speech(
+    model="vertex_ai/chirp",
+    voice="alloy",  # OpenAI voice name - automatically mapped
+    input="Hello, this is Vertex AI Text to Speech",
+    vertex_project="your-project-id",
+    vertex_location="us-central1",
+)
+response.stream_to_file(speech_file_path)
+```
+
+#### LiteLLM AI Gateway
+
+**1. Setup config.yaml**
+
+```yaml showLineNumbers title="config.yaml"
+model_list:
+  - model_name: vertex-tts
+    litellm_params:
+      model: vertex_ai/chirp
+      vertex_project: "your-project-id"
+      vertex_location: "us-central1"
+      vertex_credentials: "/path/to/service_account.json"
+```
+
+**2. Start the proxy**
+
+```bash title="Start LiteLLM Proxy"
+litellm --config /path/to/config.yaml
+```
+
+**3. Make requests**
+
+<Tabs>
+<TabItem value="curl" label="curl">
+
+```bash showLineNumbers title="Chirp3 Quick Start"
+curl http://0.0.0.0:4000/v1/audio/speech \
+  -H "Authorization: Bearer sk-1234" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "vertex-tts",
+    "voice": "alloy",
+    "input": "Hello, this is Vertex AI Text to Speech"
+  }' \
+  --output speech.mp3
+```
+
+</TabItem>
+<TabItem value="openai-sdk" label="OpenAI Python SDK">
+
+```python showLineNumbers title="Chirp3 Quick Start"
+import openai
+
+client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
+
+response = client.audio.speech.create(
+    model="vertex-tts",
+    voice="alloy",
+    input="Hello, this is Vertex AI Text to Speech",
+)
+response.stream_to_file("speech.mp3")
+```
+
+</TabItem>
+</Tabs>
+
+### Voice Mapping
+
+LiteLLM maps OpenAI voice names to Google Cloud voices. You can use either OpenAI voices or Google Cloud voices directly.
+
+| OpenAI Voice | Google Cloud Voice |
+|-------------|-------------------|
+| `alloy` | en-US-Studio-O |
+| `echo` | en-US-Studio-M |
+| `fable` | en-GB-Studio-B |
+| `onyx` | en-US-Wavenet-D |
+| `nova` | en-US-Studio-O |
+| `shimmer` | en-US-Wavenet-F |
+
+### Using Google Cloud Voices Directly
+
+#### LiteLLM Python SDK
+
+```python showLineNumbers title="Chirp3 HD Voice"
+from litellm import speech
+
+# Pass Chirp3 HD voice name directly
+response = speech(
+    model="vertex_ai/chirp",
+    voice="en-US-Chirp3-HD-Charon",
+    input="Hello with a Chirp3 HD voice",
+    vertex_project="your-project-id",
+)
+response.stream_to_file("speech.mp3")
+```
+
+```python showLineNumbers title="Voice as Dict (Multilingual)"
+from litellm import speech
+
+# Pass as dict for full control over language and voice
+response = speech(
+    model="vertex_ai/chirp",
+    voice={
+        "languageCode": "de-DE",
+        "name": "de-DE-Chirp3-HD-Charon",
+    },
+    input="Hallo, dies ist ein Test",
+    vertex_project="your-project-id",
+)
+response.stream_to_file("speech.mp3")
+```
+
+#### LiteLLM AI Gateway
+
+<Tabs>
+<TabItem value="curl" label="curl">
+
+```bash showLineNumbers title="Chirp3 HD Voice"
+curl http://0.0.0.0:4000/v1/audio/speech \
+  -H "Authorization: Bearer sk-1234" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "vertex-tts",
+    "voice": "en-US-Chirp3-HD-Charon",
+    "input": "Hello with a Chirp3 HD voice"
+  }' \
+  --output speech.mp3
+```
+
+```bash showLineNumbers title="Voice as Dict (Multilingual)"
+curl http://0.0.0.0:4000/v1/audio/speech \
+  -H "Authorization: Bearer sk-1234" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "vertex-tts",
+    "voice": {"languageCode": "de-DE", "name": "de-DE-Chirp3-HD-Charon"},
+    "input": "Hallo, dies ist ein Test"
+  }' \
+  --output speech.mp3
+```
+
+</TabItem>
+<TabItem value="openai-sdk" label="OpenAI Python SDK">
+
+```python showLineNumbers title="Chirp3 HD Voice"
+import openai
+
+client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
+
+response = client.audio.speech.create(
+    model="vertex-tts",
+    voice="en-US-Chirp3-HD-Charon",
+    input="Hello with a Chirp3 HD voice",
+)
+response.stream_to_file("speech.mp3")
+```
+
+```python showLineNumbers title="Voice as Dict (Multilingual)"
+import openai
+
+client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
+
+response = client.audio.speech.create(
+    model="vertex-tts",
+    voice={"languageCode": "de-DE", "name": "de-DE-Chirp3-HD-Charon"},
+    input="Hallo, dies ist ein Test",
+)
+response.stream_to_file("speech.mp3")
+```
+
+</TabItem>
+</Tabs>
+
+Browse available voices: [Google Cloud Text-to-Speech Console](https://console.cloud.google.com/vertex-ai/generative/speech/text-to-speech)
+
+### Passing Raw SSML
+
+LiteLLM auto-detects SSML when your input contains `<speak>` tags and passes it through unchanged.
+
+#### LiteLLM Python SDK
+
+```python showLineNumbers title="SSML Input"
+from litellm import speech
+
+ssml = """
+<speak>
+    <p>Hello, world!</p>
+    <p>This is a test of the <break strength="medium" /> text-to-speech API.</p>
+</speak>
+"""
+
+response = speech(
+    model="vertex_ai/chirp",
+    voice="en-US-Studio-O",
+    input=ssml,  # Auto-detected as SSML
+    vertex_project="your-project-id",
+)
+response.stream_to_file("speech.mp3")
+```
+
+```python showLineNumbers title="Force SSML Mode"
+from litellm import speech
+
+# Force SSML mode with use_ssml=True
+response = speech(
+    model="vertex_ai/chirp",
+    voice="en-US-Studio-O",
+    input="<speak><prosody rate='slow'>Speaking slowly</prosody></speak>",
+    use_ssml=True,
+    vertex_project="your-project-id",
+)
+response.stream_to_file("speech.mp3")
+```
+
+#### LiteLLM AI Gateway
+
+<Tabs>
+<TabItem value="curl" label="curl">
+
+```bash showLineNumbers title="SSML Input"
+curl http://0.0.0.0:4000/v1/audio/speech \
+  -H "Authorization: Bearer sk-1234" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "vertex-tts",
+    "voice": "en-US-Studio-O",
+    "input": "<speak><p>Hello!</p><break time=\"500ms\"/><p>How are you?</p></speak>"
+  }' \
+  --output speech.mp3
+```
+
+</TabItem>
+<TabItem value="openai-sdk" label="OpenAI Python SDK">
+
+```python showLineNumbers title="SSML Input"
+import openai
+
+client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
+
+ssml = """<speak><p>Hello!</p><break time="500ms"/><p>How are you?</p></speak>"""
+
+response = client.audio.speech.create(
+    model="vertex-tts",
+    voice="en-US-Studio-O",
+    input=ssml,
+)
+response.stream_to_file("speech.mp3")
+```
+
+</TabItem>
+</Tabs>
+
+### Supported Parameters
+
+| Parameter | Description | Values |
+|-----------|-------------|--------|
+| `voice` | Voice selection | OpenAI voice, Google Cloud voice name, or dict |
+| `input` | Text to convert | Plain text or SSML |
+| `speed` | Speaking rate | 0.25 to 4.0 (default: 1.0) |
+| `response_format` | Audio format | `mp3`, `opus`, `wav`, `pcm`, `flac` |
+| `use_ssml` | Force SSML mode | `True` / `False` |
+
+### Async Usage
+
+```python showLineNumbers title="Async Speech Generation"
+import asyncio
+from litellm import aspeech
+
+async def main():
+    response = await aspeech(
+        model="vertex_ai/chirp",
+        voice="alloy",
+        input="Hello from async",
+        vertex_project="your-project-id",
+    )
+    response.stream_to_file("speech.mp3")
+
+asyncio.run(main())
+```
+
+---
+
+## Gemini TTS
+
+Gemini models with audio output capabilities using the chat completions API.
+
+:::warning
+**Limitations:**
+- Only supports `pcm16` audio format
+- Streaming not yet supported
+- Must set `modalities: ["audio"]`
+:::
+
+### Quick Start
+
+#### LiteLLM Python SDK
+
+```python showLineNumbers title="Gemini TTS Quick Start"
+from litellm import completion
+import json
+
+# Load credentials
+with open('path/to/service_account.json', 'r') as file:
+    vertex_credentials = json.dumps(json.load(file))
+
+response = completion(
+    model="vertex_ai/gemini-2.5-flash-preview-tts",
+    messages=[{"role": "user", "content": "Say hello in a friendly voice"}],
+    modalities=["audio"],
+    audio={
+        "voice": "Kore",
+        "format": "pcm16"
+    },
+    vertex_credentials=vertex_credentials
+)
+print(response)
+```
+
+#### LiteLLM AI Gateway
+
+**1. Setup config.yaml**
+
+```yaml showLineNumbers title="config.yaml"
+model_list:
+  - model_name: gemini-tts
+    litellm_params:
+      model: vertex_ai/gemini-2.5-flash-preview-tts
+      vertex_project: "your-project-id"
+      vertex_location: "us-central1"
+      vertex_credentials: "/path/to/service_account.json"
+```
+
+**2. Start the proxy**
+
+```bash title="Start LiteLLM Proxy"
+litellm --config /path/to/config.yaml
+```
+
+**3. Make requests**
+
+<Tabs>
+<TabItem value="curl" label="curl">
+
+```bash showLineNumbers title="Gemini TTS Request"
+curl http://0.0.0.0:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer sk-1234" \
+  -d '{
+    "model": "gemini-tts",
+    "messages": [{"role": "user", "content": "Say hello in a friendly voice"}],
+    "modalities": ["audio"],
+    "audio": {"voice": "Kore", "format": "pcm16"}
+  }'
+```
+
+</TabItem>
+<TabItem value="openai-sdk" label="OpenAI Python SDK">
+
+```python showLineNumbers title="Gemini TTS Request"
+import openai
+
+client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
+
+response = client.chat.completions.create(
+    model="gemini-tts",
+    messages=[{"role": "user", "content": "Say hello in a friendly voice"}],
+    modalities=["audio"],
+    audio={"voice": "Kore", "format": "pcm16"},
+)
+print(response)
+```
+
+</TabItem>
+</Tabs>
+
+### Supported Models
+
+- `vertex_ai/gemini-2.5-flash-preview-tts`
+- `vertex_ai/gemini-2.5-pro-preview-tts`
+
+See [Gemini TTS documentation](https://ai.google.dev/gemini-api/docs/speech-generation) for available voices.
+
+### Advanced Usage
+
+```python showLineNumbers title="Gemini TTS with System Prompt"
+from litellm import completion
+
+response = completion(
+    model="vertex_ai/gemini-2.5-pro-preview-tts",
+    messages=[
+        {"role": "system", "content": "You are a helpful assistant that speaks clearly."},
+        {"role": "user", "content": "Explain quantum computing in simple terms"}
+    ],
+    modalities=["audio"],
+    audio={"voice": "Charon", "format": "pcm16"},
+    temperature=0.7,
+    max_tokens=150,
+    vertex_credentials=vertex_credentials
+)
+```

docs/my-website/docs/providers/watsonx.md

@@ -1,287 +0,0 @@
-import Tabs from '@theme/Tabs';
-import TabItem from '@theme/TabItem';
-
-# IBM watsonx.ai
-
-LiteLLM supports all IBM [watsonx.ai](https://watsonx.ai/) foundational models and embeddings.
-
-## Environment Variables
-```python
-os.environ["WATSONX_URL"] = ""  # (required) Base URL of your WatsonX instance
-# (required) either one of the following:
-os.environ["WATSONX_APIKEY"] = "" # IBM cloud API key
-os.environ["WATSONX_TOKEN"] = "" # IAM auth token
-# optional - can also be passed as params to completion() or embedding()
-os.environ["WATSONX_PROJECT_ID"] = "" # Project ID of your WatsonX instance
-os.environ["WATSONX_DEPLOYMENT_SPACE_ID"] = "" # ID of your deployment space to use deployed models
-os.environ["WATSONX_ZENAPIKEY"] = "" # Zen API key (use for long-term api token)
-```
-
-See [here](https://cloud.ibm.com/apidocs/watsonx-ai#api-authentication) for more information on how to get an access token to authenticate to watsonx.ai.
-
-## Usage
-
-<a target="_blank" href="https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/liteLLM_IBM_Watsonx.ipynb">
-  <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/>
-</a>
-
-```python
-import os
-from litellm import completion
-
-os.environ["WATSONX_URL"] = ""
-os.environ["WATSONX_APIKEY"] = ""
-
-## Call WATSONX `/text/chat` endpoint - supports function calling
-response = completion(
-  model="watsonx/meta-llama/llama-3-1-8b-instruct",
-  messages=[{ "content": "what is your favorite colour?","role": "user"}],
-  project_id="<my-project-id>" # or pass with os.environ["WATSONX_PROJECT_ID"]
-)
-
-## Call WATSONX `/text/generation` endpoint - not all models support /chat route. 
-response = completion(
-  model="watsonx/ibm/granite-13b-chat-v2",
-  messages=[{ "content": "what is your favorite colour?","role": "user"}],
-  project_id="<my-project-id>"
-)
-```
-
-## Usage - Streaming
-```python
-import os
-from litellm import completion
-
-os.environ["WATSONX_URL"] = ""
-os.environ["WATSONX_APIKEY"] = ""
-os.environ["WATSONX_PROJECT_ID"] = ""
-
-response = completion(
-  model="watsonx/meta-llama/llama-3-1-8b-instruct",
-  messages=[{ "content": "what is your favorite colour?","role": "user"}],
-  stream=True
-)
-for chunk in response:
-  print(chunk)
-```
-
-#### Example Streaming Output Chunk
-```json
-{
-  "choices": [
-    {
-      "finish_reason": null,
-      "index": 0,
-      "delta": {
-        "content": "I don't have a favorite color, but I do like the color blue. What's your favorite color?"
-      }
-    }
-  ],
-  "created": null,
-  "model": "watsonx/ibm/granite-13b-chat-v2",
-  "usage": {
-    "prompt_tokens": null,
-    "completion_tokens": null,
-    "total_tokens": null
-  }
-}
-```
-
-## Usage - Models in deployment spaces
-
-Models that have been deployed to a deployment space (e.g.: tuned models) can be called using the `deployment/<deployment_id>` format (where `<deployment_id>` is the ID of the deployed model in your deployment space). 
-
-The ID of your deployment space must also be set in the environment variable `WATSONX_DEPLOYMENT_SPACE_ID` or passed to the function as `space_id=<deployment_space_id>`. 
-
-```python
-import litellm
-response = litellm.completion(
-    model="watsonx/deployment/<deployment_id>",
-    messages=[{"content": "Hello, how are you?", "role": "user"}],
-    space_id="<deployment_space_id>"
-)
-```
-
-## Usage - Embeddings
-
-LiteLLM also supports making requests to IBM watsonx.ai embedding models. The credential needed for this is the same as for completion.
-
-```python
-from litellm import embedding
-
-response = embedding(
-    model="watsonx/ibm/slate-30m-english-rtrvr",
-    input=["What is the capital of France?"],
-    project_id="<my-project-id>"
-)
-print(response)
-# EmbeddingResponse(model='ibm/slate-30m-english-rtrvr', data=[{'object': 'embedding', 'index': 0, 'embedding': [-0.037463713, -0.02141933, -0.02851813, 0.015519324, ..., -0.0021367231, -0.01704561, -0.001425816, 0.0035238306]}], object='list', usage=Usage(prompt_tokens=8, total_tokens=8))
-```
-
-## OpenAI Proxy Usage 
-
-Here's how to call IBM watsonx.ai with the LiteLLM Proxy Server
-
-### 1. Save keys in your environment
-
-```bash
-export WATSONX_URL=""
-export WATSONX_APIKEY=""
-export WATSONX_PROJECT_ID=""
-```
-
-### 2. Start the proxy 
-
-<Tabs>
-<TabItem value="cli" label="CLI">
-
-```bash
-$ litellm --model watsonx/meta-llama/llama-3-8b-instruct
-
-# Server running on http://0.0.0.0:4000
-```
-
-</TabItem>
-<TabItem value="config" label="config.yaml">
-
-```yaml
-model_list:
-  - model_name: llama-3-8b
-    litellm_params:
-      # all params accepted by litellm.completion()
-      model: watsonx/meta-llama/llama-3-8b-instruct
-      api_key: "os.environ/WATSONX_API_KEY" # does os.getenv("WATSONX_API_KEY")
-```
-</TabItem>
-</Tabs>
-
-### 3. Test it
-
-
-<Tabs>
-<TabItem value="Curl" label="Curl Request">
-
-```shell
-curl --location 'http://0.0.0.0:4000/chat/completions' \
---header 'Content-Type: application/json' \
---data ' {
-      "model": "llama-3-8b",
-      "messages": [
-        {
-          "role": "user",
-          "content": "what is your favorite colour?"
-        }
-      ]
-    }
-'
-```
-</TabItem>
-<TabItem value="openai" label="OpenAI v1.0.0+">
-
-```python
-import openai
-client = openai.OpenAI(
-    api_key="anything",
-    base_url="http://0.0.0.0:4000"
-)
-
-# request sent to model set on litellm proxy, `litellm --model`
-response = client.chat.completions.create(model="llama-3-8b", messages=[
-    {
-        "role": "user",
-        "content": "what is your favorite colour?"
-    }
-])
-
-print(response)
-
-```
-</TabItem>
-<TabItem value="langchain" label="Langchain">
-
-```python
-from langchain.chat_models import ChatOpenAI
-from langchain.prompts.chat import (
-    ChatPromptTemplate,
-    HumanMessagePromptTemplate,
-    SystemMessagePromptTemplate,
-)
-from langchain.schema import HumanMessage, SystemMessage
-
-chat = ChatOpenAI(
-    openai_api_base="http://0.0.0.0:4000", # set openai_api_base to the LiteLLM Proxy
-    model = "llama-3-8b",
-    temperature=0.1
-)
-
-messages = [
-    SystemMessage(
-        content="You are a helpful assistant that im using to make a test request to."
-    ),
-    HumanMessage(
-        content="test from litellm. tell me why it's amazing in 1 sentence"
-    ),
-]
-response = chat(messages)
-
-print(response)
-```
-</TabItem>
-</Tabs>
-
-
-## Authentication
-
-### Passing credentials as parameters
-
-You can also pass the credentials as parameters to the completion and embedding functions.
-
-```python
-import os
-from litellm import completion
-
-response = completion(
-            model="watsonx/ibm/granite-13b-chat-v2",
-            messages=[{ "content": "What is your favorite color?","role": "user"}],
-            url="",
-            api_key="",
-            project_id=""
-)
-```
-
-
-## Supported IBM watsonx.ai Models
-
-Here are some examples of models available in IBM watsonx.ai that you can use with LiteLLM:
-
-| Mode Name                          | Command                                                                                  |
-|------------------------------------|------------------------------------------------------------------------------------------|
-| Flan T5 XXL                        | `completion(model=watsonx/google/flan-t5-xxl, messages=messages)`                        |
-| Flan Ul2                           | `completion(model=watsonx/google/flan-ul2, messages=messages)`                           |
-| Mt0 XXL                            | `completion(model=watsonx/bigscience/mt0-xxl, messages=messages)`                        |
-| Gpt Neox                           | `completion(model=watsonx/eleutherai/gpt-neox-20b, messages=messages)`                   |
-| Mpt 7B Instruct2                   | `completion(model=watsonx/ibm/mpt-7b-instruct2, messages=messages)`                      |
-| Starcoder                          | `completion(model=watsonx/bigcode/starcoder, messages=messages)`                         |
-| Llama 2 70B Chat                   | `completion(model=watsonx/meta-llama/llama-2-70b-chat, messages=messages)`               |
-| Llama 2 13B Chat                   | `completion(model=watsonx/meta-llama/llama-2-13b-chat, messages=messages)`               |
-| Granite 13B Instruct               | `completion(model=watsonx/ibm/granite-13b-instruct-v1, messages=messages)`               |
-| Granite 13B Chat                   | `completion(model=watsonx/ibm/granite-13b-chat-v1, messages=messages)`                   |
-| Flan T5 XL                         | `completion(model=watsonx/google/flan-t5-xl, messages=messages)`                         |
-| Granite 13B Chat V2                | `completion(model=watsonx/ibm/granite-13b-chat-v2, messages=messages)`                   |
-| Granite 13B Instruct V2            | `completion(model=watsonx/ibm/granite-13b-instruct-v2, messages=messages)`               |
-| Elyza Japanese Llama 2 7B Instruct | `completion(model=watsonx/elyza/elyza-japanese-llama-2-7b-instruct, messages=messages)`  |
-| Mixtral 8X7B Instruct V01 Q        | `completion(model=watsonx/ibm-mistralai/mixtral-8x7b-instruct-v01-q, messages=messages)` |
-
-
-For a list of all available models in watsonx.ai, see [here](https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx&locale=en&audience=wdp).
-
-
-## Supported IBM watsonx.ai Embedding Models
-
-| Model Name | Function Call                                                          |
-|------------|------------------------------------------------------------------------|
-| Slate 30m  | `embedding(model="watsonx/ibm/slate-30m-english-rtrvr", input=input)`  |
-| Slate 125m | `embedding(model="watsonx/ibm/slate-125m-english-rtrvr", input=input)` |
-
-
-For a list of all available embedding models in watsonx.ai, see [here](https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models-embed.html?context=wx).

docs/my-website/docs/providers/watsonx/audio_transcription.md

@@ -0,0 +1,57 @@
+# WatsonX Audio Transcription
+
+## Overview
+
+| Property | Details |
+|----------|---------|
+| Description | WatsonX audio transcription using Whisper models for speech-to-text |
+| Provider Route on LiteLLM | `watsonx/` |
+| Supported Operations | `/v1/audio/transcriptions` |
+| Link to Provider Doc | [IBM WatsonX.ai ↗](https://www.ibm.com/watsonx) |
+
+## Quick Start
+
+### **LiteLLM SDK**
+
+```python showLineNumbers title="transcription.py"
+import litellm
+
+response = litellm.transcription(
+    model="watsonx/whisper-large-v3-turbo",
+    file=open("audio.mp3", "rb"),
+    api_base="https://us-south.ml.cloud.ibm.com",
+    api_key="your-api-key",
+    project_id="your-project-id"
+)
+print(response.text)
+```
+
+### **LiteLLM Proxy**
+
+```yaml showLineNumbers title="config.yaml"
+model_list:
+  - model_name: whisper-large-v3-turbo
+    litellm_params:
+      model: watsonx/whisper-large-v3-turbo
+      api_key: os.environ/WATSONX_APIKEY
+      api_base: os.environ/WATSONX_URL
+      project_id: os.environ/WATSONX_PROJECT_ID
+```
+
+```bash title="Request"
+curl http://localhost:4000/v1/audio/transcriptions \
+  -H "Authorization: Bearer sk-1234" \
+  -F file="@audio.mp3" \
+  -F model="whisper-large-v3-turbo"
+```
+
+## Supported Parameters
+
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| `model` | string | Model ID (e.g., `watsonx/whisper-large-v3-turbo`) |
+| `file` | file | Audio file to transcribe |
+| `language` | string | Language code (e.g., `en`) |
+| `prompt` | string | Optional prompt to guide transcription |
+| `temperature` | float | Sampling temperature (0-1) |
+| `response_format` | string | `json`, `text`, `srt`, `verbose_json`, `vtt` |

docs/my-website/docs/providers/watsonx/index.md

@@ -0,0 +1,230 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# IBM watsonx.ai
+
+LiteLLM supports all IBM [watsonx.ai](https://watsonx.ai/) foundational models and embeddings.
+
+## Environment Variables
+```python
+os.environ["WATSONX_URL"] = ""  # (required) Base URL of your WatsonX instance
+# (required) either one of the following:
+os.environ["WATSONX_APIKEY"] = "" # IBM cloud API key
+os.environ["WATSONX_TOKEN"] = "" # IAM auth token
+# optional - can also be passed as params to completion() or embedding()
+os.environ["WATSONX_PROJECT_ID"] = "" # Project ID of your WatsonX instance
+os.environ["WATSONX_DEPLOYMENT_SPACE_ID"] = "" # ID of your deployment space to use deployed models
+os.environ["WATSONX_ZENAPIKEY"] = "" # Zen API key (use for long-term api token)
+```
+
+See [here](https://cloud.ibm.com/apidocs/watsonx-ai#api-authentication) for more information on how to get an access token to authenticate to watsonx.ai.
+
+## Usage
+
+<a target="_blank" href="https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/liteLLM_IBM_Watsonx.ipynb">
+  <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/>
+</a>
+
+```python showLineNumbers title="Chat Completion"
+import os
+from litellm import completion
+
+os.environ["WATSONX_URL"] = ""
+os.environ["WATSONX_APIKEY"] = ""
+
+response = completion(
+  model="watsonx/meta-llama/llama-3-1-8b-instruct",
+  messages=[{ "content": "what is your favorite colour?","role": "user"}],
+  project_id="<my-project-id>"
+)
+```
+
+## Usage - Streaming
+```python showLineNumbers title="Streaming"
+import os
+from litellm import completion
+
+os.environ["WATSONX_URL"] = ""
+os.environ["WATSONX_APIKEY"] = ""
+os.environ["WATSONX_PROJECT_ID"] = ""
+
+response = completion(
+  model="watsonx/meta-llama/llama-3-1-8b-instruct",
+  messages=[{ "content": "what is your favorite colour?","role": "user"}],
+  stream=True
+)
+for chunk in response:
+  print(chunk)
+```
+
+## Usage - Models in deployment spaces
+
+Models deployed to a deployment space (e.g.: tuned models) can be called using the `deployment/<deployment_id>` format.
+
+```python showLineNumbers title="Deployment Space"
+import litellm
+
+response = litellm.completion(
+    model="watsonx/deployment/<deployment_id>",
+    messages=[{"content": "Hello, how are you?", "role": "user"}],
+    space_id="<deployment_space_id>"
+)
+```
+
+## Usage - Embeddings
+
+```python showLineNumbers title="Embeddings"
+from litellm import embedding
+
+response = embedding(
+    model="watsonx/ibm/slate-30m-english-rtrvr",
+    input=["What is the capital of France?"],
+    project_id="<my-project-id>"
+)
+```
+
+## LiteLLM Proxy Usage 
+
+### 1. Save keys in your environment
+
+```bash
+export WATSONX_URL=""
+export WATSONX_APIKEY=""
+export WATSONX_PROJECT_ID=""
+```
+
+### 2. Start the proxy 
+
+<Tabs>
+<TabItem value="cli" label="CLI">
+
+```bash
+$ litellm --model watsonx/meta-llama/llama-3-8b-instruct
+```
+
+</TabItem>
+<TabItem value="config" label="config.yaml">
+
+```yaml showLineNumbers title="config.yaml"
+model_list:
+  - model_name: llama-3-8b
+    litellm_params:
+      model: watsonx/meta-llama/llama-3-8b-instruct
+      api_key: "os.environ/WATSONX_API_KEY"
+```
+</TabItem>
+</Tabs>
+
+### 3. Test it
+
+
+<Tabs>
+<TabItem value="Curl" label="Curl Request">
+
+```shell
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+--header 'Content-Type: application/json' \
+--data '{
+      "model": "llama-3-8b",
+      "messages": [
+        {
+          "role": "user",
+          "content": "what is your favorite colour?"
+        }
+      ]
+    }'
+```
+</TabItem>
+<TabItem value="openai" label="OpenAI SDK">
+
+```python showLineNumbers
+import openai
+
+client = openai.OpenAI(
+    api_key="anything",
+    base_url="http://0.0.0.0:4000"
+)
+
+response = client.chat.completions.create(
+    model="llama-3-8b", 
+    messages=[{"role": "user", "content": "what is your favorite colour?"}]
+)
+print(response)
+```
+</TabItem>
+</Tabs>
+
+
+## Supported Models
+
+| Model Name                         | Command                                                                                  |
+|------------------------------------|------------------------------------------------------------------------------------------|
+| Llama 3.1 8B Instruct              | `completion(model="watsonx/meta-llama/llama-3-1-8b-instruct", messages=messages)`        |
+| Llama 2 70B Chat                   | `completion(model="watsonx/meta-llama/llama-2-70b-chat", messages=messages)`             |
+| Granite 13B Chat V2                | `completion(model="watsonx/ibm/granite-13b-chat-v2", messages=messages)`                 |
+| Mixtral 8X7B Instruct              | `completion(model="watsonx/ibm-mistralai/mixtral-8x7b-instruct-v01-q", messages=messages)` |
+
+For all available models, see [watsonx.ai documentation](https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models.html?context=wx).
+
+## Supported Embedding Models
+
+| Model Name | Function Call                                                          |
+|------------|------------------------------------------------------------------------|
+| Slate 30m  | `embedding(model="watsonx/ibm/slate-30m-english-rtrvr", input=input)`  |
+| Slate 125m | `embedding(model="watsonx/ibm/slate-125m-english-rtrvr", input=input)` |
+
+For all available embedding models, see [watsonx.ai embedding documentation](https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models-embed.html?context=wx).
+
+
+## Advanced
+
+### Using Zen API Key
+
+You can use a Zen API key for long-term authentication instead of generating IAM tokens. Pass it either as an environment variable or as a parameter:
+
+```python
+import os
+from litellm import completion
+
+# Option 1: Set as environment variable
+os.environ["WATSONX_ZENAPIKEY"] = "your-zen-api-key"
+
+response = completion(
+    model="watsonx/ibm/granite-13b-chat-v2",
+    messages=[{"content": "What is your favorite color?", "role": "user"}],
+    project_id="your-project-id"
+)
+
+# Option 2: Pass as parameter
+response = completion(
+    model="watsonx/ibm/granite-13b-chat-v2",
+    messages=[{"content": "What is your favorite color?", "role": "user"}],
+    zen_api_key="your-zen-api-key",
+    project_id="your-project-id"
+)
+```
+
+**Using with LiteLLM Proxy via OpenAI client:**
+
+```python
+import openai
+
+client = openai.OpenAI(
+    api_key="sk-1234",  # LiteLLM proxy key
+    base_url="http://0.0.0.0:4000"
+)
+
+response = client.chat.completions.create(
+    model="watsonx/ibm/granite-3-3-8b-instruct",
+    messages=[{"role": "user", "content": "What is your favorite color?"}],
+    max_tokens=2048,
+    extra_body={
+        "project_id": "your-project-id",
+        "zen_api_key": "your-zen-api-key"
+    }
+)
+```
+
+See [IBM documentation](https://www.ibm.com/docs/en/watsonx/w-and-w/2.2.0?topic=keys-generating-zenapikey-authorization-tokens) for more information on generating Zen API keys.
+
+

docs/my-website/docs/providers/zai.md

@@ -0,0 +1,135 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Z.AI (Zhipu AI)
+https://z.ai/
+
+**We support Z.AI GLM text/chat models, just set `zai/` as a prefix when sending completion requests**
+
+## API Key
+```python
+# env variable
+os.environ['ZAI_API_KEY']
+```
+
+## Sample Usage
+```python
+from litellm import completion
+import os
+
+os.environ['ZAI_API_KEY'] = ""
+response = completion(
+    model="zai/glm-4.6",
+    messages=[
+       {"role": "user", "content": "hello from litellm"}
+   ],
+)
+print(response)
+```
+
+## Sample Usage - Streaming
+```python
+from litellm import completion
+import os
+
+os.environ['ZAI_API_KEY'] = ""
+response = completion(
+    model="zai/glm-4.6",
+    messages=[
+       {"role": "user", "content": "hello from litellm"}
+   ],
+    stream=True
+)
+
+for chunk in response:
+    print(chunk)
+```
+
+## Supported Models
+
+We support ALL Z.AI GLM models, just set `zai/` as a prefix when sending completion requests.
+
+| Model Name | Function Call | Notes |
+|------------|---------------|-------|
+| glm-4.6 | `completion(model="zai/glm-4.6", messages)` | Latest flagship model, 200K context |
+| glm-4.5 | `completion(model="zai/glm-4.5", messages)` | 128K context |
+| glm-4.5v | `completion(model="zai/glm-4.5v", messages)` | Vision model |
+| glm-4.5-x | `completion(model="zai/glm-4.5-x", messages)` | Premium tier |
+| glm-4.5-air | `completion(model="zai/glm-4.5-air", messages)` | Lightweight |
+| glm-4.5-airx | `completion(model="zai/glm-4.5-airx", messages)` | Fast lightweight |
+| glm-4-32b-0414-128k | `completion(model="zai/glm-4-32b-0414-128k", messages)` | 32B parameter model |
+| glm-4.5-flash | `completion(model="zai/glm-4.5-flash", messages)` | **FREE tier** |
+
+## Model Pricing
+
+| Model | Input ($/1M tokens) | Output ($/1M tokens) | Context Window |
+|-------|---------------------|----------------------|----------------|
+| glm-4.6 | $0.60 | $2.20 | 200K |
+| glm-4.5 | $0.60 | $2.20 | 128K |
+| glm-4.5v | $0.60 | $1.80 | 128K |
+| glm-4.5-x | $2.20 | $8.90 | 128K |
+| glm-4.5-air | $0.20 | $1.10 | 128K |
+| glm-4.5-airx | $1.10 | $4.50 | 128K |
+| glm-4-32b-0414-128k | $0.10 | $0.10 | 128K |
+| glm-4.5-flash | **FREE** | **FREE** | 128K |
+
+## Using with LiteLLM Proxy
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+from litellm import completion
+import os
+
+os.environ['ZAI_API_KEY'] = ""
+response = completion(
+    model="zai/glm-4.6",
+    messages=[{"role": "user", "content": "Hello, how are you?"}],
+)
+
+print(response.choices[0].message.content)
+```
+
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+1. Setup config.yaml
+
+```yaml
+model_list:
+  - model_name: glm-4.6
+    litellm_params:
+        model: zai/glm-4.6
+        api_key: os.environ/ZAI_API_KEY
+  - model_name: glm-4.5-flash  # Free tier
+    litellm_params:
+        model: zai/glm-4.5-flash
+        api_key: os.environ/ZAI_API_KEY
+```
+
+2. Run proxy
+
+```bash
+litellm --config config.yaml
+```
+
+3. Test it!
+
+```bash
+curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
+-H 'Content-Type: application/json' \
+-H 'Authorization: Bearer sk-1234' \
+-d '{
+    "model": "glm-4.6",
+    "messages": [
+      {
+        "role": "user",
+        "content": "Hello, how are you?"
+      }
+    ]
+}'
+```
+
+</TabItem>
+</Tabs>

docs/my-website/docs/proxy/config_settings.md

@@ -104,6 +104,7 @@ general_settings:
   disable_responses_id_security: boolean  # turn off response ID security checks that prevent users from accessing other users' responses
   enable_jwt_auth: boolean  # allow proxy admin to auth in via jwt tokens with 'litellm_proxy_admin' in claims
   enforce_user_param: boolean  # requires all openai endpoint requests to have a 'user' param
+  reject_clientside_metadata_tags: boolean  # if true, rejects requests with client-side 'metadata.tags' to prevent users from influencing budgets
   allowed_routes: ["route1", "route2"]  # list of allowed proxy API routes - a user can access. (currently JWT-Auth only)
   key_management_system: google_kms  # either google_kms or azure_kms
   master_key: string
@@ -112,7 +113,7 @@ general_settings:
 
   # Database Settings
   database_url: string
-  database_connection_pool_limit: 0  # default 100
+  database_connection_pool_limit: 0  # default 10
   database_connection_timeout: 0  # default 60s
   allow_requests_on_db_unavailable: boolean  # if true, will allow requests that can not connect to the DB to verify Virtual Key to still work 
 
@@ -201,6 +202,7 @@ router_settings:
 | disable_responses_id_security | boolean | If true, disables response ID security checks that prevent users from accessing response IDs from other users. When false (default), response IDs are encrypted with user information to ensure users can only access their own responses. Applies to /v1/responses endpoints |
 | enable_jwt_auth | boolean | allow proxy admin to auth in via jwt tokens with 'litellm_proxy_admin' in claims. [Doc on JWT Tokens](token_auth) |
 | enforce_user_param | boolean | If true, requires all OpenAI endpoint requests to have a 'user' param. [Doc on call hooks](call_hooks)|
+| reject_clientside_metadata_tags | boolean | If true, rejects requests that contain client-side 'metadata.tags' to prevent users from influencing budgets by sending different tags. Tags can only be inherited from the API key metadata. |
 | allowed_routes | array of strings | List of allowed proxy API routes a user can access [Doc on controlling allowed routes](enterprise#control-available-public-private-routes)|
 | key_management_system | string | Specifies the key management system. [Doc Secret Managers](../secret) |
 | master_key | string | The master key for the proxy [Set up Virtual Keys](virtual_keys) |
@@ -232,7 +234,7 @@ router_settings:
 | max_response_size_mb | int | The maximum size for responses in MB. LLM Responses above this size will not be sent. |
 | proxy_budget_rescheduler_min_time | int | The minimum time (in seconds) to wait before checking db for budget resets. **Default is 597 seconds** |
 | proxy_budget_rescheduler_max_time | int | The maximum time (in seconds) to wait before checking db for budget resets. **Default is 605 seconds** |
-| proxy_batch_write_at | int | Time (in seconds) to wait before batch writing spend logs to the db. **Default is 30 seconds** |
+| proxy_batch_write_at | int | Time (in seconds) to wait before batch writing spend logs to the db. **Default is 10 seconds** |
 | proxy_batch_polling_interval | int | Time (in seconds) to wait before polling a batch, to check if it's completed. **Default is 6000 seconds (1 hour)** |
 | alerting_args | dict | Args for Slack Alerting [Doc on Slack Alerting](./alerting.md) |
 | custom_key_generate | str | Custom function for key generation [Doc on custom key generation](./virtual_keys.md#custom--key-generate) |
@@ -375,6 +377,7 @@ router_settings:
 | ATHINA_API_KEY | API key for Athina service
 | ATHINA_BASE_URL | Base URL for Athina service (defaults to `https://log.athina.ai`)
 | AUTH_STRATEGY | Strategy used for authentication (e.g., OAuth, API key)
+| AUTO_REDIRECT_UI_LOGIN_TO_SSO | Flag to enable automatic redirect of UI login page to SSO when SSO is configured. Default is **true**
 | ANTHROPIC_API_KEY | API key for Anthropic service
 | ANTHROPIC_API_BASE | Base URL for Anthropic API. Default is https://api.anthropic.com
 | AWS_ACCESS_KEY_ID | Access Key ID for AWS services
@@ -473,6 +476,8 @@ router_settings:
 | DEFAULT_ALLOWED_FAILS | Maximum failures allowed before cooling down a model. Default is 3
 | DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS | Default maximum tokens for Anthropic chat completions. Default is 4096
 | DEFAULT_BATCH_SIZE | Default batch size for operations. Default is 512
+| DEFAULT_CHUNK_OVERLAP | Default chunk overlap for RAG text splitters. Default is 200
+| DEFAULT_CHUNK_SIZE | Default chunk size for RAG text splitters. Default is 1000
 | DEFAULT_CLIENT_DISCONNECT_CHECK_TIMEOUT_SECONDS | Timeout in seconds for checking client disconnection. Default is 1
 | DEFAULT_COOLDOWN_TIME_SECONDS | Duration in seconds to cooldown a model after failures. Default is 5
 | DEFAULT_CRON_JOB_LOCK_TTL_SECONDS | Time-to-live for cron job locks in seconds. Default is 60 (1 minute)
@@ -572,6 +577,8 @@ router_settings:
 | GENERIC_USER_PROVIDER_ATTRIBUTE | Attribute specifying the user's provider
 | GENERIC_USER_ROLE_ATTRIBUTE | Attribute specifying the user's role
 | GENERIC_USERINFO_ENDPOINT | Endpoint to fetch user information in generic OAuth
+| GENERIC_LOGGER_ENDPOINT | Endpoint URL for the Generic Logger callback to send logs to
+| GENERIC_LOGGER_HEADERS | JSON string of headers to include in Generic Logger callback requests
 | GEMINI_API_BASE | Base URL for Gemini API. Default is https://generativelanguage.googleapis.com
 | GALILEO_BASE_URL | Base URL for Galileo platform
 | GALILEO_PASSWORD | Password for Galileo authentication
@@ -757,7 +764,7 @@ router_settings:
 | PROMPTLAYER_API_KEY | API key for PromptLayer integration
 | PROXY_ADMIN_ID | Admin identifier for proxy server
 | PROXY_BASE_URL | Base URL for proxy service
-| PROXY_BATCH_WRITE_AT | Time in seconds to wait before batch writing spend logs to the database. Default is 30
+| PROXY_BATCH_WRITE_AT | Time in seconds to wait before batch writing spend logs to the database. Default is 10
 | PROXY_BATCH_POLLING_INTERVAL | Time in seconds to wait before polling a batch, to check if it's completed. Default is 6000s (1 hour)
 | PROXY_BUDGET_RESCHEDULER_MAX_TIME | Maximum time in seconds to wait before checking database for budget resets. Default is 605
 | PROXY_BUDGET_RESCHEDULER_MIN_TIME | Minimum time in seconds to wait before checking database for budget resets. Default is 597

docs/my-website/docs/proxy/configs.md

@@ -576,7 +576,7 @@ custom_tokenizer:
 
 ```yaml
 general_settings: 
-  database_connection_pool_limit: 100 # sets connection pool for prisma client to postgres db at 100
+  database_connection_pool_limit: 10 # sets connection pool for prisma client to postgres db (default: 10, recommended: 10-20)
   database_connection_timeout: 60 # sets a 60s timeout for any connection call to the db 
 ```
 

docs/my-website/docs/proxy/error_diagnosis.md

@@ -0,0 +1,90 @@
+# Diagnosing Errors - Provider vs Gateway
+
+Having trouble diagnosing if an error is from the **LLM Provider** (OpenAI, Anthropic, etc.) or from the **LiteLLM AI Gateway** itself? Here's how to tell.
+
+## Quick Rule
+
+**If the error contains `<Provider>Exception`, it's from the provider.**
+
+| Error Contains | Error Source |
+|----------------|--------------|
+| `AnthropicException` | Anthropic |
+| `OpenAIException` | OpenAI |
+| `AzureException` | Azure |
+| `BedrockException` | AWS Bedrock |
+| `VertexAIException` | Google Vertex AI |
+| No provider name | LiteLLM AI Gateway |
+
+## Examples
+
+### Provider Error (from AWS Bedrock)
+
+```
+{
+  "error": {
+    "message": "litellm.BadRequestError: BedrockException - {\"message\":\"The model returned the following errors: messages.1.content.0.type: Expected `thinking` or `redacted_thinking`, but found `text`.\"}",
+    "type": "invalid_request_error",
+    "param": null,
+    "code": "400"
+  }
+}
+```
+
+This error is from **AWS Bedrock** (notice `BedrockException`). The Bedrock API is rejecting the request due to invalid message format - this is not a LiteLLM issue.
+
+### Provider Error (from OpenAI)
+
+```
+{
+  "error": {
+    "message": "litellm.AuthenticationError: OpenAIException - Incorrect API key provided: <my-key>. You can find your API key at https://platform.openai.com/account/api-keys.",
+    "type": "invalid_request_error",
+    "param": null,
+    "code": "invalid_api_key"
+  }
+}
+```
+
+This error is from **OpenAI** (notice `OpenAIException`). The OpenAI API key configured in LiteLLM is invalid.
+
+### Provider Error (from Anthropic)
+
+```
+{
+  "error": {
+    "message": "litellm.InternalServerError: AnthropicException - Overloaded. Handle with `litellm.InternalServerError`.",
+    "type": "internal_server_error",
+    "param": null,
+    "code": "500"
+  }
+}
+```
+
+This error is from **Anthropic** (notice `AnthropicException`). The Anthropic API is overloaded - this is not a LiteLLM issue.
+
+### Gateway Error (from LiteLLM)
+
+```
+{
+  "error": {
+    "message": "Invalid API Key. Please check your LiteLLM API key.",
+    "type": "auth_error",
+    "param": null,
+    "code": "401"
+  }
+}
+```
+
+This error is from the **LiteLLM AI Gateway** (no provider name). Your LiteLLM virtual key is invalid.
+
+## What to do?
+
+| Error Source | Action |
+|--------------|--------|
+| Provider Error | Check the provider's status page, adjust rate limits, or retry later |
+| Gateway Error | Check your LiteLLM configuration, API keys, or [open an issue](https://github.com/BerriAI/litellm/issues) |
+
+## See Also
+
+- [Debugging](/docs/proxy/debugging) - Enable debug logs to see detailed request/response info
+- [Exception Mapping](/docs/exception_mapping) - Full list of LiteLLM exception types

docs/my-website/docs/proxy/guardrails/bedrock.md

@@ -188,6 +188,28 @@ My email is [EMAIL] and my phone number is [PHONE_NUMBER]
 
 This helps protect sensitive information while still allowing the model to understand the context of the request.
 
+## Experimental: Only Send Latest User Message
+
+When you're chaining long conversations through Bedrock guardrails, you can opt into a lighter, experimental behavior by setting `experimental_use_latest_role_message_only: true` in the guardrail's `litellm_params`. When enabled, LiteLLM only sends the most recent `user` message (or assistant output during post-call checks) to Bedrock, which:
+
+- prevents unintended blocks on older system/dev messages
+- keeps Bedrock payloads smaller, reducing latency and cost
+- applies to proxy hooks (`pre_call`, `during_call`) and the `/guardrails/apply_guardrail` testing endpoint
+
+```yaml showLineNumbers title="litellm proxy config.yaml"
+guardrails:
+  - guardrail_name: "bedrock-pre-guard"
+    litellm_params:
+      guardrail: bedrock
+      mode: "pre_call"
+      guardrailIdentifier: wf0hkdb5x07f
+      guardrailVersion: "DRAFT"
+      aws_region_name: os.environ/AWS_REGION
+      experimental_use_latest_role_message_only: true  # NEW
+```
+
+> ⚠️ This flag is currently experimental and defaults to `false` to preserve the legacy behavior (entire message history). We'll be listening to user feedback to decide if this becomes the default or rolls out more broadly.
+
 ## Disabling Exceptions on Bedrock BLOCK
 
 By default, when Bedrock guardrails block content, LiteLLM raises an HTTP 400 exception. However, you can disable this behavior by setting `disable_exception_on_block: true`. This is particularly useful when integrating with **OpenWebUI**, where exceptions can interrupt the chat flow and break the user experience.

docs/my-website/docs/proxy/guardrails/lasso_security.md

@@ -35,7 +35,7 @@ guardrails:
       guardrail: lasso
       mode: "pre_call"
       api_key: os.environ/LASSO_API_KEY
-      api_base: "https://server.lasso.security"
+      api_base: "https://server.lasso.security/gateway/v3"
   - guardrail_name: "lasso-post-guard"
     litellm_params:
       guardrail: lasso
@@ -228,7 +228,7 @@ Expected response:
 
 ## PII Masking with Lasso
 
-Lasso supports automatic PII detection and masking using the `/gateway/v1/classifix` endpoint. When enabled, sensitive information like emails, phone numbers, and other PII will be automatically masked with appropriate placeholders.
+Lasso supports automatic PII detection and masking using the `/classifix` endpoint. When enabled, sensitive information like emails, phone numbers, and other PII will be automatically masked with appropriate placeholders.
 
 ### Enabling PII Masking
 

docs/my-website/docs/proxy/guardrails/pillar_security.md

@@ -60,6 +60,8 @@ litellm_settings:
   set_verbose: true                          # Enable detailed logging
 ```
 
+**Note:** Virtual key context is **automatically passed** as headers - no additional configuration needed!
+
 ### 3. Start the Proxy
 
 ```bash
@@ -210,7 +212,7 @@ export PILLAR_API_KEY="your_api_key_here"
 export PILLAR_API_BASE="https://api.pillar.security"
 export PILLAR_ON_FLAGGED_ACTION="monitor"
 export PILLAR_FALLBACK_ON_ERROR="allow"
-export PILLAR_TIMEOUT="30.0"
+export PILLAR_TIMEOUT="5.0"
 ```
 
 ### Session Tracking

docs/my-website/docs/proxy/guardrails/tool_permission.md

@@ -1,4 +1,3 @@
-import Image from '@theme/IdealImage';
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
 
@@ -7,9 +6,34 @@ import TabItem from '@theme/TabItem';
 LiteLLM provides the LiteLLM Tool Permission Guardrail that lets you control which **tool calls** a model is allowed to invoke, using configurable allow/deny rules. This offers fine-grained, provider-agnostic control over tool execution (e.g., OpenAI Chat Completions `tool_calls`, Anthropic Messages `tool_use`, MCP tools).
 
 ## Quick Start
-### 1. Define Guardrails on your LiteLLM config.yaml 
 
-Define your guardrails under the `guardrails` section
+### LiteLLM UI
+
+#### Step 1: Select Tool Permission Guardrail
+
+Open the LiteLLM Dashboard, click **Add New Guardrail**, and choose **LiteLLM Tool Permission Guardrail**. This loads the rule builder UI.
+
+#### Step 2: Define Regex Rules
+
+1. Click **Add Rule**.
+2. Enter a unique Rule ID.
+3. Provide a regex for the tool name (e.g., `^mcp__github_.*$`).
+4. Optionally add a regex for tool type (e.g., `^function$`).
+5. Pick **Allow** or **Deny**.
+
+#### Step 3: Restrict Tool Arguments (Optional)
+
+Select **+ Restrict tool arguments** to attach regex validations to nested paths (dot + `[]` notation). This enforces that sensitive parameters (such as `arguments.to[]`) conform to pre-approved formats.
+
+#### Step 4: Choose Defaults & Actions
+
+- Set the fallback decision (`default_action`) for tools that do not hit any rule.
+- Decide how disallowed tools behave: **Block** halts the request, **Rewrite** strips forbidden tools and returns an error message inside the response.
+- Customize `violation_message_template` if you want branded error copy.
+- Save the guardrail.
+
+### LiteLLM Config.yaml Setup
+
 ```yaml
 guardrails:
   - guardrail_name: "tool-permission-guardrail"
@@ -21,16 +45,17 @@ guardrails:
           tool_name: "Bash"
           decision: "allow"
         - id: "allow_github_mcp"
-          tool_name: "mcp__github_*"
+          tool_name: "^mcp__github_.*$"
           decision: "allow"
         - id: "allow_aws_documentation"
-          tool_name: "mcp__aws-documentation_*_documentation"
+          tool_name: "^mcp__aws-documentation_.*_documentation$"
           decision: "allow"
         - id: "deny_read_commands"
           tool_name: "Read"
-          decision: "Deny"
+          decision: "deny"
         - id: "mail-domain"
-          tool_name: "send_email"
+          tool_name: "^send_email$"
+          tool_type: "^function$"
           decision: "allow"
           allowed_param_patterns:
             "to[]": "^.+@berri\\.ai$"
@@ -44,7 +69,8 @@ guardrails:
 
 ```yaml
 - id: "unique_rule_id"           # Unique identifier for the rule
-  tool_name: "pattern"           # Tool name or pattern to match
+  tool_name: "^regex$"           # Regex for tool name (optional, at least one of name/type required)
+  tool_type: "^function$"        # Regex for tool type (optional)
   decision: "allow"              # "allow" or "deny"
   allowed_param_patterns:         # Optional - regex map for argument paths (dot + [] notation)
     "path.to[].field": "^regex$"

docs/my-website/docs/proxy/pass_through.md

@@ -275,6 +275,20 @@ In this video, we'll add the Azure OpenAI Assistants API as a pass through endpo
 - Check LiteLLM proxy logs for error details
 - Verify the target API's expected request format
 
+### Allowing Team JWTs to use pass-through routes
+
+If you are using pass-through provider routes (e.g., `/anthropic/*`) and want your JWT team tokens to access these routes, add `mapped_pass_through_routes` to the `team_allowed_routes` in `litellm_jwtauth` or explicitly add the relevant route(s).
+
+Example (`proxy_server_config.yaml`):
+
+```yaml
+general_settings:
+  enable_jwt_auth: True
+  litellm_jwtauth:
+    team_ids_jwt_field: "team_ids"
+    team_allowed_routes: ["openai_routes","info_routes","mapped_pass_through_routes"]
+```
+
 ### Getting Help
 
 [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version)

docs/my-website/docs/proxy/pass_through_guardrails.md

@@ -0,0 +1,250 @@
+# Guardrails on Pass-Through Endpoints
+
+import Image from '@theme/IdealImage';
+
+## Overview
+
+| Property | Details |
+|----------|---------|
+| Description | Enable guardrail execution on LiteLLM pass-through endpoints with opt-in activation and automatic inheritance from org/team/key levels |
+| Supported Guardrails | All LiteLLM guardrails (Bedrock, Aporia, Lakera, etc.) |
+| Default Behavior | Guardrails are **disabled** on pass-through endpoints unless explicitly enabled |
+
+## Quick Start
+
+You can configure guardrails on pass-through endpoints either via the **UI** (recommended) or **config file**.
+
+### Using the UI
+
+#### 1. Navigate to Pass-Through Endpoints
+
+Go to **Models + Endpoints** → Click **+ Add Pass-Through Endpoint**
+
+<Image img={require('../../img/pt_guard1.png')} alt="Add guardrails to pass-through endpoint" />
+
+Scroll to the **Guardrails** section and select which guardrails to enforce.
+
+:::tip Default Behavior
+By default, you don't need to specify fields - LiteLLM will JSON dump the entire request/response payload and send it to the guardrail.
+:::
+
+#### 2. Target Specific Fields (Optional)
+
+<Image img={require('../../img/pt_guard2.png')} alt="Configure field-level targeting" />
+
+To check only specific fields instead of the entire payload:
+
+1. Select your guardrails
+2. In **Field Targeting (Optional)**, specify fields for each guardrail
+3. Use the quick-add buttons (`+ query`, `+ documents[*]`) or type custom JSONPath expressions
+4. **Request Fields (pre_call)**: Fields to check before sending to target API
+5. **Response Fields (post_call)**: Fields to check in the response from target API
+
+**Example**: In the screenshot above, we set `query` as a request field, so only the `query` field is sent to the guardrail instead of the entire request.
+
+---
+
+### Using Config File
+
+#### 1. Define guardrails and pass-through endpoint
+
+```yaml showLineNumbers title="config.yaml"
+guardrails:
+  - guardrail_name: "pii-guard"
+    litellm_params:
+      guardrail: bedrock
+      mode: pre_call
+      guardrailIdentifier: "your-guardrail-id"
+      guardrailVersion: "1"
+
+general_settings:
+  pass_through_endpoints:
+    - path: "/v1/rerank"
+      target: "https://api.cohere.com/v1/rerank"
+      headers:
+        Authorization: "bearer os.environ/COHERE_API_KEY"
+      guardrails:
+        pii-guard:
+```
+
+#### 2. Start proxy
+
+```bash
+litellm --config config.yaml
+```
+
+#### 3. Test request
+
+```bash
+curl -X POST "http://localhost:4000/v1/rerank" \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer sk-1234" \
+  -d '{
+    "model": "rerank-english-v3.0",
+    "query": "What is the capital of France?",
+    "documents": ["Paris is the capital of France."]
+  }'
+```
+
+---
+
+## Opt-In Behavior
+
+| Configuration | Behavior |
+|--------------|----------|
+| `guardrails` not set | No guardrails execute (default) |
+| `guardrails` set | All org/team/key + pass-through guardrails execute |
+
+When guardrails are enabled, the system collects and executes:
+- Org-level guardrails
+- Team-level guardrails  
+- Key-level guardrails
+- Pass-through specific guardrails
+
+---
+
+
+## How It Works
+
+The diagram below shows what happens when a client makes a request to `/special/rerank` - a pass-through endpoint configured with guardrails in your `config.yaml`.
+
+When guardrails are configured on a pass-through endpoint:
+1. **Pre-call guardrails** run on the request before forwarding to the target API
+2. If `request_fields` is specified (e.g., `["query"]`), only those fields are sent to the guardrail. Otherwise, the entire request payload is evaluated.
+3. The request is forwarded to the target API only if guardrails pass
+4. **Post-call guardrails** run on the response from the target API
+5. If `response_fields` is specified (e.g., `["results[*].text"]`), only those fields are evaluated. Otherwise, the entire response is checked.
+
+:::info
+If the `guardrails` block is omitted or empty in your pass-through endpoint config, the request skips the guardrail flow entirely and goes directly to the target API.
+:::
+
+```mermaid
+sequenceDiagram
+    participant Client
+    box rgb(200, 220, 255) LiteLLM Proxy
+        participant PassThrough as Pass-through Endpoint
+        participant Guardrails
+    end
+    participant Target as Target API (Cohere, etc.)
+
+    Client->>PassThrough: POST /special/rerank
+    Note over PassThrough,Guardrails: Collect passthrough + org/team/key guardrails
+    PassThrough->>Guardrails: Run pre_call (request_fields or full payload)
+    Guardrails-->>PassThrough: ✓ Pass / ✗ Block
+    PassThrough->>Target: Forward request
+    Target-->>PassThrough: Response
+    PassThrough->>Guardrails: Run post_call (response_fields or full payload)
+    Guardrails-->>PassThrough: ✓ Pass / ✗ Block
+    PassThrough-->>Client: Return response (or error)
+```
+
+---
+
+## Field-Level Targeting
+
+Target specific JSON fields instead of the entire request/response payload.
+
+```yaml showLineNumbers title="config.yaml"
+guardrails:
+  - guardrail_name: "pii-detection"
+    litellm_params:
+      guardrail: bedrock
+      mode: pre_call
+      guardrailIdentifier: "pii-guard-id"
+      guardrailVersion: "1"
+
+  - guardrail_name: "content-moderation"
+    litellm_params:
+      guardrail: bedrock
+      mode: post_call
+      guardrailIdentifier: "content-guard-id"
+      guardrailVersion: "1"
+
+general_settings:
+  pass_through_endpoints:
+    - path: "/v1/rerank"
+      target: "https://api.cohere.com/v1/rerank"
+      headers:
+        Authorization: "bearer os.environ/COHERE_API_KEY"
+      guardrails:
+        pii-detection:
+          request_fields: ["query", "documents[*].text"]
+        content-moderation:
+          response_fields: ["results[*].text"]
+```
+
+### Field Options
+
+| Field | Description |
+|-------|-------------|
+| `request_fields` | JSONPath expressions for input (pre_call) |
+| `response_fields` | JSONPath expressions for output (post_call) |
+| Neither specified | Guardrail runs on entire payload |
+
+### JSONPath Examples
+
+| Expression | Matches |
+|------------|---------|
+| `query` | Single field named `query` |
+| `documents[*].text` | All `text` fields in `documents` array |
+| `messages[*].content` | All `content` fields in `messages` array |
+
+---
+
+## Configuration Examples
+
+### Single guardrail on entire payload
+
+```yaml showLineNumbers title="config.yaml"
+guardrails:
+  - guardrail_name: "pii-detection"
+    litellm_params:
+      guardrail: bedrock
+      mode: pre_call
+      guardrailIdentifier: "your-id"
+      guardrailVersion: "1"
+
+general_settings:
+  pass_through_endpoints:
+    - path: "/v1/rerank"
+      target: "https://api.cohere.com/v1/rerank"
+      guardrails:
+        pii-detection:
+```
+
+### Multiple guardrails with mixed settings
+
+```yaml showLineNumbers title="config.yaml"
+guardrails:
+  - guardrail_name: "pii-detection"
+    litellm_params:
+      guardrail: bedrock
+      mode: pre_call
+      guardrailIdentifier: "pii-id"
+      guardrailVersion: "1"
+
+  - guardrail_name: "content-moderation"
+    litellm_params:
+      guardrail: bedrock
+      mode: post_call
+      guardrailIdentifier: "content-id"
+      guardrailVersion: "1"
+
+  - guardrail_name: "prompt-injection"
+    litellm_params:
+      guardrail: lakera
+      mode: pre_call
+      api_key: os.environ/LAKERA_API_KEY
+
+general_settings:
+  pass_through_endpoints:
+    - path: "/v1/rerank"
+      target: "https://api.cohere.com/v1/rerank"
+      guardrails:
+        pii-detection:
+          request_fields: ["input", "query"]
+        content-moderation:
+        prompt-injection:
+          request_fields: ["messages[*].content"]
+```

docs/my-website/docs/proxy/reject_clientside_metadata_tags.md

@@ -0,0 +1,120 @@
+# Reject Client-Side Metadata Tags
+
+## Overview
+
+The `reject_clientside_metadata_tags` setting allows you to prevent users from passing client-side `metadata.tags` in their API requests. This ensures that tags are only inherited from the API key metadata and cannot be overridden by users to potentially influence budget tracking or routing decisions.
+
+## Use Case
+
+This feature is particularly useful in multi-tenant scenarios where:
+- You want to enforce strict budget tracking based on API key tags
+- You want to prevent users from manipulating routing decisions by sending custom client-side tags
+- You need to ensure consistent tag-based filtering and reporting
+
+## Configuration
+
+Add the following to your `config.yaml`:
+
+```yaml
+general_settings:
+  reject_clientside_metadata_tags: true  # Default is false/null
+```
+
+## Behavior
+
+### When `reject_clientside_metadata_tags: true`
+
+**Rejected Request Example:**
+```bash
+curl -X POST http://localhost:4000/chat/completions \
+  -H "Authorization: Bearer sk-1234" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "gpt-3.5-turbo",
+    "messages": [{"role": "user", "content": "Hello"}],
+    "metadata": {
+      "tags": ["custom-tag"]  # This will be rejected
+    }
+  }'
+```
+
+**Error Response:**
+```json
+{
+  "error": {
+    "message": "Client-side 'metadata.tags' not allowed in request. 'reject_clientside_metadata_tags'=True. Tags can only be set via API key metadata.",
+    "type": "bad_request_error",
+    "param": "metadata.tags",
+    "code": 400
+  }
+}
+```
+
+**Allowed Request Example:**
+```bash
+curl -X POST http://localhost:4000/chat/completions \
+  -H "Authorization: Bearer sk-1234" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "gpt-3.5-turbo",
+    "messages": [{"role": "user", "content": "Hello"}],
+    "metadata": {
+      "custom_field": "value"  # Other metadata fields are allowed
+    }
+  }'
+```
+
+### When `reject_clientside_metadata_tags: false` or not set
+
+All requests are allowed, including those with client-side `metadata.tags`.
+
+## Setting Tags via API Key
+
+When `reject_clientside_metadata_tags` is enabled, tags should be set on the API key metadata:
+
+```bash
+curl -X POST http://localhost:4000/key/generate \
+  -H "Authorization: Bearer sk-master-key" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "metadata": {
+      "tags": ["team-a", "production"]
+    }
+  }'
+```
+
+These tags will be automatically inherited by all requests made with that API key.
+
+## Complete Example Configuration
+
+```yaml
+model_list:
+  - model_name: gpt-3.5-turbo
+    litellm_params:
+      model: gpt-3.5-turbo
+      api_key: os.environ/OPENAI_API_KEY
+
+general_settings:
+  master_key: sk-1234
+  database_url: "postgresql://user:password@localhost:5432/litellm"
+  
+  # Reject client-side tags
+  reject_clientside_metadata_tags: true
+  
+  # Optional: Also enforce user parameter
+  enforce_user_param: true
+```
+
+## Similar Features
+
+- `enforce_user_param` - Requires all requests to include a 'user' parameter
+- Tag-based routing - Use tags for intelligent request routing
+- Budget tracking - Track spending per tag
+
+## Notes
+
+- This check only applies to LLM API routes (e.g., `/chat/completions`, `/embeddings`)
+- Management endpoints (e.g., `/key/generate`) are not affected
+- The check validates that client-side `metadata.tags` is not present in the request body
+- Other metadata fields can still be passed in requests
+- Tags set on API keys will still be applied to all requests

docs/my-website/docs/proxy/token_auth.md

@@ -338,6 +338,58 @@ general_settings:
     team_allowed_routes: ["/v1/chat/completions"] # 👈 Set accepted routes
 ```
 
+### Allowing other provider routes for Teams
+
+To enable team JWT tokens to access Anthropic-style endpoints such as `/v1/messages`, update `team_allowed_routes` in your `litellm_jwtauth` configuration. `team_allowed_routes` supports the following values:
+
+- Named route groups from `LiteLLMRoutes` (e.g., `openai_routes`, `anthropic_routes`, `info_routes`, `mapped_pass_through_routes`).
+
+Below is a quick reference for the route groups you can use and example representative routes from each group. If you need the exhaustive list, see the `LiteLLMRoutes` enum in `litellm/proxy/_types.py` for the authoritative list.
+
+| Route Group | What it contains | Representative routes |
+|-------------|------------------|-----------------------|
+| `openai_routes` | OpenAI-compatible REST endpoints (chat, completion, embeddings, images, responses, models, etc.) | `/v1/chat/completions`, `/v1/completions`, `/v1/embeddings`, `/v1/images/generations`, `/v1/models` |
+| `anthropic_routes` | Anthropic-style endpoints (`/v1/messages` and related) | `/v1/messages`, `/v1/messages/count_tokens`, `/v1/skills` |
+| `mapped_pass_through_routes` | Provider-specific pass-through route prefixes (e.g., Anthropic when proxied via `/anthropic`). Use with `mapped_pass_through_routes` for provider wildcard mapping | `/anthropic/*`, `/vertex-ai/*`, `/bedrock/*` |
+| `passthrough_routes_wildcard` | Wildcard mapping for providers (e.g., `/anthropic/*`) - precomputed wildcard list used by the proxy | `/anthropic/*`, `/vllm/*` |
+| `google_routes` | Google-specific (e.g., Vertex / Batching endpoints) | `/v1beta/models/{model_name}:generateContent` |
+| `mcp_routes` | Internal MCP management endpoints | `/mcp/tools`, `/mcp/tools/call` |
+| `info_routes` | Read-only & info endpoints used by the UI | `/key/info`, `/team/info`, `/v1/models` |
+| `management_routes` | Admin-only management endpoints (create/update/delete user/team/model) | `/team/new`, `/key/generate`, `/model/new` |
+| `spend_tracking_routes` | Budget/spend related endpoints | `/spend/logs`, `/spend/keys` |
+| `public_routes` | Public and unauthenticated endpoints | `/`, `/routes`, `/.well-known/litellm-ui-config` |
+
+Note: `llm_api_routes` is the union of OpenAI, Anthropic, Google, pass-through and other LLM routes (`openai_routes + anthropic_routes + google_routes + mapped_pass_through_routes + passthrough_routes_wildcard + apply_guardrail_routes + mcp_routes + litellm_native_routes`).
+
+Defaults (what the proxy uses if you don't override them in `litellm_jwtauth`):
+
+- `admin_jwt_scope`: `litellm_proxy_admin`
+- `admin_allowed_routes` (default): `management_routes`, `spend_tracking_routes`, `global_spend_tracking_routes`, `info_routes` 
+- `team_allowed_routes` (default): `openai_routes`, `info_routes` 
+- `public_allowed_routes` (default): `public_routes`
+
+
+Example: Allow team JWTs to call Anthropic `/v1/messages` (either by route group or by explicit route string):
+
+```yaml
+general_settings:
+  enable_jwt_auth: True
+  litellm_jwtauth:
+    team_ids_jwt_field: "team_ids"
+    team_allowed_routes: ["openai_routes", "info_routes", "anthropic_routes"]
+```
+
+Or selectively allow the exact Anthropic message endpoint only:
+
+```yaml
+general_settings:
+  enable_jwt_auth: True
+  litellm_jwtauth:
+    team_ids_jwt_field: "team_ids"
+    team_allowed_routes: ["/v1/messages", "info_routes"]
+```
+
+
 ### Caching Public Keys 
 
 Control how long public keys are cached for (in seconds).
@@ -407,6 +459,72 @@ general_settings:
     user_id_upsert: true # 👈 upserts the user to db, if valid email but not in db
 ```
 
+## OIDC UserInfo Endpoint
+
+Use this when your JWT/access token doesn't contain user-identifying information. LiteLLM will call your identity provider's UserInfo endpoint to fetch user details.
+
+### When to Use
+
+- Your JWT is opaque (not self-contained) or lacks user claims
+- You need to fetch fresh user information from your identity provider
+- Your access tokens don't include email, roles, or other identifying data
+
+### Configuration
+
+```yaml title="config.yaml" showLineNumbers
+general_settings:
+  enable_jwt_auth: True
+  litellm_jwtauth:
+    # Enable OIDC UserInfo endpoint
+    oidc_userinfo_enabled: true
+    oidc_userinfo_endpoint: "https://your-idp.com/oauth2/userinfo"
+    oidc_userinfo_cache_ttl: 300  # Cache for 5 minutes (default: 300)
+    
+    # Map fields from UserInfo response
+    user_id_jwt_field: "sub"
+    user_email_jwt_field: "email"
+    user_roles_jwt_field: "roles"
+```
+
+### Flow Diagram
+
+```mermaid
+sequenceDiagram
+    participant Client
+    participant LiteLLM
+    participant IdP as Identity Provider
+
+    Client->>LiteLLM: Request with Bearer token
+    Note over LiteLLM: Check cache for UserInfo
+    
+    LiteLLM->>IdP: GET /userinfo (if not cached)<br/>Authorization: Bearer {token}
+    IdP-->>LiteLLM: User data (sub, email, roles)
+    
+    Note over LiteLLM: Cache response (TTL: 5min)<br/>Extract user_id, email, roles<br/>Perform RBAC checks
+    
+    LiteLLM-->>Client: Authorized/Denied
+```
+
+### Example: Azure AD
+
+```yaml title="config.yaml" showLineNumbers
+litellm_jwtauth:
+  oidc_userinfo_enabled: true
+  oidc_userinfo_endpoint: "https://graph.microsoft.com/oidc/userinfo"
+  user_id_jwt_field: "sub"
+  user_email_jwt_field: "email"
+```
+
+### Example: Keycloak
+
+```yaml title="config.yaml" showLineNumbers
+litellm_jwtauth:
+  oidc_userinfo_enabled: true
+  oidc_userinfo_endpoint: "https://keycloak.example.com/realms/your-realm/protocol/openid-connect/userinfo"
+  user_id_jwt_field: "sub"
+  user_roles_jwt_field: "resource_access.your-client.roles"
+```
+
 ## [BETA] Control Access with OIDC Roles
 
 Allow JWT tokens with supported roles to access the proxy.

docs/my-website/docs/rag_ingest.md

@@ -0,0 +1,305 @@
+# /rag/ingest
+
+All-in-one document ingestion pipeline: **Upload → Chunk → Embed → Vector Store**
+
+| Feature | Supported |
+|---------|-----------|
+| Logging | ✅ |
+| Supported Providers | `openai`, `bedrock`, `vertex_ai`, `gemini` |
+
+## Quick Start
+
+### OpenAI
+
+```bash showLineNumbers title="Ingest to OpenAI vector store"
+curl -X POST "http://localhost:4000/v1/rag/ingest" \
+    -H "Authorization: Bearer sk-1234" \
+    -H "Content-Type: application/json" \
+    -d "{
+        \"file\": {
+            \"filename\": \"document.txt\",
+            \"content\": \"$(base64 -i document.txt)\",
+            \"content_type\": \"text/plain\"
+        },
+        \"ingest_options\": {
+            \"vector_store\": {
+                \"custom_llm_provider\": \"openai\"
+            }
+        }
+    }"
+```
+
+### Bedrock
+
+```bash showLineNumbers title="Ingest to Bedrock Knowledge Base"
+curl -X POST "http://localhost:4000/v1/rag/ingest" \
+    -H "Authorization: Bearer sk-1234" \
+    -H "Content-Type: application/json" \
+    -d "{
+        \"file\": {
+            \"filename\": \"document.txt\",
+            \"content\": \"$(base64 -i document.txt)\",
+            \"content_type\": \"text/plain\"
+        },
+        \"ingest_options\": {
+            \"vector_store\": {
+                \"custom_llm_provider\": \"bedrock\"
+            }
+        }
+    }"
+```
+
+### Vertex AI RAG Engine
+
+```bash showLineNumbers title="Ingest to Vertex AI RAG Corpus"
+curl -X POST "http://localhost:4000/v1/rag/ingest" \
+    -H "Authorization: Bearer sk-1234" \
+    -H "Content-Type: application/json" \
+    -d "{
+        \"file\": {
+            \"filename\": \"document.txt\",
+            \"content\": \"$(base64 -i document.txt)\",
+            \"content_type\": \"text/plain\"
+        },
+        \"ingest_options\": {
+            \"vector_store\": {
+                \"custom_llm_provider\": \"vertex_ai\",
+                \"vector_store_id\": \"your-corpus-id\",
+                \"gcs_bucket\": \"your-gcs-bucket\"
+            }
+        }
+    }"
+```
+
+## Response
+
+```json
+{
+  "id": "ingest_abc123",
+  "status": "completed",
+  "vector_store_id": "vs_xyz789",
+  "file_id": "file_123"
+}
+```
+
+## Query the Vector Store
+
+After ingestion, query with `/vector_stores/{vector_store_id}/search`:
+
+```bash showLineNumbers title="Search the vector store"
+curl -X POST "http://localhost:4000/v1/vector_stores/vs_xyz789/search" \
+    -H "Authorization: Bearer sk-1234" \
+    -H "Content-Type: application/json" \
+    -d '{
+        "query": "What is the main topic?",
+        "max_num_results": 5
+    }'
+```
+
+## End-to-End Example
+
+### OpenAI
+
+#### 1. Ingest Document
+
+```bash showLineNumbers title="Step 1: Ingest"
+curl -X POST "http://localhost:4000/v1/rag/ingest" \
+    -H "Authorization: Bearer sk-1234" \
+    -H "Content-Type: application/json" \
+    -d "{
+        \"file\": {
+            \"filename\": \"test_document.txt\",
+            \"content\": \"$(base64 -i test_document.txt)\",
+            \"content_type\": \"text/plain\"
+        },
+        \"ingest_options\": {
+            \"name\": \"test-basic-ingest\",
+            \"vector_store\": {
+                \"custom_llm_provider\": \"openai\"
+            }
+        }
+    }"
+```
+
+Response:
+```json
+{
+  "id": "ingest_d834f544-fc5e-4751-902d-fb0bcc183b85",
+  "status": "completed",
+  "vector_store_id": "vs_692658d337c4819183f2ad8488d12fc9",
+  "file_id": "file-M2pJJiWH56cfUP4Fe7rJay"
+}
+```
+
+#### 2. Query
+
+```bash showLineNumbers title="Step 2: Query"
+curl -X POST "http://localhost:4000/v1/vector_stores/vs_692658d337c4819183f2ad8488d12fc9/search" \
+    -H "Authorization: Bearer sk-1234" \
+    -H "Content-Type: application/json" \
+    -d '{
+        "query": "What is LiteLLM?",
+        "custom_llm_provider": "openai"
+    }'
+```
+
+Response:
+```json
+{
+  "object": "vector_store.search_results.page",
+  "search_query": ["What is LiteLLM?"],
+  "data": [
+    {
+      "file_id": "file-M2pJJiWH56cfUP4Fe7rJay",
+      "filename": "test_document.txt",
+      "score": 0.4004629778869299,
+      "attributes": {},
+      "content": [
+        {
+          "type": "text",
+          "text": "Test document abc123 for RAG ingestion.\nThis is a sample document to test the RAG ingest API.\nLiteLLM provides a unified interface for vector stores."
+        }
+      ]
+    }
+  ],
+  "has_more": false,
+  "next_page": null
+}
+```
+
+## Request Parameters
+
+### Top-Level
+
+| Parameter | Type | Required | Description |
+|-----------|------|----------|-------------|
+| `file` | object | One of file/file_url/file_id required | Base64-encoded file |
+| `file.filename` | string | Yes | Filename with extension |
+| `file.content` | string | Yes | Base64-encoded content |
+| `file.content_type` | string | Yes | MIME type (e.g., `text/plain`) |
+| `file_url` | string | One of file/file_url/file_id required | URL to fetch file from |
+| `file_id` | string | One of file/file_url/file_id required | Existing file ID |
+| `ingest_options` | object | Yes | Pipeline configuration |
+
+### ingest_options
+
+| Parameter | Type | Required | Description |
+|-----------|------|----------|-------------|
+| `vector_store` | object | Yes | Vector store configuration |
+| `name` | string | No | Pipeline name for logging |
+
+### vector_store (OpenAI)
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `custom_llm_provider` | string | - | `"openai"` |
+| `vector_store_id` | string | auto-create | Existing vector store ID |
+
+### vector_store (Bedrock)
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `custom_llm_provider` | string | - | `"bedrock"` |
+| `vector_store_id` | string | auto-create | Existing Knowledge Base ID |
+| `wait_for_ingestion` | boolean | `false` | Wait for indexing to complete |
+| `ingestion_timeout` | integer | `300` | Timeout in seconds (if waiting) |
+| `s3_bucket` | string | auto-create | S3 bucket for documents |
+| `s3_prefix` | string | `"data/"` | S3 key prefix |
+| `embedding_model` | string | `amazon.titan-embed-text-v2:0` | Bedrock embedding model |
+| `aws_region_name` | string | `us-west-2` | AWS region |
+
+:::info Bedrock Auto-Creation
+When `vector_store_id` is omitted, LiteLLM automatically creates:
+- S3 bucket for document storage
+- OpenSearch Serverless collection
+- IAM role with required permissions
+- Bedrock Knowledge Base
+- Data Source
+:::
+
+### vector_store (Vertex AI)
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `custom_llm_provider` | string | - | `"vertex_ai"` |
+| `vector_store_id` | string | **required** | RAG corpus ID |
+| `gcs_bucket` | string | **required** | GCS bucket for file uploads |
+| `vertex_project` | string | env `VERTEXAI_PROJECT` | GCP project ID |
+| `vertex_location` | string | `us-central1` | GCP region |
+| `vertex_credentials` | string | ADC | Path to credentials JSON |
+| `wait_for_import` | boolean | `true` | Wait for import to complete |
+| `import_timeout` | integer | `600` | Timeout in seconds (if waiting) |
+
+:::info Vertex AI Prerequisites
+1. Create a RAG corpus in Vertex AI console or via API
+2. Create a GCS bucket for file uploads
+3. Authenticate via `gcloud auth application-default login`
+4. Install: `pip install 'google-cloud-aiplatform>=1.60.0'`
+:::
+
+## Input Examples
+
+### File (Base64)
+
+```json title="Request body"
+{
+  "file": {
+    "filename": "document.txt",
+    "content": "<base64-encoded-content>",
+    "content_type": "text/plain"
+  },
+  "ingest_options": {
+    "vector_store": {"custom_llm_provider": "openai"}
+  }
+}
+```
+
+### File URL
+
+```bash showLineNumbers title="Ingest from URL"
+curl -X POST "http://localhost:4000/v1/rag/ingest" \
+    -H "Authorization: Bearer sk-1234" \
+    -H "Content-Type: application/json" \
+    -d '{
+        "file_url": "https://example.com/document.pdf",
+        "ingest_options": {"vector_store": {"custom_llm_provider": "openai"}}
+    }'
+```
+
+## Chunking Strategy
+
+Control how documents are split into chunks before embedding. Specify `chunking_strategy` in `ingest_options`.
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `chunk_size` | integer | `1000` | Maximum size of each chunk |
+| `chunk_overlap` | integer | `200` | Overlap between consecutive chunks |
+
+### Vertex AI RAG Engine
+
+Vertex AI RAG Engine supports custom chunking via the `chunking_strategy` parameter. Chunks are processed server-side during import.
+
+```bash showLineNumbers title="Vertex AI with custom chunking"
+curl -X POST "http://localhost:4000/v1/rag/ingest" \
+    -H "Authorization: Bearer sk-1234" \
+    -H "Content-Type: application/json" \
+    -d "{
+        \"file\": {
+            \"filename\": \"document.txt\",
+            \"content\": \"$(base64 -i document.txt)\",
+            \"content_type\": \"text/plain\"
+        },
+        \"ingest_options\": {
+            \"chunking_strategy\": {
+                \"chunk_size\": 500,
+                \"chunk_overlap\": 100
+            },
+            \"vector_store\": {
+                \"custom_llm_provider\": \"vertex_ai\",
+                \"vector_store_id\": \"your-corpus-id\",
+                \"gcs_bucket\": \"your-gcs-bucket\"
+            }
+        }
+    }"
+```
+

docs/my-website/docs/secret_managers/cyberark.md

@@ -41,6 +41,7 @@ CYBERARK_CLIENT_KEY="path/to/client.key"
 
 # OPTIONAL
 CYBERARK_REFRESH_INTERVAL="300" # defaults to 300 seconds (5 minutes), frequency of token refresh
+CYBERARK_SSL_VERIFY="true" # defaults to true, set to "false" to disable SSL verification (for self-signed certificates)
 ```
 
 **Step 2.** Add to proxy config.yaml
@@ -172,6 +173,24 @@ If these commands work successfully against your CyberArk instance, then CyberAr
 - The `CYBERARK_API_BASE` URL is accessible from your LiteLLM instance
 - Your API key or certificates have the necessary permissions in CyberArk
 
+### SSL Certificate Errors
+
+If you encounter SSL certificate verification errors like:
+
+```
+RuntimeError: Could not authenticate to CyberArk Conjur: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed certificate in certificate chain
+```
+
+This typically occurs when your CyberArk Conjur instance uses a self-signed certificate. You can disable SSL verification by setting:
+
+```bash
+CYBERARK_SSL_VERIFY="false"
+```
+
+:::warning
+Disabling SSL verification is insecure and should only be used for testing or development environments with self-signed certificates. For production, configure your certificate chain properly or use certificate-based authentication with `CYBERARK_CLIENT_CERT` and `CYBERARK_CLIENT_KEY`.
+:::
+
 ## Video Walkthrough
 
 This video walks through using CyberArk Conjur as a secret manager with LiteLLM. We create a virtual key in the LiteLLM Admin UI and verify it exists in CyberArk. Then we rotate the secret key and verify it exists in CyberArk.

docs/my-website/docs/vector_stores/create.md

@@ -14,6 +14,7 @@ Create a vector store which can be used to store and search document chunks for
 | End-user Tracking | ✅ | |
 | Support LLM Providers (OpenAI `/vector_stores` API) | **OpenAI** | Full vector stores API support across providers |
 | Support LLM Providers (Passthrough API) | [**Azure AI**](/docs/providers/azure_ai/azure_ai_vector_stores_passthrough) | Full vector stores API support across providers |
+| Support LLM Providers (Dataset Management) | [**RAGFlow**](/docs/providers/ragflow_vector_store.md) | Dataset creation and management (search not supported) |
 
 ## Usage
 

docs/my-website/docs/vector_stores/search.md

@@ -12,7 +12,7 @@ Search a vector store for relevant chunks based on a query and file attributes f
 | Cost Tracking | ✅ | Tracked per search operation |
 | Logging | ✅ | Works across all integrations |
 | End-user Tracking | ✅ | |
-| Support LLM Providers | **OpenAI, Azure OpenAI, Bedrock, Vertex RAG Engine, Azure AI, Milvus** | Full vector stores API support across providers |
+| Support LLM Providers | **OpenAI, Azure OpenAI, Bedrock, Vertex RAG Engine, Azure AI, Milvus, Gemini** | Full vector stores API support across providers |
 
 ## Usage
 
@@ -164,6 +164,41 @@ print(response)
 
 [See full Milvus vector store documentation](../providers/milvus_vector_stores.md)
 
+</TabItem>
+
+<TabItem value="gemini-provider" label="Gemini Provider">
+
+#### Using Gemini File Search
+```python showLineNumbers title="Search Vector Store - Gemini Provider"
+import litellm
+import os
+
+# Set credentials
+os.environ["GEMINI_API_KEY"] = "your-gemini-api-key"
+
+response = await litellm.vector_stores.asearch(
+    vector_store_id="fileSearchStores/your-store-id",
+    query="What is the capital of France?",
+    custom_llm_provider="gemini",
+    max_num_results=5
+)
+print(response)
+```
+
+**With Metadata Filter:**
+```python showLineNumbers title="Search with Metadata Filter"
+response = await litellm.vector_stores.asearch(
+    vector_store_id="fileSearchStores/your-store-id",
+    query="What is LiteLLM?",
+    custom_llm_provider="gemini",
+    filters={"author": "John Doe", "category": "documentation"},
+    max_num_results=5
+)
+print(response)
+```
+
+[See full Gemini File Search documentation](../providers/gemini_file_search.md)
+
 </TabItem>
 </Tabs>
 

docs/my-website/package-lock.json

@@ -16891,9 +16891,9 @@
       }
     },
     "node_modules/node-forge": {
-      "version": "1.3.1",
-      "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.3.1.tgz",
-      "integrity": "sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==",
+      "version": "1.3.2",
+      "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.3.2.tgz",
+      "integrity": "sha512-6xKiQ+cph9KImrRh0VsjH2d8/GXA4FIMlgU4B757iI1ApvcyA9VlouP0yZJha01V+huImO+kKMU7ih+2+E14fw==",
       "license": "(BSD-3-Clause OR GPL-2.0)",
       "engines": {
         "node": ">= 6.13.0"

docs/my-website/package.json

@@ -52,13 +52,15 @@
     "webpack-dev-server": ">=5.2.1",
     "form-data": ">=4.0.4",
     "mermaid": ">=11.10.0",
-    "gray-matter": "4.0.3"
+    "gray-matter": "4.0.3",
+    "node-forge": ">=1.3.2"
   },
   "overrides": {
     "webpack-dev-server": ">=5.2.1",
     "form-data": ">=4.0.4",
     "mermaid": ">=11.10.0",
     "gray-matter": "4.0.3",
-    "glob": ">=11.1.0"
+    "glob": ">=11.1.0",
+    "node-forge": ">=1.3.2"
   }
 }

docs/my-website/release_notes/v1.80.5-stable/index.md

@@ -1,5 +1,5 @@
 ---
-title: "[PREVIEW] v1.80.5.rc.2 - Gemini 3.0 Support"
+title: "v1.80.5-stable - Gemini 3.0 Support"
 slug: "v1-80-5"
 date: 2025-11-22T10:00:00
 authors:
@@ -27,7 +27,7 @@ import TabItem from '@theme/TabItem';
 docker run \
 -e STORE_MODEL_IN_DB=True \
 -p 4000:4000 \
-ghcr.io/berriai/litellm:v1.80.5.rc.2
+ghcr.io/berriai/litellm:v1.80.5-stable
 ```
 
 </TabItem>

docs/my-website/sidebars.js

@@ -20,6 +20,16 @@ const sidebars = {
       type: "category",
       label: "Observability",
       items: [
+        {
+          type: "category",
+          label: "Contributing to Integrations",
+          items: [
+            {
+              type: "autogenerated",
+              dirName: "contribute_integration"
+            }
+          ]
+        },
         {
           type: "autogenerated",
           dirName: "observability"
@@ -35,6 +45,7 @@ const sidebars = {
           type: "category",
           "label": "Contributing to Guardrails",
           items: [
+            "adding_provider/generic_guardrail_api",
             "adding_provider/simple_guardrail_tutorial",
             "adding_provider/adding_guardrail_support",
           ]
@@ -130,6 +141,7 @@ const sidebars = {
             "proxy/quick_start",
             "proxy/cli",
             "proxy/debugging",
+            "proxy/error_diagnosis",
             "proxy/deploy",
             "proxy/health",
             "proxy/master_key_rotations",
@@ -304,6 +316,7 @@ const sidebars = {
         slug: "/supported_endpoints",
       },
       items: [
+        "a2a",
         "assistants",
         {
           type: "category",
@@ -409,9 +422,11 @@ const sidebars = {
               ]
             },
             "pass_through/vllm",
-            "proxy/pass_through"
+            "proxy/pass_through",
+            "proxy/pass_through_guardrails"
           ]
         },
+        "rag_ingest",
         "realtime",
         "rerank",
         "response_api",
@@ -507,6 +522,7 @@ const sidebars = {
             "providers/vertex_partner",
             "providers/vertex_self_deployed",
             "providers/vertex_image",
+            "providers/vertex_speech",
             "providers/vertex_batch",
             "providers/vertex_ocr",
           ]
@@ -530,6 +546,7 @@ const sidebars = {
           items: [
             "providers/bedrock",
             "providers/bedrock_embedding",
+            "providers/bedrock_imported",
             "providers/bedrock_image_gen",
             "providers/bedrock_rerank",
             "providers/bedrock_agentcore",
@@ -609,7 +626,9 @@ const sidebars = {
         "providers/ovhcloud",
         "providers/perplexity",
         "providers/petals",
+        "providers/publicai",
         "providers/predibase",
+        "providers/ragflow",
         "providers/recraft",
         "providers/replicate",
         {
@@ -631,9 +650,17 @@ const sidebars = {
         "providers/volcano",
         "providers/voyage",
         "providers/wandb_inference",
-        "providers/watsonx",
+        {
+          type: "category",
+          label: "WatsonX",
+          items: [
+            "providers/watsonx/index",
+            "providers/watsonx/audio_transcription",
+          ]
+        },
         "providers/xai",
         "providers/xinference",
+        "providers/zai",
       ],
     },
     {
@@ -798,6 +825,10 @@ const sidebars = {
           items: [
             "projects/smolagents",
             "projects/mini-swe-agent",
+            "projects/openai-agents",
+            "projects/Google ADK",
+            "projects/Agent Lightning",
+            "projects/Harbor",
             "projects/Docq.AI",
             "projects/PDL",
             "projects/OpenInterpreter",

docs/my-website/src/pages/intro.md

@@ -1,47 +0,0 @@
----
-sidebar_position: 1
----
-
-# Tutorial Intro
-
-Let's discover **Docusaurus in less than 5 minutes**.
-
-## Getting Started
-
-Get started by **creating a new site**.
-
-Or **try Docusaurus immediately** with **[docusaurus.new](https://docusaurus.new)**.
-
-### What you'll need
-
-- [Node.js](https://nodejs.org/en/download/) version 16.14 or above:
-  - When installing Node.js, you are recommended to check all checkboxes related to dependencies.
-
-## Generate a new site
-
-Generate a new Docusaurus site using the **classic template**.
-
-The classic template will automatically be added to your project after you run the command:
-
-```bash
-npm init docusaurus@latest my-website classic
-```
-
-You can type this command into Command Prompt, Powershell, Terminal, or any other integrated terminal of your code editor.
-
-The command also installs all necessary dependencies you need to run Docusaurus.
-
-## Start your site
-
-Run the development server:
-
-```bash
-cd my-website
-npm run start
-```
-
-The `cd` command changes the directory you're working with. In order to work with your newly created Docusaurus site, you'll need to navigate the terminal there.
-
-The `npm run start` command builds your website locally and serves it through a development server, ready for you to view at http://localhost:3000/.
-
-Open `docs/intro.md` (this page) and edit some lines: the site **reloads automatically** and displays your changes.

docs/my-website/src/pages/tutorial-basics/_category_.json

@@ -1,8 +0,0 @@
-{
-  "label": "Tutorial - Basics",
-  "position": 2,
-  "link": {
-    "type": "generated-index",
-    "description": "5 minutes to learn the most important Docusaurus concepts."
-  }
-}

docs/my-website/src/pages/tutorial-basics/congratulations.md

@@ -1,23 +0,0 @@
----
-sidebar_position: 6
----
-
-# Congratulations!
-
-You have just learned the **basics of Docusaurus** and made some changes to the **initial template**.
-
-Docusaurus has **much more to offer**!
-
-Have **5 more minutes**? Take a look at **[versioning](../tutorial-extras/manage-docs-versions.md)** and **[i18n](../tutorial-extras/translate-your-site.md)**.
-
-Anything **unclear** or **buggy** in this tutorial? [Please report it!](https://github.com/facebook/docusaurus/discussions/4610)
-
-## What's next?
-
-- Read the [official documentation](https://docusaurus.io/)
-- Modify your site configuration with [`docusaurus.config.js`](https://docusaurus.io/docs/api/docusaurus-config)
-- Add navbar and footer items with [`themeConfig`](https://docusaurus.io/docs/api/themes/configuration)
-- Add a custom [Design and Layout](https://docusaurus.io/docs/styling-layout)
-- Add a [search bar](https://docusaurus.io/docs/search)
-- Find inspirations in the [Docusaurus showcase](https://docusaurus.io/showcase)
-- Get involved in the [Docusaurus Community](https://docusaurus.io/community/support)

docs/my-website/src/pages/tutorial-basics/create-a-blog-post.md

@@ -1,34 +0,0 @@
----
-sidebar_position: 3
----
-
-# Create a Blog Post
-
-Docusaurus creates a **page for each blog post**, but also a **blog index page**, a **tag system**, an **RSS** feed...
-
-## Create your first Post
-
-Create a file at `blog/2021-02-28-greetings.md`:
-
-```md title="blog/2021-02-28-greetings.md"
----
-slug: greetings
-title: Greetings!
-authors:
-  - name: Joel Marcey
-    title: Co-creator of Docusaurus 1
-    url: https://github.com/JoelMarcey
-    image_url: https://github.com/JoelMarcey.png
-  - name: Sébastien Lorber
-    title: Docusaurus maintainer
-    url: https://sebastienlorber.com
-    image_url: https://github.com/slorber.png
-tags: [greetings]
----
-
-Congratulations, you have made your first post!
-
-Feel free to play around and edit this post as much you like.
-```
-
-A new blog post is now available at [http://localhost:3000/blog/greetings](http://localhost:3000/blog/greetings).

docs/my-website/src/pages/tutorial-basics/create-a-document.md

@@ -1,57 +0,0 @@
----
-sidebar_position: 2
----
-
-# Create a Document
-
-Documents are **groups of pages** connected through:
-
-- a **sidebar**
-- **previous/next navigation**
-- **versioning**
-
-## Create your first Doc
-
-Create a Markdown file at `docs/hello.md`:
-
-```md title="docs/hello.md"
-# Hello
-
-This is my **first Docusaurus document**!
-```
-
-A new document is now available at [http://localhost:3000/docs/hello](http://localhost:3000/docs/hello).
-
-## Configure the Sidebar
-
-Docusaurus automatically **creates a sidebar** from the `docs` folder.
-
-Add metadata to customize the sidebar label and position:
-
-```md title="docs/hello.md" {1-4}
----
-sidebar_label: 'Hi!'
-sidebar_position: 3
----
-
-# Hello
-
-This is my **first Docusaurus document**!
-```
-
-It is also possible to create your sidebar explicitly in `sidebars.js`:
-
-```js title="sidebars.js"
-module.exports = {
-  tutorialSidebar: [
-    'intro',
-    // highlight-next-line
-    'hello',
-    {
-      type: 'category',
-      label: 'Tutorial',
-      items: ['tutorial-basics/create-a-document'],
-    },
-  ],
-};
-```

docs/my-website/src/pages/tutorial-basics/create-a-page.md

@@ -1,43 +0,0 @@
----
-sidebar_position: 1
----
-
-# Create a Page
-
-Add **Markdown or React** files to `src/pages` to create a **standalone page**:
-
-- `src/pages/index.js` → `localhost:3000/`
-- `src/pages/foo.md` → `localhost:3000/foo`
-- `src/pages/foo/bar.js` → `localhost:3000/foo/bar`
-
-## Create your first React Page
-
-Create a file at `src/pages/my-react-page.js`:
-
-```jsx title="src/pages/my-react-page.js"
-import React from 'react';
-import Layout from '@theme/Layout';
-
-export default function MyReactPage() {
-  return (
-    <Layout>
-      <h1>My React page</h1>
-      <p>This is a React page</p>
-    </Layout>
-  );
-}
-```
-
-A new page is now available at [http://localhost:3000/my-react-page](http://localhost:3000/my-react-page).
-
-## Create your first Markdown Page
-
-Create a file at `src/pages/my-markdown-page.md`:
-
-```mdx title="src/pages/my-markdown-page.md"
-# My Markdown page
-
-This is a Markdown page
-```
-
-A new page is now available at [http://localhost:3000/my-markdown-page](http://localhost:3000/my-markdown-page).

docs/my-website/src/pages/tutorial-basics/deploy-your-site.md

@@ -1,31 +0,0 @@
----
-sidebar_position: 5
----
-
-# Deploy your site
-
-Docusaurus is a **static-site-generator** (also called **[Jamstack](https://jamstack.org/)**).
-
-It builds your site as simple **static HTML, JavaScript and CSS files**.
-
-## Build your site
-
-Build your site **for production**:
-
-```bash
-npm run build
-```
-
-The static files are generated in the `build` folder.
-
-## Deploy your site
-
-Test your production build locally:
-
-```bash
-npm run serve
-```
-
-The `build` folder is now served at [http://localhost:3000/](http://localhost:3000/).
-
-You can now deploy the `build` folder **almost anywhere** easily, **for free** or very small cost (read the **[Deployment Guide](https://docusaurus.io/docs/deployment)**).

docs/my-website/src/pages/tutorial-basics/markdown-features.mdx

@@ -1,150 +0,0 @@
----
-sidebar_position: 4
----
-
-# Markdown Features
-
-Docusaurus supports **[Markdown](https://daringfireball.net/projects/markdown/syntax)** and a few **additional features**.
-
-## Front Matter
-
-Markdown documents have metadata at the top called [Front Matter](https://jekyllrb.com/docs/front-matter/):
-
-```text title="my-doc.md"
-// highlight-start
----
-id: my-doc-id
-title: My document title
-description: My document description
-slug: /my-custom-url
----
-// highlight-end
-
-## Markdown heading
-
-Markdown text with [links](./hello.md)
-```
-
-## Links
-
-Regular Markdown links are supported, using url paths or relative file paths.
-
-```md
-Let's see how to [Create a page](/create-a-page).
-```
-
-```md
-Let's see how to [Create a page](./create-a-page.md).
-```
-
-**Result:** Let's see how to [Create a page](./create-a-page.md).
-
-## Images
-
-Regular Markdown images are supported.
-
-You can use absolute paths to reference images in the static directory (`static/img/docusaurus.png`):
-
-```md
-![Docusaurus logo](/img/docusaurus.png)
-```
-
-![Docusaurus logo](/img/docusaurus.png)
-
-You can reference images relative to the current file as well. This is particularly useful to colocate images close to the Markdown files using them:
-
-```md
-![Docusaurus logo](./img/docusaurus.png)
-```
-
-## Code Blocks
-
-Markdown code blocks are supported with Syntax highlighting.
-
-    ```jsx title="src/components/HelloDocusaurus.js"
-    function HelloDocusaurus() {
-        return (
-            <h1>Hello, Docusaurus!</h1>
-        )
-    }
-    ```
-
-```jsx title="src/components/HelloDocusaurus.js"
-function HelloDocusaurus() {
-  return <h1>Hello, Docusaurus!</h1>;
-}
-```
-
-## Admonitions
-
-Docusaurus has a special syntax to create admonitions and callouts:
-
-    :::tip My tip
-
-    Use this awesome feature option
-
-    :::
-
-    :::danger Take care
-
-    This action is dangerous
-
-    :::
-
-:::tip My tip
-
-Use this awesome feature option
-
-:::
-
-:::danger Take care
-
-This action is dangerous
-
-:::
-
-## MDX and React Components
-
-[MDX](https://mdxjs.com/) can make your documentation more **interactive** and allows using any **React components inside Markdown**:
-
-```jsx
-export const Highlight = ({children, color}) => (
-  <span
-    style={{
-      backgroundColor: color,
-      borderRadius: '20px',
-      color: '#fff',
-      padding: '10px',
-      cursor: 'pointer',
-    }}
-    onClick={() => {
-      alert(`You clicked the color ${color} with label ${children}`)
-    }}>
-    {children}
-  </span>
-);
-
-This is <Highlight color="#25c2a0">Docusaurus green</Highlight> !
-
-This is <Highlight color="#1877F2">Facebook blue</Highlight> !
-```
-
-export const Highlight = ({children, color}) => (
-  <span
-    style={{
-      backgroundColor: color,
-      borderRadius: '20px',
-      color: '#fff',
-      padding: '10px',
-      cursor: 'pointer',
-    }}
-    onClick={() => {
-      alert(`You clicked the color ${color} with label ${children}`);
-    }}>
-    {children}
-  </span>
-);
-
-This is <Highlight color="#25c2a0">Docusaurus green</Highlight> !
-
-This is <Highlight color="#1877F2">Facebook blue</Highlight> !

docs/my-website/src/pages/tutorial-extras/_category_.json

@@ -1,7 +0,0 @@
-{
-  "label": "Tutorial - Extras",
-  "position": 3,
-  "link": {
-    "type": "generated-index"
-  }
-}

docs/my-website/src/pages/tutorial-extras/manage-docs-versions.md

@@ -1,55 +0,0 @@
----
-sidebar_position: 1
----
-
-# Manage Docs Versions
-
-Docusaurus can manage multiple versions of your docs.
-
-## Create a docs version
-
-Release a version 1.0 of your project:
-
-```bash
-npm run docusaurus docs:version 1.0
-```
-
-The `docs` folder is copied into `versioned_docs/version-1.0` and `versions.json` is created.
-
-Your docs now have 2 versions:
-
-- `1.0` at `http://localhost:3000/docs/` for the version 1.0 docs
-- `current` at `http://localhost:3000/docs/next/` for the **upcoming, unreleased docs**
-
-## Add a Version Dropdown
-
-To navigate seamlessly across versions, add a version dropdown.
-
-Modify the `docusaurus.config.js` file:
-
-```js title="docusaurus.config.js"
-module.exports = {
-  themeConfig: {
-    navbar: {
-      items: [
-        // highlight-start
-        {
-          type: 'docsVersionDropdown',
-        },
-        // highlight-end
-      ],
-    },
-  },
-};
-```
-
-The docs version dropdown appears in your navbar:
-
-![Docs Version Dropdown](./img/docsVersionDropdown.png)
-
-## Update an existing version
-
-It is possible to edit versioned docs in their respective folder:
-
-- `versioned_docs/version-1.0/hello.md` updates `http://localhost:3000/docs/hello`
-- `docs/hello.md` updates `http://localhost:3000/docs/next/hello`

docs/my-website/src/pages/tutorial-extras/translate-your-site.md

@@ -1,88 +0,0 @@
----
-sidebar_position: 2
----
-
-# Translate your site
-
-Let's translate `docs/intro.md` to French.
-
-## Configure i18n
-
-Modify `docusaurus.config.js` to add support for the `fr` locale:
-
-```js title="docusaurus.config.js"
-module.exports = {
-  i18n: {
-    defaultLocale: 'en',
-    locales: ['en', 'fr'],
-  },
-};
-```
-
-## Translate a doc
-
-Copy the `docs/intro.md` file to the `i18n/fr` folder:
-
-```bash
-mkdir -p i18n/fr/docusaurus-plugin-content-docs/current/
-
-cp docs/intro.md i18n/fr/docusaurus-plugin-content-docs/current/intro.md
-```
-
-Translate `i18n/fr/docusaurus-plugin-content-docs/current/intro.md` in French.
-
-## Start your localized site
-
-Start your site on the French locale:
-
-```bash
-npm run start -- --locale fr
-```
-
-Your localized site is accessible at [http://localhost:3000/fr/](http://localhost:3000/fr/) and the `Getting Started` page is translated.
-
-:::caution
-
-In development, you can only use one locale at a same time.
-
-:::
-
-## Add a Locale Dropdown
-
-To navigate seamlessly across languages, add a locale dropdown.
-
-Modify the `docusaurus.config.js` file:
-
-```js title="docusaurus.config.js"
-module.exports = {
-  themeConfig: {
-    navbar: {
-      items: [
-        // highlight-start
-        {
-          type: 'localeDropdown',
-        },
-        // highlight-end
-      ],
-    },
-  },
-};
-```
-
-The locale dropdown now appears in your navbar:
-
-![Locale Dropdown](./img/localeDropdown.png)
-
-## Build your localized site
-
-Build your site for a specific locale:
-
-```bash
-npm run build -- --locale fr
-```
-
-Or build your site to include all the locales at once:
-
-```bash
-npm run build
-```

document.txt

@@ -0,0 +1,19 @@
+LiteLLM provides a unified interface for calling 100+ different LLM providers.
+
+Key capabilities:
+- Translate requests to provider-specific formats
+- Consistent OpenAI-compatible responses
+- Retry and fallback logic across deployments
+- Proxy server with authentication and rate limiting
+- Support for streaming, function calling, and embeddings
+
+Popular providers supported:
+- OpenAI (GPT-4, GPT-3.5)
+- Anthropic (Claude)
+- AWS Bedrock
+- Azure OpenAI
+- Google Vertex AI
+- Cohere
+- And 95+ more
+
+This allows developers to easily switch between providers without code changes.

enterprise/litellm_enterprise/proxy/enterprise_routes.py

@@ -5,14 +5,10 @@ from litellm_enterprise.enterprise_callbacks.send_emails.endpoints import (
 )
 
 from .audit_logging_endpoints import router as audit_logging_router
-from .guardrails.endpoints import router as guardrails_router
 from .management_endpoints import management_endpoints_router
 from .utils import _should_block_robots
-from .vector_stores.endpoints import router as vector_stores_router
 
 router = APIRouter()
-router.include_router(vector_stores_router)
-router.include_router(guardrails_router)
 router.include_router(email_events_router)
 router.include_router(audit_logging_router)
 router.include_router(management_endpoints_router)