Merge remote-tracking branch 'origin' into litellm_ui_callback_fix

Dockerfile

@@ -1,8 +1,8 @@
 # Base image for building
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/python:latest-dev
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base
 
 # Runtime image
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/python:latest-dev
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base
 # Builder stage
 FROM $LITELLM_BUILD_IMAGE AS builder
 
@@ -12,11 +12,9 @@ WORKDIR /app
 USER root
 
 # Install build dependencies
-RUN apk add --no-cache gcc python3-dev openssl openssl-dev
+RUN apk add --no-cache bash gcc py3-pip python3 python3-dev openssl openssl-dev
 
-
-RUN pip install --upgrade pip>=24.3.1 && \
-    pip install build
+RUN python -m pip install build
 
 # Copy the current directory contents into the container at /app
 COPY . .
@@ -48,10 +46,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
 USER root
 
 # Install runtime dependencies
-RUN apk add --no-cache openssl tzdata nodejs npm
-
-# Upgrade pip to fix CVE-2025-8869
-RUN pip install --upgrade pip>=24.3.1
+RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip
 
 WORKDIR /app
 # Copy the current directory contents into the container at /app

cookbook/mock_guardrail_server/mock_bedrock_guardrail_server.py

@@ -0,0 +1,570 @@
+#!/usr/bin/env python3
+"""
+Mock Bedrock Guardrail API Server
+
+This is a FastAPI server that mimics the AWS Bedrock Guardrail API for testing purposes.
+It follows the same API spec as the real Bedrock guardrail endpoint.
+
+Usage:
+    python mock_bedrock_guardrail_server.py
+
+The server will start on http://localhost:8080
+"""
+
+import os
+import re
+from typing import Any, Dict, List, Literal, Optional
+
+from fastapi import Depends, FastAPI, Header, HTTPException, status
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel, Field
+
+# ============================================================================
+# Request/Response Models (matching Bedrock API spec)
+# ============================================================================
+
+
+class BedrockTextContent(BaseModel):
+    text: str
+
+
+class BedrockContentItem(BaseModel):
+    text: BedrockTextContent
+
+
+class BedrockRequest(BaseModel):
+    source: Literal["INPUT", "OUTPUT"]
+    content: List[BedrockContentItem] = Field(default_factory=list)
+
+
+class BedrockGuardrailOutput(BaseModel):
+    text: Optional[str] = None
+
+
+class TopicPolicyItem(BaseModel):
+    name: str
+    type: str
+    action: Literal["BLOCKED", "NONE"]
+
+
+class TopicPolicy(BaseModel):
+    topics: List[TopicPolicyItem] = Field(default_factory=list)
+
+
+class ContentFilterItem(BaseModel):
+    type: str
+    confidence: str
+    action: Literal["BLOCKED", "NONE"]
+
+
+class ContentPolicy(BaseModel):
+    filters: List[ContentFilterItem] = Field(default_factory=list)
+
+
+class CustomWord(BaseModel):
+    match: str
+    action: Literal["BLOCKED", "NONE"]
+
+
+class WordPolicy(BaseModel):
+    customWords: List[CustomWord] = Field(default_factory=list)
+    managedWordLists: List[Dict[str, Any]] = Field(default_factory=list)
+
+
+class PiiEntity(BaseModel):
+    type: str
+    match: str
+    action: Literal["BLOCKED", "ANONYMIZED", "NONE"]
+
+
+class RegexMatch(BaseModel):
+    name: str
+    match: str
+    regex: str
+    action: Literal["BLOCKED", "ANONYMIZED", "NONE"]
+
+
+class SensitiveInformationPolicy(BaseModel):
+    piiEntities: List[PiiEntity] = Field(default_factory=list)
+    regexes: List[RegexMatch] = Field(default_factory=list)
+
+
+class ContextualGroundingFilter(BaseModel):
+    type: str
+    threshold: float
+    score: float
+    action: Literal["BLOCKED", "NONE"]
+
+
+class ContextualGroundingPolicy(BaseModel):
+    filters: List[ContextualGroundingFilter] = Field(default_factory=list)
+
+
+class Assessment(BaseModel):
+    topicPolicy: Optional[TopicPolicy] = None
+    contentPolicy: Optional[ContentPolicy] = None
+    wordPolicy: Optional[WordPolicy] = None
+    sensitiveInformationPolicy: Optional[SensitiveInformationPolicy] = None
+    contextualGroundingPolicy: Optional[ContextualGroundingPolicy] = None
+
+
+class BedrockGuardrailResponse(BaseModel):
+    usage: Dict[str, int] = Field(
+        default_factory=lambda: {"topicPolicyUnits": 1, "contentPolicyUnits": 1}
+    )
+    action: Literal["NONE", "GUARDRAIL_INTERVENED"] = "NONE"
+    outputs: List[BedrockGuardrailOutput] = Field(default_factory=list)
+    assessments: List[Assessment] = Field(default_factory=list)
+
+
+# ============================================================================
+# Mock Guardrail Configuration
+# ============================================================================
+
+
+class GuardrailConfig(BaseModel):
+    """Configuration for mock guardrail behavior"""
+
+    blocked_words: List[str] = Field(
+        default_factory=lambda: ["offensive", "inappropriate", "badword"]
+    )
+    blocked_topics: List[str] = Field(default_factory=lambda: ["violence", "illegal"])
+    pii_patterns: Dict[str, str] = Field(
+        default_factory=lambda: {
+            "EMAIL": r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b",
+            "PHONE": r"\b\d{3}[-.]?\d{3}[-.]?\d{4}\b",
+            "SSN": r"\b\d{3}-\d{2}-\d{4}\b",
+            "CREDIT_CARD": r"\b\d{4}[\s-]?\d{4}[\s-]?\d{4}[\s-]?\d{4}\b",
+        }
+    )
+    anonymize_pii: bool = True  # If True, ANONYMIZE PII; if False, BLOCK it
+    bearer_token: str = "mock-bedrock-token-12345"
+
+
+# Global config
+GUARDRAIL_CONFIG = GuardrailConfig()
+
+# ============================================================================
+# FastAPI App Setup
+# ============================================================================
+
+app = FastAPI(
+    title="Mock Bedrock Guardrail API",
+    description="Mock server mimicking AWS Bedrock Guardrail API",
+    version="1.0.0",
+)
+
+
+# ============================================================================
+# Authentication
+# ============================================================================
+
+
+async def verify_bearer_token(authorization: Optional[str] = Header(None)) -> str:
+    """
+    Verify the Bearer token from the Authorization header.
+
+    Args:
+        authorization: The Authorization header value
+
+    Returns:
+        The token if valid
+
+    Raises:
+        HTTPException: If token is missing or invalid
+    """
+    if authorization is None:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Missing Authorization header",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+
+    # Check if it's a Bearer token
+    parts = authorization.split()
+    print(f"parts: {parts}")
+    if len(parts) != 2 or parts[0].lower() != "bearer":
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Invalid Authorization header format. Expected: Bearer <token>",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+
+    token = parts[1]
+
+    # Verify token
+    if token != GUARDRAIL_CONFIG.bearer_token:
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail="Invalid bearer token",
+        )
+
+    return token
+
+
+# ============================================================================
+# Guardrail Logic
+# ============================================================================
+
+
+def check_blocked_words(text: str) -> Optional[WordPolicy]:
+    """Check if text contains blocked words"""
+    found_words = []
+    text_lower = text.lower()
+
+    for word in GUARDRAIL_CONFIG.blocked_words:
+        if word.lower() in text_lower:
+            found_words.append(CustomWord(match=word, action="BLOCKED"))
+
+    if found_words:
+        return WordPolicy(customWords=found_words)
+    return None
+
+
+def check_blocked_topics(text: str) -> Optional[TopicPolicy]:
+    """Check if text contains blocked topics"""
+    found_topics = []
+    text_lower = text.lower()
+
+    for topic in GUARDRAIL_CONFIG.blocked_topics:
+        if topic.lower() in text_lower:
+            found_topics.append(
+                TopicPolicyItem(name=topic, type=topic.upper(), action="BLOCKED")
+            )
+
+    if found_topics:
+        return TopicPolicy(topics=found_topics)
+    return None
+
+
+def check_pii(text: str) -> tuple[Optional[SensitiveInformationPolicy], str]:
+    """
+    Check for PII in text and return policy + anonymized text
+
+    Returns:
+        Tuple of (SensitiveInformationPolicy or None, anonymized_text)
+    """
+    pii_entities = []
+    anonymized_text = text
+    action = "ANONYMIZED" if GUARDRAIL_CONFIG.anonymize_pii else "BLOCKED"
+
+    for pii_type, pattern in GUARDRAIL_CONFIG.pii_patterns.items():
+        try:
+            # Compile the regex pattern with a timeout to prevent ReDoS attacks
+            compiled_pattern = re.compile(pattern)
+            matches = compiled_pattern.finditer(text)
+            for match in matches:
+                matched_text = match.group()
+                pii_entities.append(
+                    PiiEntity(type=pii_type, match=matched_text, action=action)
+                )
+
+                # Anonymize the text if configured
+                if GUARDRAIL_CONFIG.anonymize_pii:
+                    anonymized_text = anonymized_text.replace(
+                        matched_text, f"[{pii_type}_REDACTED]"
+                    )
+        except re.error:
+            # Invalid regex pattern - skip it and log a warning
+            print(f"Warning: Invalid regex pattern for PII type {pii_type}: {pattern}")
+            continue
+
+    if pii_entities:
+        return SensitiveInformationPolicy(piiEntities=pii_entities), anonymized_text
+
+    return None, text
+
+
+def process_guardrail_request(
+    request: BedrockRequest,
+) -> tuple[BedrockGuardrailResponse, List[str]]:
+    """
+    Process a guardrail request and return the response.
+
+    Returns:
+        Tuple of (response, list of output texts)
+    """
+    all_text_content = []
+    output_texts = []
+
+    # Extract all text from content items
+    for content_item in request.content:
+        if content_item.text and content_item.text.text:
+            all_text_content.append(content_item.text.text)
+
+    # Combine all text for analysis
+    combined_text = " ".join(all_text_content)
+
+    # Initialize response
+    response = BedrockGuardrailResponse()
+    assessment = Assessment()
+    has_intervention = False
+
+    # Check for blocked words
+    word_policy = check_blocked_words(combined_text)
+    if word_policy:
+        assessment.wordPolicy = word_policy
+        has_intervention = True
+
+    # Check for blocked topics
+    topic_policy = check_blocked_topics(combined_text)
+    if topic_policy:
+        assessment.topicPolicy = topic_policy
+        has_intervention = True
+
+    # Check for PII
+    for text in all_text_content:
+        pii_policy, anonymized_text = check_pii(text)
+        if pii_policy:
+            assessment.sensitiveInformationPolicy = pii_policy
+            if GUARDRAIL_CONFIG.anonymize_pii:
+                # If anonymizing, we don't block, we modify the text
+                output_texts.append(anonymized_text)
+                has_intervention = True
+            else:
+                # If not anonymizing PII, we block it
+                output_texts.append(text)
+                has_intervention = True
+        else:
+            output_texts.append(text)
+
+    # Build response
+    if has_intervention:
+        response.action = "GUARDRAIL_INTERVENED"
+        # Only add assessment if there were interventions
+        response.assessments = [assessment]
+
+    # Add outputs (modified or original text)
+    response.outputs = [BedrockGuardrailOutput(text=txt) for txt in output_texts]
+
+    return response, output_texts
+
+
+# ============================================================================
+# API Endpoints
+# ============================================================================
+
+
+@app.get("/")
+async def root():
+    """Health check endpoint"""
+    return {
+        "service": "Mock Bedrock Guardrail API",
+        "status": "running",
+        "endpoint_format": "/guardrail/{guardrailIdentifier}/version/{guardrailVersion}/apply",
+    }
+
+
+@app.get("/health")
+async def health():
+    """Health check endpoint"""
+    return {"status": "healthy"}
+
+
+@app.post(
+    "/guardrail/{guardrailIdentifier}/version/{guardrailVersion}/apply",
+    response_model=BedrockGuardrailResponse,
+)
+async def apply_guardrail(
+    guardrailIdentifier: str,
+    guardrailVersion: str,
+    request: BedrockRequest,
+    token: str = Depends(verify_bearer_token),
+) -> BedrockGuardrailResponse:
+    """
+    Apply guardrail to input or output content.
+
+    This endpoint mimics the AWS Bedrock ApplyGuardrail API.
+
+    Args:
+        guardrailIdentifier: The guardrail ID
+        guardrailVersion: The guardrail version
+        request: The guardrail request containing content to analyze
+        token: Bearer token (verified by dependency)
+
+    Returns:
+        BedrockGuardrailResponse with analysis results
+    """
+    # Process the request
+    response, output_texts = process_guardrail_request(request)
+
+    # Log the request (optional, for debugging)
+    print(f"Guardrail applied: {guardrailIdentifier} v{guardrailVersion}")
+    print(f"Source: {request.source}")
+    print(f"Action: {response.action}")
+
+    return response
+
+
+"""
+LiteLLM exposes a basic guardrail API with the text extracted from the request and sent to the guardrail API, as well as the received request body for any further processing. 
+
+This works across all LiteLLM endpoints (completion, anthropic /v1/messages, responses api, image generation, embedding, etc.)
+
+This makes it easy to support your own guardrail API without having to make a PR to LiteLLM.
+
+LiteLLM supports passing any provider specific params from LiteLLM config.yaml to the guardrail API.
+
+Example:
+
+```yaml
+guardrails:
+  - guardrail_name: "bedrock-content-guard"
+    litellm_params:
+      guardrail: generic_guardrail_api
+      mode: "pre_call"
+      api_key: os.environ/GUARDRAIL_API_KEY
+      api_base: os.environ/GUARDRAIL_API_BASE
+      additional_provider_specific_params:
+        api_version: os.environ/GUARDRAIL_API_VERSION # additional provider specific params
+```
+
+This is a beta API. Please help us improve it. 
+"""
+
+
+class LitellmBasicGuardrailRequest(BaseModel):
+    texts: List[str]
+    images: Optional[List[str]] = None
+    request_data: Dict[str, Any] = Field(default_factory=dict)
+    additional_provider_specific_params: Dict[str, Any] = Field(default_factory=dict)
+    input_type: Literal["request", "response"]
+
+
+class LitellmBasicGuardrailResponse(BaseModel):
+    action: Literal[
+        "BLOCKED", "NONE", "GUARDRAIL_INTERVENED"
+    ]  # BLOCKED = litellm will raise an error, NONE = litellm will continue, GUARDRAIL_INTERVENED = litellm will continue, but the text was modified by the guardrail
+    blocked_reason: Optional[str] = None  # only if action is BLOCKED, otherwise None
+    texts: Optional[List[str]] = None
+    images: Optional[List[str]] = None
+
+
+@app.post(
+    "/beta/litellm_basic_guardrail_api",
+    response_model=LitellmBasicGuardrailResponse,
+)
+async def beta_litellm_basic_guardrail_api(
+    request: LitellmBasicGuardrailRequest,
+) -> LitellmBasicGuardrailResponse:
+    """
+    Apply guardrail to input or output content.
+
+    This endpoint mimics the AWS Bedrock ApplyGuardrail API.
+
+    Args:
+        request: The guardrail request containing content to analyze
+        token: Bearer token (verified by dependency)
+
+    Returns:
+        LitellmBasicGuardrailResponse with analysis results
+    """
+    print(f"request: {request}")
+    if any("ishaan" in text.lower() for text in request.texts):
+        return LitellmBasicGuardrailResponse(
+            action="BLOCKED", blocked_reason="Ishaan is not allowed"
+        )
+    elif any("pii_value" in text for text in request.texts):
+        return LitellmBasicGuardrailResponse(
+            action="GUARDRAIL_INTERVENED",
+            texts=[
+                text.replace("pii_value", "pii_value_redacted")
+                for text in request.texts
+            ],
+        )
+    return LitellmBasicGuardrailResponse(action="NONE")
+
+
+@app.post("/config/update")
+async def update_config(
+    config: GuardrailConfig, token: str = Depends(verify_bearer_token)
+):
+    """
+    Update the guardrail configuration.
+
+    This is a testing endpoint to modify the mock guardrail behavior.
+
+    Args:
+        config: New guardrail configuration
+        token: Bearer token (verified by dependency)
+
+    Returns:
+        Updated configuration
+    """
+    global GUARDRAIL_CONFIG
+    GUARDRAIL_CONFIG = config
+    return {"status": "updated", "config": GUARDRAIL_CONFIG}
+
+
+@app.get("/config")
+async def get_config(token: str = Depends(verify_bearer_token)):
+    """
+    Get the current guardrail configuration.
+
+    Args:
+        token: Bearer token (verified by dependency)
+
+    Returns:
+        Current configuration
+    """
+    return GUARDRAIL_CONFIG
+
+
+# ============================================================================
+# Error Handlers
+# ============================================================================
+
+
+@app.exception_handler(HTTPException)
+async def http_exception_handler(request, exc: HTTPException):
+    """Custom error handler for HTTP exceptions"""
+    return JSONResponse(
+        status_code=exc.status_code,
+        content={"error": exc.detail},
+        headers=exc.headers,
+    )
+
+
+# ============================================================================
+# Main
+# ============================================================================
+
+if __name__ == "__main__":
+    import uvicorn
+
+    # Get configuration from environment
+    host = os.getenv("MOCK_BEDROCK_HOST", "0.0.0.0")
+    port = int(os.getenv("MOCK_BEDROCK_PORT", "8080"))
+    bearer_token = os.getenv("MOCK_BEDROCK_TOKEN", "mock-bedrock-token-12345")
+
+    # Update config with environment token
+    GUARDRAIL_CONFIG.bearer_token = bearer_token
+
+    print("=" * 80)
+    print("Mock Bedrock Guardrail API Server")
+    print("=" * 80)
+    print(f"Server starting on: http://{host}:{port}")
+    print(f"Bearer Token: {bearer_token}")
+    print(f"Endpoint: POST /guardrail/{{id}}/version/{{version}}/apply")
+    print("=" * 80)
+    print("\nExample curl command:")
+    print(
+        f"""
+curl -X POST "http://{host}:{port}/guardrail/test-guardrail/version/1/apply" \\
+  -H "Authorization: Bearer {bearer_token}" \\
+  -H "Content-Type: application/json" \\
+  -d '{{
+    "source": "INPUT",
+    "content": [
+      {{
+        "text": {{
+          "text": "Hello, my email is test@example.com"
+        }}
+      }}
+    ]
+  }}'
+    """
+    )
+    print("=" * 80)
+
+    uvicorn.run(app, host=host, port=port)

deploy/charts/litellm-helm/Chart.yaml

@@ -18,7 +18,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.4.8
+version: 0.4.9
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
@@ -33,5 +33,5 @@ dependencies:
     condition: db.deployStandalone
   - name: redis
     version: ">=18.0.0"
-    repository: oci://registry-1.docker.io/bitnamicharts 
+    repository: oci://registry-1.docker.io/bitnamicharts
     condition: redis.enabled

deploy/charts/litellm-helm/README.md

@@ -10,46 +10,48 @@
 - Helm 3.8.0+
 
 If `db.deployStandalone` is used:
+
 - PV provisioner support in the underlying infrastructure
 
 If `db.useStackgresOperator` is used (not yet implemented):
-- The Stackgres Operator must already be installed in the Kubernetes Cluster.  This chart will **not** install the operator if it is missing.
+
+- The Stackgres Operator must already be installed in the Kubernetes Cluster. This chart will **not** install the operator if it is missing.
 
 ## Parameters
 
 ### LiteLLM Proxy Deployment Settings
 
-| Name                                                       | Description                                                                                                                                                                           | Value |
-| ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- |
-| `replicaCount`                                             | The number of LiteLLM Proxy pods to be deployed                                                                                                                                       | `1`  |
-| `masterkeySecretName`                                      | The name of the Kubernetes Secret that contains the Master API Key for LiteLLM.  If not specified, use the generated secret name.                                                                                                         | N/A  |
-| `masterkeySecretKey`                                      | The key within the Kubernetes Secret that contains the Master API Key for LiteLLM.  If not specified, use `masterkey` as the key.                                                                                                         | N/A  |
-| `masterkey`                                                | The Master API Key for LiteLLM.  If not specified, a random key in the `sk-...` format is generated.                                                                                   | N/A  |
-| `environmentSecrets`                                       | An optional array of Secret object names.  The keys and values in these secrets will be presented to the LiteLLM proxy pod as environment variables.  See below for an example Secret object.  | `[]`  |
-| `environmentConfigMaps`                                       | An optional array of ConfigMap object names.  The keys and values in these configmaps will be presented to the LiteLLM proxy pod as environment variables.  See below for an example Secret object.  | `[]`  |
-| `image.repository`                                         | LiteLLM Proxy image repository                                                                                                                                                        | `ghcr.io/berriai/litellm`  |
-| `image.pullPolicy`                                         | LiteLLM Proxy image pull policy                                                                                                                                                       | `IfNotPresent`  |
-| `image.tag`                                                | Overrides the image tag whose default the latest version of LiteLLM at the time this chart was published.                                                                             | `""`  |
-| `imagePullSecrets`                                         | Registry credentials for the LiteLLM and initContainer images.                                                                                                                        | `[]`  |
-| `serviceAccount.create`                                    | Whether or not to create a Kubernetes Service Account for this deployment.  The default is `false` because LiteLLM has no need to access the Kubernetes API.                          | `false`  |
-| `service.type`                                             | Kubernetes Service type (e.g. `LoadBalancer`, `ClusterIP`, etc.)                                                                                                                      | `ClusterIP`  |
-| `service.port`                                             | TCP port that the Kubernetes Service will listen on.  Also the TCP port within the Pod that the proxy will listen on.                                                                 | `4000`  |
-| `service.loadBalancerClass`                                | Optional LoadBalancer implementation class (only used when `service.type` is `LoadBalancer`)                                                                                                          | `""`  |
-| `ingress.*`                                                | See [values.yaml](./values.yaml) for example settings                                                                                                                                 | N/A  |
-| `proxyConfigMap.create`     | When `true`, render a ConfigMap from `.Values.proxy_config` and mount it.                                                                                                                                      | `true` |
-| `proxyConfigMap.name`       | When `create=false`, name of the existing ConfigMap to mount.                                                                                                                                                  | `""`  |
-| `proxyConfigMap.key`        | Key in the ConfigMap that contains the proxy config file.                                                                                                                                                      | `"config.yaml"` |
-| `proxy_config.*`            | See [values.yaml](./values.yaml) for default settings. Rendered into the ConfigMap’s `config.yaml` only when `proxyConfigMap.create=true`. See [example_config_yaml](../../../litellm/proxy/example_config_yaml/) for configuration examples. | `N/A` |
-| `extraContainers[]`         | An array of additional containers to be deployed as sidecars alongside the LiteLLM Proxy.
-| `pdb.enabled`                   | Enable a PodDisruptionBudget for the LiteLLM proxy Deployment                                                                 | `false` |
-| `pdb.minAvailable`             | Minimum number/percentage of pods that must be available during **voluntary** disruptions (choose **one** of minAvailable/maxUnavailable) | `null`  |
-| `pdb.maxUnavailable`           | Maximum number/percentage of pods that can be unavailable during **voluntary** disruptions (choose **one** of minAvailable/maxUnavailable) | `null`  |
-| `pdb.annotations`              | Extra metadata annotations to add to the PDB                                                                                   | `{}`    |
-| `pdb.labels`                   | Extra metadata labels to add to the PDB                                                                                        | `{}`    |
+| Name                        | Description                                                                                                                                                                                                                                   | Value                     |
+| --------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------- |
+| `replicaCount`              | The number of LiteLLM Proxy pods to be deployed                                                                                                                                                                                               | `1`                       |
+| `masterkeySecretName`       | The name of the Kubernetes Secret that contains the Master API Key for LiteLLM. If not specified, use the generated secret name.                                                                                                              | N/A                       |
+| `masterkeySecretKey`        | The key within the Kubernetes Secret that contains the Master API Key for LiteLLM. If not specified, use `masterkey` as the key.                                                                                                              | N/A                       |
+| `masterkey`                 | The Master API Key for LiteLLM. If not specified, a random key in the `sk-...` format is generated.                                                                                                                                           | N/A                       |
+| `environmentSecrets`        | An optional array of Secret object names. The keys and values in these secrets will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object.                                                   | `[]`                      |
+| `environmentConfigMaps`     | An optional array of ConfigMap object names. The keys and values in these configmaps will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object.                                             | `[]`                      |
+| `image.repository`          | LiteLLM Proxy image repository                                                                                                                                                                                                                | `ghcr.io/berriai/litellm` |
+| `image.pullPolicy`          | LiteLLM Proxy image pull policy                                                                                                                                                                                                               | `IfNotPresent`            |
+| `image.tag`                 | Overrides the image tag whose default the latest version of LiteLLM at the time this chart was published.                                                                                                                                     | `""`                      |
+| `imagePullSecrets`          | Registry credentials for the LiteLLM and initContainer images.                                                                                                                                                                                | `[]`                      |
+| `serviceAccount.create`     | Whether or not to create a Kubernetes Service Account for this deployment. The default is `false` because LiteLLM has no need to access the Kubernetes API.                                                                                   | `false`                   |
+| `service.type`              | Kubernetes Service type (e.g. `LoadBalancer`, `ClusterIP`, etc.)                                                                                                                                                                              | `ClusterIP`               |
+| `service.port`              | TCP port that the Kubernetes Service will listen on. Also the TCP port within the Pod that the proxy will listen on.                                                                                                                          | `4000`                    |
+| `service.loadBalancerClass` | Optional LoadBalancer implementation class (only used when `service.type` is `LoadBalancer`)                                                                                                                                                  | `""`                      |
+| `ingress.labels`            | Additional labels for the Ingress resource                                                                                                                                                                                                    | `{}`                      |
+| `ingress.*`                 | See [values.yaml](./values.yaml) for example settings                                                                                                                                                                                         | N/A                       |
+| `proxyConfigMap.create`     | When `true`, render a ConfigMap from `.Values.proxy_config` and mount it.                                                                                                                                                                     | `true`                    |
+| `proxyConfigMap.name`       | When `create=false`, name of the existing ConfigMap to mount.                                                                                                                                                                                 | `""`                      |
+| `proxyConfigMap.key`        | Key in the ConfigMap that contains the proxy config file.                                                                                                                                                                                     | `"config.yaml"`           |
+| `proxy_config.*`            | See [values.yaml](./values.yaml) for default settings. Rendered into the ConfigMap’s `config.yaml` only when `proxyConfigMap.create=true`. See [example_config_yaml](../../../litellm/proxy/example_config_yaml/) for configuration examples. | `N/A`                     |
+| `extraContainers[]`         | An array of additional containers to be deployed as sidecars alongside the LiteLLM Proxy.                                                                                                                                                     |
+| `pdb.enabled`               | Enable a PodDisruptionBudget for the LiteLLM proxy Deployment                                                                                                                                                                                 | `false`                   |
+| `pdb.minAvailable`          | Minimum number/percentage of pods that must be available during **voluntary** disruptions (choose **one** of minAvailable/maxUnavailable)                                                                                                     | `null`                    |
+| `pdb.maxUnavailable`        | Maximum number/percentage of pods that can be unavailable during **voluntary** disruptions (choose **one** of minAvailable/maxUnavailable)                                                                                                    | `null`                    |
+| `pdb.annotations`           | Extra metadata annotations to add to the PDB                                                                                                                                                                                                  | `{}`                      |
+| `pdb.labels`                | Extra metadata labels to add to the PDB                                                                                                                                                                                                       | `{}`                      |
 
 #### Example `proxy_config` ConfigMap from values (default):
 
-
 ```
 proxyConfigMap:
   create: true
@@ -67,7 +69,6 @@ proxy_config:
 
 #### Example using existing `proxyConfigMap` instead of creating it:
 
-
 ```
 proxyConfigMap:
   create: false
@@ -77,8 +78,7 @@ proxyConfigMap:
 # proxy_config is ignored in this mode
 ```
 
-#### Example `environmentSecrets` Secret 
-
+#### Example `environmentSecrets` Secret
 
 ```
 apiVersion: v1
@@ -91,21 +91,23 @@ type: Opaque
 ```
 
 ### Database Settings
-| Name                                                       | Description                                                                                                                                                                           | Value |
-| ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- |
-| `db.useExisting`                                           | Use an existing Postgres database.  A Kubernetes Secret object must exist that contains credentials for connecting to the database.  An example secret object definition is provided below.  | `false`  |
-| `db.endpoint`                                              | If `db.useExisting` is `true`, this is the IP, Hostname or Service Name of the Postgres server to connect to.                                                                         | `localhost`  |
-| `db.database`                                              | If `db.useExisting` is `true`, the name of the existing database to connect to.                                                                                                       | `litellm`  |
-| `db.url`                                              | If `db.useExisting` is `true`, the connection url of the existing database to connect to can be overwritten with this value.                                                                                                       | `postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_HOST)/$(DATABASE_NAME)`  |
-| `db.secret.name`                                           | If `db.useExisting` is `true`, the name of the Kubernetes Secret that contains credentials.                                                                                           | `postgres`  |
-| `db.secret.usernameKey`                                    | If `db.useExisting` is `true`, the name of the key within the Kubernetes Secret that holds the username for authenticating with the Postgres instance.                                | `username`  |
-| `db.secret.passwordKey`                                    | If `db.useExisting` is `true`, the name of the key within the Kubernetes Secret that holds the password associates with the above user.                                               | `password`  |
-| `db.useStackgresOperator`                                  | Not yet implemented.                                                                                                                                                                  | `false`  |
-| `db.deployStandalone`                                      | Deploy a standalone, single instance deployment of Postgres, using the Bitnami postgresql chart.  This is useful for getting started but doesn't provide HA or (by default) data backups.  | `true`  |
-| `postgresql.*`                                             | If `db.deployStandalone` is `true`, configuration passed to the Bitnami postgresql chart.   See the [Bitnami Documentation](https://github.com/bitnami/charts/tree/main/bitnami/postgresql) for full configuration details.  See [values.yaml](./values.yaml) for the default configuration.  | See [values.yaml](./values.yaml) |
-| `postgresql.auth.*`                                        | If `db.deployStandalone` is `true`, care should be taken to ensure the default `password` and `postgres-password` values are **NOT** used.                                            | `NoTaGrEaTpAsSwOrD`  |
+
+| Name                      | Description                                                                                                                                                                                                                                                                               | Value                                                                                      |
+| ------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------ |
+| `db.useExisting`          | Use an existing Postgres database. A Kubernetes Secret object must exist that contains credentials for connecting to the database. An example secret object definition is provided below.                                                                                                 | `false`                                                                                    |
+| `db.endpoint`             | If `db.useExisting` is `true`, this is the IP, Hostname or Service Name of the Postgres server to connect to.                                                                                                                                                                             | `localhost`                                                                                |
+| `db.database`             | If `db.useExisting` is `true`, the name of the existing database to connect to.                                                                                                                                                                                                           | `litellm`                                                                                  |
+| `db.url`                  | If `db.useExisting` is `true`, the connection url of the existing database to connect to can be overwritten with this value.                                                                                                                                                              | `postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_HOST)/$(DATABASE_NAME)` |
+| `db.secret.name`          | If `db.useExisting` is `true`, the name of the Kubernetes Secret that contains credentials.                                                                                                                                                                                               | `postgres`                                                                                 |
+| `db.secret.usernameKey`   | If `db.useExisting` is `true`, the name of the key within the Kubernetes Secret that holds the username for authenticating with the Postgres instance.                                                                                                                                    | `username`                                                                                 |
+| `db.secret.passwordKey`   | If `db.useExisting` is `true`, the name of the key within the Kubernetes Secret that holds the password associates with the above user.                                                                                                                                                   | `password`                                                                                 |
+| `db.useStackgresOperator` | Not yet implemented.                                                                                                                                                                                                                                                                      | `false`                                                                                    |
+| `db.deployStandalone`     | Deploy a standalone, single instance deployment of Postgres, using the Bitnami postgresql chart. This is useful for getting started but doesn't provide HA or (by default) data backups.                                                                                                  | `true`                                                                                     |
+| `postgresql.*`            | If `db.deployStandalone` is `true`, configuration passed to the Bitnami postgresql chart. See the [Bitnami Documentation](https://github.com/bitnami/charts/tree/main/bitnami/postgresql) for full configuration details. See [values.yaml](./values.yaml) for the default configuration. | See [values.yaml](./values.yaml)                                                           |
+| `postgresql.auth.*`       | If `db.deployStandalone` is `true`, care should be taken to ensure the default `password` and `postgres-password` values are **NOT** used.                                                                                                                                                | `NoTaGrEaTpAsSwOrD`                                                                        |
 
 #### Example Postgres `db.useExisting` Secret
+
 ```yaml
 apiVersion: v1
 kind: Secret
@@ -143,7 +145,7 @@ metadata:
   name: litellm-env-secret
 type: Opaque
 data:
-  SOME_PASSWORD: cDZbUGVXeU5e0ZW    # base64 encoded
+  SOME_PASSWORD: cDZbUGVXeU5e0ZW # base64 encoded
   ANOTHER_PASSWORD: AAZbUGVXeU5e0ZB # base64 encoded
 ```
 
@@ -153,23 +155,23 @@ Source: [GitHub Gist from troyharvey](https://gist.github.com/troyharvey/4506472
 
 The migration job supports both ArgoCD and Helm hooks to ensure database migrations run at the appropriate time during deployments.
 
-| Name                                                       | Description                                                                                                                                                                           | Value |
-| ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- |
-| `migrationJob.enabled`                                     | Enable or disable the schema migration Job                                                                                                                                           | `true`  |
-| `migrationJob.backoffLimit`                                | Backoff limit for Job restarts                                                                                                                                                       | `4`  |
-| `migrationJob.ttlSecondsAfterFinished`                     | TTL for completed migration jobs                                                                                                                                                      | `120`  |
-| `migrationJob.annotations`                                 | Additional annotations for the migration job pod                                                                                                                                      | `{}`  |
-| `migrationJob.extraContainers`                             | Additional containers to run alongside the migration job                                                                                                                              | `[]`  |
-| `migrationJob.hooks.argocd.enabled`                       | Enable ArgoCD hooks for the migration job (uses PreSync hook with BeforeHookCreation delete policy)                                                                                  | `true`  |
-| `migrationJob.hooks.helm.enabled`                         | Enable Helm hooks for the migration job (uses pre-install,pre-upgrade hooks with before-hook-creation delete policy)                                                                 | `false`  |
-| `migrationJob.hooks.helm.weight`                          | Helm hook execution order (lower weights executed first). Optional - defaults to "1" if not specified.                                                                               | N/A  |
-
+| Name                                   | Description                                                                                                          | Value   |
+| -------------------------------------- | -------------------------------------------------------------------------------------------------------------------- | ------- |
+| `migrationJob.enabled`                 | Enable or disable the schema migration Job                                                                           | `true`  |
+| `migrationJob.backoffLimit`            | Backoff limit for Job restarts                                                                                       | `4`     |
+| `migrationJob.ttlSecondsAfterFinished` | TTL for completed migration jobs                                                                                     | `120`   |
+| `migrationJob.annotations`             | Additional annotations for the migration job pod                                                                     | `{}`    |
+| `migrationJob.extraContainers`         | Additional containers to run alongside the migration job                                                             | `[]`    |
+| `migrationJob.hooks.argocd.enabled`    | Enable ArgoCD hooks for the migration job (uses PreSync hook with BeforeHookCreation delete policy)                  | `true`  |
+| `migrationJob.hooks.helm.enabled`      | Enable Helm hooks for the migration job (uses pre-install,pre-upgrade hooks with before-hook-creation delete policy) | `false` |
+| `migrationJob.hooks.helm.weight`       | Helm hook execution order (lower weights executed first). Optional - defaults to "1" if not specified.               | N/A     |
 
 ## Accessing the Admin UI
+
 When browsing to the URL published per the settings in `ingress.*`, you will
-be prompted for **Admin Configuration**.  The **Proxy Endpoint** is the internal
+be prompted for **Admin Configuration**. The **Proxy Endpoint** is the internal
 (from the `litellm` pod's perspective) URL published by the `<RELEASE>-litellm`
-Kubernetes Service.  If the deployment uses the default settings for this
+Kubernetes Service. If the deployment uses the default settings for this
 service, the **Proxy Endpoint** should be set to `http://<RELEASE>-litellm:4000`.
 
 The **Proxy Key** is the value specified for `masterkey` or, if a `masterkey`
@@ -181,7 +183,8 @@ kubectl -n litellm get secret <RELEASE>-litellm-masterkey -o jsonpath="{.data.ma
 ```
 
 ## Admin UI Limitations
-At the time of writing, the Admin UI is unable to add models.  This is because
+
+At the time of writing, the Admin UI is unable to add models. This is because
 it would need to update the `config.yaml` file which is a exposed ConfigMap, and
-therefore, read-only.  This is a limitation of this helm chart, not the Admin UI
+therefore, read-only. This is a limitation of this helm chart, not the Admin UI
 itself.

deploy/charts/litellm-helm/templates/ingress.yaml

@@ -18,6 +18,9 @@ metadata:
   name: {{ $fullName }}
   labels:
     {{- include "litellm.labels" . | nindent 4 }}
+    {{- with .Values.ingress.labels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
   {{- with .Values.ingress.annotations }}
   annotations:
     {{- toYaml . | nindent 4 }}

deploy/charts/litellm-helm/tests/ingress_tests.yaml

@@ -0,0 +1,45 @@
+suite: Ingress Configuration Tests
+templates:
+  - ingress.yaml
+tests:
+  - it: should not create Ingress by default
+    asserts:
+      - hasDocuments:
+          count: 0
+
+  - it: should create Ingress when enabled
+    set:
+      ingress.enabled: true
+    asserts:
+      - hasDocuments:
+          count: 1
+      - isKind:
+          of: Ingress
+
+  - it: should add custom labels
+    set:
+      ingress.enabled: true
+      ingress.labels:
+        custom-label: "true"
+        another-label: "value"
+    asserts:
+      - isKind:
+          of: Ingress
+      - equal:
+          path: metadata.labels.custom-label
+          value: "true"
+      - equal:
+          path: metadata.labels.another-label
+          value: "value"
+
+  - it: should add annotations
+    set:
+      ingress.enabled: true
+      ingress.annotations:
+        kubernetes.io/ingress.class: "nginx"
+    asserts:
+      - isKind:
+          of: Ingress
+      - equal:
+          path: metadata.annotations["kubernetes.io/ingress.class"]
+          value: "nginx"

deploy/charts/litellm-helm/values.yaml

@@ -35,7 +35,8 @@ podAnnotations: {}
 podLabels: {}
 
 terminationGracePeriodSeconds: 90
-topologySpreadConstraints: []
+topologySpreadConstraints:
+  []
   # - maxSkew: 1
   #   topologyKey: kubernetes.io/hostname
   #   whenUnsatisfiable: DoNotSchedule
@@ -46,7 +47,8 @@ topologySpreadConstraints: []
 # At the time of writing, the litellm docker image requires write access to the
 #  filesystem on startup so that prisma can install some dependencies.
 podSecurityContext: {}
-securityContext: {}
+securityContext:
+  {}
   # capabilities:
   #   drop:
   #     - ALL
@@ -57,13 +59,15 @@ securityContext: {}
 # A list of Kubernetes Secret objects that will be exported to the LiteLLM proxy
 #  pod as environment variables.  These secrets can then be referenced in the
 #  configuration file (or "litellm" ConfigMap) with `os.environ/<Env Var Name>`
-environmentSecrets: []
+environmentSecrets:
+  []
   # - litellm-env-secret
 
 # A list of Kubernetes ConfigMap objects that will be exported to the LiteLLM proxy
 #  pod as environment variables.  The ConfigMap kv-pairs can then be referenced in the
 #  configuration file (or "litellm" ConfigMap) with `os.environ/<Env Var Name>`
-environmentConfigMaps: []
+environmentConfigMaps:
+  []
   # - litellm-env-configmap
 
 service:
@@ -82,7 +86,9 @@ separateHealthPort: 8081
 ingress:
   enabled: false
   className: "nginx"
-  annotations: {}
+  labels: {}
+  annotations:
+    {}
     # kubernetes.io/ingress.class: nginx
     # kubernetes.io/tls-acme: "true"
   hosts:
@@ -129,7 +135,8 @@ proxy_config:
   general_settings:
     master_key: os.environ/PROXY_MASTER_KEY
 
-resources: {}
+resources:
+  {}
   # We usually recommend not to specify default resources and to leave this as a conscious
   # choice for the user. This also increases chances charts run on environments with little
   # resources, such as Minikube. If you do want to specify resources, uncomment the following
@@ -231,7 +238,7 @@ migrationJob:
   #    cpu: 100m
   #    memory: 100Mi
   extraContainers: []
-  
+
   # Hook configuration
   hooks:
     argocd:
@@ -240,30 +247,30 @@ migrationJob:
       enabled: false
 
 # Additional environment variables to be added to the deployment as a map of key-value pairs
-envVars: {
-    # USE_DDTRACE: "true"
-}
+envVars: {}
 
+# USE_DDTRACE: "true"
 # Additional environment variables to be added to the deployment as a list of k8s env vars
-extraEnvVars: {
-    # - name: EXTRA_ENV_VAR
-    #   value: EXTRA_ENV_VAR_VALUE
-}
+extraEnvVars: {}
 
+# - name: EXTRA_ENV_VAR
+#   value: EXTRA_ENV_VAR_VALUE
 # Pod Disruption Budget
 pdb:
   enabled: false
   # Set exactly one of the following. If both are set, minAvailable takes precedence.
-  minAvailable: null     # e.g. "50%" or 1
-  maxUnavailable: null   # e.g. 1 or "20%"
+  minAvailable: null # e.g. "50%" or 1
+  maxUnavailable: null # e.g. 1 or "20%"
   annotations: {}
   labels: {}
 
 serviceMonitor:
   enabled: false
-  labels: {}
+  labels:
+    {}
     # test: test
-  annotations: {}
+  annotations:
+    {}
     # kubernetes.io/test: test
   interval: 15s
   scrapeTimeout: 10s
@@ -273,4 +280,4 @@ serviceMonitor:
   #   action: replace
   namespaceSelector:
     matchNames: []
-    # - test-namespace
+    # - test-namespace

docker/Dockerfile.database

@@ -1,8 +1,8 @@
 # Base image for building
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/python:latest-dev
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base
 
 # Runtime image
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/python:latest-dev
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base
 # Builder stage
 FROM $LITELLM_BUILD_IMAGE AS builder
 
@@ -13,13 +13,15 @@ USER root
 
 # Install build dependencies
 RUN apk add --no-cache \
-    build-base \
+    bash \
+    gcc \
+    py3-pip \
+    python3 \
     python3-dev \
+    openssl \
     openssl-dev
 
-
-RUN pip install --upgrade pip && \
-    pip install build
+RUN python -m pip install build
 
 # Copy the current directory contents into the container at /app
 COPY . .
@@ -46,7 +48,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
 USER root
 
 # Install runtime dependencies
-RUN apk add --no-cache openssl
+RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip
 
 WORKDIR /app
 # Copy the current directory contents into the container at /app

docker/Dockerfile.non_root

@@ -1,6 +1,6 @@
 # Base images
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/python:latest-dev
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/python:latest-dev
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base
 
 # -----------------
 # Builder Stage
@@ -10,7 +10,18 @@ WORKDIR /app
 
 # Install build dependencies including Node.js for UI build
 USER root
-RUN apk add --no-cache build-base bash nodejs npm \
+RUN apk add --no-cache \
+    python3 \
+    py3-pip \
+    clang \
+    llvm \
+    lld \
+    gcc \
+    linux-headers \
+    build-base \
+    bash \
+    nodejs \
+    npm \
   && pip install --no-cache-dir --upgrade pip build
 
 # Copy project files
@@ -62,7 +73,7 @@ WORKDIR /app
 # Install runtime dependencies
 USER root
 RUN apk upgrade --no-cache && \
-  apk add --no-cache bash libstdc++ ca-certificates openssl supervisor
+  apk add --no-cache python3 py3-pip bash openssl tzdata nodejs npm supervisor
 
 # Copy only necessary artifacts from builder stage for runtime
 COPY . .

docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md

@@ -33,7 +33,7 @@ This guide covers Anthropic's latest model (Claude Opus 4.5) and its advanced fe
 | Input Examples | Claude Opus 4.5, Sonnet 4.5 |
 | Effort Parameter | Claude Opus 4.5 only |
 
-Supported Providers: [Anthropic](../../docs/providers/anthropic), [Bedrock](../../docs/providers/bedrock), [Vertex AI](../../docs/providers/vertex_partner#vertex-ai---anthropic-claude).
+Supported Providers: [Anthropic](../../docs/providers/anthropic), [Bedrock](../../docs/providers/bedrock), [Vertex AI](../../docs/providers/vertex_partner#vertex-ai---anthropic-claude), [Azure AI](../../docs/providers/azure_ai).
 
 ## Usage
 
@@ -327,6 +327,81 @@ curl --location 'http://0.0.0.0:4000/v1/messages' \
 </TabItem>
 </Tabs>
 
+## Usage - Azure Anthropic (Azure Foundry Claude)
+
+LiteLLM funnels Azure Claude deployments through the `azure_ai/` provider so Claude Opus models on Azure Foundry keep working with Tool Search, Effort, streaming, and the rest of the advanced feature set. Point `AZURE_AI_API_BASE` to `https://<resource>.services.ai.azure.com/anthropic` (LiteLLM appends `/v1/messages` automatically) and authenticate with `AZURE_AI_API_KEY` or an Azure AD token.
+
+<Tabs>
+<TabItem value="sdk" label="LiteLLM Python SDK">
+
+```python
+import os
+from litellm import completion
+
+# Configure Azure credentials
+os.environ["AZURE_AI_API_KEY"] = "your-azure-ai-api-key"
+os.environ["AZURE_AI_API_BASE"] = "https://my-resource.services.ai.azure.com/anthropic"
+
+response = completion(
+    model="azure_ai/claude-opus-4-1",
+    messages=[{"role": "user", "content": "Explain how Azure Anthropic hosts Claude Opus differently from the public Anthropic API."}],
+    max_tokens=1200,
+    temperature=0.7,
+    stream=True,
+)
+
+for chunk in response:
+    if chunk.choices[0].delta.content:
+        print(chunk.choices[0].delta.content, end="", flush=True)
+```
+
+</TabItem>
+<TabItem value="proxy" label="LiteLLM Proxy">
+
+**1. Set environment variables**
+
+```bash
+export AZURE_AI_API_KEY="your-azure-ai-api-key"
+export AZURE_AI_API_BASE="https://my-resource.services.ai.azure.com/anthropic"
+```
+
+**2. Configure the proxy**
+
+```yaml
+model_list:
+  - model_name: claude-4-azure
+    litellm_params:
+      model: azure_ai/claude-opus-4-1
+      api_key: os.environ/AZURE_AI_API_KEY
+      api_base: os.environ/AZURE_AI_API_BASE
+```
+
+**3. Start LiteLLM**
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+**4. Test the Azure Claude route**
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+  --header 'Content-Type: application/json' \
+  --header 'Authorization: Bearer $LITELLM_KEY' \
+  --data '{
+    "model": "claude-4-azure",
+    "messages": [
+      {
+        "role": "user",
+        "content": "How do I use Claude Opus 4 via Azure Anthropic in LiteLLM?"
+      }
+    ],
+    "max_tokens": 1024
+  }'
+```
+
+</TabItem>
+</Tabs>
 
 
 ## Tool Search {#tool-search}
@@ -897,14 +972,13 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
 
 ## Effort Parameter: Control Token Usage {#effort-parameter}
 
-Controls aspects like how much effort the model puts into its response, via `output_config={"effort": ..}`.
+Control how much effort Claude puts into its response using the `reasoning_effort` parameter. This allows you to trade off between response thoroughness and token efficiency.
 
 :::info
-
-Soon, we will map OpenAI's `reasoning_effort` parameter to this.
+LiteLLM automatically maps `reasoning_effort` to Anthropic's `output_config` format and adds the required `effort-2025-11-24` beta header for Claude Opus 4.5.
 :::
 
-Potential Values for `effort` parameter: `"high"`, `"medium"`, `"low"`.
+Potential values for `reasoning_effort` parameter: `"high"`, `"medium"`, `"low"`.
 
 ### Usage Example
 
@@ -920,7 +994,7 @@ message = "Analyze the trade-offs between microservices and monolithic architect
 response_high = litellm.completion(
     model="anthropic/claude-opus-4-5-20251101",
     messages=[{"role": "user", "content": message}],
-    output_config={"effort": "high"}
+    reasoning_effort="high"
 )
 
 print("High effort response:")
@@ -931,7 +1005,7 @@ print(f"Tokens used: {response_high.usage.completion_tokens}\n")
 response_medium = litellm.completion(
     model="anthropic/claude-opus-4-5-20251101",
     messages=[{"role": "user", "content": message}],
-    output_config={"effort": "medium"}
+    reasoning_effort="medium"
 )
 
 print("Medium effort response:")
@@ -942,7 +1016,7 @@ print(f"Tokens used: {response_medium.usage.completion_tokens}\n")
 response_low = litellm.completion(
     model="anthropic/claude-opus-4-5-20251101",
     messages=[{"role": "user", "content": message}],
-    output_config={"effort": "low"}
+    reasoning_effort="low"
 )
 
 print("Low effort response:")
@@ -987,295 +1061,9 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
         "role": "user",
         "content": "Analyze the trade-offs between microservices and monolithic architectures"
       }],
-      "output_config": {
-        "effort": "high"
-      }
+      "reasoning_effort": "high"
     }
 '
 ```
 </TabItem>
 </Tabs>
-
-
-## Cost Tracking: Monitor Tool Search Usage {#cost-tracking}
-
-### Understanding Tool Search Costs
-
-Tool search operations are tracked separately in the usage object, allowing you to monitor and optimize costs.
-
-It is available in the `usage` object, under `server_tool_use.tool_search_requests`.
-
-Anthropic charges $0.0001 per tool search request. 
-
-### Tracking Example
-
-<Tabs>
-<TabItem value="sdk" label="LiteLLM Python SDK">
-
-```python
-import litellm
-
-tools = [
-    {
-        "type": "tool_search_tool_regex_20251119",
-        "name": "tool_search_tool_regex"
-    },
-    # ... 100 deferred tools
-]
-
-response = litellm.completion(
-    model="anthropic/claude-sonnet-4-5-20250929",
-    messages=[{
-        "role": "user",
-        "content": "Find and use the weather tool for San Francisco"
-    }],
-    tools=tools
-)
-
-# Standard token usage
-print("Token Usage:")
-print(f"  Input tokens:  {response.usage.prompt_tokens}")
-print(f"  Output tokens: {response.usage.completion_tokens}")
-print(f"  Total tokens:  {response.usage.total_tokens}")
-
-# Tool search specific usage
-if hasattr(response.usage, 'server_tool_use') and response.usage.server_tool_use:
-    print(f"\nTool Search Usage:")
-    print(f"  Search requests: {response.usage.server_tool_use.tool_search_requests}")
-    
-    # Calculate cost (example pricing)
-    input_cost = response.usage.prompt_tokens * 0.000003  # $3 per 1M tokens
-    output_cost = response.usage.completion_tokens * 0.000015  # $15 per 1M tokens
-    search_cost = response.usage.server_tool_use.tool_search_requests * 0.0001  # Example
-    
-    total_cost = input_cost + output_cost + search_cost
-    
-    print(f"\nCost Breakdown:")
-    print(f"  Input tokens:   ${input_cost:.6f}")
-    print(f"  Output tokens:  ${output_cost:.6f}")
-    print(f"  Tool searches:  ${search_cost:.6f}")
-    print(f"  Total:          ${total_cost:.6f}")
-```
-
-</TabItem>
-<TabItem value="proxy" label="LiteLLM Proxy">
-
-1. Setup config.yaml
-
-```yaml
-model_list:
-  - model_name: claude-4
-    litellm_params:
-      model: anthropic/claude-opus-4-5-20251101
-      api_key: os.environ/ANTHROPIC_API_KEY
-```
-
-2. Start the proxy
-
-```bash
-litellm --config /path/to/config.yaml
-```
-
-3. Test it!
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
---header 'Content-Type: application/json' \
---header 'Authorization: Bearer $LITELLM_KEY' \
---data ' {
-      "model": "claude-4",
-      "messages": [{
-        "role": "user",
-        "content": "Find and use the weather tool for San Francisco"
-      }],
-      "tools": [
-        {
-          "type": "tool_search_tool_regex_20251119",
-          "name": "tool_search_tool_regex"
-        },
-        # ... 100 deferred tools
-      ]
-    }
-'
-```
-
-Expected Response:
-
-```json
-{
-    ...,
-    "usage": {
-        ...,
-        "server_tool_use": {
-            "tool_search_requests": 1
-        }
-    }
-}
-```
-
-</TabItem>
-</Tabs>
-
-### Cost Optimization Tips
-
-1. **Keep frequently used tools non-deferred** (3-5 tools)
-2. **Use tool search for large catalogs** (10+ tools)
-3. **Monitor search requests** to identify optimization opportunities
-4. **Combine with effort parameter** for maximum efficiency
-
-
----
-
-## Combining Features {#combining-features}
-
-### The Power of Integration
-
-These features work together seamlessly. Here's a real-world example combining all of them:
-
-<Tabs>
-<TabItem value="sdk" label="LiteLLM Python SDK">
-
-```python
-import litellm
-import json
-
-# Large tool catalog with search, programmatic calling, and examples
-tools = [
-    # Enable tool search
-    {
-        "type": "tool_search_tool_regex_20251119",
-        "name": "tool_search_tool_regex"
-    },
-    # Enable programmatic calling
-    {
-        "type": "code_execution_20250825",
-        "name": "code_execution"
-    },
-    # Database tool with all features
-    {
-        "type": "function",
-        "function": {
-            "name": "query_database",
-            "description": "Execute SQL queries against the analytics database. Returns JSON array of results.",
-            "parameters": {
-                "type": "object",
-                "properties": {
-                    "sql": {
-                        "type": "string",
-                        "description": "SQL SELECT statement"
-                    },
-                    "limit": {
-                        "type": "integer",
-                        "description": "Maximum rows to return"
-                    }
-                },
-                "required": ["sql"]
-            }
-        },
-        "defer_loading": True,  # Tool search
-        "allowed_callers": ["code_execution_20250825"],  # Programmatic calling
-        "input_examples": [  # Input examples
-            {
-                "sql": "SELECT region, SUM(revenue) as total FROM sales GROUP BY region",
-                "limit": 100
-            }
-        ]
-    },
-    # ... 50 more tools with defer_loading
-]
-
-# Make request with effort control
-response = litellm.completion(
-    model="anthropic/claude-opus-4-5-20251101",
-    messages=[{
-        "role": "user",
-        "content": "Analyze sales by region for the last quarter and identify top performers"
-    }],
-    tools=tools,
-    output_config={"effort": "medium"}  # Balanced efficiency
-)
-
-# Track comprehensive usage
-print("Complete Usage Metrics:")
-print(f"  Input tokens:     {response.usage.prompt_tokens}")
-print(f"  Output tokens:    {response.usage.completion_tokens}")
-print(f"  Total tokens:     {response.usage.total_tokens}")
-
-if hasattr(response.usage, 'server_tool_use') and response.usage.server_tool_use:
-    print(f"  Tool searches:    {response.usage.server_tool_use.tool_search_requests}")
-
-print(f"\nResponse: {response.choices[0].message.content}")
-```
-
-</TabItem>
-<TabItem value="proxy" label="LiteLLM Proxy">
-
-1. Setup config.yaml
-
-```yaml
-model_list:
-  - model_name: claude-4
-    litellm_params:
-      model: anthropic/claude-opus-4-5-20251101
-      api_key: os.environ/ANTHROPIC_API_KEY
-```
-
-2. Start the proxy
-
-```bash
-litellm --config /path/to/config.yaml
-```
-
-3. Test it!
-
-```bash
-curl --location 'http://0.0.0.0:4000/chat/completions' \
---header 'Content-Type: application/json' \
---header 'Authorization: Bearer $LITELLM_KEY' \
---data ' {
-      "model": "claude-4",
-      "messages": [{
-        "role": "user",
-        "content": "Analyze sales by region for the last quarter and identify top performers"
-      }],
-      "tools": [
-        {
-          "type": "tool_search_tool_regex_20251119",
-          "name": "tool_search_tool_regex"
-        },
-        # ... 100 deferred tools
-      ],
-      "output_config": {
-        "effort": "medium"
-      }
-    }
-'
-```
-
-Expected Response:
-
-```json
-{
-    ...,
-    "usage": {
-        ...,    
-        "server_tool_use": {
-            "tool_search_requests": 1
-        }
-    }
-}
-```
-
-</TabItem>
-</Tabs>
-
-### Real-World Benefits
-
-This combination enables:
-
-1. **Massive scale** - Handle 1000+ tools efficiently
-2. **Low latency** - Programmatic calling reduces round trips
-3. **High accuracy** - Input examples ensure correct tool usage
-4. **Cost control** - Effort parameter optimizes token spend
-5. **Full visibility** - Track all usage metrics
-

docs/my-website/docs/adding_provider/generic_guardrail_api.md

@@ -0,0 +1,182 @@
+# [BETA] Generic Guardrail API - Integrate Without a PR
+
+## The Problem
+
+As a guardrail provider, integrating with LiteLLM traditionally requires:
+- Making a PR to the LiteLLM repository
+- Waiting for review and merge
+- Maintaining provider-specific code in LiteLLM's codebase
+- Updating the integration for changes to your API
+
+## The Solution
+
+The **Generic Guardrail API** lets you integrate with LiteLLM **instantly** by implementing a simple API endpoint. No PR required.
+
+### Key Benefits
+
+1. **No PR Needed** - Deploy and integrate immediately
+2. **Universal Support** - Works across ALL LiteLLM endpoints (chat, embeddings, image generation, etc.)
+3. **Simple Contract** - One endpoint, three response types
+4. **Multi-Modal Support** - Handle both text and images in requests/responses
+5. **Custom Parameters** - Pass provider-specific params via config
+6. **Full Control** - You own and maintain your guardrail API
+
+## How It Works
+
+1. LiteLLM extracts text and images from any request (chat messages, embeddings, image prompts, etc.)
+2. Sends extracted content + metadata to your API endpoint
+3. Your API responds with: `BLOCKED`, `NONE`, or `GUARDRAIL_INTERVENED`
+4. LiteLLM enforces the decision and applies any modifications
+
+## API Contract
+
+### Endpoint
+
+Implement `POST /beta/litellm_basic_guardrail_api`
+
+### Request Format
+
+```json
+{
+  "texts": ["extracted text from the request"],  // array of text strings
+  "images": ["base64_encoded_image_data"],  // optional array of images
+  "request_data": {
+    "user_api_key_hash": "hash of the litellm virtual key used",
+    "user_api_key_alias": "alias of the litellm virtual key used",
+    "user_api_key_user_id": "user id associated with the litellm virtual key used",
+    "user_api_key_user_email": "user email associated with the litellm virtual key used",
+    "user_api_key_team_id": "team id associated with the litellm virtual key used",
+    "user_api_key_team_alias": "team alias associated with the litellm virtual key used",
+    "user_api_key_end_user_id": "end user id associated with the litellm virtual key used",
+    "user_api_key_org_id": "org id associated with the litellm virtual key used"
+  },
+  "input_type": "request",  // "request" or "response"
+  "litellm_call_id": "unique_call_id",  // the call id of the individual LLM call
+  "litellm_trace_id": "trace_id",  // the trace id of the LLM call - useful if there are multiple LLM calls for the same conversation
+  "additional_provider_specific_params": {
+    // your custom params from config
+  }
+}
+```
+
+### Response Format
+
+```json
+{
+  "action": "BLOCKED" | "NONE" | "GUARDRAIL_INTERVENED",
+  "blocked_reason": "why content was blocked",  // required if action=BLOCKED
+  "texts": ["modified text"],  // optional array of modified text strings
+  "images": ["modified_base64_image"]  // optional array of modified images
+}
+```
+
+**Actions:**
+- `BLOCKED` - LiteLLM raises error and blocks request
+- `NONE` - Request proceeds unchanged  
+- `GUARDRAIL_INTERVENED` - Request proceeds with modified texts/images (provide `texts` and/or `images` fields)
+
+## LiteLLM Configuration
+
+Add to `config.yaml`:
+
+```yaml
+litellm_settings:
+  guardrails:
+    - guardrail_name: "my-guardrail"
+      litellm_params:
+        guardrail: generic_guardrail_api
+        mode: pre_call  # or post_call, during_call
+        api_base: https://your-guardrail-api.com
+        api_key: os.environ/YOUR_GUARDRAIL_API_KEY  # optional
+        additional_provider_specific_params:
+          # your custom parameters
+          threshold: 0.8
+          language: "en"
+```
+
+## Usage
+
+Users apply your guardrail by name:
+
+```python
+response = client.chat.completions.create(
+    model="gpt-4",
+    messages=[{"role": "user", "content": "hello"}],
+    guardrails=["my-guardrail"]
+)
+```
+
+Or with dynamic parameters:
+
+```python
+response = client.chat.completions.create(
+    model="gpt-4",
+    messages=[{"role": "user", "content": "hello"}],
+    guardrails=[{
+        "my-guardrail": {
+            "extra_body": {
+                "custom_threshold": 0.9
+            }
+        }
+    }]
+)
+```
+
+## Implementation Example
+
+See [mock_bedrock_guardrail_server.py](https://github.com/BerriAI/litellm/blob/main/cookbook/mock_guardrail_server/mock_bedrock_guardrail_server.py) for a complete reference implementation.
+
+**Minimal FastAPI example:**
+
+```python
+from fastapi import FastAPI
+from pydantic import BaseModel
+from typing import List, Optional, Dict, Any
+
+app = FastAPI()
+
+class GuardrailRequest(BaseModel):
+    texts: List[str]
+    images: Optional[List[str]] = None
+    request_data: Dict[str, Any]
+    input_type: str  # "request" or "response"
+    litellm_call_id: Optional[str] = None
+    litellm_trace_id: Optional[str] = None
+    additional_provider_specific_params: Dict[str, Any]
+
+class GuardrailResponse(BaseModel):
+    action: str  # BLOCKED, NONE, or GUARDRAIL_INTERVENED
+    blocked_reason: Optional[str] = None
+    texts: Optional[List[str]] = None
+    images: Optional[List[str]] = None
+
+@app.post("/beta/litellm_basic_guardrail_api")
+async def apply_guardrail(request: GuardrailRequest):
+    # Your guardrail logic here
+    for text in request.texts:
+        if "badword" in text.lower():
+            return GuardrailResponse(
+                action="BLOCKED",
+                blocked_reason="Content contains prohibited terms"
+            )
+    
+    return GuardrailResponse(action="NONE")
+```
+
+## When to Use This
+
+✅ **Use Generic Guardrail API when:**
+- You want instant integration without waiting for PRs
+- You maintain your own guardrail service
+- You need full control over updates and features
+- You want to support all LiteLLM endpoints automatically
+
+❌ **Make a PR when:**
+- You want deeper integration with LiteLLM internals
+- Your guardrail requires complex LiteLLM-specific logic
+- You want to be featured as a built-in provider
+
+## Questions?
+
+This is a **beta API**. We're actively improving it based on feedback. Open an issue or PR if you need additional capabilities.
+

docs/my-website/docs/audio_transcription.md

@@ -13,7 +13,7 @@ import TabItem from '@theme/TabItem';
 | Fallbacks | ✅ | Works between supported models |
 | Loadbalancing | ✅ | Works between supported models |
 | Guardrails | ✅ | Applies to output transcribed text (non-streaming only) |
-| Supported Providers | `openai`, `azure`, `vertex_ai`, `gemini`, `deepgram`, `groq`, `fireworks_ai` | |
+| Supported Providers | `openai`, `azure`, `vertex_ai`, `gemini`, `deepgram`, `groq`, `fireworks_ai`, `ovhcloud` | |
 
 ## Quick Start
 
@@ -126,6 +126,7 @@ transcript = client.audio.transcriptions.create(
 - [Fireworks AI](./providers/fireworks_ai.md#audio-transcription)
 - [Groq](./providers/groq.md#speech-to-text---whisper)
 - [Deepgram](./providers/deepgram.md)
+- [OVHcloud AI Endpoints](./providers/ovhcloud.md)
 
 ---
 

docs/my-website/docs/completion/knowledgebase.md

@@ -21,6 +21,7 @@ LiteLLM integrates with vector stores, allowing your models to access your organ
 - [Azure Vector Stores](https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/file-search?tabs=python#vector-stores) (Cannot be directly queried. Only available for calling in Assistants messages. We will be adding Azure AI Search Vector Store API support soon.)
 - [Vertex AI RAG API](https://cloud.google.com/vertex-ai/generative-ai/docs/rag-overview)
 - [Gemini File Search](https://ai.google.dev/gemini-api/docs/file-search)
+- [RAGFlow Datasets](/docs/providers/ragflow_vector_store.md) (Dataset management only, search not supported)
 
 ## Quick Start
 

docs/my-website/docs/embedding/supported_embedding.md

@@ -263,6 +263,8 @@ print(response)
 
 | Model Name           | Function Call                               |
 |----------------------|---------------------------------------------|
+| Amazon Nova Multimodal Embeddings | `embedding(model="bedrock/amazon.nova-2-multimodal-embeddings-v1:0", input=input)` | [Nova Docs](../providers/bedrock_embedding#amazon-nova-multimodal-embeddings) |
+| Amazon Nova (Async) | `embedding(model="bedrock/async_invoke/amazon.nova-2-multimodal-embeddings-v1:0", input=input, input_type="text", output_s3_uri="s3://bucket/")` | [Nova Async Docs](../providers/bedrock_embedding#asynchronous-embeddings-with-segmentation) |
 | Titan Embeddings - G1 | `embedding(model="amazon.titan-embed-text-v1", input=input)` |
 | Cohere Embeddings - English | `embedding(model="cohere.embed-english-v3", input=input)` |
 | Cohere Embeddings - Multilingual | `embedding(model="cohere.embed-multilingual-v3", input=input)` |

docs/my-website/docs/getting_started.md

@@ -1,108 +0,0 @@
-# Getting Started
-
-import QuickStart from '../src/components/QuickStart.js'
-
-LiteLLM simplifies LLM API calls by mapping them all to the [OpenAI ChatCompletion format](https://platform.openai.com/docs/api-reference/chat).
-
-## basic usage
-
-By default we provide a free $10 community-key to try all providers supported on LiteLLM.
-
-```python
-from litellm import completion
-
-## set ENV variables
-os.environ["OPENAI_API_KEY"] = "your-api-key"
-os.environ["COHERE_API_KEY"] = "your-api-key"
-
-messages = [{ "content": "Hello, how are you?","role": "user"}]
-
-# openai call
-response = completion(model="gpt-3.5-turbo", messages=messages)
-
-# cohere call
-response = completion("command-nightly", messages)
-```
-
-**Need a dedicated key?**
-Email us @ krrish@berri.ai
-
-Next Steps 👉 [Call all supported models - e.g. Claude-2, Llama2-70b, etc.](./proxy_api.md#supported-models)
-
-More details 👉
-
-- [Completion() function details](./completion/)
-- [Overview of supported models / providers on LiteLLM](./providers/)
-- [Search all models / providers](https://models.litellm.ai/)
-- [Build your own OpenAI proxy](https://github.com/BerriAI/liteLLM-proxy/tree/main)
-
-## streaming
-
-Same example from before. Just pass in `stream=True` in the completion args.
-
-```python
-from litellm import completion
-
-## set ENV variables
-os.environ["OPENAI_API_KEY"] = "openai key"
-os.environ["COHERE_API_KEY"] = "cohere key"
-
-messages = [{ "content": "Hello, how are you?","role": "user"}]
-
-# openai call
-response = completion(model="gpt-3.5-turbo", messages=messages, stream=True)
-
-# cohere call
-response = completion("command-nightly", messages, stream=True)
-
-print(response)
-```
-
-More details 👉
-
-- [streaming + async](./completion/stream.md)
-- [tutorial for streaming Llama2 on TogetherAI](./tutorials/TogetherAI_liteLLM.md)
-
-## exception handling
-
-LiteLLM maps exceptions across all supported providers to the OpenAI exceptions. All our exceptions inherit from OpenAI's exception types, so any error-handling you have for that, should work out of the box with LiteLLM.
-
-```python
-from openai.error import OpenAIError
-from litellm import completion
-
-os.environ["ANTHROPIC_API_KEY"] = "bad-key"
-try:
-    # some code
-    completion(model="claude-instant-1", messages=[{"role": "user", "content": "Hey, how's it going?"}])
-except OpenAIError as e:
-    print(e)
-```
-
-## Logging Observability - Log LLM Input/Output ([Docs](https://docs.litellm.ai/docs/observability/callbacks))
-
-LiteLLM exposes pre defined callbacks to send data to MLflow, Lunary, Langfuse, Helicone, Promptlayer, Traceloop, Slack
-
-```python
-from litellm import completion
-
-## set env variables for logging tools (API key set up is not required when using MLflow)
-os.environ["LUNARY_PUBLIC_KEY"] = "your-lunary-public-key" # get your public key at https://app.lunary.ai/settings
-os.environ["HELICONE_API_KEY"] = "your-helicone-key"
-os.environ["LANGFUSE_PUBLIC_KEY"] = ""
-os.environ["LANGFUSE_SECRET_KEY"] = ""
-
-os.environ["OPENAI_API_KEY"]
-
-# set callbacks
-litellm.success_callback = ["lunary", "mlflow", "langfuse", "helicone"] # log input/output to MLflow, langfuse, lunary, helicone
-
-#openai call
-response = completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}])
-```
-
-More details 👉
-
-- [exception mapping](./exception_mapping.md)
-- [retries + model fallbacks for completion()](./completion/reliable_completions.md)
-- [tutorial for model fallbacks with completion()](./tutorials/fallbacks.md)

docs/my-website/docs/observability/datadog.md

@@ -71,17 +71,19 @@ DD_SOURCE="litellm_dev"       # [OPTIONAL] your datadog source. use to different
 Send logs through a local DataDog agent (useful for containerized environments):
 
 ```shell
-DD_AGENT_HOST="localhost"         # hostname or IP of DataDog agent
-DD_AGENT_PORT="10518"             # [OPTIONAL] port of DataDog agent (default: 10518)
-DD_API_KEY="5f2d0f310***********" # [OPTIONAL] your datadog API Key (agent handles auth)
-DD_SOURCE="litellm_dev"           # [OPTIONAL] your datadog source
+LITELLM_DD_AGENT_HOST="localhost"         # hostname or IP of DataDog agent
+LITELLM_DD_AGENT_PORT="10518"             # [OPTIONAL] port of DataDog agent (default: 10518)
+DD_API_KEY="5f2d0f310***********"         # [OPTIONAL] your datadog API Key (agent handles auth)
+DD_SOURCE="litellm_dev"                   # [OPTIONAL] your datadog source
 ```
 
-When `DD_AGENT_HOST` is set, logs are sent to the agent instead of directly to DataDog API. This is useful for:
+When `LITELLM_DD_AGENT_HOST` is set, logs are sent to the agent instead of directly to DataDog API. This is useful for:
 - Centralized log shipping in containerized environments
 - Reducing direct API calls from multiple services
 - Leveraging agent-side processing and filtering
 
+**Note:** We use `LITELLM_DD_AGENT_HOST` instead of `DD_AGENT_HOST` to avoid conflicts with `ddtrace` which automatically sets `DD_AGENT_HOST` for APM tracing.
+
 **Step 3**: Start the proxy, make a test request
 
 Start proxy
@@ -191,8 +193,8 @@ LiteLLM supports customizing the following Datadog environment variables
 |---------------------|-------------|---------------|----------|
 | `DD_API_KEY` | Your Datadog API key for authentication (required for direct API, optional for agent) | None | Conditional* |
 | `DD_SITE` | Your Datadog site (e.g., "us5.datadoghq.com") (required for direct API) | None | Conditional* |
-| `DD_AGENT_HOST` | Hostname or IP of DataDog agent (e.g., "localhost"). When set, logs are sent to agent instead of direct API | None | ❌ No |
-| `DD_AGENT_PORT` | Port of DataDog agent for log intake | "10518" | ❌ No |
+| `LITELLM_DD_AGENT_HOST` | Hostname or IP of DataDog agent (e.g., "localhost"). When set, logs are sent to agent instead of direct API | None | ❌ No |
+| `LITELLM_DD_AGENT_PORT` | Port of DataDog agent for log intake | "10518" | ❌ No |
 | `DD_ENV` | Environment tag for your logs (e.g., "production", "staging") | "unknown" | ❌ No |
 | `DD_SERVICE` | Service name for your logs | "litellm-server" | ❌ No |
 | `DD_SOURCE` | Source name for your logs | "litellm" | ❌ No |
@@ -201,5 +203,5 @@ LiteLLM supports customizing the following Datadog environment variables
 | `POD_NAME` | Pod name tag (useful for Kubernetes deployments) | "unknown" | ❌ No |
 
 \* **Required when using Direct API** (default): `DD_API_KEY` and `DD_SITE` are required  
-\* **Optional when using DataDog Agent**: Set `DD_AGENT_HOST` to use agent mode; `DD_API_KEY` and `DD_SITE` are not required
+\* **Optional when using DataDog Agent**: Set `LITELLM_DD_AGENT_HOST` to use agent mode; `DD_API_KEY` and `DD_SITE` are not required
 

docs/my-website/docs/observability/phoenix_integration.md

@@ -6,7 +6,7 @@ Open source tracing and evaluation platform
 
 :::tip
 
-This is community maintained, Please make an issue if you run into a bug
+This is community maintained. Please make an issue if you run into a bug:
 https://github.com/BerriAI/litellm
 
 :::
@@ -31,19 +31,16 @@ litellm.callbacks = ["arize_phoenix"]
 import litellm
 import os
 
-os.environ["PHOENIX_API_KEY"] = "" # Necessary only using Phoenix Cloud
-os.environ["PHOENIX_COLLECTOR_HTTP_ENDPOINT"] = "" # The URL of your Phoenix OSS instance e.g. http://localhost:6006/v1/traces
-os.environ["PHOENIX_PROJECT_NAME"]="litellm" # OPTIONAL: you can configure project names, otherwise traces would go to "default" project
+# Set env variables
+os.environ["PHOENIX_API_KEY"] = "d0*****" # Set the Phoenix API key here. It is necessary only when using Phoenix Cloud.
+os.environ["PHOENIX_COLLECTOR_HTTP_ENDPOINT"] = "https://app.phoenix.arize.com/s/<space-name>/v1/traces" # Set the URL of your Phoenix OSS instance, otherwise tracer would use https://app.phoenix.arize.com/v1/traces for Phoenix Cloud.
+os.environ["PHOENIX_PROJECT_NAME"] = "litellm" # Configure the project name, otherwise traces would go to "default" project.
+os.environ['OPENAI_API_KEY'] = "fake-key" # Set the OpenAI API key here.
 
-# This defaults to https://app.phoenix.arize.com/v1/traces for Phoenix Cloud
-
-# LLM API Keys
-os.environ['OPENAI_API_KEY']=""
-
-# set arize as a callback, litellm will send the data to arize
+# Set arize_phoenix as a callback & LiteLLM will send the data to Phoenix.
 litellm.callbacks = ["arize_phoenix"]
- 
-# openai call
+
+# OpenAI call
 response = litellm.completion(
   model="gpt-3.5-turbo",
   messages=[
@@ -52,8 +49,9 @@ response = litellm.completion(
 )
 ```
 
-### Using with LiteLLM Proxy
+## Using with LiteLLM Proxy
 
+1. Setup config.yaml
 
 ```yaml
 model_list:
@@ -66,12 +64,63 @@ model_list:
 litellm_settings:
   callbacks: ["arize_phoenix"]
 
+general_settings:
+  master_key: "sk-1234"
+
 environment_variables:
     PHOENIX_API_KEY: "d0*****"
-    PHOENIX_COLLECTOR_ENDPOINT: "https://app.phoenix.arize.com/v1/traces" # OPTIONAL, for setting the GRPC endpoint
-    PHOENIX_COLLECTOR_HTTP_ENDPOINT: "https://app.phoenix.arize.com/v1/traces" # OPTIONAL, for setting the HTTP endpoint
+    PHOENIX_COLLECTOR_ENDPOINT: "https://app.phoenix.arize.com/s/<space-name>/v1/traces" # OPTIONAL - For setting the gRPC endpoint
+    PHOENIX_COLLECTOR_HTTP_ENDPOINT: "https://app.phoenix.arize.com/s/<space-name>/v1/traces" # OPTIONAL - For setting the HTTP endpoint
 ```
 
+2. Start the proxy
+
+```bash
+litellm --config config.yaml
+```
+
+3. Test it!
+
+```bash
+curl -X POST 'http://0.0.0.0:4000/chat/completions' \
+-H 'Content-Type: application/json' \
+-H 'Authorization: Bearer sk-1234' \
+-d '{ "model": "gpt-4o", "messages": [{"role": "user", "content": "Hi 👋 - i'm openai"}]}'
+```
+
+## Supported Phoenix Endpoints
+Phoenix now supports multiple deployment types. The correct endpoint depends on which version of Phoenix Cloud you are using.
+
+**Phoenix Cloud (With Spaces - New Version)**
+Use this if your Phoenix URL contains `/s/<space-name>` path.
+
+```bash
+https://app.phoenix.arize.com/s/<space-name>/v1/traces
+```
+
+**Phoenix Cloud (Legacy - Deprecated)**
+Use this only if your deployment still shows the `/legacy` pattern.
+
+```bash
+https://app.phoenix.arize.com/legacy/v1/traces
+```
+
+**Phoenix Cloud (Without Spaces - Old Version)**
+Use this if your Phoenix Cloud URL does not contain `/s/<space-name>` or `/legacy` path.
+
+```bash
+https://app.phoenix.arize.com/v1/traces
+```
+
+**Self-Hosted Phoenix (Local Instance)**
+Use this when running Phoenix on your machine or a private server.
+
+```bash
+http://localhost:6006/v1/traces
+```
+
+Depending on which Phoenix Cloud version or deployment you are using, you should set the corresponding endpoint in `PHOENIX_COLLECTOR_HTTP_ENDPOINT` or `PHOENIX_COLLECTOR_ENDPOINT`.
+
 ## Support & Talk to Founders
 
 - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version)

docs/my-website/docs/projects/Agent Lightning.md

@@ -0,0 +1,10 @@
+
+# Agent Lightning
+
+[Agent Lightning](https://github.com/microsoft/agent-lightning) is Microsoft's open-source framework for training and optimizing AI agents with Reinforcement Learning, Automatic Prompt Optimization, and Supervised Fine-tuning — with almost zero code changes.
+
+It works with any agent framework including LangChain, OpenAI Agents SDK, AutoGen, and CrewAI. Agent Lightning uses LiteLLM Proxy under the hood to route LLM requests and collect traces that power its training algorithms.
+
+- [GitHub](https://github.com/microsoft/agent-lightning)
+- [Docs](https://microsoft.github.io/agent-lightning/)
+- [arXiv Paper](https://arxiv.org/abs/2508.03680)

docs/my-website/docs/projects/Google ADK.md

@@ -0,0 +1,21 @@
+
+# Google ADK (Agent Development Kit)
+
+[Google ADK](https://github.com/google/adk-python) is an open-source, code-first Python framework for building, evaluating, and deploying sophisticated AI agents. While optimized for Gemini, ADK is model-agnostic and supports LiteLLM for using 100+ providers.
+
+```python
+from google.adk.agents.llm_agent import Agent
+from google.adk.models.lite_llm import LiteLlm
+
+root_agent = Agent(
+    model=LiteLlm(model="openai/gpt-4o"),  # Or any LiteLLM-supported model
+    name="my_agent",
+    description="An agent using LiteLLM",
+    instruction="You are a helpful assistant.",
+    tools=[your_tools],
+)
+```
+
+- [GitHub](https://github.com/google/adk-python)
+- [Documentation](https://google.github.io/adk-docs)
+- [LiteLLM Samples](https://github.com/google/adk-python/tree/main/contributing/samples/hello_world_litellm)

docs/my-website/docs/projects/Harbor.md

@@ -0,0 +1,24 @@
+
+# Harbor
+
+[Harbor](https://github.com/laude-institute/harbor) is a framework from the creators of Terminal-Bench for evaluating and optimizing agents and language models. It uses LiteLLM to call 100+ LLM providers.
+
+```bash
+# Install
+pip install harbor
+
+# Run a benchmark with any LiteLLM-supported model
+harbor run --dataset terminal-bench@2.0 \
+   --agent claude-code \
+   --model anthropic/claude-opus-4-1 \
+   --n-concurrent 4
+```
+
+Key features:
+- Evaluate agents like Claude Code, OpenHands, Codex CLI
+- Build and share benchmarks and environments
+- Run experiments in parallel across cloud providers (Daytona, Modal)
+- Generate rollouts for RL optimization
+
+- [GitHub](https://github.com/laude-institute/harbor)
+- [Documentation](https://harborframework.com/docs)

docs/my-website/docs/providers/anthropic.md

@@ -41,7 +41,8 @@ Check this in code, [here](../completion/input.md#translated-openai-params)
 "extra_headers",
 "parallel_tool_calls",
 "response_format",
-"user"
+"user",
+"reasoning_effort",
 ```
 
 :::info
@@ -49,6 +50,7 @@ Check this in code, [here](../completion/input.md#translated-openai-params)
 **Notes:**
 - Anthropic API fails requests when `max_tokens` are not passed. Due to this litellm passes `max_tokens=4096` when no `max_tokens` are passed.
 - `response_format` is fully supported for Claude Sonnet 4.5 and Opus 4.1 models (see [Structured Outputs](#structured-outputs) section)
+- `reasoning_effort` is automatically mapped to `output_config={"effort": ...}` for Claude Opus 4.5 models (see [Effort Parameter](./anthropic_effort.md))
 
 :::
 
@@ -199,6 +201,30 @@ Without `LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX`:
 With `LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX=true`:
 - Base URL `https://my-proxy.com/custom/path` → `https://my-proxy.com/custom/path` (unchanged)
 
+### Azure AI Foundry (Alternative Method)
+
+:::tip Recommended Method
+For full Azure support including Azure AD authentication, use the dedicated [Azure Anthropic provider](./azure/azure_anthropic) with `azure_ai/` prefix.
+:::
+
+As an alternative, you can use the `anthropic/` provider directly with your Azure endpoint since Azure exposes Claude using Anthropic's native API.
+
+```python
+from litellm import completion
+
+response = completion(
+    model="anthropic/claude-sonnet-4-5",
+    api_base="https://<your-resource>.services.ai.azure.com/anthropic",
+    api_key="<your-azure-api-key>",
+    messages=[{"role": "user", "content": "Hello!"}],
+)
+print(response)
+```
+
+:::info
+**Finding your Azure endpoint:** Go to Azure AI Foundry → Your deployment → Overview. Your base URL will be `https://<resource-name>.services.ai.azure.com/anthropic`
+:::
+
 ## Usage
 
 ```python

docs/my-website/docs/providers/anthropic_effort.md

@@ -9,7 +9,10 @@ Control how many tokens Claude uses when responding with the `effort` parameter,
 
 The `effort` parameter allows you to control how eager Claude is about spending tokens when responding to requests. This gives you the ability to trade off between response thoroughness and token efficiency, all with a single model.
 
-**Note**: The effort parameter is currently in beta and only supported by Claude Opus 4.5. You must include the beta header `effort-2025-11-24` when using this feature (LiteLLM automatically adds this header when `output_config` with `effort` is detected).
+**Note**: The effort parameter is currently in beta and only supported by Claude Opus 4.5. LiteLLM automatically adds the `effort-2025-11-24` beta header when:
+- `reasoning_effort` parameter is provided (for Claude Opus 4.5 only)
+
+For Claude Opus 4.5, `reasoning_effort="medium"`—both are automatically mapped to the correct format.
 
 ## How Effort Works
 
@@ -52,9 +55,7 @@ response = litellm.completion(
         "role": "user",
         "content": "Analyze the trade-offs between microservices and monolithic architectures"
     }],
-    output_config={
-        "effort": "medium"
-    }
+    reasoning_effort="medium"  # Automatically mapped to output_config for Opus 4.5
 )
 
 print(response.choices[0].message.content)
@@ -217,11 +218,14 @@ response = litellm.completion(
 
 The effort parameter is supported across all Anthropic-compatible providers:
 
-- **Standard Anthropic**: ✅ Supported (Claude Opus 4.5)
-- **Azure Anthropic**: ✅ Supported (Claude Opus 4.5)
-- **Vertex AI Anthropic**: ✅ Supported (Claude Opus 4.5)
+- **Standard Anthropic API**: ✅ Supported (Claude Opus 4.5)
+- **Azure Anthropic / Microsoft Foundry**: ✅ Supported (Claude Opus 4.5)
+- **Amazon Bedrock**: ✅ Supported (Claude Opus 4.5)
+- **Google Cloud Vertex AI**: ✅ Supported (Claude Opus 4.5)
 
-LiteLLM automatically handles the beta header injection for all providers.
+LiteLLM automatically handles:
+- Beta header injection (`effort-2025-11-24`) for all providers
+- Parameter mapping: `reasoning_effort` → `output_config={"effort": ...}` for Claude Opus 4.5
 
 ## Usage and Pricing
 
@@ -242,9 +246,12 @@ print(f"Total tokens: {response.usage.total_tokens}")
 
 ### Beta header not being added
 
-LiteLLM automatically adds the `effort-2025-11-24` beta header when `output_config` with `effort` is detected. If you're not seeing the header:
+LiteLLM automatically adds the `effort-2025-11-24` beta header when:
+- `reasoning_effort` parameter is provided (for Claude Opus 4.5 only)
 
-1. Ensure you're using `output_config` with an `effort` field
+If you're not seeing the header:
+
+1. Ensure you're using `reasoning_effort` parameter
 2. Verify the model is Claude Opus 4.5
 3. Check that LiteLLM version supports this feature
 

docs/my-website/docs/providers/anthropic_programmatic_tool_calling.md

@@ -3,7 +3,11 @@
 Programmatic tool calling allows Claude to write code that calls your tools programmatically within a code execution container, rather than requiring round trips through the model for each tool invocation. This reduces latency for multi-tool workflows and decreases token consumption by allowing Claude to filter or process data before it reaches the model's context window.
 
 :::info
-Programmatic tool calling is currently in public beta. LiteLLM automatically adds the required `advanced-tool-use-2025-11-20` beta header when it detects tools with the `allowed_callers` field.
+Programmatic tool calling is currently in public beta. LiteLLM automatically detects tools with the `allowed_callers` field and adds the appropriate beta header based on your provider:
+
+- **Anthropic API & Microsoft Foundry**: `advanced-tool-use-2025-11-20`
+- **Amazon Bedrock**: `advanced-tool-use-2025-11-20`
+- **Google Cloud Vertex AI**: Not supported
 
 This feature requires the code execution tool to be enabled.
 :::
@@ -380,13 +384,14 @@ For example, calling 10 tools directly uses ~10x the tokens of calling them prog
 
 ## Provider Support
 
-LiteLLM supports programmatic tool calling across all Anthropic-compatible providers:
+LiteLLM supports programmatic tool calling across the following Anthropic-compatible providers:
 
-- **Standard Anthropic API** (`anthropic/claude-sonnet-4-5-20250929`)
-- **Azure Anthropic** (`azure/claude-sonnet-4-5-20250929`)
-- **Vertex AI Anthropic** (`vertex_ai/claude-sonnet-4-5-20250929`)
+- **Standard Anthropic API** (`anthropic/claude-sonnet-4-5-20250929`) ✅
+- **Azure Anthropic / Microsoft Foundry** (`azure/claude-sonnet-4-5-20250929`) ✅
+- **Amazon Bedrock** (`bedrock/invoke/anthropic.claude-sonnet-4-5-20250929-v1:0`) ✅
+- **Google Cloud Vertex AI** (`vertex_ai/claude-sonnet-4-5-20250929`) ❌ Not supported
 
-The beta header is automatically added when LiteLLM detects tools with `allowed_callers` field.
+The beta header (`advanced-tool-use-2025-11-20`) is automatically added when LiteLLM detects tools with the `allowed_callers` field.
 
 ## Limitations
 

docs/my-website/docs/providers/anthropic_tool_input_examples.md

@@ -3,7 +3,13 @@
 Provide concrete examples of valid tool inputs to help Claude understand how to use your tools more effectively. This is particularly useful for complex tools with nested objects, optional parameters, or format-sensitive inputs.
 
 :::info
-Tool input examples is a beta feature. LiteLLM automatically adds the required `advanced-tool-use-2025-11-20` beta header when it detects tools with the `input_examples` field.
+Tool input examples is a beta feature. LiteLLM automatically detects tools with the `input_examples` field and adds the appropriate beta header based on your provider:
+
+- **Anthropic API & Microsoft Foundry**: `advanced-tool-use-2025-11-20`
+- **Amazon Bedrock**: `advanced-tool-use-2025-11-20` (Claude Opus 4.5 only)
+- **Google Cloud Vertex AI**: Not supported
+
+You don't need to manually specify beta headers—LiteLLM handles this automatically.
 :::
 
 ## When to Use Input Examples
@@ -378,13 +384,14 @@ Input examples work seamlessly with other Anthropic tool features:
 
 ## Provider Support
 
-LiteLLM supports input examples across all Anthropic-compatible providers:
+LiteLLM supports input examples across the following Anthropic-compatible providers:
 
-- **Standard Anthropic API** (`anthropic/claude-sonnet-4-5-20250929`)
-- **Azure Anthropic** (`azure/claude-sonnet-4-5-20250929`)
-- **Vertex AI Anthropic** (`vertex_ai/claude-sonnet-4-5-20250929`)
+- **Standard Anthropic API** (`anthropic/claude-sonnet-4-5-20250929`) ✅
+- **Azure Anthropic / Microsoft Foundry** (`azure/claude-sonnet-4-5-20250929`) ✅
+- **Amazon Bedrock** (`bedrock/invoke/anthropic.claude-opus-4-5-20251101-v1:0`) ✅ (Opus 4.5 only)
+- **Google Cloud Vertex AI** (`vertex_ai/claude-sonnet-4-5-20250929`) ❌ Not supported
 
-The beta header is automatically added when LiteLLM detects tools with `input_examples` field.
+The beta header (`advanced-tool-use-2025-11-20`) is automatically added when LiteLLM detects tools with the `input_examples` field.
 
 ## Troubleshooting
 

docs/my-website/docs/providers/anthropic_tool_search.md

@@ -290,7 +290,13 @@ response = client.chat.completions.create(
 
 ### Beta Header
 
-LiteLLM automatically adds the `advanced-tool-use-2025-11-20` beta header when tool search tools are detected. You don't need to manually specify it.
+LiteLLM automatically detects tool search tools and adds the appropriate beta header based on your provider:
+
+- **Anthropic API & Microsoft Foundry**: `advanced-tool-use-2025-11-20`
+- **Google Cloud Vertex AI**: `tool-search-tool-2025-10-19`
+- **Amazon Bedrock** (Invoke API, Opus 4.5 only): `tool-search-tool-2025-10-19`
+
+You don't need to manually specify beta headers—LiteLLM handles this automatically.
 
 ### Deferred Loading
 
@@ -387,9 +393,18 @@ If Claude references a tool that isn't in your deferred tools list, you'll get a
 - Not compatible with tool use examples
 - Requires Claude Opus 4.5 or Sonnet 4.5
 - On Bedrock, only available via invoke API (not converse API)
+- On Bedrock, only supported for Claude Opus 4.5 (not Sonnet 4.5)
+- BM25 variant (`tool_search_tool_bm25_20251119`) is not supported on Bedrock
 - Maximum 10,000 tools in catalog
 - Returns 3-5 most relevant tools per search
 
+### Bedrock-Specific Notes
+
+When using Bedrock's Invoke API:
+- The regex variant (`tool_search_tool_regex_20251119`) is automatically normalized to `tool_search_tool_regex`
+- The BM25 variant (`tool_search_tool_bm25_20251119`) is automatically filtered out as it's not supported
+- Tool search is only available for Claude Opus 4.5 models
+
 ## Additional Resources
 
 - [Anthropic Tool Search Documentation](https://docs.anthropic.com/en/docs/build-with-claude/tool-use/tool-search)

docs/my-website/docs/providers/azure/azure_anthropic.md

@@ -16,7 +16,7 @@ Azure Foundry supports the following Claude models:
 | Property | Details |
 |-------|-------|
 | Description | Claude models deployed via Microsoft Azure Foundry. Uses the same API as Anthropic's Messages API but with Azure authentication. |
-| Provider Route on LiteLLM | `azure/` (add this prefix to Claude model names - e.g. `azure/claude-sonnet-4-5`) |
+| Provider Route on LiteLLM | `azure_ai/` (add this prefix to Claude model names - e.g. `azure_ai/claude-sonnet-4-5`) |
 | Provider Doc | [Azure Foundry Claude Models ↗](https://learn.microsoft.com/en-us/azure/ai-services/foundry-models/claude) |
 | API Endpoint | `https://<resource-name>.services.ai.azure.com/anthropic/v1/messages` |
 | Supported Endpoints | `/chat/completions`, `/anthropic/v1/messages`|
@@ -68,7 +68,7 @@ os.environ["AZURE_API_BASE"] = "https://<resource-name>.services.ai.azure.com/an
 
 # Make a completion request
 response = completion(
-    model="azure/claude-sonnet-4-5",
+    model="azure_ai/claude-sonnet-4-5",
     messages=[
         {"role": "user", "content": "What are 3 things to visit in Seattle?"}
     ],
@@ -85,7 +85,7 @@ print(response)
 import litellm
 
 response = litellm.completion(
-    model="azure/claude-sonnet-4-5",
+    model="azure_ai/claude-sonnet-4-5",
     api_base="https://<resource-name>.services.ai.azure.com/anthropic",
     api_key="your-azure-api-key",
     messages=[
@@ -101,7 +101,7 @@ response = litellm.completion(
 import litellm
 
 response = litellm.completion(
-    model="azure/claude-sonnet-4-5",
+    model="azure_ai/claude-sonnet-4-5",
     api_base="https://<resource-name>.services.ai.azure.com/anthropic",
     azure_ad_token="your-azure-ad-token",
     messages=[
@@ -117,7 +117,7 @@ response = litellm.completion(
 from litellm import completion
 
 response = completion(
-    model="azure/claude-sonnet-4-5",
+    model="azure_ai/claude-sonnet-4-5",
     messages=[
         {"role": "user", "content": "Write a short story"}
     ],
@@ -136,7 +136,7 @@ for chunk in response:
 from litellm import completion
 
 response = completion(
-    model="azure/claude-sonnet-4-5",
+    model="azure_ai/claude-sonnet-4-5",
     messages=[
         {"role": "user", "content": "What's the weather in Seattle?"}
     ],
@@ -181,7 +181,7 @@ export AZURE_API_BASE="https://<resource-name>.services.ai.azure.com/anthropic"
 model_list:
   - model_name: claude-sonnet-4-5
     litellm_params:
-      model: azure/claude-sonnet-4-5
+      model: azure_ai/claude-sonnet-4-5
       api_base: https://<resource-name>.services.ai.azure.com/anthropic
       api_key: os.environ/AZURE_API_KEY
 ```
@@ -331,7 +331,7 @@ os.environ["AZURE_API_BASE"] = "https://my-resource.services.ai.azure.com/anthro
 
 # Make a request
 response = completion(
-    model="azure/claude-sonnet-4-5",
+    model="azure_ai/claude-sonnet-4-5",
     messages=[
         {"role": "system", "content": "You are a helpful assistant."},
         {"role": "user", "content": "Explain quantum computing in simple terms."}
@@ -358,7 +358,7 @@ Or pass it directly:
 
 ```python
 response = completion(
-    model="azure/claude-sonnet-4-5",
+    model="azure_ai/claude-sonnet-4-5",
     api_base="https://<resource-name>.services.ai.azure.com/anthropic",
     # ...
 )

docs/my-website/docs/providers/azure_ai.md

@@ -312,6 +312,82 @@ LiteLLM supports **ALL** azure ai models. Here's a few examples:
 | mistral-large-latest | `completion(model="azure_ai/mistral-large-latest", messages)` | 
 | AI21-Jamba-Instruct | `completion(model="azure_ai/ai21-jamba-instruct", messages)` | 
 
+## Usage - Azure Anthropic (Azure Foundry Claude)
+
+LiteLLM funnels Azure Claude deployments through the `azure_ai/` provider so Claude Opus models on Azure Foundry keep working with Tool Search, Effort, streaming, and the rest of the advanced feature set. Point `AZURE_AI_API_BASE` to `https://<resource>.services.ai.azure.com/anthropic` (LiteLLM appends `/v1/messages` automatically) and authenticate with `AZURE_AI_API_KEY` or an Azure AD token.
+
+<Tabs>
+<TabItem value="sdk" label="LiteLLM Python SDK">
+
+```python
+import os
+from litellm import completion
+
+# Configure Azure credentials
+os.environ["AZURE_AI_API_KEY"] = "your-azure-ai-api-key"
+os.environ["AZURE_AI_API_BASE"] = "https://my-resource.services.ai.azure.com/anthropic"
+
+response = completion(
+    model="azure_ai/claude-opus-4-1",
+    messages=[{"role": "user", "content": "Explain how Azure Anthropic hosts Claude Opus differently from the public Anthropic API."}],
+    max_tokens=1200,
+    temperature=0.7,
+    stream=True,
+)
+
+for chunk in response:
+    if chunk.choices[0].delta.content:
+        print(chunk.choices[0].delta.content, end="", flush=True)
+```
+
+</TabItem>
+<TabItem value="proxy" label="LiteLLM Proxy">
+
+**1. Set environment variables**
+
+```bash
+export AZURE_AI_API_KEY="your-azure-ai-api-key"
+export AZURE_AI_API_BASE="https://my-resource.services.ai.azure.com/anthropic"
+```
+
+**2. Configure the proxy**
+
+```yaml
+model_list:
+  - model_name: claude-4-azure
+    litellm_params:
+      model: azure_ai/claude-opus-4-1
+      api_key: os.environ/AZURE_AI_API_KEY
+      api_base: os.environ/AZURE_AI_API_BASE
+```
+
+**3. Start LiteLLM**
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+**4. Test the Azure Claude route**
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+  --header 'Content-Type: application/json' \
+  --header 'Authorization: Bearer $LITELLM_KEY' \
+  --data '{
+    "model": "claude-4-azure",
+    "messages": [
+      {
+        "role": "user",
+        "content": "How do I use Claude Opus 4 via Azure Anthropic in LiteLLM?"
+      }
+    ],
+    "max_tokens": 1024
+  }'
+```
+
+</TabItem>
+</Tabs>
+
 
 
 ## Rerank Endpoint
@@ -397,4 +473,5 @@ curl http://0.0.0.0:4000/rerank \
 ```
 
 </TabItem>
-</Tabs>
+</Tabs>
+

docs/my-website/docs/providers/bedrock.md

@@ -43,6 +43,8 @@ export AWS_BEARER_TOKEN_BEDROCK="your-api-key"
 
 Option 2: use the api_key parameter to pass in API key for completion, embedding, image_generation API calls.
 
+<Tabs>
+<TabItem value="sdk" label="SDK">
 ```python
 response = completion(
   model="bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
@@ -50,7 +52,17 @@ response = completion(
   api_key="your-api-key"
 )
 ```
-
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+```yaml
+model_list:
+  - model_name: bedrock-claude-3-sonnet
+    litellm_params:
+      model: bedrock/anthropic.claude-3-sonnet-20240229-v1:0
+      api_key: os.environ/AWS_BEARER_TOKEN_BEDROCK
+```
+</TabItem>
+</Tabs>
 
 ## Usage
 
@@ -1683,6 +1695,131 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
 </TabItem>
 </Tabs>
 
+## TwelveLabs Pegasus - Video Understanding
+
+TwelveLabs Pegasus 1.2 is a video understanding model that can analyze and describe video content. LiteLLM supports this model through Bedrock's `/invoke` endpoint.
+
+| Property | Details |
+|----------|---------|
+| Provider Route | `bedrock/us.twelvelabs.pegasus-1-2-v1:0`, `bedrock/eu.twelvelabs.pegasus-1-2-v1:0` |
+| Provider Documentation | [TwelveLabs Pegasus Docs ↗](https://docs.twelvelabs.io/docs/models/pegasus) |
+| Supported Parameters | `max_tokens`, `temperature`, `response_format` |
+| Media Input | S3 URI or base64-encoded video |
+
+### Supported Features
+
+- **Video Analysis**: Analyze video content from S3 or base64 input
+- **Structured Output**: Support for JSON schema response format
+- **S3 Integration**: Support for S3 video URLs with bucket owner specification
+
+### Usage with S3 Video
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python title="TwelveLabs Pegasus SDK Usage" showLineNumbers
+from litellm import completion
+import os
+
+# Set AWS credentials
+os.environ["AWS_ACCESS_KEY_ID"] = "your-aws-access-key"
+os.environ["AWS_SECRET_ACCESS_KEY"] = "your-aws-secret-key"
+os.environ["AWS_REGION_NAME"] = "us-east-1"
+
+response = completion(
+    model="bedrock/us.twelvelabs.pegasus-1-2-v1:0",
+    messages=[{"role": "user", "content": "Describe what happens in this video."}],
+    mediaSource={
+        "s3Location": {
+            "uri": "s3://your-bucket/video.mp4",
+            "bucketOwner": "123456789012",  # 12-digit AWS account ID
+        }
+    },
+    temperature=0.2
+)
+
+print(response.choices[0].message.content)
+```
+
+</TabItem>
+
+<TabItem value="proxy" label="Proxy">
+
+**1. Add to config**
+
+```yaml title="config.yaml" showLineNumbers
+model_list:
+  - model_name: pegasus-video
+    litellm_params:
+      model: bedrock/us.twelvelabs.pegasus-1-2-v1:0
+      aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
+      aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
+      aws_region_name: os.environ/AWS_REGION_NAME
+```
+
+**2. Start proxy**
+
+```bash title="Start LiteLLM Proxy" showLineNumbers
+litellm --config /path/to/config.yaml
+
+# RUNNING at http://0.0.0.0:4000
+```
+
+**3. Test it!**
+
+```bash title="Test Pegasus via Proxy" showLineNumbers
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+  --header 'Authorization: Bearer sk-1234' \
+  --header 'Content-Type: application/json' \
+  --data '{
+    "model": "pegasus-video",
+    "messages": [
+      {
+        "role": "user",
+        "content": "Describe what happens in this video."
+      }
+    ],
+    "mediaSource": {
+      "s3Location": {
+        "uri": "s3://your-bucket/video.mp4",
+        "bucketOwner": "123456789012"
+      }
+    },
+    "temperature": 0.2
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+### Usage with Base64 Video
+
+You can also pass video content directly as base64:
+
+```python title="Base64 Video Input" showLineNumbers
+from litellm import completion
+import base64
+
+# Read video file and encode to base64
+with open("video.mp4", "rb") as video_file:
+    video_base64 = base64.b64encode(video_file.read()).decode("utf-8")
+
+response = completion(
+    model="bedrock/us.twelvelabs.pegasus-1-2-v1:0",
+    messages=[{"role": "user", "content": "What is happening in this video?"}],
+    mediaSource={
+        "base64String": video_base64
+    },
+    temperature=0.2,
+)
+
+print(response.choices[0].message.content)
+```
+
+### Important Notes
+
+- **Response Format**: The model supports structured output via `response_format` with JSON schema
+
 ## Provisioned throughput models
 To use provisioned throughput Bedrock models pass 
 - `model=bedrock/<base-model>`, example `model=bedrock/anthropic.claude-v2`. Set `model` to any of the [Supported AWS models](#supported-aws-bedrock-models)
@@ -1743,6 +1880,8 @@ Here's an example of using a bedrock model with LiteLLM. For a complete list, re
 | Meta Llama 2 Chat 70b      | `completion(model='bedrock/meta.llama2-70b-chat-v1', messages=messages)`   | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` |
 | Mistral 7B Instruct        | `completion(model='bedrock/mistral.mistral-7b-instruct-v0:2', messages=messages)`   | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` |
 | Mixtral 8x7B Instruct      | `completion(model='bedrock/mistral.mixtral-8x7b-instruct-v0:1', messages=messages)`   | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` |
+| TwelveLabs Pegasus 1.2 (US) | `completion(model='bedrock/us.twelvelabs.pegasus-1-2-v1:0', messages=messages, mediaSource={...})`   | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` |
+| TwelveLabs Pegasus 1.2 (EU) | `completion(model='bedrock/eu.twelvelabs.pegasus-1-2-v1:0', messages=messages, mediaSource={...})`   | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` |
 
 
 ## Bedrock Embedding

docs/my-website/docs/providers/bedrock_embedding.md

@@ -4,7 +4,8 @@
 
 | Provider | LiteLLM Route | AWS Documentation | Cost Tracking |
 |----------|---------------|-------------------|---------------|
-| Amazon Titan | `bedrock/amazon.*` | [Amazon Titan Embeddings](https://docs.aws.amazon.com/bedrock/latest/userguide/titan-embedding-models.html) | ✅ |
+| Amazon Titan | `bedrock/amazon.titan-*` | [Amazon Titan Embeddings](https://docs.aws.amazon.com/bedrock/latest/userguide/titan-embedding-models.html) | ✅ |
+| Amazon Nova | `bedrock/amazon.nova-*` | [Amazon Nova Embeddings](https://docs.aws.amazon.com/bedrock/latest/userguide/nova-embed.html) | ✅ |
 | Cohere | `bedrock/cohere.*` | [Cohere Embeddings](https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-cohere-embed.html) | ✅ |
 | TwelveLabs | `bedrock/us.twelvelabs.*` | [TwelveLabs](https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-twelvelabs.html) | ✅ |
 
@@ -16,6 +17,7 @@ LiteLLM supports AWS Bedrock's async-invoke feature for embedding models that re
 
 | Provider | Async Invoke Route | Use Case |
 |----------|-------------------|----------|
+| Amazon Nova | `bedrock/async_invoke/amazon.nova-2-multimodal-embeddings-v1:0` | Multimodal embeddings with segmentation for long text, video, and audio |
 | TwelveLabs Marengo | `bedrock/async_invoke/us.twelvelabs.marengo-embed-2-7-v1:0` | Video, audio, image, and text embeddings |
 
 ### Required Parameters
@@ -116,7 +118,7 @@ def check_async_job_status(invocation_arn, aws_region_name="us-east-1"):
     """Check the status of an async invoke job using LiteLLM batch API"""
     try:
         response = retrieve_batch(
-            batch_id=invocation_arn,
+            batch_id=invocation_arn,  # Pass the invocation ARN here
             custom_llm_provider="bedrock",
             aws_region_name=aws_region_name
         )
@@ -128,11 +130,47 @@ def check_async_job_status(invocation_arn, aws_region_name="us-east-1"):
 # Check status
 status = check_async_job_status(invocation_arn, "us-east-1")
 if status:
-    print(f"Job Status: {status.status}")
-    print(f"Output Location: {status.output_file_id}")
+    print(f"Job Status: {status.status}")  # "in_progress", "completed", or "failed"
+    print(f"Output Location: {status.metadata['output_file_id']}")  # S3 URI where results are stored
 ```
 
-**Note:** The actual embedding results are stored in S3. The `output_file_id` from the batch status can be used to locate the results file in your S3 bucket.
+#### Polling Until Complete
+
+Here's a complete example of polling for job completion:
+
+```python
+def wait_for_async_job(invocation_arn, aws_region_name="us-east-1", max_wait=3600):
+    """Poll job status until completion"""
+    start_time = time.time()
+    
+    while True:
+        status = retrieve_batch(
+            batch_id=invocation_arn,
+            custom_llm_provider="bedrock",
+            aws_region_name=aws_region_name,
+        )
+        
+        if status.status == "completed":
+            print("✅ Job completed!")
+            return status
+        elif status.status == "failed":
+            error_msg = status.metadata.get('failure_message', 'Unknown error')
+            raise Exception(f"❌ Job failed: {error_msg}")
+        else:
+            elapsed = time.time() - start_time
+            if elapsed > max_wait:
+                raise TimeoutError(f"Job timed out after {max_wait} seconds")
+            
+            print(f"⏳ Job still processing... (elapsed: {elapsed:.0f}s)")
+            time.sleep(10)  # Wait 10 seconds before checking again
+
+# Wait for completion
+completed_status = wait_for_async_job(invocation_arn)
+output_s3_uri = completed_status.metadata['output_file_id']
+print(f"Results available at: {output_s3_uri}")
+```
+
+**Note:** The actual embedding results are stored in S3. When the job is completed, download the results from the S3 location specified in `status.metadata['output_file_id']`. The results will be in JSON/JSONL format containing the embedding vectors.
 
 ### Error Handling
 
@@ -179,7 +217,7 @@ except Exception as e:
 
 ### Limitations
 
-- Async-invoke is currently only supported for TwelveLabs Marengo models
+- Async-invoke is supported for TwelveLabs Marengo and Amazon Nova models
 - Results are stored in S3 and must be retrieved separately using the output file ID
 - Job status checking requires using LiteLLM's `retrieve_batch()` function
 - No built-in polling mechanism in LiteLLM (must implement your own status checking loop)
@@ -259,6 +297,7 @@ print(response)
 
 | Model Name           | Usage                               | Supported Additional OpenAI params |
 |----------------------|---------------------------------------------|-----|
+| **Amazon Nova Multimodal Embeddings** | `embedding(model="bedrock/amazon.nova-2-multimodal-embeddings-v1:0", input=input)` | Supports multimodal input (text, image, video, audio), multiple purposes, dimensions (256, 384, 1024, 3072) |
 | Titan Embeddings V2 | `embedding(model="bedrock/amazon.titan-embed-text-v2:0", input=input)` | [here](https://github.com/BerriAI/litellm/blob/f5905e100068e7a4d61441d7453d7cf5609c2121/litellm/llms/bedrock/embed/amazon_titan_v2_transformation.py#L59) |
 | Titan Embeddings - V1 | `embedding(model="bedrock/amazon.titan-embed-text-v1", input=input)` | [here](https://github.com/BerriAI/litellm/blob/f5905e100068e7a4d61441d7453d7cf5609c2121/litellm/llms/bedrock/embed/amazon_titan_g1_transformation.py#L53)
 | Titan Multimodal Embeddings | `embedding(model="bedrock/amazon.titan-embed-image-v1", input=input)` | [here](https://github.com/BerriAI/litellm/blob/f5905e100068e7a4d61441d7453d7cf5609c2121/litellm/llms/bedrock/embed/amazon_titan_multimodal_transformation.py#L28) |

docs/my-website/docs/providers/ovhcloud.md

@@ -311,6 +311,21 @@ response = embedding(
 print(response.data)
 ```
 
+### Audio Transcription
+
+```python
+from litellm import transcription
+
+audio_file = open("path/to/your/audio.wav", "rb")
+
+response = transcription(
+    model="ovhcloud/whisper-large-v3-turbo",
+    file=audio_file
+)
+
+print(response.text)
+```
+
 ## Usage with LiteLLM Proxy Server
 
 Here's how to call a OVHCloud AI Endpoints model with the LiteLLM Proxy Server

docs/my-website/docs/providers/ragflow.md

@@ -0,0 +1,244 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# RAGFlow
+
+Litellm supports Ragflow's chat completions APIs
+
+## Supported Features
+
+- ✅ Chat completions
+- ✅ Streaming responses
+- ✅ Both chat and agent endpoints
+- ✅ Multiple credential sources (params, env vars, litellm_params)
+- ✅ OpenAI-compatible API format
+
+
+## API Key
+
+```python
+# env variable
+os.environ['RAGFLOW_API_KEY']
+```
+
+## API Base
+
+```python
+# env variable
+os.environ['RAGFLOW_API_BASE']
+```
+
+## Overview
+
+RAGFlow provides OpenAI-compatible APIs with unique path structures that include chat and agent IDs:
+
+- **Chat endpoint**: `/api/v1/chats_openai/{chat_id}/chat/completions`
+- **Agent endpoint**: `/api/v1/agents_openai/{agent_id}/chat/completions`
+
+The model name format embeds the endpoint type and ID:
+- Chat: `ragflow/chat/{chat_id}/{model_name}`
+- Agent: `ragflow/agent/{agent_id}/{model_name}`
+
+
+## Sample Usage - Chat Endpoint
+
+```python
+from litellm import completion
+import os
+
+os.environ['RAGFLOW_API_KEY'] = "your-ragflow-api-key"
+os.environ['RAGFLOW_API_BASE'] = "http://localhost:9380"  # or your hosted URL
+
+response = completion(
+    model="ragflow/chat/my-chat-id/gpt-4o-mini",
+    messages=[{"role": "user", "content": "How does the deep doc understanding work?"}]
+)
+print(response)
+```
+
+## Sample Usage - Agent Endpoint
+
+```python
+from litellm import completion
+import os
+
+os.environ['RAGFLOW_API_KEY'] = "your-ragflow-api-key"
+os.environ['RAGFLOW_API_BASE'] = "http://localhost:9380"  # or your hosted URL
+
+response = completion(
+    model="ragflow/agent/my-agent-id/gpt-4o-mini",
+    messages=[{"role": "user", "content": "What are the key features?"}]
+)
+print(response)
+```
+
+## Sample Usage - With Parameters
+
+You can also pass `api_key` and `api_base` directly as parameters:
+
+```python
+from litellm import completion
+
+response = completion(
+    model="ragflow/chat/my-chat-id/gpt-4o-mini",
+    messages=[{"role": "user", "content": "Hello!"}],
+    api_key="your-ragflow-api-key",
+    api_base="http://localhost:9380"
+)
+print(response)
+```
+
+## Sample Usage - Streaming
+
+```python
+from litellm import completion
+import os
+
+os.environ['RAGFLOW_API_KEY'] = "your-ragflow-api-key"
+os.environ['RAGFLOW_API_BASE'] = "http://localhost:9380"
+
+response = completion(
+    model="ragflow/agent/my-agent-id/gpt-4o-mini",
+    messages=[{"role": "user", "content": "Explain RAGFlow"}],
+    stream=True
+)
+
+for chunk in response:
+    print(chunk)
+```
+
+## Model Name Format
+
+The model name must follow one of these formats:
+
+### Chat Endpoint
+```
+ragflow/chat/{chat_id}/{model_name}
+```
+
+Example: `ragflow/chat/my-chat-id/gpt-4o-mini`
+
+### Agent Endpoint
+```
+ragflow/agent/{agent_id}/{model_name}
+```
+
+Example: `ragflow/agent/my-agent-id/gpt-4o-mini`
+
+Where:
+- `{chat_id}` or `{agent_id}` is the ID of your chat or agent in RAGFlow
+- `{model_name}` is the actual model name (e.g., `gpt-4o-mini`, `gpt-4o`, etc.)
+
+## Configuration Sources
+
+LiteLLM supports multiple ways to provide credentials, checked in this order:
+
+1. **Function parameters**: `api_key="..."`, `api_base="..."`
+2. **litellm_params**: `litellm_params={"api_key": "...", "api_base": "..."}`
+3. **Environment variables**: `RAGFLOW_API_KEY`, `RAGFLOW_API_BASE`
+4. **Global litellm settings**: `litellm.api_key`, `litellm.api_base`
+
+## Usage - LiteLLM Proxy Server
+
+### 1. Save key in your environment
+
+```bash
+export RAGFLOW_API_KEY="your-ragflow-api-key"
+export RAGFLOW_API_BASE="http://localhost:9380"
+```
+
+### 2. Start the proxy
+
+<Tabs>
+<TabItem value="config" label="config.yaml">
+
+```yaml
+model_list:
+  - model_name: ragflow-chat-gpt4
+    litellm_params:
+      model: ragflow/chat/my-chat-id/gpt-4o-mini
+      api_key: os.environ/RAGFLOW_API_KEY
+      api_base: os.environ/RAGFLOW_API_BASE
+  - model_name: ragflow-agent-gpt4
+    litellm_params:
+      model: ragflow/agent/my-agent-id/gpt-4o-mini
+      api_key: os.environ/RAGFLOW_API_KEY
+      api_base: os.environ/RAGFLOW_API_BASE
+```
+
+</TabItem>
+<TabItem value="cli" label="CLI">
+
+```bash
+$ litellm --config /path/to/config.yaml
+
+# Server running on http://0.0.0.0:4000
+```
+
+</TabItem>
+</Tabs>
+
+### 3. Test it
+
+<Tabs>
+<TabItem value="Curl" label="Curl Request">
+
+```bash
+curl http://0.0.0.0:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer sk-1234" \
+  -d '{
+    "model": "ragflow-chat-gpt4",
+    "messages": [
+      {"role": "user", "content": "How does RAGFlow work?"}
+    ]
+  }'
+```
+
+</TabItem>
+<TabItem value="Python" label="Python SDK">
+
+```python
+from openai import OpenAI
+
+client = OpenAI(
+    api_key="sk-1234",  # Your LiteLLM proxy key
+    base_url="http://0.0.0.0:4000"
+)
+
+response = client.chat.completions.create(
+    model="ragflow-chat-gpt4",
+    messages=[
+        {"role": "user", "content": "How does RAGFlow work?"}
+    ]
+)
+print(response)
+```
+
+</TabItem>
+</Tabs>
+
+## API Base URL Handling
+
+The `api_base` parameter can be provided with or without `/v1` suffix. LiteLLM will automatically handle it:
+
+- `http://localhost:9380` → `http://localhost:9380/api/v1/chats_openai/{chat_id}/chat/completions`
+- `http://localhost:9380/v1` → `http://localhost:9380/api/v1/chats_openai/{chat_id}/chat/completions`
+- `http://localhost:9380/api/v1` → `http://localhost:9380/api/v1/chats_openai/{chat_id}/chat/completions`
+
+All three formats will work correctly.
+
+## Error Handling
+
+If you encounter errors:
+
+1. **Invalid model format**: Ensure your model name follows `ragflow/{chat|agent}/{id}/{model_name}` format
+2. **Missing api_base**: Provide `api_base` via parameter, environment variable, or litellm_params
+3. **Connection errors**: Verify your RAGFlow server is running and accessible at the provided `api_base`
+
+:::info
+
+For more information about passing provider-specific parameters, [go here](../completion/provider_specific_params.md)
+
+:::
+

docs/my-website/docs/providers/ragflow_vector_store.md

@@ -0,0 +1,349 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import Image from '@theme/IdealImage';
+
+# RAGFlow Vector Stores
+
+Litellm support creation and management of datasets for document processing and knowledge base management in Ragflow.
+
+| Property | Details |
+|----------|---------|
+| Description | RAGFlow datasets enable document processing, chunking, and knowledge base management for RAG applications. |
+| Provider Route on LiteLLM | `ragflow` in the litellm vector_store_registry |
+| Provider Doc | [RAGFlow API Documentation ↗](https://ragflow.io/docs) |
+| Supported Operations | Dataset Management (Create, List, Update, Delete) |
+| Search/Retrieval | ❌ Not supported (management only) |
+
+## Quick Start
+
+### LiteLLM Python SDK
+
+```python showLineNumbers title="Example using LiteLLM Python SDK"
+import os
+import litellm
+
+# Set RAGFlow credentials
+os.environ["RAGFLOW_API_KEY"] = "your-ragflow-api-key"
+os.environ["RAGFLOW_API_BASE"] = "http://localhost:9380"  # Optional, defaults to localhost:9380
+
+# Create a RAGFlow dataset
+response = litellm.vector_stores.create(
+    name="my-dataset",
+    custom_llm_provider="ragflow",
+    metadata={
+        "description": "My knowledge base dataset",
+        "embedding_model": "BAAI/bge-large-zh-v1.5@BAAI",
+        "chunk_method": "naive"
+    }
+)
+
+print(f"Created dataset ID: {response.id}")
+print(f"Dataset name: {response.name}")
+```
+
+### LiteLLM Proxy
+
+#### 1. Configure your vector_store_registry
+
+<Tabs>
+<TabItem value="config-yaml" label="config.yaml">
+
+```yaml
+model_list:
+  - model_name: gpt-4o-mini
+    litellm_params:
+      model: gpt-4o-mini
+      api_key: os.environ/OPENAI_API_KEY
+
+vector_store_registry:
+  - vector_store_name: "ragflow-knowledge-base"
+    litellm_params:
+      vector_store_id: "your-dataset-id"
+      custom_llm_provider: "ragflow"
+      api_key: os.environ/RAGFLOW_API_KEY
+      api_base: os.environ/RAGFLOW_API_BASE  # Optional
+      vector_store_description: "RAGFlow dataset for knowledge base"
+      vector_store_metadata:
+        source: "Company documentation"
+```
+
+</TabItem>
+
+<TabItem value="litellm-ui" label="LiteLLM UI">
+
+On the LiteLLM UI, Navigate to Experimental > Vector Stores > Create Vector Store. On this page you can create a vector store with a name, vector store id and credentials.
+
+<Image 
+  img={require('../../img/kb_2.png')}
+  style={{width: '50%'}}
+/>
+
+</TabItem>
+</Tabs>
+
+#### 2. Create a dataset via Proxy
+
+<Tabs>
+<TabItem value="curl" label="Curl">
+
+```bash
+curl http://localhost:4000/v1/vector_stores \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer $LITELLM_API_KEY" \
+  -d '{
+    "name": "my-ragflow-dataset",
+    "custom_llm_provider": "ragflow",
+    "metadata": {
+      "description": "Test dataset",
+      "chunk_method": "naive"
+    }
+  }'
+```
+
+</TabItem>
+
+<TabItem value="openai-sdk" label="OpenAI Python SDK">
+
+```python
+from openai import OpenAI
+
+# Initialize client with your LiteLLM proxy URL
+client = OpenAI(
+    base_url="http://localhost:4000",
+    api_key="your-litellm-api-key"
+)
+
+# Create a RAGFlow dataset
+response = client.vector_stores.create(
+    name="my-ragflow-dataset",
+    custom_llm_provider="ragflow",
+    metadata={
+        "description": "Test dataset",
+        "chunk_method": "naive"
+    }
+)
+
+print(f"Created dataset: {response.id}")
+```
+
+</TabItem>
+</Tabs>
+
+## Configuration
+
+### Environment Variables
+
+RAGFlow vector stores support configuration via environment variables:
+
+- `RAGFLOW_API_KEY` - Your RAGFlow API key (required)
+- `RAGFLOW_API_BASE` - RAGFlow API base URL (optional, defaults to `http://localhost:9380`)
+
+### Parameters
+
+You can also pass these via `litellm_params`:
+
+- `api_key` - RAGFlow API key (overrides `RAGFLOW_API_KEY` env var)
+- `api_base` - RAGFlow API base URL (overrides `RAGFLOW_API_BASE` env var)
+
+## Dataset Creation Options
+
+### Basic Dataset Creation
+
+```python
+response = litellm.vector_stores.create(
+    name="basic-dataset",
+    custom_llm_provider="ragflow"
+)
+```
+
+### Dataset with Chunk Method
+
+RAGFlow supports various chunk methods for different document types:
+
+<Tabs>
+<TabItem value="naive" label="Naive (General)">
+
+```python
+response = litellm.vector_stores.create(
+    name="general-dataset",
+    custom_llm_provider="ragflow",
+    metadata={
+        "chunk_method": "naive",
+        "parser_config": {
+            "chunk_token_num": 512,
+            "delimiter": "\n",
+            "html4excel": False,
+            "layout_recognize": "DeepDOC"
+        }
+    }
+)
+```
+
+</TabItem>
+
+<TabItem value="book" label="Book">
+
+```python
+response = litellm.vector_stores.create(
+    name="book-dataset",
+    custom_llm_provider="ragflow",
+    metadata={
+        "chunk_method": "book",
+        "parser_config": {
+            "raptor": {
+                "use_raptor": False
+            }
+        }
+    }
+)
+```
+
+</TabItem>
+
+<TabItem value="qa" label="Q&A">
+
+```python
+response = litellm.vector_stores.create(
+    name="qa-dataset",
+    custom_llm_provider="ragflow",
+    metadata={
+        "chunk_method": "qa",
+        "parser_config": {
+            "raptor": {
+                "use_raptor": False
+            }
+        }
+    }
+)
+```
+
+</TabItem>
+
+<TabItem value="paper" label="Paper">
+
+```python
+response = litellm.vector_stores.create(
+    name="paper-dataset",
+    custom_llm_provider="ragflow",
+    metadata={
+        "chunk_method": "paper",
+        "parser_config": {
+            "raptor": {
+                "use_raptor": False
+            }
+        }
+    }
+)
+```
+
+</TabItem>
+</Tabs>
+
+### Dataset with Ingestion Pipeline
+
+Instead of using a chunk method, you can use an ingestion pipeline:
+
+```python
+response = litellm.vector_stores.create(
+    name="pipeline-dataset",
+    custom_llm_provider="ragflow",
+    metadata={
+        "parse_type": 2,  # Number of parsers in your pipeline
+        "pipeline_id": "d0bebe30ae2211f0970942010a8e0005"  # 32-character hex ID
+    }
+)
+```
+
+**Note**: `chunk_method` and `pipeline_id` are mutually exclusive. Use one or the other.
+
+### Advanced Parser Configuration
+
+```python
+response = litellm.vector_stores.create(
+    name="advanced-dataset",
+    custom_llm_provider="ragflow",
+    metadata={
+        "chunk_method": "naive",
+        "description": "Advanced dataset with custom parser config",
+        "embedding_model": "BAAI/bge-large-zh-v1.5@BAAI",
+        "permission": "me",  # or "team"
+        "parser_config": {
+            "chunk_token_num": 1024,
+            "delimiter": "\n!?;。；！？",
+            "html4excel": True,
+            "layout_recognize": "DeepDOC",
+            "auto_keywords": 5,
+            "auto_questions": 3,
+            "task_page_size": 12,
+            "raptor": {
+                "use_raptor": True
+            },
+            "graphrag": {
+                "use_graphrag": False
+            }
+        }
+    }
+)
+```
+
+## Supported Chunk Methods
+
+RAGFlow supports the following chunk methods:
+
+- `naive` - General purpose (default)
+- `book` - For book documents
+- `email` - For email documents
+- `laws` - For legal documents
+- `manual` - Manual chunking
+- `one` - Single chunk
+- `paper` - For academic papers
+- `picture` - For image documents
+- `presentation` - For presentation documents
+- `qa` - Q&A format
+- `table` - For table documents
+- `tag` - Tag-based chunking
+
+## RAGFlow-Specific Parameters
+
+All RAGFlow-specific parameters should be passed via the `metadata` field:
+
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| `avatar` | string | Base64 encoding of the avatar (max 65535 chars) |
+| `description` | string | Brief description of the dataset (max 65535 chars) |
+| `embedding_model` | string | Embedding model name (e.g., "BAAI/bge-large-zh-v1.5@BAAI") |
+| `permission` | string | Access permission: "me" (default) or "team" |
+| `chunk_method` | string | Chunking method (see supported methods above) |
+| `parser_config` | object | Parser configuration (varies by chunk_method) |
+| `parse_type` | int | Number of parsers in pipeline (required with pipeline_id) |
+| `pipeline_id` | string | 32-character hex pipeline ID (required with parse_type) |
+
+## Error Handling
+
+RAGFlow returns error responses in the following format:
+
+```json
+{
+    "code": 101,
+    "message": "Dataset name 'my-dataset' already exists"
+}
+```
+
+LiteLLM automatically maps these to appropriate exceptions:
+
+- `code != 0` → Raises exception with the error message
+- Missing required fields → Raises `ValueError`
+- Mutually exclusive parameters → Raises `ValueError`
+
+## Limitations
+
+- **Search/Retrieval**: RAGFlow vector stores support dataset management only. Search operations are not supported and will raise `NotImplementedError`.
+- **List/Update/Delete**: These operations are not yet implemented through the standard vector store API. Use RAGFlow's native API endpoints directly.
+
+## Further Reading
+
+Vector Stores:
+- [Vector Store Creation](../vector_stores/create.md)
+- [Using Vector Stores with Completions](../completion/knowledgebase.md)
+- [Vector Store Registry](../completion/knowledgebase.md#vectorstoreregistry)
+

docs/my-website/docs/providers/vertex.md

@@ -2550,355 +2550,6 @@ print(response)
 </TabItem>
 </Tabs>
 
-
-## **Gemini TTS (Text-to-Speech) Audio Output**
-
-:::info
-
-LiteLLM supports Gemini TTS models on Vertex AI that can generate audio responses using the OpenAI-compatible `audio` parameter format.
-
-:::
-
-### Supported Models
-
-LiteLLM supports Gemini TTS models with audio capabilities on Vertex AI (e.g. `vertex_ai/gemini-2.5-flash-preview-tts` and `vertex_ai/gemini-2.5-pro-preview-tts`). For the complete list of available TTS models and voices, see the [official Gemini TTS documentation](https://ai.google.dev/gemini-api/docs/speech-generation).
-
-### Limitations
-
-:::warning
-
-**Important Limitations**:
-- Gemini TTS models only support the `pcm16` audio format
-- **Streaming support has not been added** to TTS models yet
-- The `modalities` parameter must be set to `['audio']` for TTS requests
-
-:::
-
-### Quick Start
-
-<Tabs>
-<TabItem value="sdk" label="SDK">
-
-```python
-from litellm import completion
-import json
-
-## GET CREDENTIALS
-file_path = 'path/to/vertex_ai_service_account.json'
-
-# Load the JSON file
-with open(file_path, 'r') as file:
-    vertex_credentials = json.load(file)
-
-# Convert to JSON string
-vertex_credentials_json = json.dumps(vertex_credentials)
-
-response = completion(
-    model="vertex_ai/gemini-2.5-flash-preview-tts",
-    messages=[{"role": "user", "content": "Say hello in a friendly voice"}],
-    modalities=["audio"],  # Required for TTS models
-    audio={
-        "voice": "Kore",
-        "format": "pcm16"  # Required: must be "pcm16"
-    },
-    vertex_credentials=vertex_credentials_json
-)
-
-print(response)
-```
-
-</TabItem>
-<TabItem value="proxy" label="PROXY">
-
-1. Setup config.yaml
-
-```yaml
-model_list:
-  - model_name: gemini-tts-flash
-    litellm_params:
-      model: vertex_ai/gemini-2.5-flash-preview-tts
-      vertex_project: "your-project-id"
-      vertex_location: "us-central1"
-      vertex_credentials: "/path/to/service_account.json"
-  - model_name: gemini-tts-pro
-    litellm_params:
-      model: vertex_ai/gemini-2.5-pro-preview-tts
-      vertex_project: "your-project-id"
-      vertex_location: "us-central1"
-      vertex_credentials: "/path/to/service_account.json"
-```
-
-2. Start proxy
-
-```bash
-litellm --config /path/to/config.yaml
-```
-
-3. Make TTS request
-
-```bash
-curl http://0.0.0.0:4000/v1/chat/completions \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer <YOUR-LITELLM-KEY>" \
-  -d '{
-    "model": "gemini-tts-flash",
-    "messages": [{"role": "user", "content": "Say hello in a friendly voice"}],
-    "modalities": ["audio"],
-    "audio": {
-      "voice": "Kore",
-      "format": "pcm16"
-    }
-  }'
-```
-
-</TabItem>
-</Tabs>
-
-### Advanced Usage
-
-You can combine TTS with other Gemini features:
-
-```python
-response = completion(
-    model="vertex_ai/gemini-2.5-pro-preview-tts",
-    messages=[
-        {"role": "system", "content": "You are a helpful assistant that speaks clearly."},
-        {"role": "user", "content": "Explain quantum computing in simple terms"}
-    ],
-    modalities=["audio"],
-    audio={
-        "voice": "Charon",
-        "format": "pcm16"
-    },
-    temperature=0.7,
-    max_tokens=150,
-    vertex_credentials=vertex_credentials_json
-)
-```
-
-For more information about Gemini's TTS capabilities and available voices, see the [official Gemini TTS documentation](https://ai.google.dev/gemini-api/docs/speech-generation).
-
-## **Text to Speech APIs**
-
-:::info
-
-LiteLLM supports calling [Vertex AI Text to Speech API](https://console.cloud.google.com/vertex-ai/generative/speech/text-to-speech) in the OpenAI text to speech API format
-
-:::
-
-
-
-### Usage - Basic
-
-<Tabs>
-<TabItem value="sdk" label="SDK">
-
-Vertex AI does not support passing a `model` param - so passing `model=vertex_ai/` is the only required param
-
-**Sync Usage**
-
-```python
-speech_file_path = Path(__file__).parent / "speech_vertex.mp3"
-response = litellm.speech(
-    model="vertex_ai/",
-    input="hello what llm guardrail do you have",
-)
-response.stream_to_file(speech_file_path)
-```
-
-**Async Usage**
-```python
-speech_file_path = Path(__file__).parent / "speech_vertex.mp3"
-response = litellm.aspeech(
-    model="vertex_ai/",
-    input="hello what llm guardrail do you have",
-)
-response.stream_to_file(speech_file_path)
-```
-
-</TabItem>
-<TabItem value="proxy" label="LiteLLM PROXY (Unified Endpoint)">
-
-1. Add model to config.yaml
-```yaml
-model_list:
-  - model_name: vertex-tts
-    litellm_params:
-      model: vertex_ai/ # Vertex AI does not support passing a `model` param - so passing `model=vertex_ai/` is the only required param
-      vertex_project: "adroit-crow-413218"
-      vertex_location: "us-central1"
-      vertex_credentials: adroit-crow-413218-a956eef1a2a8.json 
-
-litellm_settings:
-  drop_params: True
-```
-
-2. Start Proxy 
-
-```
-$ litellm --config /path/to/config.yaml
-```
-
-3. Make Request use OpenAI Python SDK
-
-
-```python
-import openai
-
-client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
-
-# see supported values for "voice" on vertex here: 
-# https://console.cloud.google.com/vertex-ai/generative/speech/text-to-speech
-response = client.audio.speech.create(
-    model = "vertex-tts",
-    input="the quick brown fox jumped over the lazy dogs",
-    voice={'languageCode': 'en-US', 'name': 'en-US-Studio-O'}
-)
-print("response from proxy", response)
-```
-
-</TabItem>
-</Tabs>
-
-
-### Usage - `ssml` as input
-
-Pass your `ssml` as input to the `input` param, if it contains `<speak>`, it will be automatically detected and passed as `ssml` to the Vertex AI API
-
-If you need to force your `input` to be passed as `ssml`, set `use_ssml=True`
-
-<Tabs>
-<TabItem value="sdk" label="SDK">
-
-Vertex AI does not support passing a `model` param - so passing `model=vertex_ai/` is the only required param
-
-
-```python
-speech_file_path = Path(__file__).parent / "speech_vertex.mp3"
-
-
-ssml = """
-<speak>
-    <p>Hello, world!</p>
-    <p>This is a test of the <break strength="medium" /> text-to-speech API.</p>
-</speak>
-"""
-
-response = litellm.speech(
-    input=ssml,
-    model="vertex_ai/test",
-    voice={
-        "languageCode": "en-UK",
-        "name": "en-UK-Studio-O",
-    },
-    audioConfig={
-        "audioEncoding": "LINEAR22",
-        "speakingRate": "10",
-    },
-)
-response.stream_to_file(speech_file_path)
-```
-
-</TabItem>
-
-<TabItem value="proxy" label="LiteLLM PROXY (Unified Endpoint)">
-
-```python
-import openai
-
-client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
-
-ssml = """
-<speak>
-    <p>Hello, world!</p>
-    <p>This is a test of the <break strength="medium" /> text-to-speech API.</p>
-</speak>
-"""
-
-# see supported values for "voice" on vertex here: 
-# https://console.cloud.google.com/vertex-ai/generative/speech/text-to-speech
-response = client.audio.speech.create(
-    model = "vertex-tts",
-    input=ssml,
-    voice={'languageCode': 'en-US', 'name': 'en-US-Studio-O'},
-)
-print("response from proxy", response)
-```
-
-</TabItem>
-</Tabs>
-
-
-### Forcing SSML Usage
-
-You can force the use of SSML by setting the `use_ssml` parameter to `True`. This is useful when you want to ensure that your input is treated as SSML, even if it doesn't contain the `<speak>` tags.
-
-Here are examples of how to force SSML usage:
-
-
-<Tabs>
-<TabItem value="sdk" label="SDK">
-
-Vertex AI does not support passing a `model` param - so passing `model=vertex_ai/` is the only required param
-
-
-```python
-speech_file_path = Path(__file__).parent / "speech_vertex.mp3"
-
-
-ssml = """
-<speak>
-    <p>Hello, world!</p>
-    <p>This is a test of the <break strength="medium" /> text-to-speech API.</p>
-</speak>
-"""
-
-response = litellm.speech(
-    input=ssml,
-    use_ssml=True,
-    model="vertex_ai/test",
-    voice={
-        "languageCode": "en-UK",
-        "name": "en-UK-Studio-O",
-    },
-    audioConfig={
-        "audioEncoding": "LINEAR22",
-        "speakingRate": "10",
-    },
-)
-response.stream_to_file(speech_file_path)
-```
-
-</TabItem>
-
-<TabItem value="proxy" label="LiteLLM PROXY (Unified Endpoint)">
-
-```python
-import openai
-
-client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
-
-ssml = """
-<speak>
-    <p>Hello, world!</p>
-    <p>This is a test of the <break strength="medium" /> text-to-speech API.</p>
-</speak>
-"""
-
-# see supported values for "voice" on vertex here: 
-# https://console.cloud.google.com/vertex-ai/generative/speech/text-to-speech
-response = client.audio.speech.create(
-    model = "vertex-tts",
-    input=ssml, # pass as None since OpenAI SDK requires this param
-    voice={'languageCode': 'en-US', 'name': 'en-US-Studio-O'},
-    extra_body={"use_ssml": True},
-)
-print("response from proxy", response)
-```
-
-</TabItem>
-</Tabs>
-
 ## **Fine Tuning APIs**
 
 

docs/my-website/docs/providers/vertex_speech.md

@@ -0,0 +1,423 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Vertex AI Text to Speech
+
+| Property | Details |
+|-------|-------|
+| Description | Google Cloud Text-to-Speech with Chirp3 HD voices and Gemini TTS |
+| Provider Route on LiteLLM | `vertex_ai/chirp` (Chirp), `vertex_ai/gemini-*-tts` (Gemini) |
+
+## Chirp3 HD Voices
+
+Google Cloud Text-to-Speech API with high-quality Chirp3 HD voices.
+
+### Quick Start
+
+#### LiteLLM Python SDK
+
+```python showLineNumbers title="Chirp3 Quick Start"
+from litellm import speech
+from pathlib import Path
+
+speech_file_path = Path(__file__).parent / "speech.mp3"
+response = speech(
+    model="vertex_ai/chirp",
+    voice="alloy",  # OpenAI voice name - automatically mapped
+    input="Hello, this is Vertex AI Text to Speech",
+    vertex_project="your-project-id",
+    vertex_location="us-central1",
+)
+response.stream_to_file(speech_file_path)
+```
+
+#### LiteLLM AI Gateway
+
+**1. Setup config.yaml**
+
+```yaml showLineNumbers title="config.yaml"
+model_list:
+  - model_name: vertex-tts
+    litellm_params:
+      model: vertex_ai/chirp
+      vertex_project: "your-project-id"
+      vertex_location: "us-central1"
+      vertex_credentials: "/path/to/service_account.json"
+```
+
+**2. Start the proxy**
+
+```bash title="Start LiteLLM Proxy"
+litellm --config /path/to/config.yaml
+```
+
+**3. Make requests**
+
+<Tabs>
+<TabItem value="curl" label="curl">
+
+```bash showLineNumbers title="Chirp3 Quick Start"
+curl http://0.0.0.0:4000/v1/audio/speech \
+  -H "Authorization: Bearer sk-1234" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "vertex-tts",
+    "voice": "alloy",
+    "input": "Hello, this is Vertex AI Text to Speech"
+  }' \
+  --output speech.mp3
+```
+
+</TabItem>
+<TabItem value="openai-sdk" label="OpenAI Python SDK">
+
+```python showLineNumbers title="Chirp3 Quick Start"
+import openai
+
+client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
+
+response = client.audio.speech.create(
+    model="vertex-tts",
+    voice="alloy",
+    input="Hello, this is Vertex AI Text to Speech",
+)
+response.stream_to_file("speech.mp3")
+```
+
+</TabItem>
+</Tabs>
+
+### Voice Mapping
+
+LiteLLM maps OpenAI voice names to Google Cloud voices. You can use either OpenAI voices or Google Cloud voices directly.
+
+| OpenAI Voice | Google Cloud Voice |
+|-------------|-------------------|
+| `alloy` | en-US-Studio-O |
+| `echo` | en-US-Studio-M |
+| `fable` | en-GB-Studio-B |
+| `onyx` | en-US-Wavenet-D |
+| `nova` | en-US-Studio-O |
+| `shimmer` | en-US-Wavenet-F |
+
+### Using Google Cloud Voices Directly
+
+#### LiteLLM Python SDK
+
+```python showLineNumbers title="Chirp3 HD Voice"
+from litellm import speech
+
+# Pass Chirp3 HD voice name directly
+response = speech(
+    model="vertex_ai/chirp",
+    voice="en-US-Chirp3-HD-Charon",
+    input="Hello with a Chirp3 HD voice",
+    vertex_project="your-project-id",
+)
+response.stream_to_file("speech.mp3")
+```
+
+```python showLineNumbers title="Voice as Dict (Multilingual)"
+from litellm import speech
+
+# Pass as dict for full control over language and voice
+response = speech(
+    model="vertex_ai/chirp",
+    voice={
+        "languageCode": "de-DE",
+        "name": "de-DE-Chirp3-HD-Charon",
+    },
+    input="Hallo, dies ist ein Test",
+    vertex_project="your-project-id",
+)
+response.stream_to_file("speech.mp3")
+```
+
+#### LiteLLM AI Gateway
+
+<Tabs>
+<TabItem value="curl" label="curl">
+
+```bash showLineNumbers title="Chirp3 HD Voice"
+curl http://0.0.0.0:4000/v1/audio/speech \
+  -H "Authorization: Bearer sk-1234" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "vertex-tts",
+    "voice": "en-US-Chirp3-HD-Charon",
+    "input": "Hello with a Chirp3 HD voice"
+  }' \
+  --output speech.mp3
+```
+
+```bash showLineNumbers title="Voice as Dict (Multilingual)"
+curl http://0.0.0.0:4000/v1/audio/speech \
+  -H "Authorization: Bearer sk-1234" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "vertex-tts",
+    "voice": {"languageCode": "de-DE", "name": "de-DE-Chirp3-HD-Charon"},
+    "input": "Hallo, dies ist ein Test"
+  }' \
+  --output speech.mp3
+```
+
+</TabItem>
+<TabItem value="openai-sdk" label="OpenAI Python SDK">
+
+```python showLineNumbers title="Chirp3 HD Voice"
+import openai
+
+client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
+
+response = client.audio.speech.create(
+    model="vertex-tts",
+    voice="en-US-Chirp3-HD-Charon",
+    input="Hello with a Chirp3 HD voice",
+)
+response.stream_to_file("speech.mp3")
+```
+
+```python showLineNumbers title="Voice as Dict (Multilingual)"
+import openai
+
+client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
+
+response = client.audio.speech.create(
+    model="vertex-tts",
+    voice={"languageCode": "de-DE", "name": "de-DE-Chirp3-HD-Charon"},
+    input="Hallo, dies ist ein Test",
+)
+response.stream_to_file("speech.mp3")
+```
+
+</TabItem>
+</Tabs>
+
+Browse available voices: [Google Cloud Text-to-Speech Console](https://console.cloud.google.com/vertex-ai/generative/speech/text-to-speech)
+
+### Passing Raw SSML
+
+LiteLLM auto-detects SSML when your input contains `<speak>` tags and passes it through unchanged.
+
+#### LiteLLM Python SDK
+
+```python showLineNumbers title="SSML Input"
+from litellm import speech
+
+ssml = """
+<speak>
+    <p>Hello, world!</p>
+    <p>This is a test of the <break strength="medium" /> text-to-speech API.</p>
+</speak>
+"""
+
+response = speech(
+    model="vertex_ai/chirp",
+    voice="en-US-Studio-O",
+    input=ssml,  # Auto-detected as SSML
+    vertex_project="your-project-id",
+)
+response.stream_to_file("speech.mp3")
+```
+
+```python showLineNumbers title="Force SSML Mode"
+from litellm import speech
+
+# Force SSML mode with use_ssml=True
+response = speech(
+    model="vertex_ai/chirp",
+    voice="en-US-Studio-O",
+    input="<speak><prosody rate='slow'>Speaking slowly</prosody></speak>",
+    use_ssml=True,
+    vertex_project="your-project-id",
+)
+response.stream_to_file("speech.mp3")
+```
+
+#### LiteLLM AI Gateway
+
+<Tabs>
+<TabItem value="curl" label="curl">
+
+```bash showLineNumbers title="SSML Input"
+curl http://0.0.0.0:4000/v1/audio/speech \
+  -H "Authorization: Bearer sk-1234" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "vertex-tts",
+    "voice": "en-US-Studio-O",
+    "input": "<speak><p>Hello!</p><break time=\"500ms\"/><p>How are you?</p></speak>"
+  }' \
+  --output speech.mp3
+```
+
+</TabItem>
+<TabItem value="openai-sdk" label="OpenAI Python SDK">
+
+```python showLineNumbers title="SSML Input"
+import openai
+
+client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
+
+ssml = """<speak><p>Hello!</p><break time="500ms"/><p>How are you?</p></speak>"""
+
+response = client.audio.speech.create(
+    model="vertex-tts",
+    voice="en-US-Studio-O",
+    input=ssml,
+)
+response.stream_to_file("speech.mp3")
+```
+
+</TabItem>
+</Tabs>
+
+### Supported Parameters
+
+| Parameter | Description | Values |
+|-----------|-------------|--------|
+| `voice` | Voice selection | OpenAI voice, Google Cloud voice name, or dict |
+| `input` | Text to convert | Plain text or SSML |
+| `speed` | Speaking rate | 0.25 to 4.0 (default: 1.0) |
+| `response_format` | Audio format | `mp3`, `opus`, `wav`, `pcm`, `flac` |
+| `use_ssml` | Force SSML mode | `True` / `False` |
+
+### Async Usage
+
+```python showLineNumbers title="Async Speech Generation"
+import asyncio
+from litellm import aspeech
+
+async def main():
+    response = await aspeech(
+        model="vertex_ai/chirp",
+        voice="alloy",
+        input="Hello from async",
+        vertex_project="your-project-id",
+    )
+    response.stream_to_file("speech.mp3")
+
+asyncio.run(main())
+```
+
+---
+
+## Gemini TTS
+
+Gemini models with audio output capabilities using the chat completions API.
+
+:::warning
+**Limitations:**
+- Only supports `pcm16` audio format
+- Streaming not yet supported
+- Must set `modalities: ["audio"]`
+:::
+
+### Quick Start
+
+#### LiteLLM Python SDK
+
+```python showLineNumbers title="Gemini TTS Quick Start"
+from litellm import completion
+import json
+
+# Load credentials
+with open('path/to/service_account.json', 'r') as file:
+    vertex_credentials = json.dumps(json.load(file))
+
+response = completion(
+    model="vertex_ai/gemini-2.5-flash-preview-tts",
+    messages=[{"role": "user", "content": "Say hello in a friendly voice"}],
+    modalities=["audio"],
+    audio={
+        "voice": "Kore",
+        "format": "pcm16"
+    },
+    vertex_credentials=vertex_credentials
+)
+print(response)
+```
+
+#### LiteLLM AI Gateway
+
+**1. Setup config.yaml**
+
+```yaml showLineNumbers title="config.yaml"
+model_list:
+  - model_name: gemini-tts
+    litellm_params:
+      model: vertex_ai/gemini-2.5-flash-preview-tts
+      vertex_project: "your-project-id"
+      vertex_location: "us-central1"
+      vertex_credentials: "/path/to/service_account.json"
+```
+
+**2. Start the proxy**
+
+```bash title="Start LiteLLM Proxy"
+litellm --config /path/to/config.yaml
+```
+
+**3. Make requests**
+
+<Tabs>
+<TabItem value="curl" label="curl">
+
+```bash showLineNumbers title="Gemini TTS Request"
+curl http://0.0.0.0:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer sk-1234" \
+  -d '{
+    "model": "gemini-tts",
+    "messages": [{"role": "user", "content": "Say hello in a friendly voice"}],
+    "modalities": ["audio"],
+    "audio": {"voice": "Kore", "format": "pcm16"}
+  }'
+```
+
+</TabItem>
+<TabItem value="openai-sdk" label="OpenAI Python SDK">
+
+```python showLineNumbers title="Gemini TTS Request"
+import openai
+
+client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
+
+response = client.chat.completions.create(
+    model="gemini-tts",
+    messages=[{"role": "user", "content": "Say hello in a friendly voice"}],
+    modalities=["audio"],
+    audio={"voice": "Kore", "format": "pcm16"},
+)
+print(response)
+```
+
+</TabItem>
+</Tabs>
+
+### Supported Models
+
+- `vertex_ai/gemini-2.5-flash-preview-tts`
+- `vertex_ai/gemini-2.5-pro-preview-tts`
+
+See [Gemini TTS documentation](https://ai.google.dev/gemini-api/docs/speech-generation) for available voices.
+
+### Advanced Usage
+
+```python showLineNumbers title="Gemini TTS with System Prompt"
+from litellm import completion
+
+response = completion(
+    model="vertex_ai/gemini-2.5-pro-preview-tts",
+    messages=[
+        {"role": "system", "content": "You are a helpful assistant that speaks clearly."},
+        {"role": "user", "content": "Explain quantum computing in simple terms"}
+    ],
+    modalities=["audio"],
+    audio={"voice": "Charon", "format": "pcm16"},
+    temperature=0.7,
+    max_tokens=150,
+    vertex_credentials=vertex_credentials
+)
+```

docs/my-website/docs/providers/watsonx/index.md

@@ -175,3 +175,56 @@ For all available models, see [watsonx.ai documentation](https://dataplatform.cl
 
 For all available embedding models, see [watsonx.ai embedding documentation](https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models-embed.html?context=wx).
 
+
+## Advanced
+
+### Using Zen API Key
+
+You can use a Zen API key for long-term authentication instead of generating IAM tokens. Pass it either as an environment variable or as a parameter:
+
+```python
+import os
+from litellm import completion
+
+# Option 1: Set as environment variable
+os.environ["WATSONX_ZENAPIKEY"] = "your-zen-api-key"
+
+response = completion(
+    model="watsonx/ibm/granite-13b-chat-v2",
+    messages=[{"content": "What is your favorite color?", "role": "user"}],
+    project_id="your-project-id"
+)
+
+# Option 2: Pass as parameter
+response = completion(
+    model="watsonx/ibm/granite-13b-chat-v2",
+    messages=[{"content": "What is your favorite color?", "role": "user"}],
+    zen_api_key="your-zen-api-key",
+    project_id="your-project-id"
+)
+```
+
+**Using with LiteLLM Proxy via OpenAI client:**
+
+```python
+import openai
+
+client = openai.OpenAI(
+    api_key="sk-1234",  # LiteLLM proxy key
+    base_url="http://0.0.0.0:4000"
+)
+
+response = client.chat.completions.create(
+    model="watsonx/ibm/granite-3-3-8b-instruct",
+    messages=[{"role": "user", "content": "What is your favorite color?"}],
+    max_tokens=2048,
+    extra_body={
+        "project_id": "your-project-id",
+        "zen_api_key": "your-zen-api-key"
+    }
+)
+```
+
+See [IBM documentation](https://www.ibm.com/docs/en/watsonx/w-and-w/2.2.0?topic=keys-generating-zenapikey-authorization-tokens) for more information on generating Zen API keys.
+
+

docs/my-website/docs/providers/zai.md

@@ -0,0 +1,135 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Z.AI (Zhipu AI)
+https://z.ai/
+
+**We support Z.AI GLM text/chat models, just set `zai/` as a prefix when sending completion requests**
+
+## API Key
+```python
+# env variable
+os.environ['ZAI_API_KEY']
+```
+
+## Sample Usage
+```python
+from litellm import completion
+import os
+
+os.environ['ZAI_API_KEY'] = ""
+response = completion(
+    model="zai/glm-4.6",
+    messages=[
+       {"role": "user", "content": "hello from litellm"}
+   ],
+)
+print(response)
+```
+
+## Sample Usage - Streaming
+```python
+from litellm import completion
+import os
+
+os.environ['ZAI_API_KEY'] = ""
+response = completion(
+    model="zai/glm-4.6",
+    messages=[
+       {"role": "user", "content": "hello from litellm"}
+   ],
+    stream=True
+)
+
+for chunk in response:
+    print(chunk)
+```
+
+## Supported Models
+
+We support ALL Z.AI GLM models, just set `zai/` as a prefix when sending completion requests.
+
+| Model Name | Function Call | Notes |
+|------------|---------------|-------|
+| glm-4.6 | `completion(model="zai/glm-4.6", messages)` | Latest flagship model, 200K context |
+| glm-4.5 | `completion(model="zai/glm-4.5", messages)` | 128K context |
+| glm-4.5v | `completion(model="zai/glm-4.5v", messages)` | Vision model |
+| glm-4.5-x | `completion(model="zai/glm-4.5-x", messages)` | Premium tier |
+| glm-4.5-air | `completion(model="zai/glm-4.5-air", messages)` | Lightweight |
+| glm-4.5-airx | `completion(model="zai/glm-4.5-airx", messages)` | Fast lightweight |
+| glm-4-32b-0414-128k | `completion(model="zai/glm-4-32b-0414-128k", messages)` | 32B parameter model |
+| glm-4.5-flash | `completion(model="zai/glm-4.5-flash", messages)` | **FREE tier** |
+
+## Model Pricing
+
+| Model | Input ($/1M tokens) | Output ($/1M tokens) | Context Window |
+|-------|---------------------|----------------------|----------------|
+| glm-4.6 | $0.60 | $2.20 | 200K |
+| glm-4.5 | $0.60 | $2.20 | 128K |
+| glm-4.5v | $0.60 | $1.80 | 128K |
+| glm-4.5-x | $2.20 | $8.90 | 128K |
+| glm-4.5-air | $0.20 | $1.10 | 128K |
+| glm-4.5-airx | $1.10 | $4.50 | 128K |
+| glm-4-32b-0414-128k | $0.10 | $0.10 | 128K |
+| glm-4.5-flash | **FREE** | **FREE** | 128K |
+
+## Using with LiteLLM Proxy
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+from litellm import completion
+import os
+
+os.environ['ZAI_API_KEY'] = ""
+response = completion(
+    model="zai/glm-4.6",
+    messages=[{"role": "user", "content": "Hello, how are you?"}],
+)
+
+print(response.choices[0].message.content)
+```
+
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+1. Setup config.yaml
+
+```yaml
+model_list:
+  - model_name: glm-4.6
+    litellm_params:
+        model: zai/glm-4.6
+        api_key: os.environ/ZAI_API_KEY
+  - model_name: glm-4.5-flash  # Free tier
+    litellm_params:
+        model: zai/glm-4.5-flash
+        api_key: os.environ/ZAI_API_KEY
+```
+
+2. Run proxy
+
+```bash
+litellm --config config.yaml
+```
+
+3. Test it!
+
+```bash
+curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
+-H 'Content-Type: application/json' \
+-H 'Authorization: Bearer sk-1234' \
+-d '{
+    "model": "glm-4.6",
+    "messages": [
+      {
+        "role": "user",
+        "content": "Hello, how are you?"
+      }
+    ]
+}'
+```
+
+</TabItem>
+</Tabs>

docs/my-website/docs/proxy/config_settings.md

@@ -113,7 +113,7 @@ general_settings:
 
   # Database Settings
   database_url: string
-  database_connection_pool_limit: 0  # default 100
+  database_connection_pool_limit: 0  # default 10
   database_connection_timeout: 0  # default 60s
   allow_requests_on_db_unavailable: boolean  # if true, will allow requests that can not connect to the DB to verify Virtual Key to still work 
 
@@ -234,7 +234,7 @@ router_settings:
 | max_response_size_mb | int | The maximum size for responses in MB. LLM Responses above this size will not be sent. |
 | proxy_budget_rescheduler_min_time | int | The minimum time (in seconds) to wait before checking db for budget resets. **Default is 597 seconds** |
 | proxy_budget_rescheduler_max_time | int | The maximum time (in seconds) to wait before checking db for budget resets. **Default is 605 seconds** |
-| proxy_batch_write_at | int | Time (in seconds) to wait before batch writing spend logs to the db. **Default is 30 seconds** |
+| proxy_batch_write_at | int | Time (in seconds) to wait before batch writing spend logs to the db. **Default is 10 seconds** |
 | proxy_batch_polling_interval | int | Time (in seconds) to wait before polling a batch, to check if it's completed. **Default is 6000 seconds (1 hour)** |
 | alerting_args | dict | Args for Slack Alerting [Doc on Slack Alerting](./alerting.md) |
 | custom_key_generate | str | Custom function for key generation [Doc on custom key generation](./virtual_keys.md#custom--key-generate) |
@@ -763,7 +763,7 @@ router_settings:
 | PROMPTLAYER_API_KEY | API key for PromptLayer integration
 | PROXY_ADMIN_ID | Admin identifier for proxy server
 | PROXY_BASE_URL | Base URL for proxy service
-| PROXY_BATCH_WRITE_AT | Time in seconds to wait before batch writing spend logs to the database. Default is 30
+| PROXY_BATCH_WRITE_AT | Time in seconds to wait before batch writing spend logs to the database. Default is 10
 | PROXY_BATCH_POLLING_INTERVAL | Time in seconds to wait before polling a batch, to check if it's completed. Default is 6000s (1 hour)
 | PROXY_BUDGET_RESCHEDULER_MAX_TIME | Maximum time in seconds to wait before checking database for budget resets. Default is 605
 | PROXY_BUDGET_RESCHEDULER_MIN_TIME | Minimum time in seconds to wait before checking database for budget resets. Default is 597

docs/my-website/docs/proxy/configs.md

@@ -576,7 +576,7 @@ custom_tokenizer:
 
 ```yaml
 general_settings: 
-  database_connection_pool_limit: 100 # sets connection pool for prisma client to postgres db at 100
+  database_connection_pool_limit: 10 # sets connection pool for prisma client to postgres db (default: 10, recommended: 10-20)
   database_connection_timeout: 60 # sets a 60s timeout for any connection call to the db 
 ```
 

docs/my-website/docs/proxy/error_diagnosis.md

@@ -0,0 +1,90 @@
+# Diagnosing Errors - Provider vs Gateway
+
+Having trouble diagnosing if an error is from the **LLM Provider** (OpenAI, Anthropic, etc.) or from the **LiteLLM AI Gateway** itself? Here's how to tell.
+
+## Quick Rule
+
+**If the error contains `<Provider>Exception`, it's from the provider.**
+
+| Error Contains | Error Source |
+|----------------|--------------|
+| `AnthropicException` | Anthropic |
+| `OpenAIException` | OpenAI |
+| `AzureException` | Azure |
+| `BedrockException` | AWS Bedrock |
+| `VertexAIException` | Google Vertex AI |
+| No provider name | LiteLLM AI Gateway |
+
+## Examples
+
+### Provider Error (from AWS Bedrock)
+
+```
+{
+  "error": {
+    "message": "litellm.BadRequestError: BedrockException - {\"message\":\"The model returned the following errors: messages.1.content.0.type: Expected `thinking` or `redacted_thinking`, but found `text`.\"}",
+    "type": "invalid_request_error",
+    "param": null,
+    "code": "400"
+  }
+}
+```
+
+This error is from **AWS Bedrock** (notice `BedrockException`). The Bedrock API is rejecting the request due to invalid message format - this is not a LiteLLM issue.
+
+### Provider Error (from OpenAI)
+
+```
+{
+  "error": {
+    "message": "litellm.AuthenticationError: OpenAIException - Incorrect API key provided: <my-key>. You can find your API key at https://platform.openai.com/account/api-keys.",
+    "type": "invalid_request_error",
+    "param": null,
+    "code": "invalid_api_key"
+  }
+}
+```
+
+This error is from **OpenAI** (notice `OpenAIException`). The OpenAI API key configured in LiteLLM is invalid.
+
+### Provider Error (from Anthropic)
+
+```
+{
+  "error": {
+    "message": "litellm.InternalServerError: AnthropicException - Overloaded. Handle with `litellm.InternalServerError`.",
+    "type": "internal_server_error",
+    "param": null,
+    "code": "500"
+  }
+}
+```
+
+This error is from **Anthropic** (notice `AnthropicException`). The Anthropic API is overloaded - this is not a LiteLLM issue.
+
+### Gateway Error (from LiteLLM)
+
+```
+{
+  "error": {
+    "message": "Invalid API Key. Please check your LiteLLM API key.",
+    "type": "auth_error",
+    "param": null,
+    "code": "401"
+  }
+}
+```
+
+This error is from the **LiteLLM AI Gateway** (no provider name). Your LiteLLM virtual key is invalid.
+
+## What to do?
+
+| Error Source | Action |
+|--------------|--------|
+| Provider Error | Check the provider's status page, adjust rate limits, or retry later |
+| Gateway Error | Check your LiteLLM configuration, API keys, or [open an issue](https://github.com/BerriAI/litellm/issues) |
+
+## See Also
+
+- [Debugging](/docs/proxy/debugging) - Enable debug logs to see detailed request/response info
+- [Exception Mapping](/docs/exception_mapping) - Full list of LiteLLM exception types

docs/my-website/docs/proxy/guardrails/bedrock.md

@@ -188,6 +188,28 @@ My email is [EMAIL] and my phone number is [PHONE_NUMBER]
 
 This helps protect sensitive information while still allowing the model to understand the context of the request.
 
+## Experimental: Only Send Latest User Message
+
+When you're chaining long conversations through Bedrock guardrails, you can opt into a lighter, experimental behavior by setting `experimental_use_latest_role_message_only: true` in the guardrail's `litellm_params`. When enabled, LiteLLM only sends the most recent `user` message (or assistant output during post-call checks) to Bedrock, which:
+
+- prevents unintended blocks on older system/dev messages
+- keeps Bedrock payloads smaller, reducing latency and cost
+- applies to proxy hooks (`pre_call`, `during_call`) and the `/guardrails/apply_guardrail` testing endpoint
+
+```yaml showLineNumbers title="litellm proxy config.yaml"
+guardrails:
+  - guardrail_name: "bedrock-pre-guard"
+    litellm_params:
+      guardrail: bedrock
+      mode: "pre_call"
+      guardrailIdentifier: wf0hkdb5x07f
+      guardrailVersion: "DRAFT"
+      aws_region_name: os.environ/AWS_REGION
+      experimental_use_latest_role_message_only: true  # NEW
+```
+
+> ⚠️ This flag is currently experimental and defaults to `false` to preserve the legacy behavior (entire message history). We'll be listening to user feedback to decide if this becomes the default or rolls out more broadly.
+
 ## Disabling Exceptions on Bedrock BLOCK
 
 By default, when Bedrock guardrails block content, LiteLLM raises an HTTP 400 exception. However, you can disable this behavior by setting `disable_exception_on_block: true`. This is particularly useful when integrating with **OpenWebUI**, where exceptions can interrupt the chat flow and break the user experience.

docs/my-website/docs/proxy/guardrails/lasso_security.md

@@ -35,7 +35,7 @@ guardrails:
       guardrail: lasso
       mode: "pre_call"
       api_key: os.environ/LASSO_API_KEY
-      api_base: "https://server.lasso.security"
+      api_base: "https://server.lasso.security/gateway/v3"
   - guardrail_name: "lasso-post-guard"
     litellm_params:
       guardrail: lasso
@@ -228,7 +228,7 @@ Expected response:
 
 ## PII Masking with Lasso
 
-Lasso supports automatic PII detection and masking using the `/gateway/v1/classifix` endpoint. When enabled, sensitive information like emails, phone numbers, and other PII will be automatically masked with appropriate placeholders.
+Lasso supports automatic PII detection and masking using the `/classifix` endpoint. When enabled, sensitive information like emails, phone numbers, and other PII will be automatically masked with appropriate placeholders.
 
 ### Enabling PII Masking
 

docs/my-website/docs/proxy/guardrails/tool_permission.md

@@ -1,4 +1,3 @@
-import Image from '@theme/IdealImage';
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
 
@@ -7,9 +6,34 @@ import TabItem from '@theme/TabItem';
 LiteLLM provides the LiteLLM Tool Permission Guardrail that lets you control which **tool calls** a model is allowed to invoke, using configurable allow/deny rules. This offers fine-grained, provider-agnostic control over tool execution (e.g., OpenAI Chat Completions `tool_calls`, Anthropic Messages `tool_use`, MCP tools).
 
 ## Quick Start
-### 1. Define Guardrails on your LiteLLM config.yaml 
 
-Define your guardrails under the `guardrails` section
+### LiteLLM UI
+
+#### Step 1: Select Tool Permission Guardrail
+
+Open the LiteLLM Dashboard, click **Add New Guardrail**, and choose **LiteLLM Tool Permission Guardrail**. This loads the rule builder UI.
+
+#### Step 2: Define Regex Rules
+
+1. Click **Add Rule**.
+2. Enter a unique Rule ID.
+3. Provide a regex for the tool name (e.g., `^mcp__github_.*$`).
+4. Optionally add a regex for tool type (e.g., `^function$`).
+5. Pick **Allow** or **Deny**.
+
+#### Step 3: Restrict Tool Arguments (Optional)
+
+Select **+ Restrict tool arguments** to attach regex validations to nested paths (dot + `[]` notation). This enforces that sensitive parameters (such as `arguments.to[]`) conform to pre-approved formats.
+
+#### Step 4: Choose Defaults & Actions
+
+- Set the fallback decision (`default_action`) for tools that do not hit any rule.
+- Decide how disallowed tools behave: **Block** halts the request, **Rewrite** strips forbidden tools and returns an error message inside the response.
+- Customize `violation_message_template` if you want branded error copy.
+- Save the guardrail.
+
+### LiteLLM Config.yaml Setup
+
 ```yaml
 guardrails:
   - guardrail_name: "tool-permission-guardrail"
@@ -21,16 +45,17 @@ guardrails:
           tool_name: "Bash"
           decision: "allow"
         - id: "allow_github_mcp"
-          tool_name: "mcp__github_*"
+          tool_name: "^mcp__github_.*$"
           decision: "allow"
         - id: "allow_aws_documentation"
-          tool_name: "mcp__aws-documentation_*_documentation"
+          tool_name: "^mcp__aws-documentation_.*_documentation$"
           decision: "allow"
         - id: "deny_read_commands"
           tool_name: "Read"
-          decision: "Deny"
+          decision: "deny"
         - id: "mail-domain"
-          tool_name: "send_email"
+          tool_name: "^send_email$"
+          tool_type: "^function$"
           decision: "allow"
           allowed_param_patterns:
             "to[]": "^.+@berri\\.ai$"
@@ -44,7 +69,8 @@ guardrails:
 
 ```yaml
 - id: "unique_rule_id"           # Unique identifier for the rule
-  tool_name: "pattern"           # Tool name or pattern to match
+  tool_name: "^regex$"           # Regex for tool name (optional, at least one of name/type required)
+  tool_type: "^function$"        # Regex for tool type (optional)
   decision: "allow"              # "allow" or "deny"
   allowed_param_patterns:         # Optional - regex map for argument paths (dot + [] notation)
     "path.to[].field": "^regex$"

docs/my-website/docs/proxy/pass_through.md

@@ -275,6 +275,20 @@ In this video, we'll add the Azure OpenAI Assistants API as a pass through endpo
 - Check LiteLLM proxy logs for error details
 - Verify the target API's expected request format
 
+### Allowing Team JWTs to use pass-through routes
+
+If you are using pass-through provider routes (e.g., `/anthropic/*`) and want your JWT team tokens to access these routes, add `mapped_pass_through_routes` to the `team_allowed_routes` in `litellm_jwtauth` or explicitly add the relevant route(s).
+
+Example (`proxy_server_config.yaml`):
+
+```yaml
+general_settings:
+  enable_jwt_auth: True
+  litellm_jwtauth:
+    team_ids_jwt_field: "team_ids"
+    team_allowed_routes: ["openai_routes","info_routes","mapped_pass_through_routes"]
+```
+
 ### Getting Help
 
 [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version)

docs/my-website/docs/proxy/token_auth.md

@@ -338,6 +338,58 @@ general_settings:
     team_allowed_routes: ["/v1/chat/completions"] # 👈 Set accepted routes
 ```
 
+### Allowing other provider routes for Teams
+
+To enable team JWT tokens to access Anthropic-style endpoints such as `/v1/messages`, update `team_allowed_routes` in your `litellm_jwtauth` configuration. `team_allowed_routes` supports the following values:
+
+- Named route groups from `LiteLLMRoutes` (e.g., `openai_routes`, `anthropic_routes`, `info_routes`, `mapped_pass_through_routes`).
+
+Below is a quick reference for the route groups you can use and example representative routes from each group. If you need the exhaustive list, see the `LiteLLMRoutes` enum in `litellm/proxy/_types.py` for the authoritative list.
+
+| Route Group | What it contains | Representative routes |
+|-------------|------------------|-----------------------|
+| `openai_routes` | OpenAI-compatible REST endpoints (chat, completion, embeddings, images, responses, models, etc.) | `/v1/chat/completions`, `/v1/completions`, `/v1/embeddings`, `/v1/images/generations`, `/v1/models` |
+| `anthropic_routes` | Anthropic-style endpoints (`/v1/messages` and related) | `/v1/messages`, `/v1/messages/count_tokens`, `/v1/skills` |
+| `mapped_pass_through_routes` | Provider-specific pass-through route prefixes (e.g., Anthropic when proxied via `/anthropic`). Use with `mapped_pass_through_routes` for provider wildcard mapping | `/anthropic/*`, `/vertex-ai/*`, `/bedrock/*` |
+| `passthrough_routes_wildcard` | Wildcard mapping for providers (e.g., `/anthropic/*`) - precomputed wildcard list used by the proxy | `/anthropic/*`, `/vllm/*` |
+| `google_routes` | Google-specific (e.g., Vertex / Batching endpoints) | `/v1beta/models/{model_name}:generateContent` |
+| `mcp_routes` | Internal MCP management endpoints | `/mcp/tools`, `/mcp/tools/call` |
+| `info_routes` | Read-only & info endpoints used by the UI | `/key/info`, `/team/info`, `/v1/models` |
+| `management_routes` | Admin-only management endpoints (create/update/delete user/team/model) | `/team/new`, `/key/generate`, `/model/new` |
+| `spend_tracking_routes` | Budget/spend related endpoints | `/spend/logs`, `/spend/keys` |
+| `public_routes` | Public and unauthenticated endpoints | `/`, `/routes`, `/.well-known/litellm-ui-config` |
+
+Note: `llm_api_routes` is the union of OpenAI, Anthropic, Google, pass-through and other LLM routes (`openai_routes + anthropic_routes + google_routes + mapped_pass_through_routes + passthrough_routes_wildcard + apply_guardrail_routes + mcp_routes + litellm_native_routes`).
+
+Defaults (what the proxy uses if you don't override them in `litellm_jwtauth`):
+
+- `admin_jwt_scope`: `litellm_proxy_admin`
+- `admin_allowed_routes` (default): `management_routes`, `spend_tracking_routes`, `global_spend_tracking_routes`, `info_routes` 
+- `team_allowed_routes` (default): `openai_routes`, `info_routes` 
+- `public_allowed_routes` (default): `public_routes`
+
+
+Example: Allow team JWTs to call Anthropic `/v1/messages` (either by route group or by explicit route string):
+
+```yaml
+general_settings:
+  enable_jwt_auth: True
+  litellm_jwtauth:
+    team_ids_jwt_field: "team_ids"
+    team_allowed_routes: ["openai_routes", "info_routes", "anthropic_routes"]
+```
+
+Or selectively allow the exact Anthropic message endpoint only:
+
+```yaml
+general_settings:
+  enable_jwt_auth: True
+  litellm_jwtauth:
+    team_ids_jwt_field: "team_ids"
+    team_allowed_routes: ["/v1/messages", "info_routes"]
+```
+
+
 ### Caching Public Keys 
 
 Control how long public keys are cached for (in seconds).
@@ -407,6 +459,72 @@ general_settings:
     user_id_upsert: true # 👈 upserts the user to db, if valid email but not in db
 ```
 
+## OIDC UserInfo Endpoint
+
+Use this when your JWT/access token doesn't contain user-identifying information. LiteLLM will call your identity provider's UserInfo endpoint to fetch user details.
+
+### When to Use
+
+- Your JWT is opaque (not self-contained) or lacks user claims
+- You need to fetch fresh user information from your identity provider
+- Your access tokens don't include email, roles, or other identifying data
+
+### Configuration
+
+```yaml title="config.yaml" showLineNumbers
+general_settings:
+  enable_jwt_auth: True
+  litellm_jwtauth:
+    # Enable OIDC UserInfo endpoint
+    oidc_userinfo_enabled: true
+    oidc_userinfo_endpoint: "https://your-idp.com/oauth2/userinfo"
+    oidc_userinfo_cache_ttl: 300  # Cache for 5 minutes (default: 300)
+    
+    # Map fields from UserInfo response
+    user_id_jwt_field: "sub"
+    user_email_jwt_field: "email"
+    user_roles_jwt_field: "roles"
+```
+
+### Flow Diagram
+
+```mermaid
+sequenceDiagram
+    participant Client
+    participant LiteLLM
+    participant IdP as Identity Provider
+
+    Client->>LiteLLM: Request with Bearer token
+    Note over LiteLLM: Check cache for UserInfo
+    
+    LiteLLM->>IdP: GET /userinfo (if not cached)<br/>Authorization: Bearer {token}
+    IdP-->>LiteLLM: User data (sub, email, roles)
+    
+    Note over LiteLLM: Cache response (TTL: 5min)<br/>Extract user_id, email, roles<br/>Perform RBAC checks
+    
+    LiteLLM-->>Client: Authorized/Denied
+```
+
+### Example: Azure AD
+
+```yaml title="config.yaml" showLineNumbers
+litellm_jwtauth:
+  oidc_userinfo_enabled: true
+  oidc_userinfo_endpoint: "https://graph.microsoft.com/oidc/userinfo"
+  user_id_jwt_field: "sub"
+  user_email_jwt_field: "email"
+```
+
+### Example: Keycloak
+
+```yaml title="config.yaml" showLineNumbers
+litellm_jwtauth:
+  oidc_userinfo_enabled: true
+  oidc_userinfo_endpoint: "https://keycloak.example.com/realms/your-realm/protocol/openid-connect/userinfo"
+  user_id_jwt_field: "sub"
+  user_roles_jwt_field: "resource_access.your-client.roles"
+```
+
 ## [BETA] Control Access with OIDC Roles
 
 Allow JWT tokens with supported roles to access the proxy.

docs/my-website/docs/vector_stores/create.md

@@ -14,6 +14,7 @@ Create a vector store which can be used to store and search document chunks for
 | End-user Tracking | ✅ | |
 | Support LLM Providers (OpenAI `/vector_stores` API) | **OpenAI** | Full vector stores API support across providers |
 | Support LLM Providers (Passthrough API) | [**Azure AI**](/docs/providers/azure_ai/azure_ai_vector_stores_passthrough) | Full vector stores API support across providers |
+| Support LLM Providers (Dataset Management) | [**RAGFlow**](/docs/providers/ragflow_vector_store.md) | Dataset creation and management (search not supported) |
 
 ## Usage
 

docs/my-website/sidebars.js

@@ -45,6 +45,7 @@ const sidebars = {
           type: "category",
           "label": "Contributing to Guardrails",
           items: [
+            "adding_provider/generic_guardrail_api",
             "adding_provider/simple_guardrail_tutorial",
             "adding_provider/adding_guardrail_support",
           ]
@@ -140,6 +141,7 @@ const sidebars = {
             "proxy/quick_start",
             "proxy/cli",
             "proxy/debugging",
+            "proxy/error_diagnosis",
             "proxy/deploy",
             "proxy/health",
             "proxy/master_key_rotations",
@@ -519,6 +521,7 @@ const sidebars = {
             "providers/vertex_partner",
             "providers/vertex_self_deployed",
             "providers/vertex_image",
+            "providers/vertex_speech",
             "providers/vertex_batch",
             "providers/vertex_ocr",
           ]
@@ -624,6 +627,7 @@ const sidebars = {
         "providers/petals",
         "providers/publicai",
         "providers/predibase",
+        "providers/ragflow",
         "providers/recraft",
         "providers/replicate",
         {
@@ -655,6 +659,7 @@ const sidebars = {
         },
         "providers/xai",
         "providers/xinference",
+        "providers/zai",
       ],
     },
     {
@@ -816,10 +821,13 @@ const sidebars = {
               "Learn how to deploy + call models from different providers on LiteLLM",
             slug: "/project",
           },
-          items: [        
+          items: [
             "projects/smolagents",
             "projects/mini-swe-agent",
             "projects/openai-agents",
+            "projects/Google ADK",
+            "projects/Agent Lightning",
+            "projects/Harbor",
             "projects/Docq.AI",
             "projects/PDL",
             "projects/OpenInterpreter",

docs/my-website/src/pages/intro.md

@@ -1,47 +0,0 @@
----
-sidebar_position: 1
----
-
-# Tutorial Intro
-
-Let's discover **Docusaurus in less than 5 minutes**.
-
-## Getting Started
-
-Get started by **creating a new site**.
-
-Or **try Docusaurus immediately** with **[docusaurus.new](https://docusaurus.new)**.
-
-### What you'll need
-
-- [Node.js](https://nodejs.org/en/download/) version 16.14 or above:
-  - When installing Node.js, you are recommended to check all checkboxes related to dependencies.
-
-## Generate a new site
-
-Generate a new Docusaurus site using the **classic template**.
-
-The classic template will automatically be added to your project after you run the command:
-
-```bash
-npm init docusaurus@latest my-website classic
-```
-
-You can type this command into Command Prompt, Powershell, Terminal, or any other integrated terminal of your code editor.
-
-The command also installs all necessary dependencies you need to run Docusaurus.
-
-## Start your site
-
-Run the development server:
-
-```bash
-cd my-website
-npm run start
-```
-
-The `cd` command changes the directory you're working with. In order to work with your newly created Docusaurus site, you'll need to navigate the terminal there.
-
-The `npm run start` command builds your website locally and serves it through a development server, ready for you to view at http://localhost:3000/.
-
-Open `docs/intro.md` (this page) and edit some lines: the site **reloads automatically** and displays your changes.

docs/my-website/src/pages/tutorial-basics/_category_.json

@@ -1,8 +0,0 @@
-{
-  "label": "Tutorial - Basics",
-  "position": 2,
-  "link": {
-    "type": "generated-index",
-    "description": "5 minutes to learn the most important Docusaurus concepts."
-  }
-}

docs/my-website/src/pages/tutorial-basics/congratulations.md

@@ -1,23 +0,0 @@
----
-sidebar_position: 6
----
-
-# Congratulations!
-
-You have just learned the **basics of Docusaurus** and made some changes to the **initial template**.
-
-Docusaurus has **much more to offer**!
-
-Have **5 more minutes**? Take a look at **[versioning](../tutorial-extras/manage-docs-versions.md)** and **[i18n](../tutorial-extras/translate-your-site.md)**.
-
-Anything **unclear** or **buggy** in this tutorial? [Please report it!](https://github.com/facebook/docusaurus/discussions/4610)
-
-## What's next?
-
-- Read the [official documentation](https://docusaurus.io/)
-- Modify your site configuration with [`docusaurus.config.js`](https://docusaurus.io/docs/api/docusaurus-config)
-- Add navbar and footer items with [`themeConfig`](https://docusaurus.io/docs/api/themes/configuration)
-- Add a custom [Design and Layout](https://docusaurus.io/docs/styling-layout)
-- Add a [search bar](https://docusaurus.io/docs/search)
-- Find inspirations in the [Docusaurus showcase](https://docusaurus.io/showcase)
-- Get involved in the [Docusaurus Community](https://docusaurus.io/community/support)

docs/my-website/src/pages/tutorial-basics/create-a-blog-post.md

@@ -1,34 +0,0 @@
----
-sidebar_position: 3
----
-
-# Create a Blog Post
-
-Docusaurus creates a **page for each blog post**, but also a **blog index page**, a **tag system**, an **RSS** feed...
-
-## Create your first Post
-
-Create a file at `blog/2021-02-28-greetings.md`:
-
-```md title="blog/2021-02-28-greetings.md"
----
-slug: greetings
-title: Greetings!
-authors:
-  - name: Joel Marcey
-    title: Co-creator of Docusaurus 1
-    url: https://github.com/JoelMarcey
-    image_url: https://github.com/JoelMarcey.png
-  - name: Sébastien Lorber
-    title: Docusaurus maintainer
-    url: https://sebastienlorber.com
-    image_url: https://github.com/slorber.png
-tags: [greetings]
----
-
-Congratulations, you have made your first post!
-
-Feel free to play around and edit this post as much you like.
-```
-
-A new blog post is now available at [http://localhost:3000/blog/greetings](http://localhost:3000/blog/greetings).

docs/my-website/src/pages/tutorial-basics/create-a-document.md

@@ -1,57 +0,0 @@
----
-sidebar_position: 2
----
-
-# Create a Document
-
-Documents are **groups of pages** connected through:
-
-- a **sidebar**
-- **previous/next navigation**
-- **versioning**
-
-## Create your first Doc
-
-Create a Markdown file at `docs/hello.md`:
-
-```md title="docs/hello.md"
-# Hello
-
-This is my **first Docusaurus document**!
-```
-
-A new document is now available at [http://localhost:3000/docs/hello](http://localhost:3000/docs/hello).
-
-## Configure the Sidebar
-
-Docusaurus automatically **creates a sidebar** from the `docs` folder.
-
-Add metadata to customize the sidebar label and position:
-
-```md title="docs/hello.md" {1-4}
----
-sidebar_label: 'Hi!'
-sidebar_position: 3
----
-
-# Hello
-
-This is my **first Docusaurus document**!
-```
-
-It is also possible to create your sidebar explicitly in `sidebars.js`:
-
-```js title="sidebars.js"
-module.exports = {
-  tutorialSidebar: [
-    'intro',
-    // highlight-next-line
-    'hello',
-    {
-      type: 'category',
-      label: 'Tutorial',
-      items: ['tutorial-basics/create-a-document'],
-    },
-  ],
-};
-```

docs/my-website/src/pages/tutorial-basics/create-a-page.md

@@ -1,43 +0,0 @@
----
-sidebar_position: 1
----
-
-# Create a Page
-
-Add **Markdown or React** files to `src/pages` to create a **standalone page**:
-
-- `src/pages/index.js` → `localhost:3000/`
-- `src/pages/foo.md` → `localhost:3000/foo`
-- `src/pages/foo/bar.js` → `localhost:3000/foo/bar`
-
-## Create your first React Page
-
-Create a file at `src/pages/my-react-page.js`:
-
-```jsx title="src/pages/my-react-page.js"
-import React from 'react';
-import Layout from '@theme/Layout';
-
-export default function MyReactPage() {
-  return (
-    <Layout>
-      <h1>My React page</h1>
-      <p>This is a React page</p>
-    </Layout>
-  );
-}
-```
-
-A new page is now available at [http://localhost:3000/my-react-page](http://localhost:3000/my-react-page).
-
-## Create your first Markdown Page
-
-Create a file at `src/pages/my-markdown-page.md`:
-
-```mdx title="src/pages/my-markdown-page.md"
-# My Markdown page
-
-This is a Markdown page
-```
-
-A new page is now available at [http://localhost:3000/my-markdown-page](http://localhost:3000/my-markdown-page).

docs/my-website/src/pages/tutorial-basics/deploy-your-site.md

@@ -1,31 +0,0 @@
----
-sidebar_position: 5
----
-
-# Deploy your site
-
-Docusaurus is a **static-site-generator** (also called **[Jamstack](https://jamstack.org/)**).
-
-It builds your site as simple **static HTML, JavaScript and CSS files**.
-
-## Build your site
-
-Build your site **for production**:
-
-```bash
-npm run build
-```
-
-The static files are generated in the `build` folder.
-
-## Deploy your site
-
-Test your production build locally:
-
-```bash
-npm run serve
-```
-
-The `build` folder is now served at [http://localhost:3000/](http://localhost:3000/).
-
-You can now deploy the `build` folder **almost anywhere** easily, **for free** or very small cost (read the **[Deployment Guide](https://docusaurus.io/docs/deployment)**).

docs/my-website/src/pages/tutorial-basics/markdown-features.mdx

@@ -1,150 +0,0 @@
----
-sidebar_position: 4
----
-
-# Markdown Features
-
-Docusaurus supports **[Markdown](https://daringfireball.net/projects/markdown/syntax)** and a few **additional features**.
-
-## Front Matter
-
-Markdown documents have metadata at the top called [Front Matter](https://jekyllrb.com/docs/front-matter/):
-
-```text title="my-doc.md"
-// highlight-start
----
-id: my-doc-id
-title: My document title
-description: My document description
-slug: /my-custom-url
----
-// highlight-end
-
-## Markdown heading
-
-Markdown text with [links](./hello.md)
-```
-
-## Links
-
-Regular Markdown links are supported, using url paths or relative file paths.
-
-```md
-Let's see how to [Create a page](/create-a-page).
-```
-
-```md
-Let's see how to [Create a page](./create-a-page.md).
-```
-
-**Result:** Let's see how to [Create a page](./create-a-page.md).
-
-## Images
-
-Regular Markdown images are supported.
-
-You can use absolute paths to reference images in the static directory (`static/img/docusaurus.png`):
-
-```md
-![Docusaurus logo](/img/docusaurus.png)
-```
-
-![Docusaurus logo](/img/docusaurus.png)
-
-You can reference images relative to the current file as well. This is particularly useful to colocate images close to the Markdown files using them:
-
-```md
-![Docusaurus logo](./img/docusaurus.png)
-```
-
-## Code Blocks
-
-Markdown code blocks are supported with Syntax highlighting.
-
-    ```jsx title="src/components/HelloDocusaurus.js"
-    function HelloDocusaurus() {
-        return (
-            <h1>Hello, Docusaurus!</h1>
-        )
-    }
-    ```
-
-```jsx title="src/components/HelloDocusaurus.js"
-function HelloDocusaurus() {
-  return <h1>Hello, Docusaurus!</h1>;
-}
-```
-
-## Admonitions
-
-Docusaurus has a special syntax to create admonitions and callouts:
-
-    :::tip My tip
-
-    Use this awesome feature option
-
-    :::
-
-    :::danger Take care
-
-    This action is dangerous
-
-    :::
-
-:::tip My tip
-
-Use this awesome feature option
-
-:::
-
-:::danger Take care
-
-This action is dangerous
-
-:::
-
-## MDX and React Components
-
-[MDX](https://mdxjs.com/) can make your documentation more **interactive** and allows using any **React components inside Markdown**:
-
-```jsx
-export const Highlight = ({children, color}) => (
-  <span
-    style={{
-      backgroundColor: color,
-      borderRadius: '20px',
-      color: '#fff',
-      padding: '10px',
-      cursor: 'pointer',
-    }}
-    onClick={() => {
-      alert(`You clicked the color ${color} with label ${children}`)
-    }}>
-    {children}
-  </span>
-);
-
-This is <Highlight color="#25c2a0">Docusaurus green</Highlight> !
-
-This is <Highlight color="#1877F2">Facebook blue</Highlight> !
-```
-
-export const Highlight = ({children, color}) => (
-  <span
-    style={{
-      backgroundColor: color,
-      borderRadius: '20px',
-      color: '#fff',
-      padding: '10px',
-      cursor: 'pointer',
-    }}
-    onClick={() => {
-      alert(`You clicked the color ${color} with label ${children}`);
-    }}>
-    {children}
-  </span>
-);
-
-This is <Highlight color="#25c2a0">Docusaurus green</Highlight> !
-
-This is <Highlight color="#1877F2">Facebook blue</Highlight> !

docs/my-website/src/pages/tutorial-extras/_category_.json

@@ -1,7 +0,0 @@
-{
-  "label": "Tutorial - Extras",
-  "position": 3,
-  "link": {
-    "type": "generated-index"
-  }
-}

docs/my-website/src/pages/tutorial-extras/manage-docs-versions.md

@@ -1,55 +0,0 @@
----
-sidebar_position: 1
----
-
-# Manage Docs Versions
-
-Docusaurus can manage multiple versions of your docs.
-
-## Create a docs version
-
-Release a version 1.0 of your project:
-
-```bash
-npm run docusaurus docs:version 1.0
-```
-
-The `docs` folder is copied into `versioned_docs/version-1.0` and `versions.json` is created.
-
-Your docs now have 2 versions:
-
-- `1.0` at `http://localhost:3000/docs/` for the version 1.0 docs
-- `current` at `http://localhost:3000/docs/next/` for the **upcoming, unreleased docs**
-
-## Add a Version Dropdown
-
-To navigate seamlessly across versions, add a version dropdown.
-
-Modify the `docusaurus.config.js` file:
-
-```js title="docusaurus.config.js"
-module.exports = {
-  themeConfig: {
-    navbar: {
-      items: [
-        // highlight-start
-        {
-          type: 'docsVersionDropdown',
-        },
-        // highlight-end
-      ],
-    },
-  },
-};
-```
-
-The docs version dropdown appears in your navbar:
-
-![Docs Version Dropdown](./img/docsVersionDropdown.png)
-
-## Update an existing version
-
-It is possible to edit versioned docs in their respective folder:
-
-- `versioned_docs/version-1.0/hello.md` updates `http://localhost:3000/docs/hello`
-- `docs/hello.md` updates `http://localhost:3000/docs/next/hello`

docs/my-website/src/pages/tutorial-extras/translate-your-site.md

@@ -1,88 +0,0 @@
----
-sidebar_position: 2
----
-
-# Translate your site
-
-Let's translate `docs/intro.md` to French.
-
-## Configure i18n
-
-Modify `docusaurus.config.js` to add support for the `fr` locale:
-
-```js title="docusaurus.config.js"
-module.exports = {
-  i18n: {
-    defaultLocale: 'en',
-    locales: ['en', 'fr'],
-  },
-};
-```
-
-## Translate a doc
-
-Copy the `docs/intro.md` file to the `i18n/fr` folder:
-
-```bash
-mkdir -p i18n/fr/docusaurus-plugin-content-docs/current/
-
-cp docs/intro.md i18n/fr/docusaurus-plugin-content-docs/current/intro.md
-```
-
-Translate `i18n/fr/docusaurus-plugin-content-docs/current/intro.md` in French.
-
-## Start your localized site
-
-Start your site on the French locale:
-
-```bash
-npm run start -- --locale fr
-```
-
-Your localized site is accessible at [http://localhost:3000/fr/](http://localhost:3000/fr/) and the `Getting Started` page is translated.
-
-:::caution
-
-In development, you can only use one locale at a same time.
-
-:::
-
-## Add a Locale Dropdown
-
-To navigate seamlessly across languages, add a locale dropdown.
-
-Modify the `docusaurus.config.js` file:
-
-```js title="docusaurus.config.js"
-module.exports = {
-  themeConfig: {
-    navbar: {
-      items: [
-        // highlight-start
-        {
-          type: 'localeDropdown',
-        },
-        // highlight-end
-      ],
-    },
-  },
-};
-```
-
-The locale dropdown now appears in your navbar:
-
-![Locale Dropdown](./img/localeDropdown.png)
-
-## Build your localized site
-
-Build your site for a specific locale:
-
-```bash
-npm run build -- --locale fr
-```
-
-Or build your site to include all the locales at once:
-
-```bash
-npm run build
-```

enterprise/litellm_enterprise/proxy/enterprise_routes.py

@@ -5,14 +5,10 @@ from litellm_enterprise.enterprise_callbacks.send_emails.endpoints import (
 )
 
 from .audit_logging_endpoints import router as audit_logging_router
-from .guardrails.endpoints import router as guardrails_router
 from .management_endpoints import management_endpoints_router
 from .utils import _should_block_robots
-from .vector_stores.endpoints import router as vector_stores_router
 
 router = APIRouter()
-router.include_router(vector_stores_router)
-router.include_router(guardrails_router)
 router.include_router(email_events_router)
 router.include_router(audit_logging_router)
 router.include_router(management_endpoints_router)

litellm/__init__.py

@@ -1125,7 +1125,7 @@ from .llms.openrouter.chat.transformation import OpenrouterConfig
 from .llms.datarobot.chat.transformation import DataRobotConfig
 from .llms.anthropic.chat.transformation import AnthropicConfig
 from .llms.anthropic.common_utils import AnthropicModelInfo
-from .llms.azure.anthropic.transformation import AzureAnthropicConfig
+from .llms.azure_ai.anthropic.transformation import AzureAnthropicConfig
 from .llms.groq.stt.transformation import GroqSTTConfig
 from .llms.anthropic.completion.transformation import AnthropicTextConfig
 from .llms.triton.completion.transformation import TritonConfig
@@ -1234,6 +1234,9 @@ from .llms.bedrock.chat.invoke_transformations.amazon_mistral_transformation imp
 from .llms.bedrock.chat.invoke_transformations.amazon_titan_transformation import (
     AmazonTitanConfig,
 )
+from .llms.bedrock.chat.invoke_transformations.amazon_twelvelabs_pegasus_transformation import (
+    AmazonTwelveLabsPegasusConfig,
+)
 from .llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
     AmazonInvokeConfig,
 )
@@ -1257,6 +1260,9 @@ from .llms.bedrock.embed.cohere_transformation import BedrockCohereEmbeddingConf
 from .llms.bedrock.embed.twelvelabs_marengo_transformation import (
     TwelveLabsMarengoEmbeddingConfig,
 )
+from .llms.bedrock.embed.amazon_nova_transformation import (
+    AmazonNovaEmbeddingConfig,
+)
 from .llms.openai.openai import OpenAIConfig, MistralEmbeddingConfig
 from .llms.openai.image_variations.transformation import OpenAIImageVariationConfig
 from .llms.deepinfra.chat.transformation import DeepInfraConfig
@@ -1371,6 +1377,7 @@ from .llms.github_copilot.chat.transformation import GithubCopilotConfig
 from .llms.github_copilot.responses.transformation import (
     GithubCopilotResponsesAPIConfig,
 )
+from .llms.github_copilot.embedding.transformation import GithubCopilotEmbeddingConfig
 from .llms.nebius.chat.transformation import NebiusConfig
 from .llms.wandb.chat.transformation import WandbConfig
 from .llms.dashscope.chat.transformation import DashScopeChatConfig
@@ -1380,6 +1387,7 @@ from .llms.docker_model_runner.chat.transformation import DockerModelRunnerChatC
 from .llms.v0.chat.transformation import V0ChatConfig
 from .llms.oci.chat.transformation import OCIChatConfig
 from .llms.morph.chat.transformation import MorphChatConfig
+from .llms.ragflow.chat.transformation import RAGFlowConfig
 from .llms.lambda_ai.chat.transformation import LambdaAIChatConfig
 from .llms.hyperbolic.chat.transformation import HyperbolicChatConfig
 from .llms.vercel_ai_gateway.chat.transformation import VercelAIGatewayConfig

litellm/batches/batch_utils.py

@@ -14,8 +14,7 @@ from litellm.utils import token_counter
 
 async def calculate_batch_cost_and_usage(
     file_content_dictionary: List[dict],
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai"],
-    model_name: Optional[str] = None,
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm"],
 ) -> Tuple[float, Usage, List[str]]:
     """
     Calculate the cost and usage of a batch
@@ -37,8 +36,7 @@ async def calculate_batch_cost_and_usage(
 
 async def _handle_completed_batch(
     batch: Batch,
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai"],
-    model_name: Optional[str] = None,
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm"],
 ) -> Tuple[float, Usage, List[str]]:
     """Helper function to process a completed batch and handle logging"""
     # Get batch results
@@ -84,8 +82,7 @@ def _get_batch_models_from_file_content(
 
 def _batch_cost_calculator(
     file_content_dictionary: List[dict],
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
-    model_name: Optional[str] = None,
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm"] = "openai",
 ) -> float:
     """
     Calculate the cost of a batch based on the output file id
@@ -186,7 +183,7 @@ def calculate_vertex_ai_batch_cost_and_usage(
 
 async def _get_batch_output_file_content_as_dictionary(
     batch: Batch,
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm"] = "openai",
 ) -> List[dict]:
     """
     Get the batch output file content as a list of dictionaries
@@ -225,7 +222,7 @@ def _get_file_content_as_dictionary(file_content: bytes) -> List[dict]:
 
 def _get_batch_job_cost_from_file_content(
     file_content_dictionary: List[dict],
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm"] = "openai",
 ) -> float:
     """
     Get the cost of a batch job from the file content
@@ -253,8 +250,7 @@ def _get_batch_job_cost_from_file_content(
 
 def _get_batch_job_total_usage_from_file_content(
     file_content_dictionary: List[dict],
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
-    model_name: Optional[str] = None,
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm"] = "openai",
 ) -> Usage:
     """
     Get the tokens of a batch job from the file content

litellm/batches/main.py

@@ -23,6 +23,7 @@ import litellm
 from litellm._logging import verbose_logger
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.llms.azure.batches.handler import AzureBatchesAPI
+from litellm.llms.bedrock.batches.handler import BedrockBatchesHandler
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
 from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
 from litellm.llms.openai.openai import OpenAIBatchesAPI
@@ -35,7 +36,11 @@ from litellm.types.llms.openai import (
     RetrieveBatchRequest,
 )
 from litellm.types.router import GenericLiteLLMParams
-from litellm.types.utils import LiteLLMBatch, LlmProviders
+from litellm.types.utils import (
+    OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS,
+    LiteLLMBatch,
+    LlmProviders,
+)
 from litellm.utils import (
     ProviderConfigManager,
     client,
@@ -100,7 +105,7 @@ async def acreate_batch(
     completion_window: Literal["24h"],
     endpoint: Literal["/v1/chat/completions", "/v1/embeddings", "/v1/completions"],
     input_file_id: str,
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock"] = "openai",
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm"] = "openai",
     metadata: Optional[Dict[str, str]] = None,
     extra_headers: Optional[Dict[str, str]] = None,
     extra_body: Optional[Dict[str, str]] = None,
@@ -148,7 +153,7 @@ def create_batch(
     completion_window: Literal["24h"],
     endpoint: Literal["/v1/chat/completions", "/v1/embeddings", "/v1/completions"],
     input_file_id: str,
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock"] = "openai",
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm"] = "openai",
     metadata: Optional[Dict[str, str]] = None,
     extra_headers: Optional[Dict[str, str]] = None,
     extra_body: Optional[Dict[str, str]] = None,
@@ -235,7 +240,7 @@ def create_batch(
             )
             return response
         api_base: Optional[str] = None
-        if custom_llm_provider == "openai":
+        if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
             # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
             api_base = (
                 optional_params.api_base
@@ -350,7 +355,7 @@ def create_batch(
 @client
 async def aretrieve_batch(
     batch_id: str,
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock"] = "openai",
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm"] = "openai",
     metadata: Optional[Dict[str, str]] = None,
     extra_headers: Optional[Dict[str, str]] = None,
     extra_body: Optional[Dict[str, str]] = None,
@@ -396,10 +401,10 @@ def _handle_retrieve_batch_providers_without_provider_config(
     litellm_params: dict,
     _retrieve_batch_request: RetrieveBatchRequest,
     _is_async: bool,
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock"] = "openai",
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm"] = "openai",
 ):
     api_base: Optional[str] = None
-    if custom_llm_provider == "openai":
+    if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
         # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
         api_base = (
             optional_params.api_base
@@ -512,7 +517,7 @@ def _handle_retrieve_batch_providers_without_provider_config(
 @client
 def retrieve_batch(
     batch_id: str,
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock"] = "openai",
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm"] = "openai",
     metadata: Optional[Dict[str, str]] = None,
     extra_headers: Optional[Dict[str, str]] = None,
     extra_body: Optional[Dict[str, str]] = None,
@@ -576,7 +581,7 @@ def retrieve_batch(
             async_kwargs = kwargs.copy()
             async_kwargs.pop("aws_region_name", None)
 
-            return _handle_async_invoke_status(
+            return BedrockBatchesHandler._handle_async_invoke_status(
                 batch_id=batch_id,
                 aws_region_name=kwargs.get("aws_region_name", "us-east-1"),
                 logging_obj=litellm_logging_obj,
@@ -644,7 +649,7 @@ def retrieve_batch(
 async def alist_batches(
     after: Optional[str] = None,
     limit: Optional[int] = None,
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
+    custom_llm_provider: Literal["openai", "azure", "hosted_vllm", "vertex_ai"] = "openai",
     metadata: Optional[Dict[str, str]] = None,
     extra_headers: Optional[Dict[str, str]] = None,
     extra_body: Optional[Dict[str, str]] = None,
@@ -687,7 +692,7 @@ async def alist_batches(
 def list_batches(
     after: Optional[str] = None,
     limit: Optional[int] = None,
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
+    custom_llm_provider: Literal["openai", "azure", "hosted_vllm", "vertex_ai"] = "openai",
     extra_headers: Optional[Dict[str, str]] = None,
     extra_body: Optional[Dict[str, str]] = None,
     **kwargs,
@@ -727,7 +732,7 @@ def list_batches(
             timeout = 600.0
 
         _is_async = kwargs.pop("alist_batches", False) is True
-        if custom_llm_provider == "openai":
+        if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
             # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
             api_base = (
                 optional_params.api_base
@@ -928,7 +933,7 @@ def cancel_batch(
 
         _is_async = kwargs.pop("acancel_batch", False) is True
         api_base: Optional[str] = None
-        if custom_llm_provider == "openai":
+        if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
             api_base = (
                 optional_params.api_base
                 or litellm.api_base
@@ -1043,30 +1048,49 @@ def _handle_async_invoke_status(
         )
 
         # Transform response to a LiteLLMBatch object
+        from litellm.types.llms.openai import BatchJobStatus
         from litellm.types.utils import LiteLLMBatch
 
+        # Normalize status to lowercase (AWS returns 'Completed', 'Failed', etc.)
+        aws_status_raw = status_response.get("status", "")
+        aws_status_lower = aws_status_raw.lower()
+        # Map AWS status values to LiteLLM expected values
+        status_mapping: dict[str, BatchJobStatus] = {
+            "completed": "completed",
+            "failed": "failed",
+            "inprogress": "in_progress",
+            "in_progress": "in_progress",
+        }
+        normalized_status: BatchJobStatus = status_mapping.get(aws_status_lower, "failed")  # Default to "failed" if unknown status
+
+        # Get output S3 URI safely
+        output_s3_uri = ""
+        try:
+            output_s3_uri = status_response["outputDataConfig"]["s3OutputDataConfig"]["s3Uri"]
+        except (KeyError, TypeError):
+            pass
+        
+        # Use BedrockBatchesConfig's timestamp parsing method (expects raw AWS status string)
+        import time
+
+        from litellm.llms.bedrock.batches.transformation import BedrockBatchesConfig
+        created_at, in_progress_at, completed_at, failed_at, _, _ = BedrockBatchesConfig()._parse_timestamps_and_status(status_response, aws_status_raw)
         result = LiteLLMBatch(
             id=status_response["invocationArn"],
             object="batch",
-            status=status_response["status"],
-            created_at=status_response["submitTime"],
-            in_progress_at=status_response["lastModifiedTime"],
-            completed_at=status_response.get("endTime"),
-            failed_at=(
-                status_response.get("endTime")
-                if status_response["status"] == "failed"
-                else None
-            ),
+            status=normalized_status,
+            created_at=created_at or int(time.time()),  # Provide default timestamp if None
+            in_progress_at=in_progress_at,
+            completed_at=completed_at,
+            failed_at=failed_at,
             request_counts=BatchRequestCounts(
                 total=1,
-                completed=1 if status_response["status"] == "completed" else 0,
-                failed=1 if status_response["status"] == "failed" else 0,
+                completed=1 if normalized_status == "completed" else 0,
+                failed=1 if normalized_status == "failed" else 0,
             ),
             metadata=dict(
                 **{
-                    "output_file_id": status_response["outputDataConfig"][
-                        "s3OutputDataConfig"
-                    ]["s3Uri"],
+                    "output_file_id": output_s3_uri,
                     "failure_message": status_response.get("failureMessage") or "",
                     "model_arn": status_response["modelArn"],
                 }

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -148,7 +148,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             if role == "system":
                 # Extract system message as instructions
                 if isinstance(content, str):
-                    instructions = content
+                    if instructions:
+                        # Concatenate multiple system prompts with a space
+                        instructions = f"{instructions} {content}"
+                    else:
+                        instructions = content
                 else:
                     input_items.append(
                         {

litellm/constants.py

@@ -1,4 +1,5 @@
 import os
+import sys
 from typing import List, Literal
 
 DEFAULT_HEALTH_CHECK_PROMPT = str(
@@ -103,6 +104,12 @@ _DEFAULT_TTL_FOR_HTTPX_CLIENTS = 3600  # 1 hour, re-use the same httpx client fo
 AIOHTTP_CONNECTOR_LIMIT = int(os.getenv("AIOHTTP_CONNECTOR_LIMIT", 0))
 AIOHTTP_KEEPALIVE_TIMEOUT = int(os.getenv("AIOHTTP_KEEPALIVE_TIMEOUT", 120))
 AIOHTTP_TTL_DNS_CACHE = int(os.getenv("AIOHTTP_TTL_DNS_CACHE", 300))
+# enable_cleanup_closed is only needed for Python versions with the SSL leak bug
+# Fixed in Python 3.12.7+ and 3.13.1+ (see https://github.com/python/cpython/pull/118960)
+# Reference: https://github.com/aio-libs/aiohttp/blob/master/aiohttp/connector.py#L74-L78
+AIOHTTP_NEEDS_CLEANUP_CLOSED = (
+    (3, 13, 0) <= sys.version_info < (3, 13, 1) or sys.version_info < (3, 12, 7)
+)
 
 # WebSocket constants
 # Default to None (unlimited) to match OpenAI's official agents SDK behavior
@@ -255,6 +262,9 @@ TOGETHER_AI_EMBEDDING_350_M = int(os.getenv("TOGETHER_AI_EMBEDDING_350_M", 350))
 QDRANT_SCALAR_QUANTILE = float(os.getenv("QDRANT_SCALAR_QUANTILE", 0.99))
 QDRANT_VECTOR_SIZE = int(os.getenv("QDRANT_VECTOR_SIZE", 1536))
 CACHED_STREAMING_CHUNK_DELAY = float(os.getenv("CACHED_STREAMING_CHUNK_DELAY", 0.02))
+AUDIO_SPEECH_CHUNK_SIZE = int(
+    os.getenv("AUDIO_SPEECH_CHUNK_SIZE", 8192)
+)  # chunk_size for audio speech streaming. Balance between latency and memory usage
 MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB = int(
     os.getenv("MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB", 512)
 )
@@ -277,10 +287,16 @@ REDACTED_BY_LITELM_STRING = "REDACTED_BY_LITELM"
 MAX_LANGFUSE_INITIALIZED_CLIENTS = int(
     os.getenv("MAX_LANGFUSE_INITIALIZED_CLIENTS", 50)
 )
-LOGGING_WORKER_CONCURRENCY = int(os.getenv("LOGGING_WORKER_CONCURRENCY", 100)) # Must be above 0
+LOGGING_WORKER_CONCURRENCY = int(
+    os.getenv("LOGGING_WORKER_CONCURRENCY", 100)
+)  # Must be above 0
 LOGGING_WORKER_MAX_QUEUE_SIZE = int(os.getenv("LOGGING_WORKER_MAX_QUEUE_SIZE", 50_000))
-LOGGING_WORKER_MAX_TIME_PER_COROUTINE = float(os.getenv("LOGGING_WORKER_MAX_TIME_PER_COROUTINE", 20.0))
-LOGGING_WORKER_CLEAR_PERCENTAGE = int(os.getenv("LOGGING_WORKER_CLEAR_PERCENTAGE", 50))  # Percentage of queue to clear (default: 50%)
+LOGGING_WORKER_MAX_TIME_PER_COROUTINE = float(
+    os.getenv("LOGGING_WORKER_MAX_TIME_PER_COROUTINE", 20.0)
+)
+LOGGING_WORKER_CLEAR_PERCENTAGE = int(
+    os.getenv("LOGGING_WORKER_CLEAR_PERCENTAGE", 50)
+)  # Percentage of queue to clear (default: 50%)
 MAX_ITERATIONS_TO_CLEAR_QUEUE = int(os.getenv("MAX_ITERATIONS_TO_CLEAR_QUEUE", 200))
 MAX_TIME_TO_CLEAR_QUEUE = float(os.getenv("MAX_TIME_TO_CLEAR_QUEUE", 5.0))
 LOGGING_WORKER_AGGRESSIVE_CLEAR_COOLDOWN_SECONDS = float(
@@ -554,6 +570,7 @@ openai_compatible_providers: List = [
     "perplexity",
     "xinference",
     "xai",
+    "zai",
     "together_ai",
     "fireworks_ai",
     "empower",
@@ -584,6 +601,7 @@ openai_compatible_providers: List = [
     "cometapi",
     "clarifai",
     "docker_model_runner",
+    "ragflow",
 ]
 openai_text_completion_compatible_providers: List = (
     [  # providers that support `/v1/completions`
@@ -856,12 +874,15 @@ BEDROCK_INVOKE_PROVIDERS_LITERAL = Literal[
     "nova",
     "deepseek_r1",
     "qwen3",
+    "twelvelabs",
+    "openai",
 ]
 
 BEDROCK_EMBEDDING_PROVIDERS_LITERAL = Literal[
     "cohere",
     "amazon",
     "twelvelabs",
+    "nova",
 ]
 
 BEDROCK_CONVERSE_MODELS = [
@@ -904,6 +925,9 @@ BEDROCK_CONVERSE_MODELS = [
     "meta.llama3-2-3b-instruct-v1:0",
     "meta.llama3-2-11b-instruct-v1:0",
     "meta.llama3-2-90b-instruct-v1:0",
+    "amazon.nova-lite-v1:0",
+    "amazon.nova-2-lite-v1:0",
+    "amazon.nova-pro-v1:0",
 ]
 
 
@@ -922,6 +946,7 @@ cohere_embedding_models: set = set(
 bedrock_embedding_models: set = set(
     [
         "amazon.titan-embed-text-v1",
+        "amazon.nova-2-multimodal-embeddings-v1:0",
         "cohere.embed-english-v3",
         "cohere.embed-multilingual-v3",
         "cohere.embed-v4:0",

litellm/files/main.py

@@ -30,7 +30,10 @@ from litellm.types.llms.openai import (
     OpenAIFileObject,
 )
 from litellm.types.router import *
-from litellm.types.utils import LlmProviders
+from litellm.types.utils import (
+    OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS,
+    LlmProviders,
+)
 from litellm.utils import (
     ProviderConfigManager,
     client,
@@ -51,7 +54,7 @@ vertex_ai_files_instance = VertexAIFilesHandler()
 async def acreate_file(
     file: FileTypes,
     purpose: Literal["assistants", "batch", "fine-tune"],
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock"] = "openai",
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm"] = "openai",
     extra_headers: Optional[Dict[str, str]] = None,
     extra_body: Optional[Dict[str, str]] = None,
     **kwargs,
@@ -95,9 +98,7 @@ async def acreate_file(
 def create_file(
     file: FileTypes,
     purpose: Literal["assistants", "batch", "fine-tune"],
-    custom_llm_provider: Optional[
-        Literal["openai", "azure", "vertex_ai", "bedrock"]
-    ] = None,
+    custom_llm_provider: Optional[Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm"]] = None,
     extra_headers: Optional[Dict[str, str]] = None,
     extra_body: Optional[Dict[str, str]] = None,
     **kwargs,
@@ -165,7 +166,7 @@ def create_file(
                 ),
                 timeout=timeout,
             )
-        elif custom_llm_provider == "openai":
+        elif custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
             # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
             api_base = (
                 optional_params.api_base
@@ -276,7 +277,7 @@ def create_file(
 @client
 async def afile_retrieve(
     file_id: str,
-    custom_llm_provider: Literal["openai", "azure"] = "openai",
+    custom_llm_provider: Literal["openai", "azure", "hosted_vllm"] = "openai",
     extra_headers: Optional[Dict[str, str]] = None,
     extra_body: Optional[Dict[str, str]] = None,
     **kwargs,
@@ -317,7 +318,7 @@ async def afile_retrieve(
 @client
 def file_retrieve(
     file_id: str,
-    custom_llm_provider: Literal["openai", "azure"] = "openai",
+    custom_llm_provider: Literal["openai", "azure", "hosted_vllm"] = "openai",
     extra_headers: Optional[Dict[str, str]] = None,
     extra_body: Optional[Dict[str, str]] = None,
     **kwargs,
@@ -347,7 +348,7 @@ def file_retrieve(
 
         _is_async = kwargs.pop("is_async", False) is True
 
-        if custom_llm_provider == "openai":
+        if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
             # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
             api_base = (
                 optional_params.api_base
@@ -514,7 +515,7 @@ def file_delete(
         elif timeout is None:
             timeout = 600.0
         _is_async = kwargs.pop("is_async", False) is True
-        if custom_llm_provider == "openai":
+        if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
             # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
             api_base = (
                 optional_params.api_base
@@ -670,7 +671,7 @@ def file_list(
             timeout = 600.0
 
         _is_async = kwargs.pop("is_async", False) is True
-        if custom_llm_provider == "openai":
+        if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
             # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
             api_base = (
                 optional_params.api_base
@@ -754,7 +755,7 @@ def file_list(
 @client
 async def afile_content(
     file_id: str,
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm"] = "openai",
     extra_headers: Optional[Dict[str, str]] = None,
     extra_body: Optional[Dict[str, str]] = None,
     **kwargs,
@@ -799,7 +800,7 @@ def file_content(
     file_id: str,
     model: Optional[str] = None,
     custom_llm_provider: Optional[
-        Union[Literal["openai", "azure", "vertex_ai"], str]
+        Union[Literal["openai", "azure", "vertex_ai", "hosted_vllm"], str]
     ] = None,
     extra_headers: Optional[Dict[str, str]] = None,
     extra_body: Optional[Dict[str, str]] = None,
@@ -846,7 +847,7 @@ def file_content(
 
         _is_async = kwargs.pop("afile_content", False) is True
 
-        if custom_llm_provider == "openai":
+        if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
             # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
             api_base = (
                 optional_params.api_base

litellm/integrations/SlackAlerting/budget_alert_types.py

@@ -50,6 +50,14 @@ class TeamBudgetAlert(BaseBudgetAlertType):
         return user_info.team_id or "default_id"
 
 
+class OrganizationBudgetAlert(BaseBudgetAlertType):
+    def get_event_message(self) -> str:
+        return "Organization Budget: "
+
+    def get_id(self, user_info: CallInfo) -> str:
+        return user_info.organization_id or "default_id"
+
+
 class TokenBudgetAlert(BaseBudgetAlertType):
     def get_event_message(self) -> str:
         return "Key Budget: "
@@ -72,6 +80,7 @@ def get_budget_alert_type(
         "soft_budget",
         "user_budget",
         "team_budget",
+        "organization_budget",
         "proxy_budget",
         "projected_limit_exceeded",
     ],
@@ -83,6 +92,7 @@ def get_budget_alert_type(
         "soft_budget": SoftBudgetAlert(),
         "user_budget": UserBudgetAlert(),
         "team_budget": TeamBudgetAlert(),
+        "organization_budget": OrganizationBudgetAlert(),
         "token_budget": TokenBudgetAlert(),
         "projected_limit_exceeded": ProjectedLimitExceededAlert(),
     }

litellm/integrations/SlackAlerting/slack_alerting.py

@@ -134,19 +134,25 @@ class SlackAlerting(CustomBatchLogger):
         if llm_router is not None:
             self.llm_router = llm_router
 
-    def _prepare_outage_value_for_cache(self, outage_value: Union[dict, ProviderRegionOutageModel, OutageModel]) -> dict:
+    def _prepare_outage_value_for_cache(
+        self, outage_value: Union[dict, ProviderRegionOutageModel, OutageModel]
+    ) -> dict:
         """
         Helper method to prepare outage value for Redis caching.
         Converts set objects to lists for JSON serialization.
         """
         # Convert to dict for processing
         cache_value = dict(outage_value)
-        
-        if "deployment_ids" in cache_value and isinstance(cache_value["deployment_ids"], set):
+
+        if "deployment_ids" in cache_value and isinstance(
+            cache_value["deployment_ids"], set
+        ):
             cache_value["deployment_ids"] = list(cache_value["deployment_ids"])
         return cache_value
 
-    def _restore_outage_value_from_cache(self, outage_value: Optional[dict]) -> Optional[dict]:
+    def _restore_outage_value_from_cache(
+        self, outage_value: Optional[dict]
+    ) -> Optional[dict]:
         """
         Helper method to restore outage value after retrieving from cache.
         Converts list objects back to sets for proper handling.
@@ -528,6 +534,7 @@ class SlackAlerting(CustomBatchLogger):
             "soft_budget",
             "user_budget",
             "team_budget",
+            "organization_budget",
             "proxy_budget",
             "projected_limit_exceeded",
         ],
@@ -1338,7 +1345,7 @@ Model Info:
             subject=email_event["subject"],
             html=email_event["html"],
         )
-        if webhook_event.event_group == "team":
+        if webhook_event.event_group == Litellm_EntityType.TEAM:
             from litellm.integrations.email_alerting import send_team_budget_alert
 
             await send_team_budget_alert(webhook_event=webhook_event)
@@ -1399,7 +1406,7 @@ Model Info:
         current_time = datetime.now().strftime("%H:%M:%S")
         _proxy_base_url = os.getenv("PROXY_BASE_URL", None)
         # Use .name if it's an enum, otherwise use as is
-        alert_type_name = getattr(alert_type, 'name', alert_type)
+        alert_type_name = getattr(alert_type, "name", alert_type)
         alert_type_formatted = f"Alert type: `{alert_type_name}`"
         if alert_type == "daily_reports" or alert_type == "new_model_added":
             formatted_message = alert_type_formatted + message

litellm/integrations/custom_guardrail.py

@@ -1,5 +1,16 @@
 from datetime import datetime
-from typing import Any, Dict, List, Optional, Type, Union, get_args
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Dict,
+    List,
+    Literal,
+    Optional,
+    Tuple,
+    Type,
+    Union,
+    get_args,
+)
 
 from litellm._logging import verbose_logger
 from litellm.caching import DualCache
@@ -9,7 +20,6 @@ from litellm.types.guardrails import (
     GuardrailEventHooks,
     LitellmParams,
     Mode,
-    PiiEntityType,
 )
 from litellm.types.llms.openai import AllMessageValues
 from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel
@@ -20,6 +30,8 @@ from litellm.types.utils import (
     StandardLoggingGuardrailInformation,
 )
 
+if TYPE_CHECKING:
+    from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 dc = DualCache()
 
 
@@ -437,30 +449,31 @@ class CustomGuardrail(CustomLogger):
 
     async def apply_guardrail(
         self,
-        text: str,
-        language: Optional[str] = None,
-        entities: Optional[List[PiiEntityType]] = None,
-        request_data: Optional[dict] = None,
-    ) -> str:
+        texts: List[str],
+        request_data: dict,
+        input_type: Literal["request", "response"],
+        logging_obj: Optional["LiteLLMLoggingObj"] = None,
+        images: Optional[List[str]] = None,
+    ) -> Tuple[List[str], Optional[List[str]]]:
         """
         Apply your guardrail logic to the given text
 
         Args:
-            text: The text to apply the guardrail to
-            language: The language of the text
-            entities: The entities to mask, optional
-            request_data: The request data dictionary to store guardrail metadata
+            texts: The texts to apply the guardrail to
+            images: The images to apply the guardrail to
+            request_data: The request data dictionary - containing user api key metadata (e.g. user_id, team_id, etc.)
+            input_type: The type of input to apply the guardrail to - "request" or "response"
 
         Any of the custom guardrails can override this method to provide custom guardrail logic
 
-        Returns the text with the guardrail applied
+        Returns the texts with the guardrail applied and the images with the guardrail applied (if any)
 
         Raises:
             Exception:
                 - If the guardrail raises an exception
 
         """
-        return text
+        return texts, images
 
     def _process_response(
         self,

litellm/integrations/datadog/datadog.py

@@ -65,11 +65,11 @@ class DataDogLogger(
         `DD_SITE` - your datadog site, example = `"us5.datadoghq.com"`
 
         Optional environment variables (DataDog Agent):
-        `DD_AGENT_HOST` - hostname or IP of DataDog agent, example = `"localhost"`
-        `DD_AGENT_PORT` - port of DataDog agent (default: 10518 for logs)
+        `LITELLM_DD_AGENT_HOST` - hostname or IP of DataDog agent, example = `"localhost"`
+        `LITELLM_DD_AGENT_PORT` - port of DataDog agent (default: 10518 for logs)
         
-        Note: If DD_AGENT_HOST is set, logs will be sent to the agent instead of directly to DataDog API.
-        In this case, DD_API_KEY and DD_SITE are not required (agent handles authentication).
+        Note: We use LITELLM_DD_AGENT_HOST instead of DD_AGENT_HOST to avoid conflicts
+        with ddtrace which automatically sets DD_AGENT_HOST for APM tracing.
         """
         try:
             verbose_logger.debug("Datadog: in init datadog logger")
@@ -85,7 +85,8 @@ class DataDogLogger(
             )
             
             # Configure DataDog endpoint (Agent or Direct API)
-            dd_agent_host = os.getenv("DD_AGENT_HOST")
+            # Use LITELLM_DD_AGENT_HOST to avoid conflicts with ddtrace's DD_AGENT_HOST
+            dd_agent_host = os.getenv("LITELLM_DD_AGENT_HOST")
             if dd_agent_host:
                 self._configure_dd_agent(dd_agent_host=dd_agent_host)
             else:
@@ -127,7 +128,7 @@ class DataDogLogger(
         Args:
             dd_agent_host: Hostname or IP of DataDog agent
         """
-        dd_agent_port = os.getenv("DD_AGENT_PORT", "10518")  # default port for logs
+        dd_agent_port = os.getenv("LITELLM_DD_AGENT_PORT", "10518")  # default port for logs
         self.intake_url = f"http://{dd_agent_host}:{dd_agent_port}/api/v2/logs"
         self.DD_API_KEY = os.getenv("DD_API_KEY")  # Optional when using agent
         verbose_logger.debug(f"Datadog: Using DD Agent at {self.intake_url}")

litellm/litellm_core_utils/README.md

@@ -9,4 +9,5 @@ Core files:
 - `default_encoding.py`: code for loading the default encoding (tiktoken)
 - `get_llm_provider_logic.py`: code for inferring the LLM provider from a given model name. 
 - `duration_parser.py`: code for parsing durations - e.g. "1d", "1mo", "10s"
+- `api_route_to_call_types.py`: mapping of API routes to their corresponding CallTypes (e.g., `/chat/completions` -> [acompletion, completion])
 

litellm/litellm_core_utils/api_route_to_call_types.py

@@ -0,0 +1,38 @@
+"""
+Dictionary mapping API routes to their corresponding CallTypes in LiteLLM.
+
+This dictionary maps each API endpoint to the CallTypes that can be used for that route.
+Each route can have both async (prefixed with 'a') and sync call types.
+"""
+
+from litellm.types.utils import API_ROUTE_TO_CALL_TYPES, CallTypes
+
+
+def get_call_types_for_route(route: str) -> list:
+    """
+    Get the list of CallTypes for a given API route.
+
+    Args:
+        route: API route path (e.g., "/chat/completions")
+
+    Returns:
+        List of CallTypes for that route, or empty list if route not found
+    """
+    return API_ROUTE_TO_CALL_TYPES.get(route, [])
+
+
+def get_routes_for_call_type(call_type: CallTypes) -> list:
+    """
+    Get all routes that use a specific CallType.
+
+    Args:
+        call_type: The CallType to search for
+
+    Returns:
+        List of routes that use this CallType
+    """
+    routes = []
+    for route, types in API_ROUTE_TO_CALL_TYPES.items():
+        if call_type in types:
+            routes.append(route)
+    return routes

litellm/litellm_core_utils/exception_mapping_utils.py

@@ -77,6 +77,7 @@ class ExceptionCheckers:
             "model's maximum context limit",
             "is longer than the model's context length",
             "input tokens exceed the configured limit",
+            "`inputs` tokens + `max_new_tokens` must be",
         ]
         for substring in known_exception_substrings:
             if substring in _error_str_lowercase:

litellm/litellm_core_utils/get_llm_provider_logic.py

@@ -22,17 +22,16 @@ def _is_non_openai_azure_model(model: str) -> bool:
     return False
 
 
-def _is_azure_anthropic_model(model: str) -> Optional[str]:
+def _is_azure_claude_model(model: str) -> bool:
+    """
+    Check if a model name contains 'claude' (case-insensitive).
+    Used to detect Claude models that need Anthropic-specific handling.
+    """
     try:
-        model_parts = model.split("/", 1)
-        if len(model_parts) > 1:
-            model_name = model_parts[1].lower()
-            # Check if model name contains claude
-            if "claude" in model_name or model_name.startswith("claude"):
-                return model_parts[1]  # Return model name without "azure/" prefix
+        model_lower = model.lower()
+        return "claude" in model_lower or model_lower.startswith("claude")
     except Exception:
-        pass
-    return None
+        return False
 
 
 def handle_cohere_chat_model_custom_llm_provider(
@@ -136,11 +135,6 @@ def get_llm_provider(  # noqa: PLR0915
         # AZURE AI-Studio Logic - Azure AI Studio supports AZURE/Cohere
         # If User passes azure/command-r-plus -> we should send it to cohere_chat/command-r-plus
         if model.split("/", 1)[0] == "azure":
-            # Check if it's an Azure Anthropic model (claude models)
-            azure_anthropic_model = _is_azure_anthropic_model(model)
-            if azure_anthropic_model:
-                custom_llm_provider = "azure_anthropic"
-                return azure_anthropic_model, custom_llm_provider, dynamic_api_key, api_base
             if _is_non_openai_azure_model(model):
                 custom_llm_provider = "openai"
                 return model, custom_llm_provider, dynamic_api_key, api_base
@@ -668,6 +662,13 @@ def _get_openai_compatible_provider_info(  # noqa: PLR0915
         ) = litellm.XAIChatConfig()._get_openai_compatible_provider_info(
             api_base, api_key
         )
+    elif custom_llm_provider == "zai":
+        api_base = (
+            api_base
+            or get_secret_str("ZAI_API_BASE")
+            or "https://api.z.ai/api/paas/v4"
+        )
+        dynamic_api_key = api_key or get_secret_str("ZAI_API_KEY")
     elif custom_llm_provider == "together_ai":
         api_base = (
             api_base
@@ -839,6 +840,16 @@ def _get_openai_compatible_provider_info(  # noqa: PLR0915
         ) = litellm.ClarifaiConfig()._get_openai_compatible_provider_info(
             api_base, api_key
         )
+    elif custom_llm_provider == "ragflow":
+        full_model = f"ragflow/{model}"
+        (
+            api_base,
+            dynamic_api_key,
+            _,
+        ) = litellm.RAGFlowConfig()._get_openai_compatible_provider_info(
+            full_model, api_base, api_key, "ragflow"
+        )
+        model = full_model
 
     if api_base is not None and not isinstance(api_base, str):
         raise Exception("api base needs to be a string. api_base={}".format(api_base))

litellm/litellm_core_utils/get_supported_openai_params.py

@@ -266,6 +266,15 @@ def get_supported_openai_params(  # noqa: PLR0915
                     model=model
                 )
             )
+    elif custom_llm_provider == "ovhcloud":
+        if request_type == "transcription":
+            from litellm.llms.ovhcloud.audio_transcription.transformation import (
+                OVHCloudAudioTranscriptionConfig,
+            )
+
+            return OVHCloudAudioTranscriptionConfig().get_supported_openai_params(
+                model=model
+            )
     elif custom_llm_provider == "elevenlabs":
         if request_type == "transcription":
             from litellm.llms.elevenlabs.audio_transcription.transformation import (

litellm/litellm_core_utils/streaming_handler.py

@@ -96,9 +96,9 @@ class CustomStreamWrapper:
 
         self.system_fingerprint: Optional[str] = None
         self.received_finish_reason: Optional[str] = None
-        self.intermittent_finish_reason: Optional[str] = (
-            None  # finish reasons that show up mid-stream
-        )
+        self.intermittent_finish_reason: Optional[
+            str
+        ] = None  # finish reasons that show up mid-stream
         self.special_tokens = [
             "<|assistant|>",
             "<|system|>",
@@ -735,7 +735,7 @@ class CustomStreamWrapper:
                 and completion_obj["function_call"] is not None
             )
             or (
-                 "tool_calls" in model_response.choices[0].delta
+                "tool_calls" in model_response.choices[0].delta
                 and model_response.choices[0].delta["tool_calls"] is not None
             )
             or (
@@ -889,7 +889,6 @@ class CustomStreamWrapper:
                 ## check if openai/azure chunk
                 original_chunk = response_obj.get("original_chunk", None)
                 if original_chunk:
-
                     if len(original_chunk.choices) > 0:
                         choices = []
                         for choice in original_chunk.choices:
@@ -906,7 +905,6 @@ class CustomStreamWrapper:
                         print_verbose(f"choices in streaming: {choices}")
                         setattr(model_response, "choices", choices)
                     else:
-
                         return
                     model_response.system_fingerprint = (
                         original_chunk.system_fingerprint
@@ -1435,9 +1433,9 @@ class CustomStreamWrapper:
                             _json_delta = delta.model_dump()
                             print_verbose(f"_json_delta: {_json_delta}")
                             if "role" not in _json_delta or _json_delta["role"] is None:
-                                _json_delta["role"] = (
-                                    "assistant"  # mistral's api returns role as None
-                                )
+                                _json_delta[
+                                    "role"
+                                ] = "assistant"  # mistral's api returns role as None
                             if "tool_calls" in _json_delta and isinstance(
                                 _json_delta["tool_calls"], list
                             ):
@@ -1533,7 +1531,7 @@ class CustomStreamWrapper:
     async def _call_post_streaming_deployment_hook(self, chunk):
         """
         Call the post-call streaming deployment hook for callbacks.
-        
+
         This allows callbacks to modify streaming chunks before they're returned.
         """
         try:
@@ -1544,15 +1542,17 @@ class CustomStreamWrapper:
             # Get request kwargs from logging object
             request_data = self.logging_obj.model_call_details
             call_type_str = self.logging_obj.call_type
-            
+
             try:
                 typed_call_type = CallTypes(call_type_str)
             except ValueError:
                 typed_call_type = None
-            
+
             # Call hooks for all callbacks
             for callback in litellm.callbacks:
-                if isinstance(callback, CustomLogger) and hasattr(callback, "async_post_call_streaming_deployment_hook"):
+                if isinstance(callback, CustomLogger) and hasattr(
+                    callback, "async_post_call_streaming_deployment_hook"
+                ):
                     result = await callback.async_post_call_streaming_deployment_hook(
                         request_data=request_data,
                         response_chunk=chunk,
@@ -1560,11 +1560,14 @@ class CustomStreamWrapper:
                     )
                     if result is not None:
                         chunk = result
-            
+
             return chunk
         except Exception as e:
             from litellm._logging import verbose_logger
-            verbose_logger.exception(f"Error in post-call streaming deployment hook: {str(e)}")
+
+            verbose_logger.exception(
+                f"Error in post-call streaming deployment hook: {str(e)}"
+            )
             return chunk
 
     def cache_streaming_response(self, processed_chunk, cache_hit: bool):
@@ -1687,7 +1690,7 @@ class CustomStreamWrapper:
                         response, "usage"
                     ):  # remove usage from chunk, only send on final chunk
                         # Convert the object to a dictionary
-                        obj_dict = response.dict()
+                        obj_dict = response.model_dump()
 
                         # Remove an attribute (e.g., 'attr2')
                         if "usage" in obj_dict:
@@ -1852,7 +1855,7 @@ class CustomStreamWrapper:
                         processed_chunk, "usage"
                     ):  # remove usage from chunk, only send on final chunk
                         # Convert the object to a dictionary
-                        obj_dict = processed_chunk.dict()
+                        obj_dict = processed_chunk.model_dump()
 
                         # Remove an attribute (e.g., 'attr2')
                         if "usage" in obj_dict:
@@ -1872,11 +1875,15 @@ class CustomStreamWrapper:
                     if self.sent_last_chunk is True and self.stream_options is None:
                         usage = calculate_total_usage(chunks=self.chunks)
                         processed_chunk._hidden_params["usage"] = usage
-                    
+
                     # Call post-call streaming deployment hook for final chunk
                     if self.sent_last_chunk is True:
-                        processed_chunk = await self._call_post_streaming_deployment_hook(processed_chunk)
-                    
+                        processed_chunk = (
+                            await self._call_post_streaming_deployment_hook(
+                                processed_chunk
+                            )
+                        )
+
                     return processed_chunk
                 raise StopAsyncIteration
             else:  # temporary patch for non-aiohttp async calls
@@ -1890,9 +1897,9 @@ class CustomStreamWrapper:
                         chunk = next(self.completion_stream)
                     if chunk is not None and chunk != b"":
                         print_verbose(f"PROCESSED CHUNK PRE CHUNK CREATOR: {chunk}")
-                        processed_chunk: Optional[ModelResponseStream] = (
-                            self.chunk_creator(chunk=chunk)
-                        )
+                        processed_chunk: Optional[
+                            ModelResponseStream
+                        ] = self.chunk_creator(chunk=chunk)
                         print_verbose(
                             f"PROCESSED CHUNK POST CHUNK CREATOR: {processed_chunk}"
                         )

litellm/llms/anthropic/chat/guardrail_translation/handler.py

@@ -12,8 +12,7 @@ Pattern Overview:
 4. Apply guardrail responses back to the original structure
 """
 
-import asyncio
-from typing import TYPE_CHECKING, Any, Coroutine, Dict, List, Optional, Tuple, cast
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, cast
 
 from litellm._logging import verbose_proxy_logger
 from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
@@ -50,30 +49,40 @@ class AnthropicMessagesHandler(BaseTranslation):
         if messages is None:
             return data
 
-        tasks: List[Coroutine[Any, Any, str]] = []
+        texts_to_check: List[str] = []
+        images_to_check: List[str] = []
         task_mappings: List[Tuple[int, Optional[int]]] = []
-        # Track (message_index, content_index) for each task
+        # Track (message_index, content_index) for each text
         # content_index is None for string content, int for list content
 
-        # Step 1: Extract all text content and create guardrail tasks
+        # Step 1: Extract all text content and images
         for msg_idx, message in enumerate(messages):
-            await self._extract_input_text_and_create_tasks(
+            self._extract_input_text_and_images(
                 message=message,
                 msg_idx=msg_idx,
-                tasks=tasks,
+                texts_to_check=texts_to_check,
+                images_to_check=images_to_check,
                 task_mappings=task_mappings,
-                guardrail_to_apply=guardrail_to_apply,
             )
 
-        # Step 2: Run all guardrail tasks in parallel
-        responses = await asyncio.gather(*tasks)
+        # Step 2: Apply guardrail to all texts in batch
+        if texts_to_check:
+            guardrailed_texts, guardrailed_images = (
+                await guardrail_to_apply.apply_guardrail(
+                    texts=texts_to_check,
+                    request_data=data,
+                    input_type="request",
+                    images=images_to_check if images_to_check else None,
+                    logging_obj=litellm_logging_obj,
+                )
+            )
 
-        # Step 3: Map guardrail responses back to original message structure
-        await self._apply_guardrail_responses_to_input(
-            messages=messages,
-            responses=responses,
-            task_mappings=task_mappings,
-        )
+            # Step 3: Map guardrail responses back to original message structure
+            await self._apply_guardrail_responses_to_input(
+                messages=messages,
+                responses=guardrailed_texts,
+                task_mappings=task_mappings,
+            )
 
         verbose_proxy_logger.debug(
             "Anthropic Messages: Processed input messages: %s", messages
@@ -81,18 +90,18 @@ class AnthropicMessagesHandler(BaseTranslation):
 
         return data
 
-    async def _extract_input_text_and_create_tasks(
+    def _extract_input_text_and_images(
         self,
         message: Dict[str, Any],
         msg_idx: int,
-        tasks: List,
+        texts_to_check: List[str],
+        images_to_check: List[str],
         task_mappings: List[Tuple[int, Optional[int]]],
-        guardrail_to_apply: "CustomGuardrail",
     ) -> None:
         """
-        Extract text content from a message and create guardrail tasks.
+        Extract text content and images from a message.
 
-        Override this method to customize text extraction logic.
+        Override this method to customize text/image extraction logic.
         """
         content = message.get("content", None)
         if content is None:
@@ -100,17 +109,26 @@ class AnthropicMessagesHandler(BaseTranslation):
 
         if isinstance(content, str):
             # Simple string content
-            tasks.append(guardrail_to_apply.apply_guardrail(text=content))
+            texts_to_check.append(content)
             task_mappings.append((msg_idx, None))
 
         elif isinstance(content, list):
             # List content (e.g., multimodal with text and images)
             for content_idx, content_item in enumerate(content):
+                # Extract text
                 text_str = content_item.get("text", None)
-                if text_str is None:
-                    continue
-                tasks.append(guardrail_to_apply.apply_guardrail(text=text_str))
-                task_mappings.append((msg_idx, int(content_idx)))
+                if text_str is not None:
+                    texts_to_check.append(text_str)
+                    task_mappings.append((msg_idx, int(content_idx)))
+
+                # Extract images
+                if content_item.get("type") == "image":
+                    source = content_item.get("source", {})
+                    if isinstance(source, dict):
+                        # Could be base64 or url
+                        data = source.get("data")
+                        if data:
+                            images_to_check.append(data)
 
     async def _apply_guardrail_responses_to_input(
         self,
@@ -147,6 +165,7 @@ class AnthropicMessagesHandler(BaseTranslation):
         response: "AnthropicMessagesResponse",
         guardrail_to_apply: "CustomGuardrail",
         litellm_logging_obj: Optional[Any] = None,
+        user_api_key_dict: Optional[Any] = None,
     ) -> Any:
         """
         Process output response by applying guardrails to text content.
@@ -154,6 +173,8 @@ class AnthropicMessagesHandler(BaseTranslation):
         Args:
             response: Anthropic MessagesResponse object
             guardrail_to_apply: The guardrail instance to apply
+            litellm_logging_obj: Optional logging object
+            user_api_key_dict: User API key metadata to pass to guardrails
 
         Returns:
             Modified response with guardrail applied to content
@@ -168,35 +189,56 @@ class AnthropicMessagesHandler(BaseTranslation):
             )
             return response
 
-        tasks: List[Coroutine[Any, Any, str]] = []
+        texts_to_check: List[str] = []
+        images_to_check: List[str] = []
         task_mappings: List[Tuple[int, Optional[int]]] = []
-        # Track (choice_index, content_index) for each task
+        # Track (content_index, None) for each text
 
         response_content = response.get("content", [])
         if not response_content:
             return response
-        # Step 1: Extract all text content from response choices
+
+        # Step 1: Extract all text content from response
         for content_idx, content_block in enumerate(response_content):
             # Check if this is a text block by checking the 'type' field
             if isinstance(content_block, dict) and content_block.get("type") == "text":
                 # Cast to dict to handle the union type properly
-                await self._extract_output_text_and_create_tasks(
+                self._extract_output_text_and_images(
                     content_block=cast(Dict[str, Any], content_block),
                     content_idx=content_idx,
-                    tasks=tasks,
+                    texts_to_check=texts_to_check,
+                    images_to_check=images_to_check,
                     task_mappings=task_mappings,
-                    guardrail_to_apply=guardrail_to_apply,
                 )
 
-        # Step 2: Run all guardrail tasks in parallel
-        responses = await asyncio.gather(*tasks)
+        # Step 2: Apply guardrail to all texts in batch
+        if texts_to_check:
+            # Create a request_data dict with response info and user API key metadata
+            request_data: dict = {"response": response}
 
-        # Step 3: Map guardrail responses back to original response structure
-        await self._apply_guardrail_responses_to_output(
-            response=response,
-            responses=responses,
-            task_mappings=task_mappings,
-        )
+            # Add user API key metadata with prefixed keys
+            user_metadata = self.transform_user_api_key_dict_to_metadata(
+                user_api_key_dict
+            )
+            if user_metadata:
+                request_data["litellm_metadata"] = user_metadata
+
+            guardrailed_texts, guardrailed_images = (
+                await guardrail_to_apply.apply_guardrail(
+                    texts=texts_to_check,
+                    request_data=request_data,
+                    input_type="response",
+                    images=images_to_check if images_to_check else None,
+                    logging_obj=litellm_logging_obj,
+                )
+            )
+
+            # Step 3: Map guardrail responses back to original response structure
+            await self._apply_guardrail_responses_to_output(
+                response=response,
+                responses=guardrailed_texts,
+                task_mappings=task_mappings,
+            )
 
         verbose_proxy_logger.debug(
             "Anthropic Messages: Processed output response: %s", response
@@ -221,23 +263,23 @@ class AnthropicMessagesHandler(BaseTranslation):
                     return True
         return False
 
-    async def _extract_output_text_and_create_tasks(
+    def _extract_output_text_and_images(
         self,
         content_block: Dict[str, Any],
         content_idx: int,
-        tasks: List,
+        texts_to_check: List[str],
+        images_to_check: List[str],
         task_mappings: List[Tuple[int, Optional[int]]],
-        guardrail_to_apply: "CustomGuardrail",
     ) -> None:
         """
-        Extract text content from a response choice and create guardrail tasks.
+        Extract text content and images from a response content block.
 
-        Override this method to customize text extraction logic.
+        Override this method to customize text/image extraction logic.
         """
         content_text = content_block.get("text")
         if content_text and isinstance(content_text, str):
             # Simple string content
-            tasks.append(guardrail_to_apply.apply_guardrail(text=content_text))
+            texts_to_check.append(content_text)
             task_mappings.append((content_idx, None))
 
     async def _apply_guardrail_responses_to_output(

litellm/llms/anthropic/chat/transformation.py

@@ -119,6 +119,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
     def get_config(cls):
         return super().get_config()
 
+    def _is_claude_opus_4_5(self, model: str) -> bool:
+        """Check if the model is Claude Opus 4.5."""
+        return "opus-4-5" in model.lower() or "opus_4_5" in model.lower()
+
     def get_supported_openai_params(self, model: str):
         params = [
             "stream",
@@ -626,7 +630,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
 
         return hosted_web_search_tool
 
-    def map_openai_params(
+    def map_openai_params(  # noqa: PLR0915
         self,
         non_default_params: dict,
         optional_params: dict,
@@ -712,9 +716,14 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
             if param == "thinking":
                 optional_params["thinking"] = value
             elif param == "reasoning_effort" and isinstance(value, str):
-                optional_params["thinking"] = AnthropicConfig._map_reasoning_effort(
-                    value
-                )
+                # For Claude Opus 4.5, map reasoning_effort to output_config
+                if self._is_claude_opus_4_5(model):
+                    optional_params["output_config"] = {"effort": value}
+                else:
+                    # For other models, map to thinking parameter
+                    optional_params["thinking"] = AnthropicConfig._map_reasoning_effort(
+                        value
+                    )
             elif param == "web_search_options" and isinstance(value, dict):
                 hosted_web_search_tool = self.map_web_search_tool(
                     cast(OpenAIWebSearchOptions, value)

litellm/llms/anthropic/common_utils.py

@@ -151,15 +151,22 @@ class AnthropicModelInfo(BaseLLMModelInfo):
         
         return False
     
-    def is_effort_used(self, optional_params: Optional[dict]) -> bool:
+    def is_effort_used(self, optional_params: Optional[dict], model: Optional[str] = None) -> bool:
         """
-        Check if effort parameter is being used via output_config.
+        Check if effort parameter is being used.
         
-        Returns True if output_config with effort field is present.
+        Returns True if effort-related parameters are present.
         """
         if not optional_params:
             return False
         
+        # Check if reasoning_effort is provided for Claude Opus 4.5
+        if model and ("opus-4-5" in model.lower() or "opus_4_5" in model.lower()):
+            reasoning_effort = optional_params.get("reasoning_effort")
+            if reasoning_effort and isinstance(reasoning_effort, str):
+                return True
+        
+        # Check if output_config is directly provided
         output_config = optional_params.get("output_config")
         if output_config and isinstance(output_config, dict):
             effort = output_config.get("effort")
@@ -193,6 +200,49 @@ class AnthropicModelInfo(BaseLLMModelInfo):
             computer_tool_version, "computer-use-2024-10-22"  # Default fallback
         )
 
+    def get_anthropic_beta_list(
+        self,
+        model: str,
+        optional_params: Optional[dict] = None,
+        computer_tool_used: Optional[str] = None,
+        prompt_caching_set: bool = False,
+        file_id_used: bool = False,
+        mcp_server_used: bool = False,
+    ) -> List[str]:
+        """
+        Get list of common beta headers based on the features that are active.
+        
+        Returns:
+            List of beta header strings
+        """
+        from litellm.types.llms.anthropic import (
+            ANTHROPIC_EFFORT_BETA_HEADER,
+        )
+        
+        betas = []
+        
+        # Detect features
+        effort_used = self.is_effort_used(optional_params, model)
+        
+        if effort_used:
+            betas.append(ANTHROPIC_EFFORT_BETA_HEADER)  # effort-2025-11-24
+        
+        if computer_tool_used:
+            beta_header = self.get_computer_tool_beta_header(computer_tool_used)
+            betas.append(beta_header)
+        
+        if prompt_caching_set:
+            betas.append("prompt-caching-2024-07-31")
+        
+        if file_id_used:
+            betas.append("files-api-2025-04-14")
+            betas.append("code-execution-2025-05-22")
+        
+        if mcp_server_used:
+            betas.append("mcp-client-2025-04-04")
+        
+        return list(set(betas))
+
     def get_anthropic_headers(
         self,
         api_key: str,
@@ -278,7 +328,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
         tool_search_used = self.is_tool_search_used(tools=tools)
         programmatic_tool_calling_used = self.is_programmatic_tool_calling_used(tools=tools)
         input_examples_used = self.is_input_examples_used(tools=tools)
-        effort_used = self.is_effort_used(optional_params=optional_params)
+        effort_used = self.is_effort_used(optional_params=optional_params, model=model)
         user_anthropic_beta_headers = self._get_user_anthropic_beta_headers(
             anthropic_beta_header=headers.get("anthropic-beta")
         )

litellm/llms/anthropic/skills/readme.md

@@ -1,17 +1,279 @@
-# Anthropic Skills API
+# Anthropic Skills API Integration
 
-This folder maintains the integration for the Anthropic Skills API.
+This module provides comprehensive support for the Anthropic Skills API through LiteLLM.
 
-You can do the following with the Anthropic Skills API:
+## Features
 
-1. Create a new skill
-2. List all skills
-3. Get a skill
-4. Delete a skill
+The Skills API allows you to:
+- **Create skills**: Define reusable AI capabilities
+- **List skills**: Browse all available skills
+- **Get skills**: Retrieve detailed information about a specific skill
+- **Delete skills**: Remove skills that are no longer needed
 
+## Quick Start
 
-Versions:
-    - Create Skill Version
-    - List Skill Versions
-    - Get Skill Version
-    - Delete Skill Version
+### Prerequisites
+
+Set your Anthropic API key:
+```python
+import os
+os.environ["ANTHROPIC_API_KEY"] = "your-api-key-here"
+```
+
+### Basic Usage
+
+#### Create a Skill
+
+```python
+import litellm
+
+# Create a skill with files
+# Note: All files must be in the same top-level directory
+# and must include a SKILL.md file at the root
+skill = litellm.create_skill(
+    files=[
+        # List of file objects to upload
+        # Must include SKILL.md
+    ],
+    display_title="Python Code Generator",
+    custom_llm_provider="anthropic"
+)
+print(f"Created skill: {skill.id}")
+
+# Asynchronous version
+skill = await litellm.acreate_skill(
+    files=[...],  # Your files here
+    display_title="Python Code Generator",
+    custom_llm_provider="anthropic"
+)
+```
+
+#### List Skills
+
+```python
+# List all skills
+skills = litellm.list_skills(
+    custom_llm_provider="anthropic"
+)
+
+for skill in skills.data:
+    print(f"{skill.display_title}: {skill.id}")
+
+# With pagination and filtering
+skills = litellm.list_skills(
+    limit=20,
+    source="custom",  # Filter by 'custom' or 'anthropic'
+    custom_llm_provider="anthropic"
+)
+
+# Get next page if available
+if skills.has_more:
+    next_page = litellm.list_skills(
+        page=skills.next_page,
+        custom_llm_provider="anthropic"
+    )
+```
+
+#### Get a Skill
+
+```python
+skill = litellm.get_skill(
+    skill_id="skill_abc123",
+    custom_llm_provider="anthropic"
+)
+
+print(f"Skill: {skill.display_title}")
+print(f"Created: {skill.created_at}")
+print(f"Latest version: {skill.latest_version}")
+print(f"Source: {skill.source}")
+```
+
+#### Delete a Skill
+
+```python
+result = litellm.delete_skill(
+    skill_id="skill_abc123",
+    custom_llm_provider="anthropic"
+)
+
+print(f"Deleted skill {result.id}, type: {result.type}")
+```
+
+## API Reference
+
+### `create_skill()`
+
+Create a new skill.
+
+**Parameters:**
+- `files` (List[Any], optional): Files to upload for the skill. All files must be in the same top-level directory and must include a SKILL.md file at the root.
+- `display_title` (str, optional): Display title for the skill
+- `custom_llm_provider` (str, optional): Provider name (default: "anthropic")
+- `extra_headers` (dict, optional): Additional HTTP headers
+- `timeout` (float, optional): Request timeout
+
+**Returns:**
+- `Skill`: The created skill object
+
+**Async version:** `acreate_skill()`
+
+### `list_skills()`
+
+List all skills.
+
+**Parameters:**
+- `limit` (int, optional): Number of results to return per page (max 100, default 20)
+- `page` (str, optional): Pagination token for fetching a specific page of results
+- `source` (str, optional): Filter skills by source ('custom' or 'anthropic')
+- `custom_llm_provider` (str, optional): Provider name (default: "anthropic")
+- `extra_headers` (dict, optional): Additional HTTP headers
+- `timeout` (float, optional): Request timeout
+
+**Returns:**
+- `ListSkillsResponse`: Object containing a list of skills and pagination info
+
+**Async version:** `alist_skills()`
+
+### `get_skill()`
+
+Get a specific skill by ID.
+
+**Parameters:**
+- `skill_id` (str, required): The skill ID
+- `custom_llm_provider` (str, optional): Provider name (default: "anthropic")
+- `extra_headers` (dict, optional): Additional HTTP headers
+- `timeout` (float, optional): Request timeout
+
+**Returns:**
+- `Skill`: The requested skill object
+
+**Async version:** `aget_skill()`
+
+### `delete_skill()`
+
+Delete a skill.
+
+**Parameters:**
+- `skill_id` (str, required): The skill ID to delete
+- `custom_llm_provider` (str, optional): Provider name (default: "anthropic")
+- `extra_headers` (dict, optional): Additional HTTP headers
+- `timeout` (float, optional): Request timeout
+
+**Returns:**
+- `DeleteSkillResponse`: Object with `id` and `type` fields
+
+**Async version:** `adelete_skill()`
+
+## Response Types
+
+### `Skill`
+
+Represents a skill from the Anthropic Skills API.
+
+**Fields:**
+- `id` (str): Unique identifier
+- `created_at` (str): ISO 8601 timestamp
+- `display_title` (str, optional): Display title
+- `latest_version` (str, optional): Latest version identifier
+- `source` (str): Source ("custom" or "anthropic")
+- `type` (str): Object type (always "skill")
+- `updated_at` (str): ISO 8601 timestamp
+
+### `ListSkillsResponse`
+
+Response from listing skills.
+
+**Fields:**
+- `data` (List[Skill]): List of skills
+- `next_page` (str, optional): Pagination token for the next page
+- `has_more` (bool): Whether more skills are available
+
+### `DeleteSkillResponse`
+
+Response from deleting a skill.
+
+**Fields:**
+- `id` (str): The deleted skill ID
+- `type` (str): Deleted object type (always "skill_deleted")
+
+## Architecture
+
+The Skills API implementation follows LiteLLM's standard patterns:
+
+1. **Type Definitions** (`litellm/types/llms/anthropic_skills.py`)
+   - Pydantic models for request/response types
+   - TypedDict definitions for request parameters
+
+2. **Base Configuration** (`litellm/llms/base_llm/skills/transformation.py`)
+   - Abstract base class `BaseSkillsAPIConfig`
+   - Defines transformation interface for provider-specific implementations
+
+3. **Provider Implementation** (`litellm/llms/anthropic/skills/transformation.py`)
+   - `AnthropicSkillsConfig` - Anthropic-specific transformations
+   - Handles API authentication, URL construction, and response mapping
+
+4. **Main Handler** (`litellm/skills/main.py`)
+   - Public API functions (sync and async)
+   - Request validation and routing
+   - Error handling
+
+5. **HTTP Handlers** (`litellm/llms/custom_httpx/llm_http_handler.py`)
+   - Low-level HTTP request/response handling
+   - Connection pooling and retry logic
+
+## Beta API Support
+
+The Skills API is in beta. The beta header (`skills-2025-10-02`) is automatically added by the Anthropic provider configuration. You can customize it if needed:
+
+```python
+skill = litellm.create_skill(
+    display_title="My Skill",
+    extra_headers={
+        "anthropic-beta": "skills-2025-10-02"  # Or any other beta version
+    },
+    custom_llm_provider="anthropic"
+)
+```
+
+The default beta version is configured in `litellm.constants.ANTHROPIC_SKILLS_API_BETA_VERSION`.
+
+## Error Handling
+
+All Skills API functions follow LiteLLM's standard error handling:
+
+```python
+import litellm
+
+try:
+    skill = litellm.create_skill(
+        display_title="My Skill",
+        custom_llm_provider="anthropic"
+    )
+except litellm.exceptions.AuthenticationError as e:
+    print(f"Authentication failed: {e}")
+except litellm.exceptions.RateLimitError as e:
+    print(f"Rate limit exceeded: {e}")
+except litellm.exceptions.APIError as e:
+    print(f"API error: {e}")
+```
+
+## Contributing
+
+To add support for Skills API to a new provider:
+
+1. Create provider-specific configuration class inheriting from `BaseSkillsAPIConfig`
+2. Implement all abstract methods for request/response transformations
+3. Register the config in `ProviderConfigManager.get_provider_skills_api_config()`
+4. Add appropriate tests
+
+## Related Documentation
+
+- [Anthropic Skills API Documentation](https://platform.claude.com/docs/en/api/beta/skills/create)
+- [LiteLLM Responses API](../../../responses/)
+- [Provider Configuration System](../../base_llm/)
+
+## Support
+
+For issues or questions:
+- GitHub Issues: https://github.com/BerriAI/litellm/issues
+- Discord: https://discord.gg/wuPM9dRgDw

litellm/llms/azure/realtime/handler.py

@@ -12,6 +12,7 @@ from ....litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
 from ....litellm_core_utils.realtime_streaming import RealTimeStreaming
 from ....llms.custom_httpx.http_handler import get_shared_realtime_ssl_context
 from ..azure import AzureChatCompletion
+from litellm._logging import verbose_proxy_logger
 
 # BACKEND_WS_URL = "ws://localhost:8080/v1/realtime?model=gpt-4o-realtime-preview-2024-10-01"
 
@@ -51,18 +52,18 @@ class AzureOpenAIRealtime(AzureChatCompletion):
 
         Examples:
             beta/default: "wss://.../openai/realtime?api-version=2024-10-01-preview&deployment=gpt-4o-realtime-preview"
-            GA/v1:        "wss://.../openai/v1/realtime?api-version=2024-10-01-preview&deployment=gpt-4o-realtime-preview"
+            GA/v1:        "wss://.../openai/v1/realtime?model=gpt-realtime-deployment"
         """
         api_base = api_base.replace("https://", "wss://")
 
         # Determine path based on realtime_protocol
         if realtime_protocol in ("GA", "v1"):
-            path = "/openai/v1/realtime"
+            path = "/openai/v1/realtime" 
+            return f"{api_base}{path}?model={model}"
         else:
             # Default to beta path for backwards compatibility
             path = "/openai/realtime"
-
-        return f"{api_base}{path}?api-version={api_version}&deployment={model}"
+            return f"{api_base}{path}?api-version={api_version}&deployment={model}"
 
     async def async_realtime(
         self,
@@ -107,4 +108,5 @@ class AzureOpenAIRealtime(AzureChatCompletion):
         except websockets.exceptions.InvalidStatusCode as e:  # type: ignore
             await websocket.close(code=e.status_code, reason=str(e))
         except Exception:
+            verbose_proxy_logger.exception("Error in AzureOpenAIRealtime.async_realtime")
             pass

litellm/llms/azure_ai/anthropic/handler.py

@@ -7,7 +7,6 @@ from typing import TYPE_CHECKING, Callable, Union
 
 import httpx
 
-import litellm
 from litellm.llms.anthropic.chat.handler import AnthropicChatCompletion
 from litellm.llms.custom_httpx.http_handler import (
     AsyncHTTPHandler,
@@ -55,7 +54,6 @@ class AzureAnthropicChatCompletion(AnthropicChatCompletion):
         Completion method that uses Azure authentication instead of Anthropic's x-api-key.
         All other logic is the same as AnthropicChatCompletion.
         """
-        from litellm.utils import ProviderConfigManager
 
         optional_params = copy.deepcopy(optional_params)
         stream = optional_params.pop("stream", None)
@@ -64,8 +62,10 @@ class AzureAnthropicChatCompletion(AnthropicChatCompletion):
         _is_function_call = False
         messages = copy.deepcopy(messages)
 
-        # Use AzureAnthropicConfig instead of AnthropicConfig
-        headers = AzureAnthropicConfig().validate_environment(
+        # Use AzureAnthropicConfig for both azure_anthropic and azure_ai Claude models
+        config = AzureAnthropicConfig()
+        
+        headers = config.validate_environment(
             api_key=api_key,
             headers=headers,
             model=model,
@@ -74,15 +74,6 @@ class AzureAnthropicChatCompletion(AnthropicChatCompletion):
             litellm_params=litellm_params,
         )
 
-        config = ProviderConfigManager.get_provider_chat_config(
-            model=model,
-            provider=litellm.types.utils.LlmProviders(custom_llm_provider),
-        )
-        if config is None:
-            raise ValueError(
-                f"Provider config not found for model: {model} and provider: {custom_llm_provider}"
-            )
-
         data = config.transform_request(
             model=model,
             messages=messages,
@@ -183,7 +174,7 @@ class AzureAnthropicChatCompletion(AnthropicChatCompletion):
                 return CustomStreamWrapper(
                     completion_stream=completion_stream,
                     model=model,
-                    custom_llm_provider="azure_anthropic",
+                    custom_llm_provider="azure_ai",
                     logging_obj=logging_obj,
                     _response_headers=process_anthropic_headers(response_headers),
                 )

litellm/llms/azure_ai/anthropic/transformation.py

@@ -21,7 +21,7 @@ class AzureAnthropicConfig(AnthropicConfig):
 
     @property
     def custom_llm_provider(self) -> Optional[str]:
-        return "azure_anthropic"
+        return "azure_ai"
 
     def validate_environment(
         self,
@@ -94,3 +94,29 @@ class AzureAnthropicConfig(AnthropicConfig):
 
         return headers
 
+    def transform_request(
+        self,
+        model: str,
+        messages: List[AllMessageValues],
+        optional_params: dict,
+        litellm_params: dict,
+        headers: dict,
+    ) -> dict:
+        """
+        Transform request using parent AnthropicConfig, then remove extra_body if present.
+        Azure Anthropic doesn't support extra_body parameter.
+        """
+        # Call parent transform_request
+        data = super().transform_request(
+            model=model,
+            messages=messages,
+            optional_params=optional_params,
+            litellm_params=litellm_params,
+            headers=headers,
+        )
+        
+        # Remove extra_body if present (Azure Anthropic doesn't support it)
+        data.pop("extra_body", None)
+        
+        return data
+

litellm/llms/base_llm/guardrail_translation/base_translation.py

@@ -1,12 +1,57 @@
 from abc import ABC, abstractmethod
-from typing import TYPE_CHECKING, Any, Optional
+from typing import TYPE_CHECKING, Any, Dict, Optional
 
 if TYPE_CHECKING:
     from litellm.integrations.custom_guardrail import CustomGuardrail
     from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+    from litellm.proxy._types import UserAPIKeyAuth
 
 
 class BaseTranslation(ABC):
+    @staticmethod
+    def transform_user_api_key_dict_to_metadata(
+        user_api_key_dict: Optional[Any],
+    ) -> Dict[str, Any]:
+        """
+        Transform user_api_key_dict to a metadata dict with prefixed keys.
+
+        Converts keys like 'user_id' to 'user_api_key_user_id' to clearly indicate
+        the source of the metadata.
+
+        Args:
+            user_api_key_dict: UserAPIKeyAuth object or dict with user information
+
+        Returns:
+            Dict with keys prefixed with 'user_api_key_'
+        """
+        if user_api_key_dict is None:
+            return {}
+
+        # Convert to dict if it's a Pydantic object
+        user_dict = (
+            user_api_key_dict.model_dump()
+            if hasattr(user_api_key_dict, "model_dump")
+            else user_api_key_dict
+        )
+
+        if not isinstance(user_dict, dict):
+            return {}
+
+        # Transform keys to be prefixed with 'user_api_key_'
+        transformed = {}
+        for key, value in user_dict.items():
+            # Skip None values and internal fields
+            if value is None or key.startswith("_"):
+                continue
+
+            # If key already has the prefix, use as-is, otherwise add prefix
+            if key.startswith("user_api_key_"):
+                transformed[key] = value
+            else:
+                transformed[f"user_api_key_{key}"] = value
+
+        return transformed
+
     @abstractmethod
     async def process_input_messages(
         self,
@@ -14,6 +59,11 @@ class BaseTranslation(ABC):
         guardrail_to_apply: "CustomGuardrail",
         litellm_logging_obj: Optional["LiteLLMLoggingObj"] = None,
     ) -> Any:
+        """
+        Process input messages with guardrails.
+
+        Note: user_api_key_dict metadata should be available in the data dict.
+        """
         pass
 
     @abstractmethod
@@ -22,5 +72,29 @@ class BaseTranslation(ABC):
         response: Any,
         guardrail_to_apply: "CustomGuardrail",
         litellm_logging_obj: Optional["LiteLLMLoggingObj"] = None,
+        user_api_key_dict: Optional["UserAPIKeyAuth"] = None,
     ) -> Any:
+        """
+        Process output response with guardrails.
+
+        Args:
+            response: The response object from the LLM
+            guardrail_to_apply: The guardrail instance to apply
+            litellm_logging_obj: Optional logging object
+            user_api_key_dict: User API key metadata (passed separately since response doesn't contain it)
+        """
         pass
+
+    async def process_output_streaming_response(
+        self,
+        response: Any,
+        guardrail_to_apply: "CustomGuardrail",
+        litellm_logging_obj: Optional["LiteLLMLoggingObj"] = None,
+        user_api_key_dict: Optional["UserAPIKeyAuth"] = None,
+    ) -> Any:
+        """
+        Process output streaming response with guardrails.
+
+        Optional to override in subclasses.
+        """
+        return response

litellm/llms/bedrock/base_aws_llm.py

@@ -353,6 +353,10 @@ class BaseAWSLLM:
             model_id = BaseAWSLLM._get_model_id_from_model_with_spec(
                 model_id, spec="deepseek_r1"
             )
+        elif provider == "openai" and "openai/" in model_id:
+            model_id = BaseAWSLLM._get_model_id_from_model_with_spec(
+                model_id, spec="openai"
+            )
         return model_id
 
     @staticmethod
@@ -387,9 +391,16 @@ class BaseAWSLLM:
         Handles scenarios like:
         1. model=cohere.embed-english-v3:0 -> Returns `cohere`
         2. model=amazon.titan-embed-text-v1 -> Returns `amazon`
-        3. model=us.twelvelabs.marengo-embed-2-7-v1:0 -> Returns `twelvelabs`
-        4. model=twelvelabs.marengo-embed-2-7-v1:0 -> Returns `twelvelabs`
+        3. model=amazon.nova-2-multimodal-embeddings-v1:0 -> Returns `nova`
+        4. model=us.twelvelabs.marengo-embed-2-7-v1:0 -> Returns `twelvelabs`
+        5. model=twelvelabs.marengo-embed-2-7-v1:0 -> Returns `twelvelabs`
         """
+        # Special case: Check for "nova" in model name first (before "amazon")
+        # This handles amazon.nova-* models
+        if "nova" in model.lower():
+            if "nova" in get_args(BEDROCK_EMBEDDING_PROVIDERS_LITERAL):
+                return cast(BEDROCK_EMBEDDING_PROVIDERS_LITERAL, "nova")
+        
         # Handle regional models like us.twelvelabs.marengo-embed-2-7-v1:0
         if "." in model:
             parts = model.split(".")

litellm/llms/bedrock/batches/handler.py

@@ -0,0 +1,96 @@
+from openai.types.batch import BatchRequestCounts
+from openai.types.batch import Metadata as OpenAIBatchMetadata
+
+from litellm.types.utils import LiteLLMBatch
+
+
+class BedrockBatchesHandler:
+    """
+    Handler for Bedrock Batches.
+
+    Specific providers/models needed some special handling.
+
+    E.g. Twelve Labs Embedding Async Invoke
+    """
+    @staticmethod
+    def _handle_async_invoke_status(
+        batch_id: str, aws_region_name: str, logging_obj=None, **kwargs
+    ) -> "LiteLLMBatch":
+        """
+        Handle async invoke status check for AWS Bedrock.
+
+        This is for Twelve Labs Embedding Async Invoke.
+
+        Args:
+            batch_id: The async invoke ARN
+            aws_region_name: AWS region name
+            **kwargs: Additional parameters
+
+        Returns:
+            dict: Status information including status, output_file_id (S3 URL), etc.
+        """
+        import asyncio
+
+        from litellm.llms.bedrock.embed.embedding import BedrockEmbedding
+
+        async def _async_get_status():
+            # Create embedding handler instance
+            embedding_handler = BedrockEmbedding()
+
+            # Get the status of the async invoke job
+            status_response = await embedding_handler._get_async_invoke_status(
+                invocation_arn=batch_id,
+                aws_region_name=aws_region_name,
+                logging_obj=logging_obj,
+                **kwargs,
+            )
+
+            # Transform response to a LiteLLMBatch object
+            from litellm.types.utils import LiteLLMBatch
+
+            openai_batch_metadata: OpenAIBatchMetadata = {
+                "output_file_id": status_response["outputDataConfig"][
+                    "s3OutputDataConfig"
+                ]["s3Uri"],
+                "failure_message": status_response.get("failureMessage") or "",
+                "model_arn": status_response["modelArn"],
+            }
+
+            result = LiteLLMBatch(
+                id=status_response["invocationArn"],
+                object="batch",
+                status=status_response["status"],
+                created_at=status_response["submitTime"],
+                in_progress_at=status_response["lastModifiedTime"],
+                completed_at=status_response.get("endTime"),
+                failed_at=status_response.get("endTime")
+                if status_response["status"] == "failed"
+                else None,
+                request_counts=BatchRequestCounts(
+                    total=1,
+                    completed=1 if status_response["status"] == "completed" else 0,
+                    failed=1 if status_response["status"] == "failed" else 0,
+                ),
+                metadata=openai_batch_metadata,
+                completion_window="24h",
+                endpoint="/v1/embeddings",
+                input_file_id="",
+            )
+
+            return result
+
+        # Since this function is called from within an async context via run_in_executor,
+        # we need to create a new event loop in a thread to avoid conflicts
+        import concurrent.futures
+
+        def run_in_thread():
+            new_loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(new_loop)
+            try:
+                return new_loop.run_until_complete(_async_get_status())
+            finally:
+                new_loop.close()
+
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            future = executor.submit(run_in_thread)
+            return future.result()

litellm/llms/bedrock/chat/converse_handler.py

@@ -29,6 +29,7 @@ def make_sync_call(
     logging_obj: LiteLLMLoggingObject,
     json_mode: Optional[bool] = False,
     fake_stream: bool = False,
+    stream_chunk_size: int = 1024,
 ):
     if client is None:
         client = _get_httpx_client()  # Create a new client if none provided
@@ -66,7 +67,7 @@ def make_sync_call(
         )
     else:
         decoder = AWSEventStreamDecoder(model=model)
-        completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=1024))
+        completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=stream_chunk_size))
 
     # LOGGING
     logging_obj.post_call(
@@ -102,6 +103,7 @@ class BedrockConverseLLM(BaseAWSLLM):
         fake_stream: bool = False,
         json_mode: Optional[bool] = False,
         api_key: Optional[str] = None,
+        stream_chunk_size: int = 1024,
     ) -> CustomStreamWrapper:
         request_data = await litellm.AmazonConverseConfig()._async_transform_request(
             model=model,
@@ -143,6 +145,7 @@ class BedrockConverseLLM(BaseAWSLLM):
             logging_obj=logging_obj,
             fake_stream=fake_stream,
             json_mode=json_mode,
+            stream_chunk_size=stream_chunk_size,
         )
         streaming_response = CustomStreamWrapper(
             completion_stream=completion_stream,
@@ -260,6 +263,7 @@ class BedrockConverseLLM(BaseAWSLLM):
     ):
         ## SETUP ##
         stream = optional_params.pop("stream", None)
+        stream_chunk_size = optional_params.pop("stream_chunk_size", 1024)
         unencoded_model_id = optional_params.pop("model_id", None)
         fake_stream = optional_params.pop("fake_stream", False)
         json_mode = optional_params.get("json_mode", False)
@@ -356,7 +360,8 @@ class BedrockConverseLLM(BaseAWSLLM):
                     json_mode=json_mode,
                     fake_stream=fake_stream,
                     credentials=credentials,
-                    api_key=api_key
+                    api_key=api_key,
+                    stream_chunk_size=stream_chunk_size,
                 )  # type: ignore
             ### ASYNC COMPLETION
             return self.async_completion(
@@ -433,6 +438,7 @@ class BedrockConverseLLM(BaseAWSLLM):
                 logging_obj=logging_obj,
                 json_mode=json_mode,
                 fake_stream=fake_stream,
+                stream_chunk_size=stream_chunk_size,
             )
             streaming_response = CustomStreamWrapper(
                 completion_stream=completion_stream,

litellm/llms/bedrock/chat/converse_transformation.py

@@ -246,6 +246,93 @@ class AmazonConverseConfig(BaseConfig):
                     llm_provider="bedrock",
                 )
 
+    def _is_nova_lite_2_model(self, model: str) -> bool:
+        """
+        Check if the model is a Nova Lite 2 model that supports reasoningConfig.
+
+        Nova Lite 2 models use a different reasoning configuration structure compared to
+        Anthropic's thinking parameter and GPT-OSS's reasoning_effort parameter.
+
+        Supported models:
+        - amazon.nova-2-lite-v1:0
+        - us.amazon.nova-2-lite-v1:0
+        - eu.amazon.nova-2-lite-v1:0
+        - apac.amazon.nova-2-lite-v1:0
+
+        Args:
+            model: The model identifier
+
+        Returns:
+            True if the model is a Nova Lite 2 model, False otherwise
+
+        Examples:
+            >>> config = AmazonConverseConfig()
+            >>> config._is_nova_lite_2_model("amazon.nova-2-lite-v1:0")
+            True
+            >>> config._is_nova_lite_2_model("us.amazon.nova-2-lite-v1:0")
+            True
+            >>> config._is_nova_lite_2_model("amazon.nova-pro-1-5-v1:0")
+            False
+            >>> config._is_nova_lite_2_model("amazon.nova-pro-v1:0")
+            False
+        """
+        # Remove regional prefix if present (us., eu., apac.)
+        model_without_region = model
+        for prefix in ["us.", "eu.", "apac."]:
+            if model.startswith(prefix):
+                model_without_region = model[len(prefix) :]
+                break
+
+        # Check if the model is specifically Nova Lite 2
+        return "nova-2-lite" in model_without_region
+
+    def _transform_reasoning_effort_to_reasoning_config(
+        self, reasoning_effort: str
+    ) -> dict:
+        """
+        Transform reasoning_effort parameter to Nova 2 reasoningConfig structure.
+
+        Nova 2 models use a reasoningConfig structure in additionalModelRequestFields
+        that differs from both Anthropic's thinking parameter and GPT-OSS's reasoning_effort.
+
+        Args:
+            reasoning_effort: The reasoning effort level, must be "low" or "high"
+
+        Returns:
+            dict: A dictionary containing the reasoningConfig structure:
+                {
+                    "reasoningConfig": {
+                        "type": "enabled",
+                        "maxReasoningEffort": "low" | "medium" |"high"
+                    }
+                }
+
+        Raises:
+            BadRequestError: If reasoning_effort is not "low", "medium" or "high"
+
+        Examples:
+            >>> config = AmazonConverseConfig()
+            >>> config._transform_reasoning_effort_to_reasoning_config("high")
+            {'reasoningConfig': {'type': 'enabled', 'maxReasoningEffort': 'high'}}
+            >>> config._transform_reasoning_effort_to_reasoning_config("low")
+            {'reasoningConfig': {'type': 'enabled', 'maxReasoningEffort': 'low'}}
+        """
+        valid_values = ["low", "medium", "high"]
+        if reasoning_effort not in valid_values:
+            raise litellm.exceptions.BadRequestError(
+                message=f"Invalid reasoning_effort value '{reasoning_effort}' for Nova 2 models. "
+                f"Supported values: {valid_values}",
+                model="amazon.nova-2-lite-v1:0",
+                llm_provider="bedrock_converse",
+            )
+
+        return {
+            "reasoningConfig": {
+                "type": "enabled",
+                "maxReasoningEffort": reasoning_effort,
+            }
+        }
+
     def get_supported_openai_params(self, model: str) -> List[str]:
         from litellm.utils import supports_function_calling
 
@@ -299,6 +386,10 @@ class AmazonConverseConfig(BaseConfig):
 
         if "gpt-oss" in model:
             supported_params.append("reasoning_effort")
+        elif self._is_nova_lite_2_model(model):
+            # Nova Lite 2 models support reasoning_effort (transformed to reasoningConfig)
+            # These models use a different reasoning structure than Anthropic's thinking parameter
+            supported_params.append("reasoning_effort")
         elif (
             "claude-3-7" in model
             or "claude-sonnet-4" in model
@@ -564,6 +655,12 @@ class AmazonConverseConfig(BaseConfig):
                     # GPT-OSS models: keep reasoning_effort as-is
                     # It will be passed through to additionalModelRequestFields
                     optional_params["reasoning_effort"] = value
+                elif self._is_nova_lite_2_model(model):
+                    # Nova Lite 2 models: transform to reasoningConfig
+                    reasoning_config = (
+                        self._transform_reasoning_effort_to_reasoning_config(value)
+                    )
+                    optional_params.update(reasoning_config)
                 else:
                     # Anthropic and other models: convert to thinking parameter
                     optional_params["thinking"] = AnthropicConfig._map_reasoning_effort(
@@ -574,8 +671,9 @@ class AmazonConverseConfig(BaseConfig):
                     self._validate_request_metadata(value)  # type: ignore
                     optional_params["requestMetadata"] = value
 
-        # Only update thinking tokens for non-GPT-OSS models
-        if "gpt-oss" not in model:
+        # Only update thinking tokens for non-GPT-OSS models and non-Nova-Lite-2 models
+        # Nova Lite 2 handles token budgeting differently through reasoningConfig
+        if "gpt-oss" not in model and not self._is_nova_lite_2_model(model):
             self.update_optional_params_with_thinking_tokens(
                 non_default_params=non_default_params, optional_params=optional_params
             )
@@ -829,11 +927,21 @@ class AmazonConverseConfig(BaseConfig):
             user_betas = get_anthropic_beta_from_headers(headers)
             anthropic_beta_list.extend(user_betas)
 
+        # Filter out tool search tools - Bedrock Converse API doesn't support them
+        filtered_tools = []
+        if original_tools:
+            for tool in original_tools:
+                tool_type = tool.get("type", "")
+                if tool_type in ("tool_search_tool_regex_20251119", "tool_search_tool_bm25_20251119"):
+                    # Tool search not supported in Converse API - skip it
+                    continue
+                filtered_tools.append(tool)
+
         # Only separate tools if computer use tools are actually present
-        if original_tools and self.is_computer_use_tool_used(original_tools, model):
+        if filtered_tools and self.is_computer_use_tool_used(filtered_tools, model):
             # Separate computer use tools from regular function tools
             computer_use_tools, regular_tools = self._separate_computer_use_tools(
-                original_tools, model
+                filtered_tools, model
             )
 
             # Process regular function tools using existing logic
@@ -849,7 +957,7 @@ class AmazonConverseConfig(BaseConfig):
                 additional_request_params["tools"] = transformed_computer_tools
         else:
             # No computer use tools, process all tools as regular tools
-            bedrock_tools = _bedrock_tools_pt(original_tools)
+            bedrock_tools = _bedrock_tools_pt(filtered_tools)
 
         # Set anthropic_beta in additional_request_params if we have any beta features
         if anthropic_beta_list:

litellm/llms/bedrock/chat/invoke_handler.py

@@ -73,6 +73,9 @@ bedrock_tool_name_mappings: InMemoryCache = InMemoryCache(
     max_size_in_memory=50, default_ttl=600
 )
 from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig
+from litellm.llms.bedrock.chat.invoke_transformations.amazon_openai_transformation import (
+    AmazonBedrockOpenAIConfig,
+)
 
 converse_config = AmazonConverseConfig()
 
@@ -189,6 +192,7 @@ async def make_call(
     fake_stream: bool = False,
     json_mode: Optional[bool] = False,
     bedrock_invoke_provider: Optional[litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL] = None,
+    stream_chunk_size: int = 1024,
 ):
     try:
         if client is None:
@@ -232,7 +236,7 @@ async def make_call(
                 json_mode=json_mode,
             )
             completion_stream = decoder.aiter_bytes(
-                response.aiter_bytes(chunk_size=1024)
+                response.aiter_bytes(chunk_size=stream_chunk_size)
             )
         elif bedrock_invoke_provider == "deepseek_r1":
             decoder = AmazonDeepSeekR1StreamDecoder(
@@ -240,12 +244,12 @@ async def make_call(
                 sync_stream=False,
             )
             completion_stream = decoder.aiter_bytes(
-                response.aiter_bytes(chunk_size=1024)
+                response.aiter_bytes(chunk_size=stream_chunk_size)
             )
         else:
             decoder = AWSEventStreamDecoder(model=model)
             completion_stream = decoder.aiter_bytes(
-                response.aiter_bytes(chunk_size=1024)
+                response.aiter_bytes(chunk_size=stream_chunk_size)
             )
 
         # LOGGING
@@ -278,6 +282,7 @@ def make_sync_call(
     fake_stream: bool = False,
     json_mode: Optional[bool] = False,
     bedrock_invoke_provider: Optional[litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL] = None,
+    stream_chunk_size: int = 1024,
 ):
     try:
         if client is None:
@@ -318,16 +323,16 @@ def make_sync_call(
                 sync_stream=True,
                 json_mode=json_mode,
             )
-            completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=1024))
+            completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=stream_chunk_size))
         elif bedrock_invoke_provider == "deepseek_r1":
             decoder = AmazonDeepSeekR1StreamDecoder(
                 model=model,
                 sync_stream=True,
             )
-            completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=1024))
+            completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=stream_chunk_size))
         else:
             decoder = AWSEventStreamDecoder(model=model)
-            completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=1024))
+            completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=stream_chunk_size))
 
         # LOGGING
         logging_obj.post_call(
@@ -401,6 +406,10 @@ class BedrockLLM(BaseAWSLLM):
             prompt = prompt_factory(
                 model=model, messages=messages, custom_llm_provider="bedrock"
             )
+        elif provider == "openai":
+            # OpenAI uses messages directly, no prompt conversion needed
+            # Return empty prompt as it won't be used
+            prompt = ""
         elif provider == "cohere":
             prompt, chat_history = cohere_message_pt(messages=messages)
         else:
@@ -578,6 +587,30 @@ class BedrockLLM(BaseAWSLLM):
                 )
             elif provider == "meta" or provider == "llama":
                 outputText = completion_response["generation"]
+            elif provider == "openai":
+                # OpenAI imported models use OpenAI Chat Completions format
+                if "choices" in completion_response and len(completion_response["choices"]) > 0:
+                    choice = completion_response["choices"][0]
+                    if "message" in choice:
+                        outputText = choice["message"].get("content")
+                    elif "text" in choice:  # fallback for completion format
+                        outputText = choice["text"]
+                    
+                    # Set finish reason
+                    if "finish_reason" in choice:
+                        model_response.choices[0].finish_reason = map_finish_reason(
+                            choice["finish_reason"]
+                        )
+                    
+                    # Set usage if available
+                    if "usage" in completion_response:
+                        usage = completion_response["usage"]
+                        _usage = litellm.Usage(
+                            prompt_tokens=usage.get("prompt_tokens", 0),
+                            completion_tokens=usage.get("completion_tokens", 0),
+                            total_tokens=usage.get("total_tokens", 0),
+                        )
+                        setattr(model_response, "usage", _usage)
             elif provider == "mistral":
                 outputText = completion_response["outputs"][0]["text"]
                 model_response.choices[0].finish_reason = completion_response[
@@ -698,6 +731,7 @@ class BedrockLLM(BaseAWSLLM):
 
         ## SETUP ##
         stream = optional_params.pop("stream", None)
+        stream_chunk_size = optional_params.pop("stream_chunk_size", 1024)
 
         provider = self.get_bedrock_invoke_provider(model)
         modelId = self.get_bedrock_model_id(
@@ -895,6 +929,20 @@ class BedrockLLM(BaseAWSLLM):
                 ):  # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in
                     inference_params[k] = v
             data = json.dumps({"prompt": prompt, **inference_params})
+        elif provider == "openai":
+            ## OpenAI imported models use OpenAI Chat Completions format (messages-based)
+            # Use AmazonBedrockOpenAIConfig for proper OpenAI transformation
+            openai_config = AmazonBedrockOpenAIConfig()
+            supported_params = openai_config.get_supported_openai_params(model=model)
+            
+            # Filter to only supported OpenAI params
+            filtered_params = {
+                k: v for k, v in inference_params.items() 
+                if k in supported_params
+            }
+            
+            # OpenAI uses messages format, not prompt
+            data = json.dumps({"messages": messages, **filtered_params})
         else:
             ## LOGGING
             logging_obj.pre_call(
@@ -958,6 +1006,7 @@ class BedrockLLM(BaseAWSLLM):
                     headers=prepped.headers,
                     timeout=timeout,
                     client=client,
+                    stream_chunk_size=stream_chunk_size,
                 )  # type: ignore
             ### ASYNC COMPLETION
             return self.async_completion(
@@ -1003,7 +1052,7 @@ class BedrockLLM(BaseAWSLLM):
 
             decoder = AWSEventStreamDecoder(model=model)
 
-            completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=1024))
+            completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=stream_chunk_size))
             streaming_response = CustomStreamWrapper(
                 completion_stream=completion_stream,
                 model=model,
@@ -1123,6 +1172,7 @@ class BedrockLLM(BaseAWSLLM):
         logger_fn=None,
         headers={},
         client: Optional[AsyncHTTPHandler] = None,
+        stream_chunk_size: int = 1024,
     ) -> CustomStreamWrapper:
         # The call is not made here; instead, we prepare the necessary objects for the stream.
 
@@ -1138,6 +1188,7 @@ class BedrockLLM(BaseAWSLLM):
                 messages=messages,
                 logging_obj=logging_obj,
                 fake_stream=True if "ai21" in api_base else False,
+                stream_chunk_size=stream_chunk_size,
             ),
             model=model,
             custom_llm_provider="bedrock",

litellm/llms/bedrock/chat/invoke_transformations/amazon_twelvelabs_pegasus_transformation.py

@@ -0,0 +1,280 @@
+"""
+Transforms OpenAI-style requests into TwelveLabs Pegasus 1.2 requests for Bedrock.
+
+Reference:
+https://docs.twelvelabs.io/docs/models/pegasus
+"""
+
+import json
+import time
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
+
+import httpx
+
+import litellm
+from litellm._logging import verbose_logger
+from litellm.litellm_core_utils.core_helpers import map_finish_reason
+from litellm.llms.base_llm.base_utils import type_to_response_format_param
+from litellm.llms.base_llm.chat.transformation import BaseConfig
+from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
+    AmazonInvokeConfig,
+)
+from litellm.llms.bedrock.common_utils import BedrockError
+from litellm.types.llms.openai import AllMessageValues
+from litellm.types.utils import ModelResponse, Usage
+from litellm.utils import get_base64_str
+
+if TYPE_CHECKING:
+    from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
+
+    LiteLLMLoggingObj = _LiteLLMLoggingObj
+else:
+    LiteLLMLoggingObj = Any
+
+
+class AmazonTwelveLabsPegasusConfig(AmazonInvokeConfig, BaseConfig):
+    """
+    Handles transforming OpenAI-style requests into Bedrock InvokeModel requests for
+    `twelvelabs.pegasus-1-2-v1:0`.
+
+    Pegasus 1.2 requires an `inputPrompt` and a `mediaSource` that either references
+    an S3 object or a base64-encoded clip. Optional OpenAI params (temperature,
+    response_format, max_tokens) are translated to the TwelveLabs schema.
+    """
+
+    def get_supported_openai_params(self, model: str) -> List[str]:
+        return [
+            "max_tokens",
+            "max_completion_tokens",
+            "temperature",
+            "response_format",
+        ]
+
+    def map_openai_params(
+        self,
+        non_default_params: dict,
+        optional_params: dict,
+        model: str,
+        drop_params: bool,
+    ) -> dict:
+        for param, value in non_default_params.items():
+            if param in {"max_tokens", "max_completion_tokens"}:
+                optional_params["maxOutputTokens"] = value
+            if param == "temperature":
+                optional_params["temperature"] = value
+            if param == "response_format":
+                optional_params["responseFormat"] = self._normalize_response_format(
+                    value
+                )
+        return optional_params
+
+    def _normalize_response_format(self, value: Any) -> Any:
+        """Normalize response_format to TwelveLabs format.
+        
+        TwelveLabs expects:
+        {
+            "jsonSchema": {...}
+        }
+        
+        But OpenAI format is:
+        {
+            "type": "json_schema",
+            "json_schema": {
+                "name": "...",
+                "schema": {...}
+            }
+        }
+        """
+        if isinstance(value, dict):
+            # If it has json_schema field, extract and transform it
+            if "json_schema" in value:
+                json_schema = value["json_schema"]
+                # Extract the schema if nested
+                if isinstance(json_schema, dict) and "schema" in json_schema:
+                    return {"jsonSchema": json_schema["schema"]}
+                # Otherwise use json_schema directly
+                return {"jsonSchema": json_schema}
+            # If it already has jsonSchema, return as is
+            if "jsonSchema" in value:
+                return value
+            # Otherwise return the dict as is
+            return value
+        return type_to_response_format_param(response_format=value) or value
+
+    def transform_request(
+        self,
+        model: str,
+        messages: List[AllMessageValues],
+        optional_params: dict,
+        litellm_params: dict,
+        headers: dict,
+    ) -> dict:
+        input_prompt = self._convert_messages_to_prompt(messages=messages)
+        request_data: Dict[str, Any] = {"inputPrompt": input_prompt}
+
+        media_source = self._build_media_source(optional_params)
+        if media_source is not None:
+            request_data["mediaSource"] = media_source
+
+        # Handle temperature and maxOutputTokens
+        for key in ("temperature", "maxOutputTokens"):
+            if key in optional_params:
+                request_data[key] = optional_params.get(key)
+        
+        # Handle responseFormat - transform to TwelveLabs format
+        if "responseFormat" in optional_params:
+            response_format = optional_params["responseFormat"]
+            transformed_format = self._normalize_response_format(response_format)
+            if transformed_format:
+                request_data["responseFormat"] = transformed_format
+        
+        return request_data
+
+    def _build_media_source(self, optional_params: dict) -> Optional[dict]:
+        direct_source = optional_params.get("mediaSource") or optional_params.get(
+            "media_source"
+        )
+        if isinstance(direct_source, dict):
+            return direct_source
+
+        base64_input = optional_params.get("video_base64") or optional_params.get(
+            "base64_string"
+        )
+        if base64_input:
+            return {"base64String": get_base64_str(base64_input)}
+
+        s3_uri = (
+            optional_params.get("video_s3_uri")
+            or optional_params.get("s3_uri")
+            or optional_params.get("media_source_s3_uri")
+        )
+        if s3_uri:
+            s3_location = {"uri": s3_uri}
+            bucket_owner = (
+                optional_params.get("video_s3_bucket_owner")
+                or optional_params.get("s3_bucket_owner")
+                or optional_params.get("media_source_bucket_owner")
+            )
+            if bucket_owner:
+                s3_location["bucketOwner"] = bucket_owner
+            return {"s3Location": s3_location}
+        return None
+
+    def _convert_messages_to_prompt(self, messages: List[AllMessageValues]) -> str:
+        prompt_parts: List[str] = []
+        for message in messages:
+            role = message.get("role", "user")
+            content = message.get("content", "")
+            if isinstance(content, list):
+                text_fragments = []
+                for item in content:
+                    if isinstance(item, dict):
+                        item_type = item.get("type")
+                        if item_type == "text":
+                            text_fragments.append(item.get("text", ""))
+                        elif item_type == "image_url":
+                            text_fragments.append("<image>")
+                        elif item_type == "video_url":
+                            text_fragments.append("<video>")
+                        elif item_type == "audio_url":
+                            text_fragments.append("<audio>")
+                    elif isinstance(item, str):
+                        text_fragments.append(item)
+                content = " ".join(text_fragments)
+            prompt_parts.append(f"{role}: {content}")
+        return "\n".join(part for part in prompt_parts if part).strip()
+
+    def transform_response(
+        self,
+        model: str,
+        raw_response: httpx.Response,
+        model_response: ModelResponse,
+        logging_obj: LiteLLMLoggingObj,
+        request_data: dict,
+        messages: List[AllMessageValues],
+        optional_params: dict,
+        litellm_params: dict,
+        encoding: Any,
+        api_key: Optional[str] = None,
+        json_mode: Optional[bool] = None,
+    ) -> ModelResponse:
+        """
+        Transform TwelveLabs Pegasus response to LiteLLM format.
+        
+        TwelveLabs response format:
+        {
+            "message": "...",
+            "finishReason": "stop" | "length"
+        }
+        
+        LiteLLM format:
+        ModelResponse with choices[0].message.content and finish_reason
+        """
+        try:
+            completion_response = raw_response.json()
+        except Exception as e:
+            raise BedrockError(
+                message=f"Error parsing response: {raw_response.text}, error: {str(e)}",
+                status_code=raw_response.status_code,
+            )
+        
+        verbose_logger.debug(
+            "twelvelabs pegasus response: %s",
+            json.dumps(completion_response, indent=4, default=str),
+        )
+        
+        # Extract message content
+        message_content = completion_response.get("message", "")
+        
+        # Extract finish reason and map to LiteLLM format
+        finish_reason_raw = completion_response.get("finishReason", "stop")
+        finish_reason = map_finish_reason(finish_reason_raw)
+        
+        # Set the response content
+        try:
+            if (
+                message_content
+                and hasattr(model_response.choices[0], "message")
+                and getattr(model_response.choices[0].message, "tool_calls", None) is None
+            ):
+                model_response.choices[0].message.content = message_content  # type: ignore
+                model_response.choices[0].finish_reason = finish_reason
+            else:
+                raise Exception("Unable to set message content")
+        except Exception as e:
+            raise BedrockError(
+                message=f"Error setting response content: {str(e)}. Response: {completion_response}",
+                status_code=raw_response.status_code,
+            )
+        
+        # Calculate usage from headers
+        bedrock_input_tokens = raw_response.headers.get(
+            "x-amzn-bedrock-input-token-count", None
+        )
+        bedrock_output_tokens = raw_response.headers.get(
+            "x-amzn-bedrock-output-token-count", None
+        )
+        
+        prompt_tokens = int(
+            bedrock_input_tokens or litellm.token_counter(messages=messages)
+        )
+        
+        completion_tokens = int(
+            bedrock_output_tokens
+            or litellm.token_counter(
+                text=model_response.choices[0].message.content,  # type: ignore
+                count_response_tokens=True,
+            )
+        )
+        
+        model_response.created = int(time.time())
+        model_response.model = model
+        usage = Usage(
+            prompt_tokens=prompt_tokens,
+            completion_tokens=completion_tokens,
+            total_tokens=prompt_tokens + completion_tokens,
+        )
+        setattr(model_response, "usage", usage)
+        
+        return model_response
+

litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py

@@ -7,6 +7,7 @@ from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation
     AmazonInvokeConfig,
 )
 from litellm.llms.bedrock.common_utils import get_anthropic_beta_from_headers
+from litellm.types.llms.anthropic import ANTHROPIC_TOOL_SEARCH_BETA_HEADER
 from litellm.types.llms.openai import AllMessageValues
 from litellm.types.utils import ModelResponse
 
@@ -76,6 +77,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
             for k, v in optional_params.items()
             if k not in self.aws_authentication_params
         }
+        filtered_params = self._normalize_bedrock_tool_search_tools(filtered_params)
         
         _anthropic_request = AnthropicConfig.transform_request(
             self,
@@ -91,13 +93,62 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
         if "anthropic_version" not in _anthropic_request:
             _anthropic_request["anthropic_version"] = self.anthropic_version
 
-        # Handle anthropic_beta from user headers
-        anthropic_beta_list = get_anthropic_beta_from_headers(headers)
-        if anthropic_beta_list:
-            _anthropic_request["anthropic_beta"] = anthropic_beta_list
+        tools = optional_params.get("tools")
+        tool_search_used = self.is_tool_search_used(tools)
+        programmatic_tool_calling_used = self.is_programmatic_tool_calling_used(tools)
+        input_examples_used = self.is_input_examples_used(tools)
+
+        beta_set = set(get_anthropic_beta_from_headers(headers))
+        auto_betas = self.get_anthropic_beta_list(
+            model=model,
+            optional_params=optional_params,
+            computer_tool_used=self.is_computer_tool_used(tools),
+            prompt_caching_set=False, 
+            file_id_used=self.is_file_id_used(messages),
+            mcp_server_used=self.is_mcp_server_used(optional_params.get("mcp_servers")),
+        )
+        beta_set.update(auto_betas)
+
+        if (
+            tool_search_used
+            and not (programmatic_tool_calling_used or input_examples_used)
+        ):
+            beta_set.discard(ANTHROPIC_TOOL_SEARCH_BETA_HEADER)
+            if "opus-4" in model.lower() or "opus_4" in model.lower():
+                beta_set.add("tool-search-tool-2025-10-19")
+
+        if beta_set:
+            _anthropic_request["anthropic_beta"] = list(beta_set)
 
         return _anthropic_request
 
+    def _normalize_bedrock_tool_search_tools(self, optional_params: dict) -> dict:
+        """
+        Convert tool search entries to the format supported by the Bedrock Invoke API.
+        """
+        tools = optional_params.get("tools")
+        if not tools or not isinstance(tools, list):
+            return optional_params
+
+        normalized_tools = []
+        for tool in tools:
+            tool_type = tool.get("type")
+            if tool_type == "tool_search_tool_bm25_20251119":
+                # Bedrock Invoke does not support the BM25 variant, so skip it.
+                continue
+            if tool_type == "tool_search_tool_regex_20251119":
+                normalized_tool = tool.copy()
+                normalized_tool["type"] = "tool_search_tool_regex"
+                normalized_tool["name"] = normalized_tool.get(
+                    "name", "tool_search_tool_regex"
+                )
+                normalized_tools.append(normalized_tool)
+                continue
+            normalized_tools.append(tool)
+
+        optional_params["tools"] = normalized_tools
+        return optional_params
+
     def transform_response(
         self,
         model: str,

litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py

@@ -250,6 +250,23 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
                 ):  # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in
                     inference_params[k] = v
             request_data = {"prompt": prompt, **inference_params}
+        elif provider == "twelvelabs":
+            return litellm.AmazonTwelveLabsPegasusConfig().transform_request(
+                model=model,
+                messages=messages,
+                optional_params=optional_params,
+                litellm_params=litellm_params,
+                headers=headers,
+            )
+        elif provider == "openai":
+            # OpenAI imported models use OpenAI Chat Completions format
+            return litellm.AmazonBedrockOpenAIConfig().transform_request(
+                model=model,
+                messages=messages,
+                optional_params=optional_params,
+                litellm_params=litellm_params,
+                headers=headers,
+            )
         else:
             raise BedrockError(
                 status_code=404,
@@ -321,6 +338,20 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
                     litellm_params=litellm_params,
                     encoding=encoding,
                 )
+            elif provider == "twelvelabs":
+                return litellm.AmazonTwelveLabsPegasusConfig().transform_response(
+                    model=model,
+                    raw_response=raw_response,
+                    model_response=model_response,
+                    logging_obj=logging_obj,
+                    request_data=request_data,
+                    messages=messages,
+                    optional_params=optional_params,
+                    litellm_params=litellm_params,
+                    encoding=encoding,
+                    api_key=api_key,
+                    json_mode=json_mode,
+                )
             elif provider == "ai21":
                 outputText = (
                     completion_response.get("completions")[0].get("data").get("text")

litellm/llms/bedrock/common_utils.py

@@ -616,6 +616,8 @@ def get_bedrock_chat_config(model: str):
         return litellm.AmazonInvokeNovaConfig()
     elif bedrock_invoke_provider == "qwen3":
         return litellm.AmazonQwen3Config()
+    elif bedrock_invoke_provider == "twelvelabs":
+        return litellm.AmazonTwelveLabsPegasusConfig()
     else:
         return litellm.AmazonInvokeConfig()
 

litellm/llms/bedrock/embed/amazon_nova_transformation.py

@@ -0,0 +1,260 @@
+"""
+Transformation logic from OpenAI /v1/embeddings format to Bedrock Amazon Nova /invoke and /async-invoke format.
+
+Why separate file? Make it easy to see how transformation works
+
+Supports:
+- Synchronous embeddings (SINGLE_EMBEDDING)
+- Asynchronous embeddings with segmentation (SEGMENTED_EMBEDDING)
+- Multimodal inputs: text, image, video, audio
+- Multiple embedding purposes and dimensions
+
+Docs - https://docs.aws.amazon.com/bedrock/latest/userguide/nova-embed.html
+"""
+
+from typing import List, Optional
+
+from litellm.types.utils import Embedding, EmbeddingResponse, Usage
+
+
+class AmazonNovaEmbeddingConfig:
+    """
+    Reference: https://docs.aws.amazon.com/bedrock/latest/userguide/nova-embed.html
+    
+    Amazon Nova Multimodal Embeddings supports:
+    - Text, image, video, and audio inputs
+    - Synchronous (InvokeModel) and asynchronous (StartAsyncInvoke) APIs
+    - Multiple embedding purposes and dimensions
+    """
+
+    def __init__(self) -> None:
+        pass
+
+    def get_supported_openai_params(self) -> List[str]:
+        return [
+            "dimensions",
+        ]
+
+    def map_openai_params(
+        self, non_default_params: dict, optional_params: dict
+    ) -> dict:
+        """Map OpenAI-style parameters to Nova parameters."""
+        for k, v in non_default_params.items():
+            if k == "dimensions":
+                # Map OpenAI dimensions to Nova embedding_dimension
+                optional_params["embedding_dimension"] = v
+            elif k in self.get_supported_openai_params():
+                optional_params[k] = v
+        return optional_params
+
+    def _transform_request(
+        self,
+        input: str,
+        inference_params: dict,
+        async_invoke_route: bool = False,
+        model_id: Optional[str] = None,
+        output_s3_uri: Optional[str] = None,
+    ) -> dict:
+        """
+        Transform OpenAI-style input to Nova format.
+        
+        Only handles OpenAI params (dimensions). All other Nova-specific params
+        should be passed via inference_params and will be passed through as-is.
+        
+        Args:
+            input: The input text or media reference
+            inference_params: Additional parameters (will be passed through)
+            async_invoke_route: Whether this is for async invoke
+            model_id: Model ID (for async invoke)
+            output_s3_uri: S3 URI for output (for async invoke)
+        
+        Returns:
+            dict: Nova embedding request
+        """
+        # Determine task type
+        task_type = "SEGMENTED_EMBEDDING" if async_invoke_route else "SINGLE_EMBEDDING"
+        
+        # Build the base request structure
+        request: dict = {
+            "schemaVersion": "nova-multimodal-embed-v1",
+            "taskType": task_type,
+        }
+        
+        # Start with inference_params (user-provided params)
+        embedding_params = inference_params.copy()
+        
+        embedding_params.pop("output_s3_uri", None)
+        
+        # Map OpenAI dimensions to embeddingDimension if provided
+        if "dimensions" in embedding_params:
+            embedding_params["embeddingDimension"] = embedding_params.pop("dimensions")
+        elif "embedding_dimension" in embedding_params:
+            embedding_params["embeddingDimension"] = embedding_params.pop("embedding_dimension")
+        
+        # Add required embeddingPurpose if not provided (required by Nova API)
+        if "embeddingPurpose" not in embedding_params:
+            embedding_params["embeddingPurpose"] = "GENERIC_INDEX"
+        
+        # Add required embeddingDimension if not provided (required by Nova API)
+        if "embeddingDimension" not in embedding_params:
+            embedding_params["embeddingDimension"] = 3072
+        
+        # For text input, add basic text structure if user hasn't provided text/image/video/audio
+        if "text" not in embedding_params and "image" not in embedding_params and "video" not in embedding_params and "audio" not in embedding_params:
+            # Default to text if no modality specified
+            if input.startswith("s3://"):
+                embedding_params["text"] = {
+                    "source": {"s3Location": {"uri": input}},
+                    "truncationMode": "END"  # Required by Nova API
+                }
+            else:
+                embedding_params["text"] = {
+                    "value": input,
+                    "truncationMode": "END"  # Required by Nova API
+                }
+        
+        # Set the embedding params in the request
+        if task_type == "SINGLE_EMBEDDING":
+            request["singleEmbeddingParams"] = embedding_params
+        else:
+            request["segmentedEmbeddingParams"] = embedding_params
+        
+        # For async invoke, wrap in the async invoke format
+        if async_invoke_route and model_id:
+            return self._wrap_async_invoke_request(
+                model_input=request,
+                model_id=model_id,
+                output_s3_uri=output_s3_uri,
+            )
+        
+        return request
+
+    def _wrap_async_invoke_request(
+        self,
+        model_input: dict,
+        model_id: str,
+        output_s3_uri: Optional[str] = None,
+    ) -> dict:
+        """
+        Wrap the transformed request in the AWS Bedrock async invoke format.
+        
+        Args:
+            model_input: The transformed Nova embedding request
+            model_id: The model identifier (without async_invoke prefix)
+            output_s3_uri: S3 URI for output data config
+        
+        Returns:
+            dict: The wrapped async invoke request
+        """
+        import urllib.parse
+
+        # Clean the model ID
+        unquoted_model_id = urllib.parse.unquote(model_id)
+        if unquoted_model_id.startswith("async_invoke/"):
+            unquoted_model_id = unquoted_model_id.replace("async_invoke/", "")
+        
+        # Validate that the S3 URI is not empty
+        if not output_s3_uri or output_s3_uri.strip() == "":
+            raise ValueError("output_s3_uri is required for async invoke requests")
+        
+        return {
+            "modelId": unquoted_model_id,
+            "modelInput": model_input,
+            "outputDataConfig": {
+                "s3OutputDataConfig": {
+                    "s3Uri": output_s3_uri
+                }
+            },
+        }
+
+    def _transform_response(
+        self, response_list: List[dict], model: str
+    ) -> EmbeddingResponse:
+        """
+        Transform Nova response to OpenAI format.
+        
+        Nova response format:
+        {
+            "embeddings": [
+                {
+                    "embeddingType": "TEXT" | "IMAGE" | "VIDEO" | "AUDIO" | "AUDIO_VIDEO_COMBINED",
+                    "embedding": [0.1, 0.2, ...],
+                    "truncatedCharLength": 100  # Optional, only for text
+                }
+            ]
+        }
+        """
+        embeddings: List[Embedding] = []
+        total_tokens = 0
+        
+        for response in response_list:
+            # Nova response has an "embeddings" array
+            if "embeddings" in response and isinstance(response["embeddings"], list):
+                for item in response["embeddings"]:
+                    if "embedding" in item:
+                        embedding = Embedding(
+                            embedding=item["embedding"],
+                            index=len(embeddings),
+                            object="embedding",
+                        )
+                        embeddings.append(embedding)
+                        
+                        # Estimate token count
+                        # For text, use truncatedCharLength if available
+                        if "truncatedCharLength" in item:
+                            total_tokens += item["truncatedCharLength"] // 4
+                        else:
+                            # Rough estimate based on embedding dimension
+                            total_tokens += len(item["embedding"]) // 4
+            elif "embedding" in response:
+                # Direct embedding response (fallback)
+                embedding = Embedding(
+                    embedding=response["embedding"],
+                    index=len(embeddings),
+                    object="embedding",
+                )
+                embeddings.append(embedding)
+                total_tokens += len(response["embedding"]) // 4
+        
+        usage = Usage(prompt_tokens=total_tokens, total_tokens=total_tokens)
+        
+        return EmbeddingResponse(data=embeddings, model=model, usage=usage)
+
+    def _transform_async_invoke_response(
+        self, response: dict, model: str
+    ) -> EmbeddingResponse:
+        """
+        Transform async invoke response (invocation ARN) to OpenAI format.
+        
+        AWS async invoke returns:
+        {
+            "invocationArn": "arn:aws:bedrock:us-east-1:123456789012:async-invoke/abc123"
+        }
+        
+        We transform this to a job-like embedding response with the ARN in hidden params.
+        """
+        invocation_arn = response.get("invocationArn", "")
+        
+        # Create a placeholder embedding object for the job
+        embedding = Embedding(
+            embedding=[],  # Empty embedding for async jobs
+            index=0,
+            object="embedding",
+        )
+        
+        # Create usage object (empty for async jobs)
+        usage = Usage(prompt_tokens=0, total_tokens=0)
+        
+        # Create hidden params with job ID
+        from litellm.types.llms.base import HiddenParams
+        
+        hidden_params = HiddenParams()
+        setattr(hidden_params, "_invocation_arn", invocation_arn)
+        
+        return EmbeddingResponse(
+            data=[embedding],
+            model=model,
+            usage=usage,
+            hidden_params=hidden_params,
+        )
+

litellm/llms/bedrock/embed/embedding.py

@@ -27,6 +27,7 @@ from litellm.types.utils import EmbeddingResponse, LlmProviders
 
 from ..base_aws_llm import BaseAWSLLM
 from ..common_utils import BedrockError
+from .amazon_nova_transformation import AmazonNovaEmbeddingConfig
 from .amazon_titan_g1_transformation import AmazonTitanG1Config
 from .amazon_titan_multimodal_transformation import (
     AmazonTitanMultimodalEmbeddingG1Config,
@@ -175,6 +176,12 @@ class BedrockEmbedding(BaseAWSLLM):
                         response=response_list[0], model=model
                     )
                 )
+            elif provider == "nova":
+                returned_response = (
+                    AmazonNovaEmbeddingConfig()._transform_async_invoke_response(
+                        response=response_list[0], model=model
+                    )
+                )
             else:
                 # For other providers, create a generic async response
                 invocation_arn = response_list[0].get("invocationArn", "")
@@ -222,6 +229,10 @@ class BedrockEmbedding(BaseAWSLLM):
                         response_list=response_list, model=model
                     )
                 )
+            elif provider == "nova":
+                returned_response = AmazonNovaEmbeddingConfig()._transform_response(
+                    response_list=response_list, model=model
+                )
 
         ##########################################################
         # Validate returned response
@@ -366,7 +377,7 @@ class BedrockEmbedding(BaseAWSLLM):
             is_async_invoke=is_async_invoke,
         )
 
-    def embeddings(
+    def embeddings(  # noqa: PLR0915
         self,
         model: str,
         input: List[str],
@@ -467,6 +478,17 @@ class BedrockEmbedding(BaseAWSLLM):
                     )
                 )
                 batch_data.append(twelvelabs_request)
+        elif provider == "nova":
+            batch_data = []
+            for i in input:
+                nova_request = AmazonNovaEmbeddingConfig()._transform_request(
+                    input=i,
+                    inference_params=inference_params,
+                    async_invoke_route=has_async_invoke,
+                    model_id=modelId,
+                    output_s3_uri=inference_params.get("output_s3_uri"),
+                )
+                batch_data.append(nova_request)
 
         ### SET RUNTIME ENDPOINT ###
         endpoint_url, proxy_endpoint_url = self.get_runtime_endpoint(
@@ -581,22 +603,39 @@ class BedrockEmbedding(BaseAWSLLM):
             aws_region_name=aws_region_name,
         )
 
-        # Construct the status check URL
-        status_url = f"{endpoint_url}/async-invoke/{invocation_arn}"
 
-        # Prepare headers
+        from urllib.parse import quote
+
+        # Encode the ARN for use in URL path
+        encoded_arn = quote(invocation_arn, safe="")
+        status_url = f"{endpoint_url.rstrip('/')}/async-invoke/{encoded_arn}"
+
+        # Prepare headers for GET request
         headers = {"Content-Type": "application/json"}
 
-        # Get AWS signed headers
-        prepped = self.get_request_headers(  # type: ignore
-            credentials=credentials,
-            aws_region_name=aws_region_name,
-            extra_headers=None,
-            endpoint_url=status_url,
-            data="",  # GET request, no body
+        # Use AWSRequest directly for GET requests (get_request_headers hardcodes POST)
+        try:
+            from botocore.auth import SigV4Auth
+            from botocore.awsrequest import AWSRequest
+        except ImportError:
+            raise ImportError(
+                "Missing boto3 to call bedrock. Run 'pip install boto3'."
+            )
+
+        # Create AWSRequest with GET method and encoded URL
+        request = AWSRequest(
+            method="GET",
+            url=status_url,
+            data=None,  # GET request, no body
             headers=headers,
-            api_key=None,
         )
+        
+        # Sign the request - SigV4Auth will create canonical string from request URL
+        sigv4 = SigV4Auth(credentials, "bedrock", aws_region_name)
+        sigv4.add_auth(request)
+        
+        # Prepare the request
+        prepped = request.prepare()
 
         # LOGGING
         if logging_obj is not None:

litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py

@@ -1,7 +1,18 @@
-from typing import TYPE_CHECKING, Any, AsyncIterator, Dict, List, Optional, Tuple, Union
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    AsyncIterator,
+    Dict,
+    List,
+    Optional,
+    Tuple,
+    Union,
+    cast,
+)
 
 import httpx
 
+from litellm.llms.anthropic.common_utils import AnthropicModelInfo
 from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
     AnthropicMessagesConfig,
 )
@@ -13,6 +24,8 @@ from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation
     AmazonInvokeConfig,
 )
 from litellm.llms.bedrock.common_utils import get_anthropic_beta_from_headers
+from litellm.types.llms.anthropic import ANTHROPIC_TOOL_SEARCH_BETA_HEADER
+from litellm.types.llms.openai import AllMessageValues
 from litellm.types.router import GenericLiteLLMParams
 from litellm.types.utils import GenericStreamingChunk
 from litellm.types.utils import GenericStreamingChunk as GChunk
@@ -129,10 +142,39 @@ class AmazonAnthropicClaudeMessagesConfig(
         if "model" in anthropic_messages_request:
             anthropic_messages_request.pop("model", None)
             
-        # 4. Handle anthropic_beta from user headers
-        anthropic_beta_list = get_anthropic_beta_from_headers(headers)
-        if anthropic_beta_list:
-            anthropic_messages_request["anthropic_beta"] = anthropic_beta_list
+        # 4. AUTO-INJECT beta headers based on features used
+        anthropic_model_info = AnthropicModelInfo()
+        tools = anthropic_messages_optional_request_params.get("tools")
+        messages_typed = cast(List[AllMessageValues], messages)
+        tool_search_used = anthropic_model_info.is_tool_search_used(tools)
+        programmatic_tool_calling_used = anthropic_model_info.is_programmatic_tool_calling_used(
+            tools
+        )
+        input_examples_used = anthropic_model_info.is_input_examples_used(tools)
+
+        beta_set = set(get_anthropic_beta_from_headers(headers))
+        auto_betas = anthropic_model_info.get_anthropic_beta_list(
+            model=model,
+            optional_params=anthropic_messages_optional_request_params,
+            computer_tool_used=anthropic_model_info.is_computer_tool_used(tools),
+            prompt_caching_set=False,
+            file_id_used=anthropic_model_info.is_file_id_used(messages_typed),
+            mcp_server_used=anthropic_model_info.is_mcp_server_used(
+                anthropic_messages_optional_request_params.get("mcp_servers")
+            ),
+        )
+        beta_set.update(auto_betas)
+
+        if (
+            tool_search_used
+            and not (programmatic_tool_calling_used or input_examples_used)
+        ):
+            beta_set.discard(ANTHROPIC_TOOL_SEARCH_BETA_HEADER)
+            if "opus-4" in model.lower() or "opus_4" in model.lower():
+                beta_set.add("tool-search-tool-2025-10-19")
+
+        if beta_set:
+            anthropic_messages_request["anthropic_beta"] = list(beta_set)
             
         return anthropic_messages_request