Merge remote-tracking branch 'origin' into litellm_ui_cred_refresh

cookbook/mock_guardrail_server/mock_bedrock_guardrail_server.py

@@ -0,0 +1,570 @@
+#!/usr/bin/env python3
+"""
+Mock Bedrock Guardrail API Server
+
+This is a FastAPI server that mimics the AWS Bedrock Guardrail API for testing purposes.
+It follows the same API spec as the real Bedrock guardrail endpoint.
+
+Usage:
+    python mock_bedrock_guardrail_server.py
+
+The server will start on http://localhost:8080
+"""
+
+import os
+import re
+from typing import Any, Dict, List, Literal, Optional
+
+from fastapi import Depends, FastAPI, Header, HTTPException, status
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel, Field
+
+# ============================================================================
+# Request/Response Models (matching Bedrock API spec)
+# ============================================================================
+
+
+class BedrockTextContent(BaseModel):
+    text: str
+
+
+class BedrockContentItem(BaseModel):
+    text: BedrockTextContent
+
+
+class BedrockRequest(BaseModel):
+    source: Literal["INPUT", "OUTPUT"]
+    content: List[BedrockContentItem] = Field(default_factory=list)
+
+
+class BedrockGuardrailOutput(BaseModel):
+    text: Optional[str] = None
+
+
+class TopicPolicyItem(BaseModel):
+    name: str
+    type: str
+    action: Literal["BLOCKED", "NONE"]
+
+
+class TopicPolicy(BaseModel):
+    topics: List[TopicPolicyItem] = Field(default_factory=list)
+
+
+class ContentFilterItem(BaseModel):
+    type: str
+    confidence: str
+    action: Literal["BLOCKED", "NONE"]
+
+
+class ContentPolicy(BaseModel):
+    filters: List[ContentFilterItem] = Field(default_factory=list)
+
+
+class CustomWord(BaseModel):
+    match: str
+    action: Literal["BLOCKED", "NONE"]
+
+
+class WordPolicy(BaseModel):
+    customWords: List[CustomWord] = Field(default_factory=list)
+    managedWordLists: List[Dict[str, Any]] = Field(default_factory=list)
+
+
+class PiiEntity(BaseModel):
+    type: str
+    match: str
+    action: Literal["BLOCKED", "ANONYMIZED", "NONE"]
+
+
+class RegexMatch(BaseModel):
+    name: str
+    match: str
+    regex: str
+    action: Literal["BLOCKED", "ANONYMIZED", "NONE"]
+
+
+class SensitiveInformationPolicy(BaseModel):
+    piiEntities: List[PiiEntity] = Field(default_factory=list)
+    regexes: List[RegexMatch] = Field(default_factory=list)
+
+
+class ContextualGroundingFilter(BaseModel):
+    type: str
+    threshold: float
+    score: float
+    action: Literal["BLOCKED", "NONE"]
+
+
+class ContextualGroundingPolicy(BaseModel):
+    filters: List[ContextualGroundingFilter] = Field(default_factory=list)
+
+
+class Assessment(BaseModel):
+    topicPolicy: Optional[TopicPolicy] = None
+    contentPolicy: Optional[ContentPolicy] = None
+    wordPolicy: Optional[WordPolicy] = None
+    sensitiveInformationPolicy: Optional[SensitiveInformationPolicy] = None
+    contextualGroundingPolicy: Optional[ContextualGroundingPolicy] = None
+
+
+class BedrockGuardrailResponse(BaseModel):
+    usage: Dict[str, int] = Field(
+        default_factory=lambda: {"topicPolicyUnits": 1, "contentPolicyUnits": 1}
+    )
+    action: Literal["NONE", "GUARDRAIL_INTERVENED"] = "NONE"
+    outputs: List[BedrockGuardrailOutput] = Field(default_factory=list)
+    assessments: List[Assessment] = Field(default_factory=list)
+
+
+# ============================================================================
+# Mock Guardrail Configuration
+# ============================================================================
+
+
+class GuardrailConfig(BaseModel):
+    """Configuration for mock guardrail behavior"""
+
+    blocked_words: List[str] = Field(
+        default_factory=lambda: ["offensive", "inappropriate", "badword"]
+    )
+    blocked_topics: List[str] = Field(default_factory=lambda: ["violence", "illegal"])
+    pii_patterns: Dict[str, str] = Field(
+        default_factory=lambda: {
+            "EMAIL": r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b",
+            "PHONE": r"\b\d{3}[-.]?\d{3}[-.]?\d{4}\b",
+            "SSN": r"\b\d{3}-\d{2}-\d{4}\b",
+            "CREDIT_CARD": r"\b\d{4}[\s-]?\d{4}[\s-]?\d{4}[\s-]?\d{4}\b",
+        }
+    )
+    anonymize_pii: bool = True  # If True, ANONYMIZE PII; if False, BLOCK it
+    bearer_token: str = "mock-bedrock-token-12345"
+
+
+# Global config
+GUARDRAIL_CONFIG = GuardrailConfig()
+
+# ============================================================================
+# FastAPI App Setup
+# ============================================================================
+
+app = FastAPI(
+    title="Mock Bedrock Guardrail API",
+    description="Mock server mimicking AWS Bedrock Guardrail API",
+    version="1.0.0",
+)
+
+
+# ============================================================================
+# Authentication
+# ============================================================================
+
+
+async def verify_bearer_token(authorization: Optional[str] = Header(None)) -> str:
+    """
+    Verify the Bearer token from the Authorization header.
+
+    Args:
+        authorization: The Authorization header value
+
+    Returns:
+        The token if valid
+
+    Raises:
+        HTTPException: If token is missing or invalid
+    """
+    if authorization is None:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Missing Authorization header",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+
+    # Check if it's a Bearer token
+    parts = authorization.split()
+    print(f"parts: {parts}")
+    if len(parts) != 2 or parts[0].lower() != "bearer":
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Invalid Authorization header format. Expected: Bearer <token>",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+
+    token = parts[1]
+
+    # Verify token
+    if token != GUARDRAIL_CONFIG.bearer_token:
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail="Invalid bearer token",
+        )
+
+    return token
+
+
+# ============================================================================
+# Guardrail Logic
+# ============================================================================
+
+
+def check_blocked_words(text: str) -> Optional[WordPolicy]:
+    """Check if text contains blocked words"""
+    found_words = []
+    text_lower = text.lower()
+
+    for word in GUARDRAIL_CONFIG.blocked_words:
+        if word.lower() in text_lower:
+            found_words.append(CustomWord(match=word, action="BLOCKED"))
+
+    if found_words:
+        return WordPolicy(customWords=found_words)
+    return None
+
+
+def check_blocked_topics(text: str) -> Optional[TopicPolicy]:
+    """Check if text contains blocked topics"""
+    found_topics = []
+    text_lower = text.lower()
+
+    for topic in GUARDRAIL_CONFIG.blocked_topics:
+        if topic.lower() in text_lower:
+            found_topics.append(
+                TopicPolicyItem(name=topic, type=topic.upper(), action="BLOCKED")
+            )
+
+    if found_topics:
+        return TopicPolicy(topics=found_topics)
+    return None
+
+
+def check_pii(text: str) -> tuple[Optional[SensitiveInformationPolicy], str]:
+    """
+    Check for PII in text and return policy + anonymized text
+
+    Returns:
+        Tuple of (SensitiveInformationPolicy or None, anonymized_text)
+    """
+    pii_entities = []
+    anonymized_text = text
+    action = "ANONYMIZED" if GUARDRAIL_CONFIG.anonymize_pii else "BLOCKED"
+
+    for pii_type, pattern in GUARDRAIL_CONFIG.pii_patterns.items():
+        try:
+            # Compile the regex pattern with a timeout to prevent ReDoS attacks
+            compiled_pattern = re.compile(pattern)
+            matches = compiled_pattern.finditer(text)
+            for match in matches:
+                matched_text = match.group()
+                pii_entities.append(
+                    PiiEntity(type=pii_type, match=matched_text, action=action)
+                )
+
+                # Anonymize the text if configured
+                if GUARDRAIL_CONFIG.anonymize_pii:
+                    anonymized_text = anonymized_text.replace(
+                        matched_text, f"[{pii_type}_REDACTED]"
+                    )
+        except re.error:
+            # Invalid regex pattern - skip it and log a warning
+            print(f"Warning: Invalid regex pattern for PII type {pii_type}: {pattern}")
+            continue
+
+    if pii_entities:
+        return SensitiveInformationPolicy(piiEntities=pii_entities), anonymized_text
+
+    return None, text
+
+
+def process_guardrail_request(
+    request: BedrockRequest,
+) -> tuple[BedrockGuardrailResponse, List[str]]:
+    """
+    Process a guardrail request and return the response.
+
+    Returns:
+        Tuple of (response, list of output texts)
+    """
+    all_text_content = []
+    output_texts = []
+
+    # Extract all text from content items
+    for content_item in request.content:
+        if content_item.text and content_item.text.text:
+            all_text_content.append(content_item.text.text)
+
+    # Combine all text for analysis
+    combined_text = " ".join(all_text_content)
+
+    # Initialize response
+    response = BedrockGuardrailResponse()
+    assessment = Assessment()
+    has_intervention = False
+
+    # Check for blocked words
+    word_policy = check_blocked_words(combined_text)
+    if word_policy:
+        assessment.wordPolicy = word_policy
+        has_intervention = True
+
+    # Check for blocked topics
+    topic_policy = check_blocked_topics(combined_text)
+    if topic_policy:
+        assessment.topicPolicy = topic_policy
+        has_intervention = True
+
+    # Check for PII
+    for text in all_text_content:
+        pii_policy, anonymized_text = check_pii(text)
+        if pii_policy:
+            assessment.sensitiveInformationPolicy = pii_policy
+            if GUARDRAIL_CONFIG.anonymize_pii:
+                # If anonymizing, we don't block, we modify the text
+                output_texts.append(anonymized_text)
+                has_intervention = True
+            else:
+                # If not anonymizing PII, we block it
+                output_texts.append(text)
+                has_intervention = True
+        else:
+            output_texts.append(text)
+
+    # Build response
+    if has_intervention:
+        response.action = "GUARDRAIL_INTERVENED"
+        # Only add assessment if there were interventions
+        response.assessments = [assessment]
+
+    # Add outputs (modified or original text)
+    response.outputs = [BedrockGuardrailOutput(text=txt) for txt in output_texts]
+
+    return response, output_texts
+
+
+# ============================================================================
+# API Endpoints
+# ============================================================================
+
+
+@app.get("/")
+async def root():
+    """Health check endpoint"""
+    return {
+        "service": "Mock Bedrock Guardrail API",
+        "status": "running",
+        "endpoint_format": "/guardrail/{guardrailIdentifier}/version/{guardrailVersion}/apply",
+    }
+
+
+@app.get("/health")
+async def health():
+    """Health check endpoint"""
+    return {"status": "healthy"}
+
+
+@app.post(
+    "/guardrail/{guardrailIdentifier}/version/{guardrailVersion}/apply",
+    response_model=BedrockGuardrailResponse,
+)
+async def apply_guardrail(
+    guardrailIdentifier: str,
+    guardrailVersion: str,
+    request: BedrockRequest,
+    token: str = Depends(verify_bearer_token),
+) -> BedrockGuardrailResponse:
+    """
+    Apply guardrail to input or output content.
+
+    This endpoint mimics the AWS Bedrock ApplyGuardrail API.
+
+    Args:
+        guardrailIdentifier: The guardrail ID
+        guardrailVersion: The guardrail version
+        request: The guardrail request containing content to analyze
+        token: Bearer token (verified by dependency)
+
+    Returns:
+        BedrockGuardrailResponse with analysis results
+    """
+    # Process the request
+    response, output_texts = process_guardrail_request(request)
+
+    # Log the request (optional, for debugging)
+    print(f"Guardrail applied: {guardrailIdentifier} v{guardrailVersion}")
+    print(f"Source: {request.source}")
+    print(f"Action: {response.action}")
+
+    return response
+
+
+"""
+LiteLLM exposes a basic guardrail API with the text extracted from the request and sent to the guardrail API, as well as the received request body for any further processing. 
+
+This works across all LiteLLM endpoints (completion, anthropic /v1/messages, responses api, image generation, embedding, etc.)
+
+This makes it easy to support your own guardrail API without having to make a PR to LiteLLM.
+
+LiteLLM supports passing any provider specific params from LiteLLM config.yaml to the guardrail API.
+
+Example:
+
+```yaml
+guardrails:
+  - guardrail_name: "bedrock-content-guard"
+    litellm_params:
+      guardrail: generic_guardrail_api
+      mode: "pre_call"
+      api_key: os.environ/GUARDRAIL_API_KEY
+      api_base: os.environ/GUARDRAIL_API_BASE
+      additional_provider_specific_params:
+        api_version: os.environ/GUARDRAIL_API_VERSION # additional provider specific params
+```
+
+This is a beta API. Please help us improve it. 
+"""
+
+
+class LitellmBasicGuardrailRequest(BaseModel):
+    texts: List[str]
+    images: Optional[List[str]] = None
+    request_data: Dict[str, Any] = Field(default_factory=dict)
+    additional_provider_specific_params: Dict[str, Any] = Field(default_factory=dict)
+    input_type: Literal["request", "response"]
+
+
+class LitellmBasicGuardrailResponse(BaseModel):
+    action: Literal[
+        "BLOCKED", "NONE", "GUARDRAIL_INTERVENED"
+    ]  # BLOCKED = litellm will raise an error, NONE = litellm will continue, GUARDRAIL_INTERVENED = litellm will continue, but the text was modified by the guardrail
+    blocked_reason: Optional[str] = None  # only if action is BLOCKED, otherwise None
+    texts: Optional[List[str]] = None
+    images: Optional[List[str]] = None
+
+
+@app.post(
+    "/beta/litellm_basic_guardrail_api",
+    response_model=LitellmBasicGuardrailResponse,
+)
+async def beta_litellm_basic_guardrail_api(
+    request: LitellmBasicGuardrailRequest,
+) -> LitellmBasicGuardrailResponse:
+    """
+    Apply guardrail to input or output content.
+
+    This endpoint mimics the AWS Bedrock ApplyGuardrail API.
+
+    Args:
+        request: The guardrail request containing content to analyze
+        token: Bearer token (verified by dependency)
+
+    Returns:
+        LitellmBasicGuardrailResponse with analysis results
+    """
+    print(f"request: {request}")
+    if any("ishaan" in text.lower() for text in request.texts):
+        return LitellmBasicGuardrailResponse(
+            action="BLOCKED", blocked_reason="Ishaan is not allowed"
+        )
+    elif any("pii_value" in text for text in request.texts):
+        return LitellmBasicGuardrailResponse(
+            action="GUARDRAIL_INTERVENED",
+            texts=[
+                text.replace("pii_value", "pii_value_redacted")
+                for text in request.texts
+            ],
+        )
+    return LitellmBasicGuardrailResponse(action="NONE")
+
+
+@app.post("/config/update")
+async def update_config(
+    config: GuardrailConfig, token: str = Depends(verify_bearer_token)
+):
+    """
+    Update the guardrail configuration.
+
+    This is a testing endpoint to modify the mock guardrail behavior.
+
+    Args:
+        config: New guardrail configuration
+        token: Bearer token (verified by dependency)
+
+    Returns:
+        Updated configuration
+    """
+    global GUARDRAIL_CONFIG
+    GUARDRAIL_CONFIG = config
+    return {"status": "updated", "config": GUARDRAIL_CONFIG}
+
+
+@app.get("/config")
+async def get_config(token: str = Depends(verify_bearer_token)):
+    """
+    Get the current guardrail configuration.
+
+    Args:
+        token: Bearer token (verified by dependency)
+
+    Returns:
+        Current configuration
+    """
+    return GUARDRAIL_CONFIG
+
+
+# ============================================================================
+# Error Handlers
+# ============================================================================
+
+
+@app.exception_handler(HTTPException)
+async def http_exception_handler(request, exc: HTTPException):
+    """Custom error handler for HTTP exceptions"""
+    return JSONResponse(
+        status_code=exc.status_code,
+        content={"error": exc.detail},
+        headers=exc.headers,
+    )
+
+
+# ============================================================================
+# Main
+# ============================================================================
+
+if __name__ == "__main__":
+    import uvicorn
+
+    # Get configuration from environment
+    host = os.getenv("MOCK_BEDROCK_HOST", "0.0.0.0")
+    port = int(os.getenv("MOCK_BEDROCK_PORT", "8080"))
+    bearer_token = os.getenv("MOCK_BEDROCK_TOKEN", "mock-bedrock-token-12345")
+
+    # Update config with environment token
+    GUARDRAIL_CONFIG.bearer_token = bearer_token
+
+    print("=" * 80)
+    print("Mock Bedrock Guardrail API Server")
+    print("=" * 80)
+    print(f"Server starting on: http://{host}:{port}")
+    print(f"Bearer Token: {bearer_token}")
+    print(f"Endpoint: POST /guardrail/{{id}}/version/{{version}}/apply")
+    print("=" * 80)
+    print("\nExample curl command:")
+    print(
+        f"""
+curl -X POST "http://{host}:{port}/guardrail/test-guardrail/version/1/apply" \\
+  -H "Authorization: Bearer {bearer_token}" \\
+  -H "Content-Type: application/json" \\
+  -d '{{
+    "source": "INPUT",
+    "content": [
+      {{
+        "text": {{
+          "text": "Hello, my email is test@example.com"
+        }}
+      }}
+    ]
+  }}'
+    """
+    )
+    print("=" * 80)
+
+    uvicorn.run(app, host=host, port=port)

docker/Dockerfile.non_root

@@ -10,7 +10,16 @@ WORKDIR /app
 
 # Install build dependencies including Node.js for UI build
 USER root
-RUN apk add --no-cache build-base bash nodejs npm \
+RUN apk add --no-cache \
+    clang \
+    llvm \
+    lld \
+    gcc \
+    linux-headers \
+    build-base \
+    bash \
+    nodejs \
+    npm \
   && pip install --no-cache-dir --upgrade pip build
 
 # Copy project files

docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md

@@ -33,7 +33,7 @@ This guide covers Anthropic's latest model (Claude Opus 4.5) and its advanced fe
 | Input Examples | Claude Opus 4.5, Sonnet 4.5 |
 | Effort Parameter | Claude Opus 4.5 only |
 
-Supported Providers: [Anthropic](../../docs/providers/anthropic), [Bedrock](../../docs/providers/bedrock), [Vertex AI](../../docs/providers/vertex_partner#vertex-ai---anthropic-claude).
+Supported Providers: [Anthropic](../../docs/providers/anthropic), [Bedrock](../../docs/providers/bedrock), [Vertex AI](../../docs/providers/vertex_partner#vertex-ai---anthropic-claude), [Azure AI](../../docs/providers/azure_ai).
 
 ## Usage
 

docs/my-website/docs/adding_provider/generic_guardrail_api.md

@@ -0,0 +1,182 @@
+# [BETA] Generic Guardrail API - Integrate Without a PR
+
+## The Problem
+
+As a guardrail provider, integrating with LiteLLM traditionally requires:
+- Making a PR to the LiteLLM repository
+- Waiting for review and merge
+- Maintaining provider-specific code in LiteLLM's codebase
+- Updating the integration for changes to your API
+
+## The Solution
+
+The **Generic Guardrail API** lets you integrate with LiteLLM **instantly** by implementing a simple API endpoint. No PR required.
+
+### Key Benefits
+
+1. **No PR Needed** - Deploy and integrate immediately
+2. **Universal Support** - Works across ALL LiteLLM endpoints (chat, embeddings, image generation, etc.)
+3. **Simple Contract** - One endpoint, three response types
+4. **Multi-Modal Support** - Handle both text and images in requests/responses
+5. **Custom Parameters** - Pass provider-specific params via config
+6. **Full Control** - You own and maintain your guardrail API
+
+## How It Works
+
+1. LiteLLM extracts text and images from any request (chat messages, embeddings, image prompts, etc.)
+2. Sends extracted content + metadata to your API endpoint
+3. Your API responds with: `BLOCKED`, `NONE`, or `GUARDRAIL_INTERVENED`
+4. LiteLLM enforces the decision and applies any modifications
+
+## API Contract
+
+### Endpoint
+
+Implement `POST /beta/litellm_basic_guardrail_api`
+
+### Request Format
+
+```json
+{
+  "texts": ["extracted text from the request"],  // array of text strings
+  "images": ["base64_encoded_image_data"],  // optional array of images
+  "request_data": {
+    "user_api_key_hash": "hash of the litellm virtual key used",
+    "user_api_key_alias": "alias of the litellm virtual key used",
+    "user_api_key_user_id": "user id associated with the litellm virtual key used",
+    "user_api_key_user_email": "user email associated with the litellm virtual key used",
+    "user_api_key_team_id": "team id associated with the litellm virtual key used",
+    "user_api_key_team_alias": "team alias associated with the litellm virtual key used",
+    "user_api_key_end_user_id": "end user id associated with the litellm virtual key used",
+    "user_api_key_org_id": "org id associated with the litellm virtual key used"
+  },
+  "input_type": "request",  // "request" or "response"
+  "litellm_call_id": "unique_call_id",  // the call id of the individual LLM call
+  "litellm_trace_id": "trace_id",  // the trace id of the LLM call - useful if there are multiple LLM calls for the same conversation
+  "additional_provider_specific_params": {
+    // your custom params from config
+  }
+}
+```
+
+### Response Format
+
+```json
+{
+  "action": "BLOCKED" | "NONE" | "GUARDRAIL_INTERVENED",
+  "blocked_reason": "why content was blocked",  // required if action=BLOCKED
+  "texts": ["modified text"],  // optional array of modified text strings
+  "images": ["modified_base64_image"]  // optional array of modified images
+}
+```
+
+**Actions:**
+- `BLOCKED` - LiteLLM raises error and blocks request
+- `NONE` - Request proceeds unchanged  
+- `GUARDRAIL_INTERVENED` - Request proceeds with modified texts/images (provide `texts` and/or `images` fields)
+
+## LiteLLM Configuration
+
+Add to `config.yaml`:
+
+```yaml
+litellm_settings:
+  guardrails:
+    - guardrail_name: "my-guardrail"
+      litellm_params:
+        guardrail: generic_guardrail_api
+        mode: pre_call  # or post_call, during_call
+        api_base: https://your-guardrail-api.com
+        api_key: os.environ/YOUR_GUARDRAIL_API_KEY  # optional
+        additional_provider_specific_params:
+          # your custom parameters
+          threshold: 0.8
+          language: "en"
+```
+
+## Usage
+
+Users apply your guardrail by name:
+
+```python
+response = client.chat.completions.create(
+    model="gpt-4",
+    messages=[{"role": "user", "content": "hello"}],
+    guardrails=["my-guardrail"]
+)
+```
+
+Or with dynamic parameters:
+
+```python
+response = client.chat.completions.create(
+    model="gpt-4",
+    messages=[{"role": "user", "content": "hello"}],
+    guardrails=[{
+        "my-guardrail": {
+            "extra_body": {
+                "custom_threshold": 0.9
+            }
+        }
+    }]
+)
+```
+
+## Implementation Example
+
+See [mock_bedrock_guardrail_server.py](https://github.com/BerriAI/litellm/blob/main/cookbook/mock_guardrail_server/mock_bedrock_guardrail_server.py) for a complete reference implementation.
+
+**Minimal FastAPI example:**
+
+```python
+from fastapi import FastAPI
+from pydantic import BaseModel
+from typing import List, Optional, Dict, Any
+
+app = FastAPI()
+
+class GuardrailRequest(BaseModel):
+    texts: List[str]
+    images: Optional[List[str]] = None
+    request_data: Dict[str, Any]
+    input_type: str  # "request" or "response"
+    litellm_call_id: Optional[str] = None
+    litellm_trace_id: Optional[str] = None
+    additional_provider_specific_params: Dict[str, Any]
+
+class GuardrailResponse(BaseModel):
+    action: str  # BLOCKED, NONE, or GUARDRAIL_INTERVENED
+    blocked_reason: Optional[str] = None
+    texts: Optional[List[str]] = None
+    images: Optional[List[str]] = None
+
+@app.post("/beta/litellm_basic_guardrail_api")
+async def apply_guardrail(request: GuardrailRequest):
+    # Your guardrail logic here
+    for text in request.texts:
+        if "badword" in text.lower():
+            return GuardrailResponse(
+                action="BLOCKED",
+                blocked_reason="Content contains prohibited terms"
+            )
+    
+    return GuardrailResponse(action="NONE")
+```
+
+## When to Use This
+
+✅ **Use Generic Guardrail API when:**
+- You want instant integration without waiting for PRs
+- You maintain your own guardrail service
+- You need full control over updates and features
+- You want to support all LiteLLM endpoints automatically
+
+❌ **Make a PR when:**
+- You want deeper integration with LiteLLM internals
+- Your guardrail requires complex LiteLLM-specific logic
+- You want to be featured as a built-in provider
+
+## Questions?
+
+This is a **beta API**. We're actively improving it based on feedback. Open an issue or PR if you need additional capabilities.
+

docs/my-website/docs/audio_transcription.md

@@ -13,7 +13,7 @@ import TabItem from '@theme/TabItem';
 | Fallbacks | ✅ | Works between supported models |
 | Loadbalancing | ✅ | Works between supported models |
 | Guardrails | ✅ | Applies to output transcribed text (non-streaming only) |
-| Supported Providers | `openai`, `azure`, `vertex_ai`, `gemini`, `deepgram`, `groq`, `fireworks_ai` | |
+| Supported Providers | `openai`, `azure`, `vertex_ai`, `gemini`, `deepgram`, `groq`, `fireworks_ai`, `ovhcloud` | |
 
 ## Quick Start
 
@@ -126,6 +126,7 @@ transcript = client.audio.transcriptions.create(
 - [Fireworks AI](./providers/fireworks_ai.md#audio-transcription)
 - [Groq](./providers/groq.md#speech-to-text---whisper)
 - [Deepgram](./providers/deepgram.md)
+- [OVHcloud AI Endpoints](./providers/ovhcloud.md)
 
 ---
 

docs/my-website/docs/providers/anthropic.md

@@ -201,6 +201,30 @@ Without `LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX`:
 With `LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX=true`:
 - Base URL `https://my-proxy.com/custom/path` → `https://my-proxy.com/custom/path` (unchanged)
 
+### Azure AI Foundry (Alternative Method)
+
+:::tip Recommended Method
+For full Azure support including Azure AD authentication, use the dedicated [Azure Anthropic provider](./azure/azure_anthropic) with `azure_ai/` prefix.
+:::
+
+As an alternative, you can use the `anthropic/` provider directly with your Azure endpoint since Azure exposes Claude using Anthropic's native API.
+
+```python
+from litellm import completion
+
+response = completion(
+    model="anthropic/claude-sonnet-4-5",
+    api_base="https://<your-resource>.services.ai.azure.com/anthropic",
+    api_key="<your-azure-api-key>",
+    messages=[{"role": "user", "content": "Hello!"}],
+)
+print(response)
+```
+
+:::info
+**Finding your Azure endpoint:** Go to Azure AI Foundry → Your deployment → Overview. Your base URL will be `https://<resource-name>.services.ai.azure.com/anthropic`
+:::
+
 ## Usage
 
 ```python

docs/my-website/docs/providers/azure_ai.md

@@ -312,6 +312,82 @@ LiteLLM supports **ALL** azure ai models. Here's a few examples:
 | mistral-large-latest | `completion(model="azure_ai/mistral-large-latest", messages)` | 
 | AI21-Jamba-Instruct | `completion(model="azure_ai/ai21-jamba-instruct", messages)` | 
 
+## Usage - Azure Anthropic (Azure Foundry Claude)
+
+LiteLLM funnels Azure Claude deployments through the `azure_ai/` provider so Claude Opus models on Azure Foundry keep working with Tool Search, Effort, streaming, and the rest of the advanced feature set. Point `AZURE_AI_API_BASE` to `https://<resource>.services.ai.azure.com/anthropic` (LiteLLM appends `/v1/messages` automatically) and authenticate with `AZURE_AI_API_KEY` or an Azure AD token.
+
+<Tabs>
+<TabItem value="sdk" label="LiteLLM Python SDK">
+
+```python
+import os
+from litellm import completion
+
+# Configure Azure credentials
+os.environ["AZURE_AI_API_KEY"] = "your-azure-ai-api-key"
+os.environ["AZURE_AI_API_BASE"] = "https://my-resource.services.ai.azure.com/anthropic"
+
+response = completion(
+    model="azure_ai/claude-opus-4-1",
+    messages=[{"role": "user", "content": "Explain how Azure Anthropic hosts Claude Opus differently from the public Anthropic API."}],
+    max_tokens=1200,
+    temperature=0.7,
+    stream=True,
+)
+
+for chunk in response:
+    if chunk.choices[0].delta.content:
+        print(chunk.choices[0].delta.content, end="", flush=True)
+```
+
+</TabItem>
+<TabItem value="proxy" label="LiteLLM Proxy">
+
+**1. Set environment variables**
+
+```bash
+export AZURE_AI_API_KEY="your-azure-ai-api-key"
+export AZURE_AI_API_BASE="https://my-resource.services.ai.azure.com/anthropic"
+```
+
+**2. Configure the proxy**
+
+```yaml
+model_list:
+  - model_name: claude-4-azure
+    litellm_params:
+      model: azure_ai/claude-opus-4-1
+      api_key: os.environ/AZURE_AI_API_KEY
+      api_base: os.environ/AZURE_AI_API_BASE
+```
+
+**3. Start LiteLLM**
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+**4. Test the Azure Claude route**
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+  --header 'Content-Type: application/json' \
+  --header 'Authorization: Bearer $LITELLM_KEY' \
+  --data '{
+    "model": "claude-4-azure",
+    "messages": [
+      {
+        "role": "user",
+        "content": "How do I use Claude Opus 4 via Azure Anthropic in LiteLLM?"
+      }
+    ],
+    "max_tokens": 1024
+  }'
+```
+
+</TabItem>
+</Tabs>
+
 
 
 ## Rerank Endpoint
@@ -397,4 +473,5 @@ curl http://0.0.0.0:4000/rerank \
 ```
 
 </TabItem>
-</Tabs>
+</Tabs>
+

docs/my-website/docs/providers/ovhcloud.md

@@ -311,6 +311,21 @@ response = embedding(
 print(response.data)
 ```
 
+### Audio Transcription
+
+```python
+from litellm import transcription
+
+audio_file = open("path/to/your/audio.wav", "rb")
+
+response = transcription(
+    model="ovhcloud/whisper-large-v3-turbo",
+    file=audio_file
+)
+
+print(response.text)
+```
+
 ## Usage with LiteLLM Proxy Server
 
 Here's how to call a OVHCloud AI Endpoints model with the LiteLLM Proxy Server

docs/my-website/docs/providers/vertex.md

@@ -2550,355 +2550,6 @@ print(response)
 </TabItem>
 </Tabs>
 
-
-## **Gemini TTS (Text-to-Speech) Audio Output**
-
-:::info
-
-LiteLLM supports Gemini TTS models on Vertex AI that can generate audio responses using the OpenAI-compatible `audio` parameter format.
-
-:::
-
-### Supported Models
-
-LiteLLM supports Gemini TTS models with audio capabilities on Vertex AI (e.g. `vertex_ai/gemini-2.5-flash-preview-tts` and `vertex_ai/gemini-2.5-pro-preview-tts`). For the complete list of available TTS models and voices, see the [official Gemini TTS documentation](https://ai.google.dev/gemini-api/docs/speech-generation).
-
-### Limitations
-
-:::warning
-
-**Important Limitations**:
-- Gemini TTS models only support the `pcm16` audio format
-- **Streaming support has not been added** to TTS models yet
-- The `modalities` parameter must be set to `['audio']` for TTS requests
-
-:::
-
-### Quick Start
-
-<Tabs>
-<TabItem value="sdk" label="SDK">
-
-```python
-from litellm import completion
-import json
-
-## GET CREDENTIALS
-file_path = 'path/to/vertex_ai_service_account.json'
-
-# Load the JSON file
-with open(file_path, 'r') as file:
-    vertex_credentials = json.load(file)
-
-# Convert to JSON string
-vertex_credentials_json = json.dumps(vertex_credentials)
-
-response = completion(
-    model="vertex_ai/gemini-2.5-flash-preview-tts",
-    messages=[{"role": "user", "content": "Say hello in a friendly voice"}],
-    modalities=["audio"],  # Required for TTS models
-    audio={
-        "voice": "Kore",
-        "format": "pcm16"  # Required: must be "pcm16"
-    },
-    vertex_credentials=vertex_credentials_json
-)
-
-print(response)
-```
-
-</TabItem>
-<TabItem value="proxy" label="PROXY">
-
-1. Setup config.yaml
-
-```yaml
-model_list:
-  - model_name: gemini-tts-flash
-    litellm_params:
-      model: vertex_ai/gemini-2.5-flash-preview-tts
-      vertex_project: "your-project-id"
-      vertex_location: "us-central1"
-      vertex_credentials: "/path/to/service_account.json"
-  - model_name: gemini-tts-pro
-    litellm_params:
-      model: vertex_ai/gemini-2.5-pro-preview-tts
-      vertex_project: "your-project-id"
-      vertex_location: "us-central1"
-      vertex_credentials: "/path/to/service_account.json"
-```
-
-2. Start proxy
-
-```bash
-litellm --config /path/to/config.yaml
-```
-
-3. Make TTS request
-
-```bash
-curl http://0.0.0.0:4000/v1/chat/completions \
-  -H "Content-Type: application/json" \
-  -H "Authorization: Bearer <YOUR-LITELLM-KEY>" \
-  -d '{
-    "model": "gemini-tts-flash",
-    "messages": [{"role": "user", "content": "Say hello in a friendly voice"}],
-    "modalities": ["audio"],
-    "audio": {
-      "voice": "Kore",
-      "format": "pcm16"
-    }
-  }'
-```
-
-</TabItem>
-</Tabs>
-
-### Advanced Usage
-
-You can combine TTS with other Gemini features:
-
-```python
-response = completion(
-    model="vertex_ai/gemini-2.5-pro-preview-tts",
-    messages=[
-        {"role": "system", "content": "You are a helpful assistant that speaks clearly."},
-        {"role": "user", "content": "Explain quantum computing in simple terms"}
-    ],
-    modalities=["audio"],
-    audio={
-        "voice": "Charon",
-        "format": "pcm16"
-    },
-    temperature=0.7,
-    max_tokens=150,
-    vertex_credentials=vertex_credentials_json
-)
-```
-
-For more information about Gemini's TTS capabilities and available voices, see the [official Gemini TTS documentation](https://ai.google.dev/gemini-api/docs/speech-generation).
-
-## **Text to Speech APIs**
-
-:::info
-
-LiteLLM supports calling [Vertex AI Text to Speech API](https://console.cloud.google.com/vertex-ai/generative/speech/text-to-speech) in the OpenAI text to speech API format
-
-:::
-
-
-
-### Usage - Basic
-
-<Tabs>
-<TabItem value="sdk" label="SDK">
-
-Vertex AI does not support passing a `model` param - so passing `model=vertex_ai/` is the only required param
-
-**Sync Usage**
-
-```python
-speech_file_path = Path(__file__).parent / "speech_vertex.mp3"
-response = litellm.speech(
-    model="vertex_ai/",
-    input="hello what llm guardrail do you have",
-)
-response.stream_to_file(speech_file_path)
-```
-
-**Async Usage**
-```python
-speech_file_path = Path(__file__).parent / "speech_vertex.mp3"
-response = litellm.aspeech(
-    model="vertex_ai/",
-    input="hello what llm guardrail do you have",
-)
-response.stream_to_file(speech_file_path)
-```
-
-</TabItem>
-<TabItem value="proxy" label="LiteLLM PROXY (Unified Endpoint)">
-
-1. Add model to config.yaml
-```yaml
-model_list:
-  - model_name: vertex-tts
-    litellm_params:
-      model: vertex_ai/ # Vertex AI does not support passing a `model` param - so passing `model=vertex_ai/` is the only required param
-      vertex_project: "adroit-crow-413218"
-      vertex_location: "us-central1"
-      vertex_credentials: adroit-crow-413218-a956eef1a2a8.json 
-
-litellm_settings:
-  drop_params: True
-```
-
-2. Start Proxy 
-
-```
-$ litellm --config /path/to/config.yaml
-```
-
-3. Make Request use OpenAI Python SDK
-
-
-```python
-import openai
-
-client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
-
-# see supported values for "voice" on vertex here: 
-# https://console.cloud.google.com/vertex-ai/generative/speech/text-to-speech
-response = client.audio.speech.create(
-    model = "vertex-tts",
-    input="the quick brown fox jumped over the lazy dogs",
-    voice={'languageCode': 'en-US', 'name': 'en-US-Studio-O'}
-)
-print("response from proxy", response)
-```
-
-</TabItem>
-</Tabs>
-
-
-### Usage - `ssml` as input
-
-Pass your `ssml` as input to the `input` param, if it contains `<speak>`, it will be automatically detected and passed as `ssml` to the Vertex AI API
-
-If you need to force your `input` to be passed as `ssml`, set `use_ssml=True`
-
-<Tabs>
-<TabItem value="sdk" label="SDK">
-
-Vertex AI does not support passing a `model` param - so passing `model=vertex_ai/` is the only required param
-
-
-```python
-speech_file_path = Path(__file__).parent / "speech_vertex.mp3"
-
-
-ssml = """
-<speak>
-    <p>Hello, world!</p>
-    <p>This is a test of the <break strength="medium" /> text-to-speech API.</p>
-</speak>
-"""
-
-response = litellm.speech(
-    input=ssml,
-    model="vertex_ai/test",
-    voice={
-        "languageCode": "en-UK",
-        "name": "en-UK-Studio-O",
-    },
-    audioConfig={
-        "audioEncoding": "LINEAR22",
-        "speakingRate": "10",
-    },
-)
-response.stream_to_file(speech_file_path)
-```
-
-</TabItem>
-
-<TabItem value="proxy" label="LiteLLM PROXY (Unified Endpoint)">
-
-```python
-import openai
-
-client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
-
-ssml = """
-<speak>
-    <p>Hello, world!</p>
-    <p>This is a test of the <break strength="medium" /> text-to-speech API.</p>
-</speak>
-"""
-
-# see supported values for "voice" on vertex here: 
-# https://console.cloud.google.com/vertex-ai/generative/speech/text-to-speech
-response = client.audio.speech.create(
-    model = "vertex-tts",
-    input=ssml,
-    voice={'languageCode': 'en-US', 'name': 'en-US-Studio-O'},
-)
-print("response from proxy", response)
-```
-
-</TabItem>
-</Tabs>
-
-
-### Forcing SSML Usage
-
-You can force the use of SSML by setting the `use_ssml` parameter to `True`. This is useful when you want to ensure that your input is treated as SSML, even if it doesn't contain the `<speak>` tags.
-
-Here are examples of how to force SSML usage:
-
-
-<Tabs>
-<TabItem value="sdk" label="SDK">
-
-Vertex AI does not support passing a `model` param - so passing `model=vertex_ai/` is the only required param
-
-
-```python
-speech_file_path = Path(__file__).parent / "speech_vertex.mp3"
-
-
-ssml = """
-<speak>
-    <p>Hello, world!</p>
-    <p>This is a test of the <break strength="medium" /> text-to-speech API.</p>
-</speak>
-"""
-
-response = litellm.speech(
-    input=ssml,
-    use_ssml=True,
-    model="vertex_ai/test",
-    voice={
-        "languageCode": "en-UK",
-        "name": "en-UK-Studio-O",
-    },
-    audioConfig={
-        "audioEncoding": "LINEAR22",
-        "speakingRate": "10",
-    },
-)
-response.stream_to_file(speech_file_path)
-```
-
-</TabItem>
-
-<TabItem value="proxy" label="LiteLLM PROXY (Unified Endpoint)">
-
-```python
-import openai
-
-client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
-
-ssml = """
-<speak>
-    <p>Hello, world!</p>
-    <p>This is a test of the <break strength="medium" /> text-to-speech API.</p>
-</speak>
-"""
-
-# see supported values for "voice" on vertex here: 
-# https://console.cloud.google.com/vertex-ai/generative/speech/text-to-speech
-response = client.audio.speech.create(
-    model = "vertex-tts",
-    input=ssml, # pass as None since OpenAI SDK requires this param
-    voice={'languageCode': 'en-US', 'name': 'en-US-Studio-O'},
-    extra_body={"use_ssml": True},
-)
-print("response from proxy", response)
-```
-
-</TabItem>
-</Tabs>
-
 ## **Fine Tuning APIs**
 
 

docs/my-website/docs/providers/vertex_speech.md

@@ -0,0 +1,423 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Vertex AI Text to Speech
+
+| Property | Details |
+|-------|-------|
+| Description | Google Cloud Text-to-Speech with Chirp3 HD voices and Gemini TTS |
+| Provider Route on LiteLLM | `vertex_ai/chirp` (Chirp), `vertex_ai/gemini-*-tts` (Gemini) |
+
+## Chirp3 HD Voices
+
+Google Cloud Text-to-Speech API with high-quality Chirp3 HD voices.
+
+### Quick Start
+
+#### LiteLLM Python SDK
+
+```python showLineNumbers title="Chirp3 Quick Start"
+from litellm import speech
+from pathlib import Path
+
+speech_file_path = Path(__file__).parent / "speech.mp3"
+response = speech(
+    model="vertex_ai/chirp",
+    voice="alloy",  # OpenAI voice name - automatically mapped
+    input="Hello, this is Vertex AI Text to Speech",
+    vertex_project="your-project-id",
+    vertex_location="us-central1",
+)
+response.stream_to_file(speech_file_path)
+```
+
+#### LiteLLM AI Gateway
+
+**1. Setup config.yaml**
+
+```yaml showLineNumbers title="config.yaml"
+model_list:
+  - model_name: vertex-tts
+    litellm_params:
+      model: vertex_ai/chirp
+      vertex_project: "your-project-id"
+      vertex_location: "us-central1"
+      vertex_credentials: "/path/to/service_account.json"
+```
+
+**2. Start the proxy**
+
+```bash title="Start LiteLLM Proxy"
+litellm --config /path/to/config.yaml
+```
+
+**3. Make requests**
+
+<Tabs>
+<TabItem value="curl" label="curl">
+
+```bash showLineNumbers title="Chirp3 Quick Start"
+curl http://0.0.0.0:4000/v1/audio/speech \
+  -H "Authorization: Bearer sk-1234" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "vertex-tts",
+    "voice": "alloy",
+    "input": "Hello, this is Vertex AI Text to Speech"
+  }' \
+  --output speech.mp3
+```
+
+</TabItem>
+<TabItem value="openai-sdk" label="OpenAI Python SDK">
+
+```python showLineNumbers title="Chirp3 Quick Start"
+import openai
+
+client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
+
+response = client.audio.speech.create(
+    model="vertex-tts",
+    voice="alloy",
+    input="Hello, this is Vertex AI Text to Speech",
+)
+response.stream_to_file("speech.mp3")
+```
+
+</TabItem>
+</Tabs>
+
+### Voice Mapping
+
+LiteLLM maps OpenAI voice names to Google Cloud voices. You can use either OpenAI voices or Google Cloud voices directly.
+
+| OpenAI Voice | Google Cloud Voice |
+|-------------|-------------------|
+| `alloy` | en-US-Studio-O |
+| `echo` | en-US-Studio-M |
+| `fable` | en-GB-Studio-B |
+| `onyx` | en-US-Wavenet-D |
+| `nova` | en-US-Studio-O |
+| `shimmer` | en-US-Wavenet-F |
+
+### Using Google Cloud Voices Directly
+
+#### LiteLLM Python SDK
+
+```python showLineNumbers title="Chirp3 HD Voice"
+from litellm import speech
+
+# Pass Chirp3 HD voice name directly
+response = speech(
+    model="vertex_ai/chirp",
+    voice="en-US-Chirp3-HD-Charon",
+    input="Hello with a Chirp3 HD voice",
+    vertex_project="your-project-id",
+)
+response.stream_to_file("speech.mp3")
+```
+
+```python showLineNumbers title="Voice as Dict (Multilingual)"
+from litellm import speech
+
+# Pass as dict for full control over language and voice
+response = speech(
+    model="vertex_ai/chirp",
+    voice={
+        "languageCode": "de-DE",
+        "name": "de-DE-Chirp3-HD-Charon",
+    },
+    input="Hallo, dies ist ein Test",
+    vertex_project="your-project-id",
+)
+response.stream_to_file("speech.mp3")
+```
+
+#### LiteLLM AI Gateway
+
+<Tabs>
+<TabItem value="curl" label="curl">
+
+```bash showLineNumbers title="Chirp3 HD Voice"
+curl http://0.0.0.0:4000/v1/audio/speech \
+  -H "Authorization: Bearer sk-1234" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "vertex-tts",
+    "voice": "en-US-Chirp3-HD-Charon",
+    "input": "Hello with a Chirp3 HD voice"
+  }' \
+  --output speech.mp3
+```
+
+```bash showLineNumbers title="Voice as Dict (Multilingual)"
+curl http://0.0.0.0:4000/v1/audio/speech \
+  -H "Authorization: Bearer sk-1234" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "vertex-tts",
+    "voice": {"languageCode": "de-DE", "name": "de-DE-Chirp3-HD-Charon"},
+    "input": "Hallo, dies ist ein Test"
+  }' \
+  --output speech.mp3
+```
+
+</TabItem>
+<TabItem value="openai-sdk" label="OpenAI Python SDK">
+
+```python showLineNumbers title="Chirp3 HD Voice"
+import openai
+
+client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
+
+response = client.audio.speech.create(
+    model="vertex-tts",
+    voice="en-US-Chirp3-HD-Charon",
+    input="Hello with a Chirp3 HD voice",
+)
+response.stream_to_file("speech.mp3")
+```
+
+```python showLineNumbers title="Voice as Dict (Multilingual)"
+import openai
+
+client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
+
+response = client.audio.speech.create(
+    model="vertex-tts",
+    voice={"languageCode": "de-DE", "name": "de-DE-Chirp3-HD-Charon"},
+    input="Hallo, dies ist ein Test",
+)
+response.stream_to_file("speech.mp3")
+```
+
+</TabItem>
+</Tabs>
+
+Browse available voices: [Google Cloud Text-to-Speech Console](https://console.cloud.google.com/vertex-ai/generative/speech/text-to-speech)
+
+### Passing Raw SSML
+
+LiteLLM auto-detects SSML when your input contains `<speak>` tags and passes it through unchanged.
+
+#### LiteLLM Python SDK
+
+```python showLineNumbers title="SSML Input"
+from litellm import speech
+
+ssml = """
+<speak>
+    <p>Hello, world!</p>
+    <p>This is a test of the <break strength="medium" /> text-to-speech API.</p>
+</speak>
+"""
+
+response = speech(
+    model="vertex_ai/chirp",
+    voice="en-US-Studio-O",
+    input=ssml,  # Auto-detected as SSML
+    vertex_project="your-project-id",
+)
+response.stream_to_file("speech.mp3")
+```
+
+```python showLineNumbers title="Force SSML Mode"
+from litellm import speech
+
+# Force SSML mode with use_ssml=True
+response = speech(
+    model="vertex_ai/chirp",
+    voice="en-US-Studio-O",
+    input="<speak><prosody rate='slow'>Speaking slowly</prosody></speak>",
+    use_ssml=True,
+    vertex_project="your-project-id",
+)
+response.stream_to_file("speech.mp3")
+```
+
+#### LiteLLM AI Gateway
+
+<Tabs>
+<TabItem value="curl" label="curl">
+
+```bash showLineNumbers title="SSML Input"
+curl http://0.0.0.0:4000/v1/audio/speech \
+  -H "Authorization: Bearer sk-1234" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "vertex-tts",
+    "voice": "en-US-Studio-O",
+    "input": "<speak><p>Hello!</p><break time=\"500ms\"/><p>How are you?</p></speak>"
+  }' \
+  --output speech.mp3
+```
+
+</TabItem>
+<TabItem value="openai-sdk" label="OpenAI Python SDK">
+
+```python showLineNumbers title="SSML Input"
+import openai
+
+client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
+
+ssml = """<speak><p>Hello!</p><break time="500ms"/><p>How are you?</p></speak>"""
+
+response = client.audio.speech.create(
+    model="vertex-tts",
+    voice="en-US-Studio-O",
+    input=ssml,
+)
+response.stream_to_file("speech.mp3")
+```
+
+</TabItem>
+</Tabs>
+
+### Supported Parameters
+
+| Parameter | Description | Values |
+|-----------|-------------|--------|
+| `voice` | Voice selection | OpenAI voice, Google Cloud voice name, or dict |
+| `input` | Text to convert | Plain text or SSML |
+| `speed` | Speaking rate | 0.25 to 4.0 (default: 1.0) |
+| `response_format` | Audio format | `mp3`, `opus`, `wav`, `pcm`, `flac` |
+| `use_ssml` | Force SSML mode | `True` / `False` |
+
+### Async Usage
+
+```python showLineNumbers title="Async Speech Generation"
+import asyncio
+from litellm import aspeech
+
+async def main():
+    response = await aspeech(
+        model="vertex_ai/chirp",
+        voice="alloy",
+        input="Hello from async",
+        vertex_project="your-project-id",
+    )
+    response.stream_to_file("speech.mp3")
+
+asyncio.run(main())
+```
+
+---
+
+## Gemini TTS
+
+Gemini models with audio output capabilities using the chat completions API.
+
+:::warning
+**Limitations:**
+- Only supports `pcm16` audio format
+- Streaming not yet supported
+- Must set `modalities: ["audio"]`
+:::
+
+### Quick Start
+
+#### LiteLLM Python SDK
+
+```python showLineNumbers title="Gemini TTS Quick Start"
+from litellm import completion
+import json
+
+# Load credentials
+with open('path/to/service_account.json', 'r') as file:
+    vertex_credentials = json.dumps(json.load(file))
+
+response = completion(
+    model="vertex_ai/gemini-2.5-flash-preview-tts",
+    messages=[{"role": "user", "content": "Say hello in a friendly voice"}],
+    modalities=["audio"],
+    audio={
+        "voice": "Kore",
+        "format": "pcm16"
+    },
+    vertex_credentials=vertex_credentials
+)
+print(response)
+```
+
+#### LiteLLM AI Gateway
+
+**1. Setup config.yaml**
+
+```yaml showLineNumbers title="config.yaml"
+model_list:
+  - model_name: gemini-tts
+    litellm_params:
+      model: vertex_ai/gemini-2.5-flash-preview-tts
+      vertex_project: "your-project-id"
+      vertex_location: "us-central1"
+      vertex_credentials: "/path/to/service_account.json"
+```
+
+**2. Start the proxy**
+
+```bash title="Start LiteLLM Proxy"
+litellm --config /path/to/config.yaml
+```
+
+**3. Make requests**
+
+<Tabs>
+<TabItem value="curl" label="curl">
+
+```bash showLineNumbers title="Gemini TTS Request"
+curl http://0.0.0.0:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer sk-1234" \
+  -d '{
+    "model": "gemini-tts",
+    "messages": [{"role": "user", "content": "Say hello in a friendly voice"}],
+    "modalities": ["audio"],
+    "audio": {"voice": "Kore", "format": "pcm16"}
+  }'
+```
+
+</TabItem>
+<TabItem value="openai-sdk" label="OpenAI Python SDK">
+
+```python showLineNumbers title="Gemini TTS Request"
+import openai
+
+client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
+
+response = client.chat.completions.create(
+    model="gemini-tts",
+    messages=[{"role": "user", "content": "Say hello in a friendly voice"}],
+    modalities=["audio"],
+    audio={"voice": "Kore", "format": "pcm16"},
+)
+print(response)
+```
+
+</TabItem>
+</Tabs>
+
+### Supported Models
+
+- `vertex_ai/gemini-2.5-flash-preview-tts`
+- `vertex_ai/gemini-2.5-pro-preview-tts`
+
+See [Gemini TTS documentation](https://ai.google.dev/gemini-api/docs/speech-generation) for available voices.
+
+### Advanced Usage
+
+```python showLineNumbers title="Gemini TTS with System Prompt"
+from litellm import completion
+
+response = completion(
+    model="vertex_ai/gemini-2.5-pro-preview-tts",
+    messages=[
+        {"role": "system", "content": "You are a helpful assistant that speaks clearly."},
+        {"role": "user", "content": "Explain quantum computing in simple terms"}
+    ],
+    modalities=["audio"],
+    audio={"voice": "Charon", "format": "pcm16"},
+    temperature=0.7,
+    max_tokens=150,
+    vertex_credentials=vertex_credentials
+)
+```

docs/my-website/docs/providers/watsonx/index.md

@@ -175,3 +175,56 @@ For all available models, see [watsonx.ai documentation](https://dataplatform.cl
 
 For all available embedding models, see [watsonx.ai embedding documentation](https://dataplatform.cloud.ibm.com/docs/content/wsj/analyze-data/fm-models-embed.html?context=wx).
 
+
+## Advanced
+
+### Using Zen API Key
+
+You can use a Zen API key for long-term authentication instead of generating IAM tokens. Pass it either as an environment variable or as a parameter:
+
+```python
+import os
+from litellm import completion
+
+# Option 1: Set as environment variable
+os.environ["WATSONX_ZENAPIKEY"] = "your-zen-api-key"
+
+response = completion(
+    model="watsonx/ibm/granite-13b-chat-v2",
+    messages=[{"content": "What is your favorite color?", "role": "user"}],
+    project_id="your-project-id"
+)
+
+# Option 2: Pass as parameter
+response = completion(
+    model="watsonx/ibm/granite-13b-chat-v2",
+    messages=[{"content": "What is your favorite color?", "role": "user"}],
+    zen_api_key="your-zen-api-key",
+    project_id="your-project-id"
+)
+```
+
+**Using with LiteLLM Proxy via OpenAI client:**
+
+```python
+import openai
+
+client = openai.OpenAI(
+    api_key="sk-1234",  # LiteLLM proxy key
+    base_url="http://0.0.0.0:4000"
+)
+
+response = client.chat.completions.create(
+    model="watsonx/ibm/granite-3-3-8b-instruct",
+    messages=[{"role": "user", "content": "What is your favorite color?"}],
+    max_tokens=2048,
+    extra_body={
+        "project_id": "your-project-id",
+        "zen_api_key": "your-zen-api-key"
+    }
+)
+```
+
+See [IBM documentation](https://www.ibm.com/docs/en/watsonx/w-and-w/2.2.0?topic=keys-generating-zenapikey-authorization-tokens) for more information on generating Zen API keys.
+
+

docs/my-website/docs/providers/zai.md

@@ -0,0 +1,135 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Z.AI (Zhipu AI)
+https://z.ai/
+
+**We support Z.AI GLM text/chat models, just set `zai/` as a prefix when sending completion requests**
+
+## API Key
+```python
+# env variable
+os.environ['ZAI_API_KEY']
+```
+
+## Sample Usage
+```python
+from litellm import completion
+import os
+
+os.environ['ZAI_API_KEY'] = ""
+response = completion(
+    model="zai/glm-4.6",
+    messages=[
+       {"role": "user", "content": "hello from litellm"}
+   ],
+)
+print(response)
+```
+
+## Sample Usage - Streaming
+```python
+from litellm import completion
+import os
+
+os.environ['ZAI_API_KEY'] = ""
+response = completion(
+    model="zai/glm-4.6",
+    messages=[
+       {"role": "user", "content": "hello from litellm"}
+   ],
+    stream=True
+)
+
+for chunk in response:
+    print(chunk)
+```
+
+## Supported Models
+
+We support ALL Z.AI GLM models, just set `zai/` as a prefix when sending completion requests.
+
+| Model Name | Function Call | Notes |
+|------------|---------------|-------|
+| glm-4.6 | `completion(model="zai/glm-4.6", messages)` | Latest flagship model, 200K context |
+| glm-4.5 | `completion(model="zai/glm-4.5", messages)` | 128K context |
+| glm-4.5v | `completion(model="zai/glm-4.5v", messages)` | Vision model |
+| glm-4.5-x | `completion(model="zai/glm-4.5-x", messages)` | Premium tier |
+| glm-4.5-air | `completion(model="zai/glm-4.5-air", messages)` | Lightweight |
+| glm-4.5-airx | `completion(model="zai/glm-4.5-airx", messages)` | Fast lightweight |
+| glm-4-32b-0414-128k | `completion(model="zai/glm-4-32b-0414-128k", messages)` | 32B parameter model |
+| glm-4.5-flash | `completion(model="zai/glm-4.5-flash", messages)` | **FREE tier** |
+
+## Model Pricing
+
+| Model | Input ($/1M tokens) | Output ($/1M tokens) | Context Window |
+|-------|---------------------|----------------------|----------------|
+| glm-4.6 | $0.60 | $2.20 | 200K |
+| glm-4.5 | $0.60 | $2.20 | 128K |
+| glm-4.5v | $0.60 | $1.80 | 128K |
+| glm-4.5-x | $2.20 | $8.90 | 128K |
+| glm-4.5-air | $0.20 | $1.10 | 128K |
+| glm-4.5-airx | $1.10 | $4.50 | 128K |
+| glm-4-32b-0414-128k | $0.10 | $0.10 | 128K |
+| glm-4.5-flash | **FREE** | **FREE** | 128K |
+
+## Using with LiteLLM Proxy
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+from litellm import completion
+import os
+
+os.environ['ZAI_API_KEY'] = ""
+response = completion(
+    model="zai/glm-4.6",
+    messages=[{"role": "user", "content": "Hello, how are you?"}],
+)
+
+print(response.choices[0].message.content)
+```
+
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+1. Setup config.yaml
+
+```yaml
+model_list:
+  - model_name: glm-4.6
+    litellm_params:
+        model: zai/glm-4.6
+        api_key: os.environ/ZAI_API_KEY
+  - model_name: glm-4.5-flash  # Free tier
+    litellm_params:
+        model: zai/glm-4.5-flash
+        api_key: os.environ/ZAI_API_KEY
+```
+
+2. Run proxy
+
+```bash
+litellm --config config.yaml
+```
+
+3. Test it!
+
+```bash
+curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
+-H 'Content-Type: application/json' \
+-H 'Authorization: Bearer sk-1234' \
+-d '{
+    "model": "glm-4.6",
+    "messages": [
+      {
+        "role": "user",
+        "content": "Hello, how are you?"
+      }
+    ]
+}'
+```
+
+</TabItem>
+</Tabs>

docs/my-website/docs/proxy/config_settings.md

@@ -113,7 +113,7 @@ general_settings:
 
   # Database Settings
   database_url: string
-  database_connection_pool_limit: 0  # default 100
+  database_connection_pool_limit: 0  # default 10
   database_connection_timeout: 0  # default 60s
   allow_requests_on_db_unavailable: boolean  # if true, will allow requests that can not connect to the DB to verify Virtual Key to still work 
 
@@ -234,7 +234,7 @@ router_settings:
 | max_response_size_mb | int | The maximum size for responses in MB. LLM Responses above this size will not be sent. |
 | proxy_budget_rescheduler_min_time | int | The minimum time (in seconds) to wait before checking db for budget resets. **Default is 597 seconds** |
 | proxy_budget_rescheduler_max_time | int | The maximum time (in seconds) to wait before checking db for budget resets. **Default is 605 seconds** |
-| proxy_batch_write_at | int | Time (in seconds) to wait before batch writing spend logs to the db. **Default is 30 seconds** |
+| proxy_batch_write_at | int | Time (in seconds) to wait before batch writing spend logs to the db. **Default is 10 seconds** |
 | proxy_batch_polling_interval | int | Time (in seconds) to wait before polling a batch, to check if it's completed. **Default is 6000 seconds (1 hour)** |
 | alerting_args | dict | Args for Slack Alerting [Doc on Slack Alerting](./alerting.md) |
 | custom_key_generate | str | Custom function for key generation [Doc on custom key generation](./virtual_keys.md#custom--key-generate) |
@@ -763,7 +763,7 @@ router_settings:
 | PROMPTLAYER_API_KEY | API key for PromptLayer integration
 | PROXY_ADMIN_ID | Admin identifier for proxy server
 | PROXY_BASE_URL | Base URL for proxy service
-| PROXY_BATCH_WRITE_AT | Time in seconds to wait before batch writing spend logs to the database. Default is 30
+| PROXY_BATCH_WRITE_AT | Time in seconds to wait before batch writing spend logs to the database. Default is 10
 | PROXY_BATCH_POLLING_INTERVAL | Time in seconds to wait before polling a batch, to check if it's completed. Default is 6000s (1 hour)
 | PROXY_BUDGET_RESCHEDULER_MAX_TIME | Maximum time in seconds to wait before checking database for budget resets. Default is 605
 | PROXY_BUDGET_RESCHEDULER_MIN_TIME | Minimum time in seconds to wait before checking database for budget resets. Default is 597

docs/my-website/docs/proxy/configs.md

@@ -576,7 +576,7 @@ custom_tokenizer:
 
 ```yaml
 general_settings: 
-  database_connection_pool_limit: 100 # sets connection pool for prisma client to postgres db at 100
+  database_connection_pool_limit: 10 # sets connection pool for prisma client to postgres db (default: 10, recommended: 10-20)
   database_connection_timeout: 60 # sets a 60s timeout for any connection call to the db 
 ```
 

docs/my-website/docs/proxy/error_diagnosis.md

@@ -0,0 +1,90 @@
+# Diagnosing Errors - Provider vs Gateway
+
+Having trouble diagnosing if an error is from the **LLM Provider** (OpenAI, Anthropic, etc.) or from the **LiteLLM AI Gateway** itself? Here's how to tell.
+
+## Quick Rule
+
+**If the error contains `<Provider>Exception`, it's from the provider.**
+
+| Error Contains | Error Source |
+|----------------|--------------|
+| `AnthropicException` | Anthropic |
+| `OpenAIException` | OpenAI |
+| `AzureException` | Azure |
+| `BedrockException` | AWS Bedrock |
+| `VertexAIException` | Google Vertex AI |
+| No provider name | LiteLLM AI Gateway |
+
+## Examples
+
+### Provider Error (from AWS Bedrock)
+
+```
+{
+  "error": {
+    "message": "litellm.BadRequestError: BedrockException - {\"message\":\"The model returned the following errors: messages.1.content.0.type: Expected `thinking` or `redacted_thinking`, but found `text`.\"}",
+    "type": "invalid_request_error",
+    "param": null,
+    "code": "400"
+  }
+}
+```
+
+This error is from **AWS Bedrock** (notice `BedrockException`). The Bedrock API is rejecting the request due to invalid message format - this is not a LiteLLM issue.
+
+### Provider Error (from OpenAI)
+
+```
+{
+  "error": {
+    "message": "litellm.AuthenticationError: OpenAIException - Incorrect API key provided: <my-key>. You can find your API key at https://platform.openai.com/account/api-keys.",
+    "type": "invalid_request_error",
+    "param": null,
+    "code": "invalid_api_key"
+  }
+}
+```
+
+This error is from **OpenAI** (notice `OpenAIException`). The OpenAI API key configured in LiteLLM is invalid.
+
+### Provider Error (from Anthropic)
+
+```
+{
+  "error": {
+    "message": "litellm.InternalServerError: AnthropicException - Overloaded. Handle with `litellm.InternalServerError`.",
+    "type": "internal_server_error",
+    "param": null,
+    "code": "500"
+  }
+}
+```
+
+This error is from **Anthropic** (notice `AnthropicException`). The Anthropic API is overloaded - this is not a LiteLLM issue.
+
+### Gateway Error (from LiteLLM)
+
+```
+{
+  "error": {
+    "message": "Invalid API Key. Please check your LiteLLM API key.",
+    "type": "auth_error",
+    "param": null,
+    "code": "401"
+  }
+}
+```
+
+This error is from the **LiteLLM AI Gateway** (no provider name). Your LiteLLM virtual key is invalid.
+
+## What to do?
+
+| Error Source | Action |
+|--------------|--------|
+| Provider Error | Check the provider's status page, adjust rate limits, or retry later |
+| Gateway Error | Check your LiteLLM configuration, API keys, or [open an issue](https://github.com/BerriAI/litellm/issues) |
+
+## See Also
+
+- [Debugging](/docs/proxy/debugging) - Enable debug logs to see detailed request/response info
+- [Exception Mapping](/docs/exception_mapping) - Full list of LiteLLM exception types

docs/my-website/docs/proxy/guardrails/bedrock.md

@@ -188,6 +188,28 @@ My email is [EMAIL] and my phone number is [PHONE_NUMBER]
 
 This helps protect sensitive information while still allowing the model to understand the context of the request.
 
+## Experimental: Only Send Latest User Message
+
+When you're chaining long conversations through Bedrock guardrails, you can opt into a lighter, experimental behavior by setting `experimental_use_latest_role_message_only: true` in the guardrail's `litellm_params`. When enabled, LiteLLM only sends the most recent `user` message (or assistant output during post-call checks) to Bedrock, which:
+
+- prevents unintended blocks on older system/dev messages
+- keeps Bedrock payloads smaller, reducing latency and cost
+- applies to proxy hooks (`pre_call`, `during_call`) and the `/guardrails/apply_guardrail` testing endpoint
+
+```yaml showLineNumbers title="litellm proxy config.yaml"
+guardrails:
+  - guardrail_name: "bedrock-pre-guard"
+    litellm_params:
+      guardrail: bedrock
+      mode: "pre_call"
+      guardrailIdentifier: wf0hkdb5x07f
+      guardrailVersion: "DRAFT"
+      aws_region_name: os.environ/AWS_REGION
+      experimental_use_latest_role_message_only: true  # NEW
+```
+
+> ⚠️ This flag is currently experimental and defaults to `false` to preserve the legacy behavior (entire message history). We'll be listening to user feedback to decide if this becomes the default or rolls out more broadly.
+
 ## Disabling Exceptions on Bedrock BLOCK
 
 By default, when Bedrock guardrails block content, LiteLLM raises an HTTP 400 exception. However, you can disable this behavior by setting `disable_exception_on_block: true`. This is particularly useful when integrating with **OpenWebUI**, where exceptions can interrupt the chat flow and break the user experience.

docs/my-website/docs/proxy/guardrails/lasso_security.md

@@ -35,7 +35,7 @@ guardrails:
       guardrail: lasso
       mode: "pre_call"
       api_key: os.environ/LASSO_API_KEY
-      api_base: "https://server.lasso.security"
+      api_base: "https://server.lasso.security/gateway/v3"
   - guardrail_name: "lasso-post-guard"
     litellm_params:
       guardrail: lasso
@@ -228,7 +228,7 @@ Expected response:
 
 ## PII Masking with Lasso
 
-Lasso supports automatic PII detection and masking using the `/gateway/v1/classifix` endpoint. When enabled, sensitive information like emails, phone numbers, and other PII will be automatically masked with appropriate placeholders.
+Lasso supports automatic PII detection and masking using the `/classifix` endpoint. When enabled, sensitive information like emails, phone numbers, and other PII will be automatically masked with appropriate placeholders.
 
 ### Enabling PII Masking
 

docs/my-website/docs/proxy/guardrails/tool_permission.md

@@ -1,4 +1,3 @@
-import Image from '@theme/IdealImage';
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
 
@@ -14,8 +13,6 @@ LiteLLM provides the LiteLLM Tool Permission Guardrail that lets you control whi
 
 Open the LiteLLM Dashboard, click **Add New Guardrail**, and choose **LiteLLM Tool Permission Guardrail**. This loads the rule builder UI.
 
-<Image img={require('../../../img/create_guard_tool_permission.png')} alt="Configure tool permission guardrail in LiteLLM UI" />
-
 #### Step 2: Define Regex Rules
 
 1. Click **Add Rule**.
@@ -24,8 +21,6 @@ Open the LiteLLM Dashboard, click **Add New Guardrail**, and choose **LiteLLM To
 4. Optionally add a regex for tool type (e.g., `^function$`).
 5. Pick **Allow** or **Deny**.
 
-<Image img={require('../../../img/create_rule_tool_permission.png')} alt="Configure tool permission guardrail in LiteLLM UI" />
-
 #### Step 3: Restrict Tool Arguments (Optional)
 
 Select **+ Restrict tool arguments** to attach regex validations to nested paths (dot + `[]` notation). This enforces that sensitive parameters (such as `arguments.to[]`) conform to pre-approved formats.

docs/my-website/docs/proxy/pass_through.md

@@ -275,6 +275,20 @@ In this video, we'll add the Azure OpenAI Assistants API as a pass through endpo
 - Check LiteLLM proxy logs for error details
 - Verify the target API's expected request format
 
+### Allowing Team JWTs to use pass-through routes
+
+If you are using pass-through provider routes (e.g., `/anthropic/*`) and want your JWT team tokens to access these routes, add `mapped_pass_through_routes` to the `team_allowed_routes` in `litellm_jwtauth` or explicitly add the relevant route(s).
+
+Example (`proxy_server_config.yaml`):
+
+```yaml
+general_settings:
+  enable_jwt_auth: True
+  litellm_jwtauth:
+    team_ids_jwt_field: "team_ids"
+    team_allowed_routes: ["openai_routes","info_routes","mapped_pass_through_routes"]
+```
+
 ### Getting Help
 
 [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version)

docs/my-website/docs/proxy/token_auth.md

@@ -338,6 +338,58 @@ general_settings:
     team_allowed_routes: ["/v1/chat/completions"] # 👈 Set accepted routes
 ```
 
+### Allowing other provider routes for Teams
+
+To enable team JWT tokens to access Anthropic-style endpoints such as `/v1/messages`, update `team_allowed_routes` in your `litellm_jwtauth` configuration. `team_allowed_routes` supports the following values:
+
+- Named route groups from `LiteLLMRoutes` (e.g., `openai_routes`, `anthropic_routes`, `info_routes`, `mapped_pass_through_routes`).
+
+Below is a quick reference for the route groups you can use and example representative routes from each group. If you need the exhaustive list, see the `LiteLLMRoutes` enum in `litellm/proxy/_types.py` for the authoritative list.
+
+| Route Group | What it contains | Representative routes |
+|-------------|------------------|-----------------------|
+| `openai_routes` | OpenAI-compatible REST endpoints (chat, completion, embeddings, images, responses, models, etc.) | `/v1/chat/completions`, `/v1/completions`, `/v1/embeddings`, `/v1/images/generations`, `/v1/models` |
+| `anthropic_routes` | Anthropic-style endpoints (`/v1/messages` and related) | `/v1/messages`, `/v1/messages/count_tokens`, `/v1/skills` |
+| `mapped_pass_through_routes` | Provider-specific pass-through route prefixes (e.g., Anthropic when proxied via `/anthropic`). Use with `mapped_pass_through_routes` for provider wildcard mapping | `/anthropic/*`, `/vertex-ai/*`, `/bedrock/*` |
+| `passthrough_routes_wildcard` | Wildcard mapping for providers (e.g., `/anthropic/*`) - precomputed wildcard list used by the proxy | `/anthropic/*`, `/vllm/*` |
+| `google_routes` | Google-specific (e.g., Vertex / Batching endpoints) | `/v1beta/models/{model_name}:generateContent` |
+| `mcp_routes` | Internal MCP management endpoints | `/mcp/tools`, `/mcp/tools/call` |
+| `info_routes` | Read-only & info endpoints used by the UI | `/key/info`, `/team/info`, `/v1/models` |
+| `management_routes` | Admin-only management endpoints (create/update/delete user/team/model) | `/team/new`, `/key/generate`, `/model/new` |
+| `spend_tracking_routes` | Budget/spend related endpoints | `/spend/logs`, `/spend/keys` |
+| `public_routes` | Public and unauthenticated endpoints | `/`, `/routes`, `/.well-known/litellm-ui-config` |
+
+Note: `llm_api_routes` is the union of OpenAI, Anthropic, Google, pass-through and other LLM routes (`openai_routes + anthropic_routes + google_routes + mapped_pass_through_routes + passthrough_routes_wildcard + apply_guardrail_routes + mcp_routes + litellm_native_routes`).
+
+Defaults (what the proxy uses if you don't override them in `litellm_jwtauth`):
+
+- `admin_jwt_scope`: `litellm_proxy_admin`
+- `admin_allowed_routes` (default): `management_routes`, `spend_tracking_routes`, `global_spend_tracking_routes`, `info_routes` 
+- `team_allowed_routes` (default): `openai_routes`, `info_routes` 
+- `public_allowed_routes` (default): `public_routes`
+
+
+Example: Allow team JWTs to call Anthropic `/v1/messages` (either by route group or by explicit route string):
+
+```yaml
+general_settings:
+  enable_jwt_auth: True
+  litellm_jwtauth:
+    team_ids_jwt_field: "team_ids"
+    team_allowed_routes: ["openai_routes", "info_routes", "anthropic_routes"]
+```
+
+Or selectively allow the exact Anthropic message endpoint only:
+
+```yaml
+general_settings:
+  enable_jwt_auth: True
+  litellm_jwtauth:
+    team_ids_jwt_field: "team_ids"
+    team_allowed_routes: ["/v1/messages", "info_routes"]
+```
+
+
 ### Caching Public Keys 
 
 Control how long public keys are cached for (in seconds).
@@ -407,6 +459,72 @@ general_settings:
     user_id_upsert: true # 👈 upserts the user to db, if valid email but not in db
 ```
 
+## OIDC UserInfo Endpoint
+
+Use this when your JWT/access token doesn't contain user-identifying information. LiteLLM will call your identity provider's UserInfo endpoint to fetch user details.
+
+### When to Use
+
+- Your JWT is opaque (not self-contained) or lacks user claims
+- You need to fetch fresh user information from your identity provider
+- Your access tokens don't include email, roles, or other identifying data
+
+### Configuration
+
+```yaml title="config.yaml" showLineNumbers
+general_settings:
+  enable_jwt_auth: True
+  litellm_jwtauth:
+    # Enable OIDC UserInfo endpoint
+    oidc_userinfo_enabled: true
+    oidc_userinfo_endpoint: "https://your-idp.com/oauth2/userinfo"
+    oidc_userinfo_cache_ttl: 300  # Cache for 5 minutes (default: 300)
+    
+    # Map fields from UserInfo response
+    user_id_jwt_field: "sub"
+    user_email_jwt_field: "email"
+    user_roles_jwt_field: "roles"
+```
+
+### Flow Diagram
+
+```mermaid
+sequenceDiagram
+    participant Client
+    participant LiteLLM
+    participant IdP as Identity Provider
+
+    Client->>LiteLLM: Request with Bearer token
+    Note over LiteLLM: Check cache for UserInfo
+    
+    LiteLLM->>IdP: GET /userinfo (if not cached)<br/>Authorization: Bearer {token}
+    IdP-->>LiteLLM: User data (sub, email, roles)
+    
+    Note over LiteLLM: Cache response (TTL: 5min)<br/>Extract user_id, email, roles<br/>Perform RBAC checks
+    
+    LiteLLM-->>Client: Authorized/Denied
+```
+
+### Example: Azure AD
+
+```yaml title="config.yaml" showLineNumbers
+litellm_jwtauth:
+  oidc_userinfo_enabled: true
+  oidc_userinfo_endpoint: "https://graph.microsoft.com/oidc/userinfo"
+  user_id_jwt_field: "sub"
+  user_email_jwt_field: "email"
+```
+
+### Example: Keycloak
+
+```yaml title="config.yaml" showLineNumbers
+litellm_jwtauth:
+  oidc_userinfo_enabled: true
+  oidc_userinfo_endpoint: "https://keycloak.example.com/realms/your-realm/protocol/openid-connect/userinfo"
+  user_id_jwt_field: "sub"
+  user_roles_jwt_field: "resource_access.your-client.roles"
+```
+
 ## [BETA] Control Access with OIDC Roles
 
 Allow JWT tokens with supported roles to access the proxy.

docs/my-website/sidebars.js

@@ -45,6 +45,7 @@ const sidebars = {
           type: "category",
           "label": "Contributing to Guardrails",
           items: [
+            "adding_provider/generic_guardrail_api",
             "adding_provider/simple_guardrail_tutorial",
             "adding_provider/adding_guardrail_support",
           ]
@@ -140,6 +141,7 @@ const sidebars = {
             "proxy/quick_start",
             "proxy/cli",
             "proxy/debugging",
+            "proxy/error_diagnosis",
             "proxy/deploy",
             "proxy/health",
             "proxy/master_key_rotations",
@@ -519,6 +521,7 @@ const sidebars = {
             "providers/vertex_partner",
             "providers/vertex_self_deployed",
             "providers/vertex_image",
+            "providers/vertex_speech",
             "providers/vertex_batch",
             "providers/vertex_ocr",
           ]
@@ -655,6 +658,7 @@ const sidebars = {
         },
         "providers/xai",
         "providers/xinference",
+        "providers/zai",
       ],
     },
     {

enterprise/litellm_enterprise/proxy/enterprise_routes.py

@@ -5,14 +5,10 @@ from litellm_enterprise.enterprise_callbacks.send_emails.endpoints import (
 )
 
 from .audit_logging_endpoints import router as audit_logging_router
-from .guardrails.endpoints import router as guardrails_router
 from .management_endpoints import management_endpoints_router
 from .utils import _should_block_robots
-from .vector_stores.endpoints import router as vector_stores_router
 
 router = APIRouter()
-router.include_router(vector_stores_router)
-router.include_router(guardrails_router)
 router.include_router(email_events_router)
 router.include_router(audit_logging_router)
 router.include_router(management_endpoints_router)

litellm/__init__.py

@@ -1377,6 +1377,7 @@ from .llms.github_copilot.chat.transformation import GithubCopilotConfig
 from .llms.github_copilot.responses.transformation import (
     GithubCopilotResponsesAPIConfig,
 )
+from .llms.github_copilot.embedding.transformation import GithubCopilotEmbeddingConfig
 from .llms.nebius.chat.transformation import NebiusConfig
 from .llms.wandb.chat.transformation import WandbConfig
 from .llms.dashscope.chat.transformation import DashScopeChatConfig

litellm/batches/batch_utils.py

@@ -14,8 +14,7 @@ from litellm.utils import token_counter
 
 async def calculate_batch_cost_and_usage(
     file_content_dictionary: List[dict],
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai"],
-    model_name: Optional[str] = None,
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm"],
 ) -> Tuple[float, Usage, List[str]]:
     """
     Calculate the cost and usage of a batch
@@ -37,8 +36,7 @@ async def calculate_batch_cost_and_usage(
 
 async def _handle_completed_batch(
     batch: Batch,
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai"],
-    model_name: Optional[str] = None,
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm"],
 ) -> Tuple[float, Usage, List[str]]:
     """Helper function to process a completed batch and handle logging"""
     # Get batch results
@@ -84,8 +82,7 @@ def _get_batch_models_from_file_content(
 
 def _batch_cost_calculator(
     file_content_dictionary: List[dict],
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
-    model_name: Optional[str] = None,
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm"] = "openai",
 ) -> float:
     """
     Calculate the cost of a batch based on the output file id
@@ -186,7 +183,7 @@ def calculate_vertex_ai_batch_cost_and_usage(
 
 async def _get_batch_output_file_content_as_dictionary(
     batch: Batch,
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm"] = "openai",
 ) -> List[dict]:
     """
     Get the batch output file content as a list of dictionaries
@@ -225,7 +222,7 @@ def _get_file_content_as_dictionary(file_content: bytes) -> List[dict]:
 
 def _get_batch_job_cost_from_file_content(
     file_content_dictionary: List[dict],
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm"] = "openai",
 ) -> float:
     """
     Get the cost of a batch job from the file content
@@ -253,8 +250,7 @@ def _get_batch_job_cost_from_file_content(
 
 def _get_batch_job_total_usage_from_file_content(
     file_content_dictionary: List[dict],
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
-    model_name: Optional[str] = None,
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm"] = "openai",
 ) -> Usage:
     """
     Get the tokens of a batch job from the file content

litellm/batches/main.py

@@ -18,11 +18,14 @@ from typing import Any, Coroutine, Dict, Literal, Optional, Union, cast
 
 import httpx
 from openai.types.batch import BatchRequestCounts
+from openai.types.batch import Metadata
+from openai.types.batch import Metadata as OpenAIBatchMetadata
 
 import litellm
 from litellm._logging import verbose_logger
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.llms.azure.batches.handler import AzureBatchesAPI
+from litellm.llms.bedrock.batches.handler import BedrockBatchesHandler
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
 from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
 from litellm.llms.openai.openai import OpenAIBatchesAPI
@@ -35,7 +38,11 @@ from litellm.types.llms.openai import (
     RetrieveBatchRequest,
 )
 from litellm.types.router import GenericLiteLLMParams
-from litellm.types.utils import LiteLLMBatch, LlmProviders
+from litellm.types.utils import (
+    OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS,
+    LiteLLMBatch,
+    LlmProviders,
+)
 from litellm.utils import (
     ProviderConfigManager,
     client,
@@ -100,7 +107,7 @@ async def acreate_batch(
     completion_window: Literal["24h"],
     endpoint: Literal["/v1/chat/completions", "/v1/embeddings", "/v1/completions"],
     input_file_id: str,
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock"] = "openai",
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm"] = "openai",
     metadata: Optional[Dict[str, str]] = None,
     extra_headers: Optional[Dict[str, str]] = None,
     extra_body: Optional[Dict[str, str]] = None,
@@ -148,7 +155,7 @@ def create_batch(
     completion_window: Literal["24h"],
     endpoint: Literal["/v1/chat/completions", "/v1/embeddings", "/v1/completions"],
     input_file_id: str,
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock"] = "openai",
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm"] = "openai",
     metadata: Optional[Dict[str, str]] = None,
     extra_headers: Optional[Dict[str, str]] = None,
     extra_body: Optional[Dict[str, str]] = None,
@@ -235,7 +242,7 @@ def create_batch(
             )
             return response
         api_base: Optional[str] = None
-        if custom_llm_provider == "openai":
+        if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
             # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
             api_base = (
                 optional_params.api_base
@@ -350,7 +357,7 @@ def create_batch(
 @client
 async def aretrieve_batch(
     batch_id: str,
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock"] = "openai",
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm"] = "openai",
     metadata: Optional[Dict[str, str]] = None,
     extra_headers: Optional[Dict[str, str]] = None,
     extra_body: Optional[Dict[str, str]] = None,
@@ -396,10 +403,10 @@ def _handle_retrieve_batch_providers_without_provider_config(
     litellm_params: dict,
     _retrieve_batch_request: RetrieveBatchRequest,
     _is_async: bool,
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock"] = "openai",
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm"] = "openai",
 ):
     api_base: Optional[str] = None
-    if custom_llm_provider == "openai":
+    if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
         # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
         api_base = (
             optional_params.api_base
@@ -512,7 +519,7 @@ def _handle_retrieve_batch_providers_without_provider_config(
 @client
 def retrieve_batch(
     batch_id: str,
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock"] = "openai",
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm"] = "openai",
     metadata: Optional[Dict[str, str]] = None,
     extra_headers: Optional[Dict[str, str]] = None,
     extra_body: Optional[Dict[str, str]] = None,
@@ -576,7 +583,7 @@ def retrieve_batch(
             async_kwargs = kwargs.copy()
             async_kwargs.pop("aws_region_name", None)
 
-            return _handle_async_invoke_status(
+            return BedrockBatchesHandler._handle_async_invoke_status(
                 batch_id=batch_id,
                 aws_region_name=kwargs.get("aws_region_name", "us-east-1"),
                 logging_obj=litellm_logging_obj,
@@ -644,7 +651,7 @@ def retrieve_batch(
 async def alist_batches(
     after: Optional[str] = None,
     limit: Optional[int] = None,
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
+    custom_llm_provider: Literal["openai", "azure", "hosted_vllm", "vertex_ai"] = "openai",
     metadata: Optional[Dict[str, str]] = None,
     extra_headers: Optional[Dict[str, str]] = None,
     extra_body: Optional[Dict[str, str]] = None,
@@ -687,7 +694,7 @@ async def alist_batches(
 def list_batches(
     after: Optional[str] = None,
     limit: Optional[int] = None,
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
+    custom_llm_provider: Literal["openai", "azure", "hosted_vllm", "vertex_ai"] = "openai",
     extra_headers: Optional[Dict[str, str]] = None,
     extra_body: Optional[Dict[str, str]] = None,
     **kwargs,
@@ -727,7 +734,7 @@ def list_batches(
             timeout = 600.0
 
         _is_async = kwargs.pop("alist_batches", False) is True
-        if custom_llm_provider == "openai":
+        if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
             # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
             api_base = (
                 optional_params.api_base
@@ -928,7 +935,7 @@ def cancel_batch(
 
         _is_async = kwargs.pop("acancel_batch", False) is True
         api_base: Optional[str] = None
-        if custom_llm_provider == "openai":
+        if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
             api_base = (
                 optional_params.api_base
                 or litellm.api_base
@@ -1043,19 +1050,20 @@ def _handle_async_invoke_status(
         )
 
         # Transform response to a LiteLLMBatch object
+        from litellm.types.llms.openai import BatchJobStatus
         from litellm.types.utils import LiteLLMBatch
 
         # Normalize status to lowercase (AWS returns 'Completed', 'Failed', etc.)
         aws_status_raw = status_response.get("status", "")
         aws_status_lower = aws_status_raw.lower()
         # Map AWS status values to LiteLLM expected values
-        status_mapping = {
+        status_mapping: dict[str, BatchJobStatus] = {
             "completed": "completed",
             "failed": "failed",
             "inprogress": "in_progress",
             "in_progress": "in_progress",
         }
-        normalized_status = status_mapping.get(aws_status_lower, aws_status_lower)
+        normalized_status: BatchJobStatus = status_mapping.get(aws_status_lower, "failed")  # Default to "failed" if unknown status
 
         # Get output S3 URI safely
         output_s3_uri = ""
@@ -1065,13 +1073,15 @@ def _handle_async_invoke_status(
             pass
         
         # Use BedrockBatchesConfig's timestamp parsing method (expects raw AWS status string)
+        import time
+
         from litellm.llms.bedrock.batches.transformation import BedrockBatchesConfig
         created_at, in_progress_at, completed_at, failed_at, _, _ = BedrockBatchesConfig()._parse_timestamps_and_status(status_response, aws_status_raw)
         result = LiteLLMBatch(
             id=status_response["invocationArn"],
             object="batch",
             status=normalized_status,
-            created_at=created_at,
+            created_at=created_at or int(time.time()),  # Provide default timestamp if None
             in_progress_at=in_progress_at,
             completed_at=completed_at,
             failed_at=failed_at,

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -148,7 +148,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             if role == "system":
                 # Extract system message as instructions
                 if isinstance(content, str):
-                    instructions = content
+                    if instructions:
+                        # Concatenate multiple system prompts with a space
+                        instructions = f"{instructions} {content}"
+                    else:
+                        instructions = content
                 else:
                     input_items.append(
                         {

litellm/constants.py

@@ -555,6 +555,7 @@ openai_compatible_providers: List = [
     "perplexity",
     "xinference",
     "xai",
+    "zai",
     "together_ai",
     "fireworks_ai",
     "empower",
@@ -858,6 +859,7 @@ BEDROCK_INVOKE_PROVIDERS_LITERAL = Literal[
     "deepseek_r1",
     "qwen3",
     "twelvelabs",
+    "openai"
 ]
 
 BEDROCK_EMBEDDING_PROVIDERS_LITERAL = Literal[

litellm/files/main.py

@@ -30,7 +30,10 @@ from litellm.types.llms.openai import (
     OpenAIFileObject,
 )
 from litellm.types.router import *
-from litellm.types.utils import LlmProviders
+from litellm.types.utils import (
+    OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS,
+    LlmProviders,
+)
 from litellm.utils import (
     ProviderConfigManager,
     client,
@@ -51,7 +54,7 @@ vertex_ai_files_instance = VertexAIFilesHandler()
 async def acreate_file(
     file: FileTypes,
     purpose: Literal["assistants", "batch", "fine-tune"],
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock"] = "openai",
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm"] = "openai",
     extra_headers: Optional[Dict[str, str]] = None,
     extra_body: Optional[Dict[str, str]] = None,
     **kwargs,
@@ -95,9 +98,7 @@ async def acreate_file(
 def create_file(
     file: FileTypes,
     purpose: Literal["assistants", "batch", "fine-tune"],
-    custom_llm_provider: Optional[
-        Literal["openai", "azure", "vertex_ai", "bedrock"]
-    ] = None,
+    custom_llm_provider: Optional[Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm"]] = None,
     extra_headers: Optional[Dict[str, str]] = None,
     extra_body: Optional[Dict[str, str]] = None,
     **kwargs,
@@ -165,7 +166,7 @@ def create_file(
                 ),
                 timeout=timeout,
             )
-        elif custom_llm_provider == "openai":
+        elif custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
             # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
             api_base = (
                 optional_params.api_base
@@ -276,7 +277,7 @@ def create_file(
 @client
 async def afile_retrieve(
     file_id: str,
-    custom_llm_provider: Literal["openai", "azure"] = "openai",
+    custom_llm_provider: Literal["openai", "azure", "hosted_vllm"] = "openai",
     extra_headers: Optional[Dict[str, str]] = None,
     extra_body: Optional[Dict[str, str]] = None,
     **kwargs,
@@ -317,7 +318,7 @@ async def afile_retrieve(
 @client
 def file_retrieve(
     file_id: str,
-    custom_llm_provider: Literal["openai", "azure"] = "openai",
+    custom_llm_provider: Literal["openai", "azure", "hosted_vllm"] = "openai",
     extra_headers: Optional[Dict[str, str]] = None,
     extra_body: Optional[Dict[str, str]] = None,
     **kwargs,
@@ -347,7 +348,7 @@ def file_retrieve(
 
         _is_async = kwargs.pop("is_async", False) is True
 
-        if custom_llm_provider == "openai":
+        if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
             # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
             api_base = (
                 optional_params.api_base
@@ -514,7 +515,7 @@ def file_delete(
         elif timeout is None:
             timeout = 600.0
         _is_async = kwargs.pop("is_async", False) is True
-        if custom_llm_provider == "openai":
+        if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
             # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
             api_base = (
                 optional_params.api_base
@@ -670,7 +671,7 @@ def file_list(
             timeout = 600.0
 
         _is_async = kwargs.pop("is_async", False) is True
-        if custom_llm_provider == "openai":
+        if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
             # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
             api_base = (
                 optional_params.api_base
@@ -754,7 +755,7 @@ def file_list(
 @client
 async def afile_content(
     file_id: str,
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm"] = "openai",
     extra_headers: Optional[Dict[str, str]] = None,
     extra_body: Optional[Dict[str, str]] = None,
     **kwargs,
@@ -799,7 +800,7 @@ def file_content(
     file_id: str,
     model: Optional[str] = None,
     custom_llm_provider: Optional[
-        Union[Literal["openai", "azure", "vertex_ai"], str]
+        Union[Literal["openai", "azure", "vertex_ai", "hosted_vllm"], str]
     ] = None,
     extra_headers: Optional[Dict[str, str]] = None,
     extra_body: Optional[Dict[str, str]] = None,
@@ -846,7 +847,7 @@ def file_content(
 
         _is_async = kwargs.pop("afile_content", False) is True
 
-        if custom_llm_provider == "openai":
+        if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
             # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
             api_base = (
                 optional_params.api_base

litellm/integrations/custom_guardrail.py

@@ -1,5 +1,16 @@
 from datetime import datetime
-from typing import Any, Dict, List, Optional, Type, Union, get_args
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Dict,
+    List,
+    Literal,
+    Optional,
+    Tuple,
+    Type,
+    Union,
+    get_args,
+)
 
 from litellm._logging import verbose_logger
 from litellm.caching import DualCache
@@ -20,6 +31,8 @@ from litellm.types.utils import (
     StandardLoggingGuardrailInformation,
 )
 
+if TYPE_CHECKING:
+    from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 dc = DualCache()
 
 
@@ -437,30 +450,31 @@ class CustomGuardrail(CustomLogger):
 
     async def apply_guardrail(
         self,
-        text: str,
-        language: Optional[str] = None,
-        entities: Optional[List[PiiEntityType]] = None,
-        request_data: Optional[dict] = None,
-    ) -> str:
+        texts: List[str],
+        request_data: dict,
+        input_type: Literal["request", "response"],
+        logging_obj: Optional["LiteLLMLoggingObj"] = None,
+        images: Optional[List[str]] = None,
+    ) -> Tuple[List[str], Optional[List[str]]]:
         """
         Apply your guardrail logic to the given text
 
         Args:
-            text: The text to apply the guardrail to
-            language: The language of the text
-            entities: The entities to mask, optional
-            request_data: The request data dictionary to store guardrail metadata
+            texts: The texts to apply the guardrail to
+            images: The images to apply the guardrail to
+            request_data: The request data dictionary - containing user api key metadata (e.g. user_id, team_id, etc.)
+            input_type: The type of input to apply the guardrail to - "request" or "response"
 
         Any of the custom guardrails can override this method to provide custom guardrail logic
 
-        Returns the text with the guardrail applied
+        Returns the texts with the guardrail applied and the images with the guardrail applied (if any)
 
         Raises:
             Exception:
                 - If the guardrail raises an exception
 
         """
-        return text
+        return texts, images
 
     def _process_response(
         self,

litellm/litellm_core_utils/exception_mapping_utils.py

@@ -77,6 +77,7 @@ class ExceptionCheckers:
             "model's maximum context limit",
             "is longer than the model's context length",
             "input tokens exceed the configured limit",
+            "`inputs` tokens + `max_new_tokens` must be",
         ]
         for substring in known_exception_substrings:
             if substring in _error_str_lowercase:

litellm/litellm_core_utils/get_llm_provider_logic.py

@@ -662,6 +662,13 @@ def _get_openai_compatible_provider_info(  # noqa: PLR0915
         ) = litellm.XAIChatConfig()._get_openai_compatible_provider_info(
             api_base, api_key
         )
+    elif custom_llm_provider == "zai":
+        api_base = (
+            api_base
+            or get_secret_str("ZAI_API_BASE")
+            or "https://api.z.ai/api/paas/v4"
+        )
+        dynamic_api_key = api_key or get_secret_str("ZAI_API_KEY")
     elif custom_llm_provider == "together_ai":
         api_base = (
             api_base

litellm/litellm_core_utils/get_supported_openai_params.py

@@ -266,6 +266,15 @@ def get_supported_openai_params(  # noqa: PLR0915
                     model=model
                 )
             )
+    elif custom_llm_provider == "ovhcloud":
+        if request_type == "transcription":
+            from litellm.llms.ovhcloud.audio_transcription.transformation import (
+                OVHCloudAudioTranscriptionConfig,
+            )
+
+            return OVHCloudAudioTranscriptionConfig().get_supported_openai_params(
+                model=model
+            )
     elif custom_llm_provider == "elevenlabs":
         if request_type == "transcription":
             from litellm.llms.elevenlabs.audio_transcription.transformation import (

litellm/llms/anthropic/chat/guardrail_translation/handler.py

@@ -12,8 +12,7 @@ Pattern Overview:
 4. Apply guardrail responses back to the original structure
 """
 
-import asyncio
-from typing import TYPE_CHECKING, Any, Coroutine, Dict, List, Optional, Tuple, cast
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, cast
 
 from litellm._logging import verbose_proxy_logger
 from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
@@ -50,30 +49,40 @@ class AnthropicMessagesHandler(BaseTranslation):
         if messages is None:
             return data
 
-        tasks: List[Coroutine[Any, Any, str]] = []
+        texts_to_check: List[str] = []
+        images_to_check: List[str] = []
         task_mappings: List[Tuple[int, Optional[int]]] = []
-        # Track (message_index, content_index) for each task
+        # Track (message_index, content_index) for each text
         # content_index is None for string content, int for list content
 
-        # Step 1: Extract all text content and create guardrail tasks
+        # Step 1: Extract all text content and images
         for msg_idx, message in enumerate(messages):
-            await self._extract_input_text_and_create_tasks(
+            self._extract_input_text_and_images(
                 message=message,
                 msg_idx=msg_idx,
-                tasks=tasks,
+                texts_to_check=texts_to_check,
+                images_to_check=images_to_check,
                 task_mappings=task_mappings,
-                guardrail_to_apply=guardrail_to_apply,
             )
 
-        # Step 2: Run all guardrail tasks in parallel
-        responses = await asyncio.gather(*tasks)
+        # Step 2: Apply guardrail to all texts in batch
+        if texts_to_check:
+            guardrailed_texts, guardrailed_images = (
+                await guardrail_to_apply.apply_guardrail(
+                    texts=texts_to_check,
+                    request_data=data,
+                    input_type="request",
+                    images=images_to_check if images_to_check else None,
+                    logging_obj=litellm_logging_obj,
+                )
+            )
 
-        # Step 3: Map guardrail responses back to original message structure
-        await self._apply_guardrail_responses_to_input(
-            messages=messages,
-            responses=responses,
-            task_mappings=task_mappings,
-        )
+            # Step 3: Map guardrail responses back to original message structure
+            await self._apply_guardrail_responses_to_input(
+                messages=messages,
+                responses=guardrailed_texts,
+                task_mappings=task_mappings,
+            )
 
         verbose_proxy_logger.debug(
             "Anthropic Messages: Processed input messages: %s", messages
@@ -81,18 +90,18 @@ class AnthropicMessagesHandler(BaseTranslation):
 
         return data
 
-    async def _extract_input_text_and_create_tasks(
+    def _extract_input_text_and_images(
         self,
         message: Dict[str, Any],
         msg_idx: int,
-        tasks: List,
+        texts_to_check: List[str],
+        images_to_check: List[str],
         task_mappings: List[Tuple[int, Optional[int]]],
-        guardrail_to_apply: "CustomGuardrail",
     ) -> None:
         """
-        Extract text content from a message and create guardrail tasks.
+        Extract text content and images from a message.
 
-        Override this method to customize text extraction logic.
+        Override this method to customize text/image extraction logic.
         """
         content = message.get("content", None)
         if content is None:
@@ -100,17 +109,26 @@ class AnthropicMessagesHandler(BaseTranslation):
 
         if isinstance(content, str):
             # Simple string content
-            tasks.append(guardrail_to_apply.apply_guardrail(text=content))
+            texts_to_check.append(content)
             task_mappings.append((msg_idx, None))
 
         elif isinstance(content, list):
             # List content (e.g., multimodal with text and images)
             for content_idx, content_item in enumerate(content):
+                # Extract text
                 text_str = content_item.get("text", None)
-                if text_str is None:
-                    continue
-                tasks.append(guardrail_to_apply.apply_guardrail(text=text_str))
-                task_mappings.append((msg_idx, int(content_idx)))
+                if text_str is not None:
+                    texts_to_check.append(text_str)
+                    task_mappings.append((msg_idx, int(content_idx)))
+
+                # Extract images
+                if content_item.get("type") == "image":
+                    source = content_item.get("source", {})
+                    if isinstance(source, dict):
+                        # Could be base64 or url
+                        data = source.get("data")
+                        if data:
+                            images_to_check.append(data)
 
     async def _apply_guardrail_responses_to_input(
         self,
@@ -147,6 +165,7 @@ class AnthropicMessagesHandler(BaseTranslation):
         response: "AnthropicMessagesResponse",
         guardrail_to_apply: "CustomGuardrail",
         litellm_logging_obj: Optional[Any] = None,
+        user_api_key_dict: Optional[Any] = None,
     ) -> Any:
         """
         Process output response by applying guardrails to text content.
@@ -154,6 +173,8 @@ class AnthropicMessagesHandler(BaseTranslation):
         Args:
             response: Anthropic MessagesResponse object
             guardrail_to_apply: The guardrail instance to apply
+            litellm_logging_obj: Optional logging object
+            user_api_key_dict: User API key metadata to pass to guardrails
 
         Returns:
             Modified response with guardrail applied to content
@@ -168,35 +189,56 @@ class AnthropicMessagesHandler(BaseTranslation):
             )
             return response
 
-        tasks: List[Coroutine[Any, Any, str]] = []
+        texts_to_check: List[str] = []
+        images_to_check: List[str] = []
         task_mappings: List[Tuple[int, Optional[int]]] = []
-        # Track (choice_index, content_index) for each task
+        # Track (content_index, None) for each text
 
         response_content = response.get("content", [])
         if not response_content:
             return response
-        # Step 1: Extract all text content from response choices
+
+        # Step 1: Extract all text content from response
         for content_idx, content_block in enumerate(response_content):
             # Check if this is a text block by checking the 'type' field
             if isinstance(content_block, dict) and content_block.get("type") == "text":
                 # Cast to dict to handle the union type properly
-                await self._extract_output_text_and_create_tasks(
+                self._extract_output_text_and_images(
                     content_block=cast(Dict[str, Any], content_block),
                     content_idx=content_idx,
-                    tasks=tasks,
+                    texts_to_check=texts_to_check,
+                    images_to_check=images_to_check,
                     task_mappings=task_mappings,
-                    guardrail_to_apply=guardrail_to_apply,
                 )
 
-        # Step 2: Run all guardrail tasks in parallel
-        responses = await asyncio.gather(*tasks)
+        # Step 2: Apply guardrail to all texts in batch
+        if texts_to_check:
+            # Create a request_data dict with response info and user API key metadata
+            request_data: dict = {"response": response}
 
-        # Step 3: Map guardrail responses back to original response structure
-        await self._apply_guardrail_responses_to_output(
-            response=response,
-            responses=responses,
-            task_mappings=task_mappings,
-        )
+            # Add user API key metadata with prefixed keys
+            user_metadata = self.transform_user_api_key_dict_to_metadata(
+                user_api_key_dict
+            )
+            if user_metadata:
+                request_data["litellm_metadata"] = user_metadata
+
+            guardrailed_texts, guardrailed_images = (
+                await guardrail_to_apply.apply_guardrail(
+                    texts=texts_to_check,
+                    request_data=request_data,
+                    input_type="response",
+                    images=images_to_check if images_to_check else None,
+                    logging_obj=litellm_logging_obj,
+                )
+            )
+
+            # Step 3: Map guardrail responses back to original response structure
+            await self._apply_guardrail_responses_to_output(
+                response=response,
+                responses=guardrailed_texts,
+                task_mappings=task_mappings,
+            )
 
         verbose_proxy_logger.debug(
             "Anthropic Messages: Processed output response: %s", response
@@ -221,23 +263,23 @@ class AnthropicMessagesHandler(BaseTranslation):
                     return True
         return False
 
-    async def _extract_output_text_and_create_tasks(
+    def _extract_output_text_and_images(
         self,
         content_block: Dict[str, Any],
         content_idx: int,
-        tasks: List,
+        texts_to_check: List[str],
+        images_to_check: List[str],
         task_mappings: List[Tuple[int, Optional[int]]],
-        guardrail_to_apply: "CustomGuardrail",
     ) -> None:
         """
-        Extract text content from a response choice and create guardrail tasks.
+        Extract text content and images from a response content block.
 
-        Override this method to customize text extraction logic.
+        Override this method to customize text/image extraction logic.
         """
         content_text = content_block.get("text")
         if content_text and isinstance(content_text, str):
             # Simple string content
-            tasks.append(guardrail_to_apply.apply_guardrail(text=content_text))
+            texts_to_check.append(content_text)
             task_mappings.append((content_idx, None))
 
     async def _apply_guardrail_responses_to_output(

litellm/llms/anthropic/skills/readme.md

@@ -1,17 +1,279 @@
-# Anthropic Skills API
+# Anthropic Skills API Integration
 
-This folder maintains the integration for the Anthropic Skills API.
+This module provides comprehensive support for the Anthropic Skills API through LiteLLM.
 
-You can do the following with the Anthropic Skills API:
+## Features
 
-1. Create a new skill
-2. List all skills
-3. Get a skill
-4. Delete a skill
+The Skills API allows you to:
+- **Create skills**: Define reusable AI capabilities
+- **List skills**: Browse all available skills
+- **Get skills**: Retrieve detailed information about a specific skill
+- **Delete skills**: Remove skills that are no longer needed
 
+## Quick Start
 
-Versions:
-    - Create Skill Version
-    - List Skill Versions
-    - Get Skill Version
-    - Delete Skill Version
+### Prerequisites
+
+Set your Anthropic API key:
+```python
+import os
+os.environ["ANTHROPIC_API_KEY"] = "your-api-key-here"
+```
+
+### Basic Usage
+
+#### Create a Skill
+
+```python
+import litellm
+
+# Create a skill with files
+# Note: All files must be in the same top-level directory
+# and must include a SKILL.md file at the root
+skill = litellm.create_skill(
+    files=[
+        # List of file objects to upload
+        # Must include SKILL.md
+    ],
+    display_title="Python Code Generator",
+    custom_llm_provider="anthropic"
+)
+print(f"Created skill: {skill.id}")
+
+# Asynchronous version
+skill = await litellm.acreate_skill(
+    files=[...],  # Your files here
+    display_title="Python Code Generator",
+    custom_llm_provider="anthropic"
+)
+```
+
+#### List Skills
+
+```python
+# List all skills
+skills = litellm.list_skills(
+    custom_llm_provider="anthropic"
+)
+
+for skill in skills.data:
+    print(f"{skill.display_title}: {skill.id}")
+
+# With pagination and filtering
+skills = litellm.list_skills(
+    limit=20,
+    source="custom",  # Filter by 'custom' or 'anthropic'
+    custom_llm_provider="anthropic"
+)
+
+# Get next page if available
+if skills.has_more:
+    next_page = litellm.list_skills(
+        page=skills.next_page,
+        custom_llm_provider="anthropic"
+    )
+```
+
+#### Get a Skill
+
+```python
+skill = litellm.get_skill(
+    skill_id="skill_abc123",
+    custom_llm_provider="anthropic"
+)
+
+print(f"Skill: {skill.display_title}")
+print(f"Created: {skill.created_at}")
+print(f"Latest version: {skill.latest_version}")
+print(f"Source: {skill.source}")
+```
+
+#### Delete a Skill
+
+```python
+result = litellm.delete_skill(
+    skill_id="skill_abc123",
+    custom_llm_provider="anthropic"
+)
+
+print(f"Deleted skill {result.id}, type: {result.type}")
+```
+
+## API Reference
+
+### `create_skill()`
+
+Create a new skill.
+
+**Parameters:**
+- `files` (List[Any], optional): Files to upload for the skill. All files must be in the same top-level directory and must include a SKILL.md file at the root.
+- `display_title` (str, optional): Display title for the skill
+- `custom_llm_provider` (str, optional): Provider name (default: "anthropic")
+- `extra_headers` (dict, optional): Additional HTTP headers
+- `timeout` (float, optional): Request timeout
+
+**Returns:**
+- `Skill`: The created skill object
+
+**Async version:** `acreate_skill()`
+
+### `list_skills()`
+
+List all skills.
+
+**Parameters:**
+- `limit` (int, optional): Number of results to return per page (max 100, default 20)
+- `page` (str, optional): Pagination token for fetching a specific page of results
+- `source` (str, optional): Filter skills by source ('custom' or 'anthropic')
+- `custom_llm_provider` (str, optional): Provider name (default: "anthropic")
+- `extra_headers` (dict, optional): Additional HTTP headers
+- `timeout` (float, optional): Request timeout
+
+**Returns:**
+- `ListSkillsResponse`: Object containing a list of skills and pagination info
+
+**Async version:** `alist_skills()`
+
+### `get_skill()`
+
+Get a specific skill by ID.
+
+**Parameters:**
+- `skill_id` (str, required): The skill ID
+- `custom_llm_provider` (str, optional): Provider name (default: "anthropic")
+- `extra_headers` (dict, optional): Additional HTTP headers
+- `timeout` (float, optional): Request timeout
+
+**Returns:**
+- `Skill`: The requested skill object
+
+**Async version:** `aget_skill()`
+
+### `delete_skill()`
+
+Delete a skill.
+
+**Parameters:**
+- `skill_id` (str, required): The skill ID to delete
+- `custom_llm_provider` (str, optional): Provider name (default: "anthropic")
+- `extra_headers` (dict, optional): Additional HTTP headers
+- `timeout` (float, optional): Request timeout
+
+**Returns:**
+- `DeleteSkillResponse`: Object with `id` and `type` fields
+
+**Async version:** `adelete_skill()`
+
+## Response Types
+
+### `Skill`
+
+Represents a skill from the Anthropic Skills API.
+
+**Fields:**
+- `id` (str): Unique identifier
+- `created_at` (str): ISO 8601 timestamp
+- `display_title` (str, optional): Display title
+- `latest_version` (str, optional): Latest version identifier
+- `source` (str): Source ("custom" or "anthropic")
+- `type` (str): Object type (always "skill")
+- `updated_at` (str): ISO 8601 timestamp
+
+### `ListSkillsResponse`
+
+Response from listing skills.
+
+**Fields:**
+- `data` (List[Skill]): List of skills
+- `next_page` (str, optional): Pagination token for the next page
+- `has_more` (bool): Whether more skills are available
+
+### `DeleteSkillResponse`
+
+Response from deleting a skill.
+
+**Fields:**
+- `id` (str): The deleted skill ID
+- `type` (str): Deleted object type (always "skill_deleted")
+
+## Architecture
+
+The Skills API implementation follows LiteLLM's standard patterns:
+
+1. **Type Definitions** (`litellm/types/llms/anthropic_skills.py`)
+   - Pydantic models for request/response types
+   - TypedDict definitions for request parameters
+
+2. **Base Configuration** (`litellm/llms/base_llm/skills/transformation.py`)
+   - Abstract base class `BaseSkillsAPIConfig`
+   - Defines transformation interface for provider-specific implementations
+
+3. **Provider Implementation** (`litellm/llms/anthropic/skills/transformation.py`)
+   - `AnthropicSkillsConfig` - Anthropic-specific transformations
+   - Handles API authentication, URL construction, and response mapping
+
+4. **Main Handler** (`litellm/skills/main.py`)
+   - Public API functions (sync and async)
+   - Request validation and routing
+   - Error handling
+
+5. **HTTP Handlers** (`litellm/llms/custom_httpx/llm_http_handler.py`)
+   - Low-level HTTP request/response handling
+   - Connection pooling and retry logic
+
+## Beta API Support
+
+The Skills API is in beta. The beta header (`skills-2025-10-02`) is automatically added by the Anthropic provider configuration. You can customize it if needed:
+
+```python
+skill = litellm.create_skill(
+    display_title="My Skill",
+    extra_headers={
+        "anthropic-beta": "skills-2025-10-02"  # Or any other beta version
+    },
+    custom_llm_provider="anthropic"
+)
+```
+
+The default beta version is configured in `litellm.constants.ANTHROPIC_SKILLS_API_BETA_VERSION`.
+
+## Error Handling
+
+All Skills API functions follow LiteLLM's standard error handling:
+
+```python
+import litellm
+
+try:
+    skill = litellm.create_skill(
+        display_title="My Skill",
+        custom_llm_provider="anthropic"
+    )
+except litellm.exceptions.AuthenticationError as e:
+    print(f"Authentication failed: {e}")
+except litellm.exceptions.RateLimitError as e:
+    print(f"Rate limit exceeded: {e}")
+except litellm.exceptions.APIError as e:
+    print(f"API error: {e}")
+```
+
+## Contributing
+
+To add support for Skills API to a new provider:
+
+1. Create provider-specific configuration class inheriting from `BaseSkillsAPIConfig`
+2. Implement all abstract methods for request/response transformations
+3. Register the config in `ProviderConfigManager.get_provider_skills_api_config()`
+4. Add appropriate tests
+
+## Related Documentation
+
+- [Anthropic Skills API Documentation](https://platform.claude.com/docs/en/api/beta/skills/create)
+- [LiteLLM Responses API](../../../responses/)
+- [Provider Configuration System](../../base_llm/)
+
+## Support
+
+For issues or questions:
+- GitHub Issues: https://github.com/BerriAI/litellm/issues
+- Discord: https://discord.gg/wuPM9dRgDw

litellm/llms/azure/realtime/handler.py

@@ -12,6 +12,7 @@ from ....litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
 from ....litellm_core_utils.realtime_streaming import RealTimeStreaming
 from ....llms.custom_httpx.http_handler import get_shared_realtime_ssl_context
 from ..azure import AzureChatCompletion
+from litellm._logging import verbose_proxy_logger
 
 # BACKEND_WS_URL = "ws://localhost:8080/v1/realtime?model=gpt-4o-realtime-preview-2024-10-01"
 
@@ -51,18 +52,18 @@ class AzureOpenAIRealtime(AzureChatCompletion):
 
         Examples:
             beta/default: "wss://.../openai/realtime?api-version=2024-10-01-preview&deployment=gpt-4o-realtime-preview"
-            GA/v1:        "wss://.../openai/v1/realtime?api-version=2024-10-01-preview&deployment=gpt-4o-realtime-preview"
+            GA/v1:        "wss://.../openai/v1/realtime?model=gpt-realtime-deployment"
         """
         api_base = api_base.replace("https://", "wss://")
 
         # Determine path based on realtime_protocol
         if realtime_protocol in ("GA", "v1"):
-            path = "/openai/v1/realtime"
+            path = "/openai/v1/realtime" 
+            return f"{api_base}{path}?model={model}"
         else:
             # Default to beta path for backwards compatibility
             path = "/openai/realtime"
-
-        return f"{api_base}{path}?api-version={api_version}&deployment={model}"
+            return f"{api_base}{path}?api-version={api_version}&deployment={model}"
 
     async def async_realtime(
         self,
@@ -107,4 +108,5 @@ class AzureOpenAIRealtime(AzureChatCompletion):
         except websockets.exceptions.InvalidStatusCode as e:  # type: ignore
             await websocket.close(code=e.status_code, reason=str(e))
         except Exception:
+            verbose_proxy_logger.exception("Error in AzureOpenAIRealtime.async_realtime")
             pass

litellm/llms/base_llm/guardrail_translation/base_translation.py

@@ -1,12 +1,57 @@
 from abc import ABC, abstractmethod
-from typing import TYPE_CHECKING, Any, Optional
+from typing import TYPE_CHECKING, Any, Dict, Optional
 
 if TYPE_CHECKING:
     from litellm.integrations.custom_guardrail import CustomGuardrail
     from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+    from litellm.proxy._types import UserAPIKeyAuth
 
 
 class BaseTranslation(ABC):
+    @staticmethod
+    def transform_user_api_key_dict_to_metadata(
+        user_api_key_dict: Optional[Any],
+    ) -> Dict[str, Any]:
+        """
+        Transform user_api_key_dict to a metadata dict with prefixed keys.
+
+        Converts keys like 'user_id' to 'user_api_key_user_id' to clearly indicate
+        the source of the metadata.
+
+        Args:
+            user_api_key_dict: UserAPIKeyAuth object or dict with user information
+
+        Returns:
+            Dict with keys prefixed with 'user_api_key_'
+        """
+        if user_api_key_dict is None:
+            return {}
+
+        # Convert to dict if it's a Pydantic object
+        user_dict = (
+            user_api_key_dict.model_dump()
+            if hasattr(user_api_key_dict, "model_dump")
+            else user_api_key_dict
+        )
+
+        if not isinstance(user_dict, dict):
+            return {}
+
+        # Transform keys to be prefixed with 'user_api_key_'
+        transformed = {}
+        for key, value in user_dict.items():
+            # Skip None values and internal fields
+            if value is None or key.startswith("_"):
+                continue
+
+            # If key already has the prefix, use as-is, otherwise add prefix
+            if key.startswith("user_api_key_"):
+                transformed[key] = value
+            else:
+                transformed[f"user_api_key_{key}"] = value
+
+        return transformed
+
     @abstractmethod
     async def process_input_messages(
         self,
@@ -14,6 +59,11 @@ class BaseTranslation(ABC):
         guardrail_to_apply: "CustomGuardrail",
         litellm_logging_obj: Optional["LiteLLMLoggingObj"] = None,
     ) -> Any:
+        """
+        Process input messages with guardrails.
+
+        Note: user_api_key_dict metadata should be available in the data dict.
+        """
         pass
 
     @abstractmethod
@@ -22,5 +72,15 @@ class BaseTranslation(ABC):
         response: Any,
         guardrail_to_apply: "CustomGuardrail",
         litellm_logging_obj: Optional["LiteLLMLoggingObj"] = None,
+        user_api_key_dict: Optional["UserAPIKeyAuth"] = None,
     ) -> Any:
+        """
+        Process output response with guardrails.
+
+        Args:
+            response: The response object from the LLM
+            guardrail_to_apply: The guardrail instance to apply
+            litellm_logging_obj: Optional logging object
+            user_api_key_dict: User API key metadata (passed separately since response doesn't contain it)
+        """
         pass

litellm/llms/bedrock/base_aws_llm.py

@@ -353,6 +353,10 @@ class BaseAWSLLM:
             model_id = BaseAWSLLM._get_model_id_from_model_with_spec(
                 model_id, spec="deepseek_r1"
             )
+        elif provider == "openai" and "openai/" in model_id:
+            model_id = BaseAWSLLM._get_model_id_from_model_with_spec(
+                model_id, spec="openai"
+            )
         return model_id
 
     @staticmethod

litellm/llms/bedrock/batches/handler.py

@@ -0,0 +1,96 @@
+from openai.types.batch import BatchRequestCounts
+from openai.types.batch import Metadata as OpenAIBatchMetadata
+
+from litellm.types.utils import LiteLLMBatch
+
+
+class BedrockBatchesHandler:
+    """
+    Handler for Bedrock Batches.
+
+    Specific providers/models needed some special handling.
+
+    E.g. Twelve Labs Embedding Async Invoke
+    """
+    @staticmethod
+    def _handle_async_invoke_status(
+        batch_id: str, aws_region_name: str, logging_obj=None, **kwargs
+    ) -> "LiteLLMBatch":
+        """
+        Handle async invoke status check for AWS Bedrock.
+
+        This is for Twelve Labs Embedding Async Invoke.
+
+        Args:
+            batch_id: The async invoke ARN
+            aws_region_name: AWS region name
+            **kwargs: Additional parameters
+
+        Returns:
+            dict: Status information including status, output_file_id (S3 URL), etc.
+        """
+        import asyncio
+
+        from litellm.llms.bedrock.embed.embedding import BedrockEmbedding
+
+        async def _async_get_status():
+            # Create embedding handler instance
+            embedding_handler = BedrockEmbedding()
+
+            # Get the status of the async invoke job
+            status_response = await embedding_handler._get_async_invoke_status(
+                invocation_arn=batch_id,
+                aws_region_name=aws_region_name,
+                logging_obj=logging_obj,
+                **kwargs,
+            )
+
+            # Transform response to a LiteLLMBatch object
+            from litellm.types.utils import LiteLLMBatch
+
+            openai_batch_metadata: OpenAIBatchMetadata = {
+                "output_file_id": status_response["outputDataConfig"][
+                    "s3OutputDataConfig"
+                ]["s3Uri"],
+                "failure_message": status_response.get("failureMessage") or "",
+                "model_arn": status_response["modelArn"],
+            }
+
+            result = LiteLLMBatch(
+                id=status_response["invocationArn"],
+                object="batch",
+                status=status_response["status"],
+                created_at=status_response["submitTime"],
+                in_progress_at=status_response["lastModifiedTime"],
+                completed_at=status_response.get("endTime"),
+                failed_at=status_response.get("endTime")
+                if status_response["status"] == "failed"
+                else None,
+                request_counts=BatchRequestCounts(
+                    total=1,
+                    completed=1 if status_response["status"] == "completed" else 0,
+                    failed=1 if status_response["status"] == "failed" else 0,
+                ),
+                metadata=openai_batch_metadata,
+                completion_window="24h",
+                endpoint="/v1/embeddings",
+                input_file_id="",
+            )
+
+            return result
+
+        # Since this function is called from within an async context via run_in_executor,
+        # we need to create a new event loop in a thread to avoid conflicts
+        import concurrent.futures
+
+        def run_in_thread():
+            new_loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(new_loop)
+            try:
+                return new_loop.run_until_complete(_async_get_status())
+            finally:
+                new_loop.close()
+
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            future = executor.submit(run_in_thread)
+            return future.result()

litellm/llms/bedrock/chat/invoke_handler.py

@@ -73,6 +73,9 @@ bedrock_tool_name_mappings: InMemoryCache = InMemoryCache(
     max_size_in_memory=50, default_ttl=600
 )
 from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig
+from litellm.llms.bedrock.chat.invoke_transformations.amazon_openai_transformation import (
+    AmazonBedrockOpenAIConfig,
+)
 
 converse_config = AmazonConverseConfig()
 
@@ -401,6 +404,10 @@ class BedrockLLM(BaseAWSLLM):
             prompt = prompt_factory(
                 model=model, messages=messages, custom_llm_provider="bedrock"
             )
+        elif provider == "openai":
+            # OpenAI uses messages directly, no prompt conversion needed
+            # Return empty prompt as it won't be used
+            prompt = ""
         elif provider == "cohere":
             prompt, chat_history = cohere_message_pt(messages=messages)
         else:
@@ -578,6 +585,30 @@ class BedrockLLM(BaseAWSLLM):
                 )
             elif provider == "meta" or provider == "llama":
                 outputText = completion_response["generation"]
+            elif provider == "openai":
+                # OpenAI imported models use OpenAI Chat Completions format
+                if "choices" in completion_response and len(completion_response["choices"]) > 0:
+                    choice = completion_response["choices"][0]
+                    if "message" in choice:
+                        outputText = choice["message"].get("content")
+                    elif "text" in choice:  # fallback for completion format
+                        outputText = choice["text"]
+                    
+                    # Set finish reason
+                    if "finish_reason" in choice:
+                        model_response.choices[0].finish_reason = map_finish_reason(
+                            choice["finish_reason"]
+                        )
+                    
+                    # Set usage if available
+                    if "usage" in completion_response:
+                        usage = completion_response["usage"]
+                        _usage = litellm.Usage(
+                            prompt_tokens=usage.get("prompt_tokens", 0),
+                            completion_tokens=usage.get("completion_tokens", 0),
+                            total_tokens=usage.get("total_tokens", 0),
+                        )
+                        setattr(model_response, "usage", _usage)
             elif provider == "mistral":
                 outputText = completion_response["outputs"][0]["text"]
                 model_response.choices[0].finish_reason = completion_response[
@@ -895,6 +926,20 @@ class BedrockLLM(BaseAWSLLM):
                 ):  # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in
                     inference_params[k] = v
             data = json.dumps({"prompt": prompt, **inference_params})
+        elif provider == "openai":
+            ## OpenAI imported models use OpenAI Chat Completions format (messages-based)
+            # Use AmazonBedrockOpenAIConfig for proper OpenAI transformation
+            openai_config = AmazonBedrockOpenAIConfig()
+            supported_params = openai_config.get_supported_openai_params(model=model)
+            
+            # Filter to only supported OpenAI params
+            filtered_params = {
+                k: v for k, v in inference_params.items() 
+                if k in supported_params
+            }
+            
+            # OpenAI uses messages format, not prompt
+            data = json.dumps({"messages": messages, **filtered_params})
         else:
             ## LOGGING
             logging_obj.pre_call(

litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py

@@ -258,6 +258,15 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
                 litellm_params=litellm_params,
                 headers=headers,
             )
+        elif provider == "openai":
+            # OpenAI imported models use OpenAI Chat Completions format
+            return litellm.AmazonBedrockOpenAIConfig().transform_request(
+                model=model,
+                messages=messages,
+                optional_params=optional_params,
+                litellm_params=litellm_params,
+                headers=headers,
+            )
         else:
             raise BedrockError(
                 status_code=404,

litellm/llms/cohere/rerank/guardrail_translation/handler.py

@@ -49,14 +49,19 @@ class CohereRerankHandler(BaseTranslation):
         # Process query only
         query = data.get("query")
         if query is not None and isinstance(query, str):
-            guardrailed_query = await guardrail_to_apply.apply_guardrail(text=query)
-            data["query"] = guardrailed_query
+            guardrailed_texts, _ = await guardrail_to_apply.apply_guardrail(
+                texts=[query],
+                request_data=data,
+                input_type="request",
+                logging_obj=litellm_logging_obj,
+            )
+            data["query"] = guardrailed_texts[0] if guardrailed_texts else query
 
             verbose_proxy_logger.debug(
                 "Rerank: Applied guardrail to query. "
                 "Original length: %d, New length: %d",
                 len(query),
-                len(guardrailed_query),
+                len(data["query"]),
             )
         else:
             verbose_proxy_logger.debug(
@@ -70,6 +75,7 @@ class CohereRerankHandler(BaseTranslation):
         response: "RerankResponse",
         guardrail_to_apply: "CustomGuardrail",
         litellm_logging_obj: Optional[Any] = None,
+        user_api_key_dict: Optional[Any] = None,
     ) -> Any:
         """
         Process output response - not applicable for rerank.
@@ -81,6 +87,8 @@ class CohereRerankHandler(BaseTranslation):
         Args:
             response: Rerank response object with rankings
             guardrail_to_apply: The guardrail instance (unused)
+            litellm_logging_obj: Optional logging object (unused)
+            user_api_key_dict: User API key metadata (unused)
 
         Returns:
             Unmodified response (rankings don't need text guardrails)

litellm/llms/custom_httpx/aiohttp_transport.py

@@ -82,9 +82,7 @@ class AiohttpResponseStream(httpx.AsyncByteStream):
 
     async def __aiter__(self) -> typing.AsyncIterator[bytes]:
         try:
-            async for chunk in self._aiohttp_response.content.iter_chunked(
-                self.CHUNK_SIZE
-            ):
+            async for chunk in self._aiohttp_response.content.iter_chunked(self.CHUNK_SIZE):
                 yield chunk
         except (
             aiohttp.ClientPayloadError,
@@ -120,16 +118,13 @@ class AiohttpResponseStream(httpx.AsyncByteStream):
 
 
 class AiohttpTransport(httpx.AsyncBaseTransport):
-    def __init__(
-        self, client: Union[ClientSession, Callable[[], ClientSession]]
-    ) -> None:
+    def __init__(self, client: Union[ClientSession, Callable[[], ClientSession]]) -> None:
         self.client = client
 
         #########################################################
         # Class variables for proxy settings
         #########################################################
-        self.proxy: Optional[str] = None
-        self.checked_proxy_env_settings: bool = False
+        self.proxy_cache: Dict[str, Optional[str]] = {}
 
     async def aclose(self) -> None:
         if isinstance(self.client, ClientSession):
@@ -184,11 +179,7 @@ class LiteLLMAiohttpTransport(AiohttpTransport):
             current_loop = asyncio.get_running_loop()
 
             # If session is from a different or closed loop, recreate it
-            if (
-                session_loop is None
-                or session_loop != current_loop
-                or session_loop.is_closed()
-            ):
+            if session_loop is None or session_loop != current_loop or session_loop.is_closed():
                 # Close old session to prevent leaks
                 old_session = self.client
                 try:
@@ -215,7 +206,7 @@ class LiteLLMAiohttpTransport(AiohttpTransport):
                 self.client = ClientSession()
 
         return self.client
-    
+
     async def _make_aiohttp_request(
         self,
         client_session: ClientSession,
@@ -226,20 +217,20 @@ class LiteLLMAiohttpTransport(AiohttpTransport):
     ) -> ClientResponse:
         """
         Helper function to make an aiohttp request with the given parameters.
-        
+
         Args:
             client_session: The aiohttp ClientSession to use
             request: The httpx Request to send
             timeout: Timeout settings dict with 'connect', 'read', 'pool' keys
             proxy: Optional proxy URL
             sni_hostname: Optional SNI hostname for SSL
-            
+
         Returns:
             ClientResponse from aiohttp
         """
         from aiohttp import ClientTimeout
         from yarl import URL as YarlURL
-        
+
         try:
             data = request.content
         except httpx.RequestNotRead:
@@ -262,9 +253,9 @@ class LiteLLMAiohttpTransport(AiohttpTransport):
             proxy=proxy,
             server_hostname=sni_hostname,
         ).__aenter__()
-        
+
         return response
-    
+
     async def handle_async_request(
         self,
         request: httpx.Request,
@@ -297,7 +288,7 @@ class LiteLLMAiohttpTransport(AiohttpTransport):
                 else:
                     self.client = ClientSession()
                 client_session = self.client
-                
+
                 # Retry the request with the new session
                 with map_aiohttp_exceptions():
                     response = await self._make_aiohttp_request(
@@ -317,45 +308,41 @@ class LiteLLMAiohttpTransport(AiohttpTransport):
             content=AiohttpResponseStream(response),
             request=request,
         )
-    
 
     async def _get_proxy_settings(self, request: httpx.Request):
         proxy = None
-        if not (
-            litellm.disable_aiohttp_trust_env
-            or str_to_bool(os.getenv("DISABLE_AIOHTTP_TRUST_ENV", "False"))
-        ):
+        if not (litellm.disable_aiohttp_trust_env or str_to_bool(os.getenv("DISABLE_AIOHTTP_TRUST_ENV", "False"))):
             try:
                 proxy = self._proxy_from_env(request.url)
             except Exception as e:  # pragma: no cover - best effort
                 verbose_logger.debug(f"Error reading proxy env: {e}")
 
         return proxy
-    
 
     def _proxy_from_env(self, url: httpx.URL) -> typing.Optional[str]:
         """
         Return proxy URL from env for the given request URL
 
         Only check the proxy env settings once, this is a costly operation for CPU % usage
-        
+
         ."""
         #########################################################
         # Check if we've already checked the proxy env settings
         #########################################################
-        if self.checked_proxy_env_settings is True:
-            return self.proxy
-        
-        #########################################################
-        # set self.checked_proxy_env_settings to True
-        #########################################################
-        self.checked_proxy_env_settings = True
+        proxy_cache_key = url.host
+
+        if proxy_cache_key in self.proxy_cache:
+            return self.proxy_cache[proxy_cache_key]
+
         proxies = urllib.request.getproxies()
         if urllib.request.proxy_bypass(url.host):
-            return None
+            proxy_url = None
+        else:
+            proxy = proxies.get(url.scheme) or proxies.get("all")
+            if proxy and "://" not in proxy:
+                proxy = f"http://{proxy}"
+            proxy_url = proxy
 
-        proxy = proxies.get(url.scheme) or proxies.get("all")
-        if proxy and "://" not in proxy:
-            proxy = f"http://{proxy}"
-        self.proxy = proxy
-        return self.proxy
+        self.proxy_cache[proxy_cache_key] = proxy_url
+
+        return proxy_url

litellm/llms/custom_httpx/llm_http_handler.py

@@ -1804,15 +1804,21 @@ class BaseLLMHTTPHandler:
             Optional[litellm.types.utils.ProviderSpecificHeader],
             kwargs.get("provider_specific_header", None),
         )
-        extra_headers = ProviderSpecificHeaderUtils.get_provider_specific_headers(
+        provider_specific_headers = ProviderSpecificHeaderUtils.get_provider_specific_headers(
             provider_specific_header=provider_specific_header,
             custom_llm_provider=custom_llm_provider,
         )
         forwarded_headers = kwargs.get("headers", None)
-        if forwarded_headers and extra_headers:
-            merged_headers = {**forwarded_headers, **extra_headers}
-        else:
-            merged_headers = forwarded_headers or extra_headers
+        # Also check for extra_headers in kwargs (from config or direct calls)
+        extra_headers_from_kwargs = kwargs.get("extra_headers", None)
+        # Merge all header sources: forwarded < extra_headers < provider_specific
+        merged_headers = {}
+        if forwarded_headers:
+            merged_headers.update(forwarded_headers)
+        if extra_headers_from_kwargs:
+            merged_headers.update(extra_headers_from_kwargs)
+        if provider_specific_headers:
+            merged_headers.update(provider_specific_headers)
         (
             headers,
             api_base,

litellm/llms/github_copilot/chat/transformation.py

@@ -5,12 +5,10 @@ from litellm.llms.openai.openai import OpenAIConfig
 from litellm.types.llms.openai import AllMessageValues
 
 from ..authenticator import Authenticator
-from ..common_utils import GetAPIKeyError
+from ..common_utils import GetAPIKeyError, GITHUB_COPILOT_API_BASE
 
 
 class GithubCopilotConfig(OpenAIConfig):
-    GITHUB_COPILOT_API_BASE = "https://api.githubcopilot.com/"
-
     def __init__(
         self,
         api_key: Optional[str] = None,
@@ -28,7 +26,7 @@ class GithubCopilotConfig(OpenAIConfig):
         custom_llm_provider: str,
     ) -> Tuple[Optional[str], Optional[str], str]:
         dynamic_api_base = (
-            self.authenticator.get_api_base() or self.GITHUB_COPILOT_API_BASE
+            self.authenticator.get_api_base() or GITHUB_COPILOT_API_BASE
         )
         try:
             dynamic_api_key = self.authenticator.get_api_key()

litellm/llms/github_copilot/common_utils.py

@@ -2,11 +2,18 @@
 Constants for Copilot integration
 """
 from typing import Optional, Union
+from uuid import uuid4
 
 import httpx
 
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 
+# Constants
+COPILOT_VERSION = "0.26.7"
+EDITOR_PLUGIN_VERSION = f"copilot-chat/{COPILOT_VERSION}"
+USER_AGENT = f"GitHubCopilotChat/{COPILOT_VERSION}"
+API_VERSION = "2025-04-01"
+GITHUB_COPILOT_API_BASE = "https://api.githubcopilot.com"
 
 class GithubCopilotError(BaseLLMException):
     def __init__(
@@ -46,3 +53,23 @@ class RefreshAPIKeyError(GithubCopilotError):
 
 class GetAPIKeyError(GithubCopilotError):
     pass
+
+
+def get_copilot_default_headers(api_key: str) -> dict:
+    """
+    Get default headers for GitHub Copilot Responses API.
+
+    Based on copilot-api's header configuration.
+    """
+    return {
+        "Authorization": f"Bearer {api_key}",
+        "content-type": "application/json",
+        "copilot-integration-id": "vscode-chat",
+        "editor-version": "vscode/1.95.0",  # Fixed version for stability
+        "editor-plugin-version": EDITOR_PLUGIN_VERSION,
+        "user-agent": USER_AGENT,
+        "openai-intent": "conversation-panel",
+        "x-github-api-version": API_VERSION,
+        "x-request-id": str(uuid4()),
+        "x-vscode-user-agent-library-version": "electron-fetch",
+    }

litellm/llms/github_copilot/embedding/transformation.py

@@ -0,0 +1,192 @@
+"""
+GitHub Copilot Embedding API Configuration.
+
+This module provides the configuration for GitHub Copilot's Embedding API.
+
+Implementation based on analysis of the copilot-api project by caozhiyuan:
+https://github.com/caozhiyuan/copilot-api
+"""
+from typing import TYPE_CHECKING, Any, Optional
+
+import httpx
+
+from litellm._logging import verbose_logger
+from litellm.exceptions import AuthenticationError
+from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig
+from litellm.types.llms.openai import AllEmbeddingInputValues
+from litellm.types.utils import EmbeddingResponse
+from litellm.utils import convert_to_model_response_object
+
+from ..authenticator import Authenticator
+from ..common_utils import (
+    GetAPIKeyError,
+    GITHUB_COPILOT_API_BASE,
+    get_copilot_default_headers,
+)
+
+if TYPE_CHECKING:
+    from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
+
+    LiteLLMLoggingObj = _LiteLLMLoggingObj
+else:
+    LiteLLMLoggingObj = Any
+
+
+class GithubCopilotEmbeddingConfig(BaseEmbeddingConfig):
+    """
+    Configuration for GitHub Copilot's Embedding API.
+
+    Reference: https://api.githubcopilot.com/embeddings
+    """
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.authenticator = Authenticator()
+
+    def validate_environment(
+        self,
+        headers: dict,
+        model: str,
+        messages: list,
+        optional_params: dict,
+        litellm_params: dict,
+        api_key: Optional[str] = None,
+        api_base: Optional[str] = None,
+    ) -> dict:
+        """
+        Validate environment and set up headers for GitHub Copilot API.
+        """
+        try:
+            # Get GitHub Copilot API key via OAuth
+            api_key = self.authenticator.get_api_key()
+
+            if not api_key:
+                raise AuthenticationError(
+                    model=model,
+                    llm_provider="github_copilot",
+                    message="GitHub Copilot API key is required. Please authenticate via OAuth Device Flow.",
+                )
+
+            # Get default headers
+            default_headers = get_copilot_default_headers(api_key)
+
+            # Merge with existing headers (user's extra_headers take priority)
+            merged_headers = {**default_headers, **headers}
+
+            verbose_logger.debug(
+                f"GitHub Copilot Embedding API: Successfully configured headers for model {model}"
+            )
+
+            return merged_headers
+
+        except GetAPIKeyError as e:
+            raise AuthenticationError(
+                model=model,
+                llm_provider="github_copilot",
+                message=str(e),
+            )
+
+    def get_complete_url(
+        self,
+        api_base: Optional[str],
+        api_key: Optional[str],
+        model: str,
+        optional_params: dict,
+        litellm_params: dict,
+        stream: Optional[bool] = None,
+    ) -> str:
+        """
+        Get the complete URL for GitHub Copilot Embedding API endpoint.
+        """
+        # Use provided api_base or fall back to authenticator's base or default
+        api_base = (
+            self.authenticator.get_api_base()
+            or api_base
+            or GITHUB_COPILOT_API_BASE
+        )
+
+        # Remove trailing slashes
+        api_base = api_base.rstrip("/")
+
+        # Return the embeddings endpoint
+        return f"{api_base}/embeddings"
+
+    def transform_embedding_request(
+        self,
+        model: str,
+        input: AllEmbeddingInputValues,
+        optional_params: dict,
+        headers: dict,
+    ) -> dict:
+        """
+        Transform embedding request to GitHub Copilot format.
+        """
+        
+        # Ensure input is a list
+        if isinstance(input, str):
+            input = [input]
+
+        # Strip 'github_copilot/' prefix if present
+        if model.startswith("github_copilot/"):
+            model = model.replace("github_copilot/", "", 1)
+
+        return {
+            "model": model,
+            "input": input,
+            **optional_params,
+        }
+
+    def transform_embedding_response(
+        self,
+        model: str,
+        raw_response: httpx.Response,
+        model_response: EmbeddingResponse,
+        logging_obj: LiteLLMLoggingObj,
+        api_key: Optional[str],
+        request_data: dict,
+        optional_params: dict,
+        litellm_params: dict,
+    ) -> EmbeddingResponse:
+        """
+        Transform embedding response from GitHub Copilot format.
+        """
+        logging_obj.post_call(original_response=raw_response.text)
+        
+        # GitHub Copilot returns standard OpenAI-compatible embedding response
+        response_json = raw_response.json()
+        
+        return convert_to_model_response_object(
+            response_object=response_json,
+            model_response_object=model_response,
+            response_type="embedding",
+        )
+
+    def get_supported_openai_params(self, model: str) -> list:
+        return [
+            "timeout",
+            "dimensions",
+            "encoding_format",
+            "user",
+        ]
+
+    def map_openai_params(
+        self,
+        non_default_params: dict,
+        optional_params: dict,
+        model: str,
+        drop_params: bool,
+    ) -> dict:
+        for param, value in non_default_params.items():
+            if param in self.get_supported_openai_params(model):
+                optional_params[param] = value
+        return optional_params
+
+    def get_error_class(
+        self, error_message: str, status_code: int, headers: Any
+    ) -> Any:
+        from litellm.llms.openai.openai import OpenAIConfig
+
+        return OpenAIConfig().get_error_class(
+            error_message=error_message, status_code=status_code, headers=headers
+        )
+

litellm/llms/github_copilot/responses/transformation.py

@@ -8,7 +8,6 @@ Implementation based on analysis of the copilot-api project by caozhiyuan:
 https://github.com/caozhiyuan/copilot-api
 """
 from typing import TYPE_CHECKING, Any, Dict, Optional, Union
-from uuid import uuid4
 
 from litellm._logging import verbose_logger
 from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
@@ -22,7 +21,11 @@ from litellm.types.router import GenericLiteLLMParams
 from litellm.types.utils import LlmProviders
 
 from ..authenticator import Authenticator
-from ..common_utils import GetAPIKeyError
+from ..common_utils import (
+    GetAPIKeyError,
+    GITHUB_COPILOT_API_BASE,
+    get_copilot_default_headers,
+)
 
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
@@ -31,12 +34,6 @@ if TYPE_CHECKING:
 else:
     LiteLLMLoggingObj = Any
 
-# GitHub Copilot API Constants (from copilot-api)
-COPILOT_VERSION = "0.26.7"
-EDITOR_PLUGIN_VERSION = f"copilot-chat/{COPILOT_VERSION}"
-USER_AGENT = f"GitHubCopilotChat/{COPILOT_VERSION}"
-API_VERSION = "2025-04-01"
-
 
 class GithubCopilotResponsesAPIConfig(OpenAIResponsesAPIConfig):
     """
@@ -55,8 +52,6 @@ class GithubCopilotResponsesAPIConfig(OpenAIResponsesAPIConfig):
     Reference: https://api.githubcopilot.com/
     """
 
-    GITHUB_COPILOT_API_BASE = "https://api.githubcopilot.com"
-
     def __init__(self) -> None:
         super().__init__()
         self.authenticator = Authenticator()
@@ -119,7 +114,7 @@ class GithubCopilotResponsesAPIConfig(OpenAIResponsesAPIConfig):
                 )
 
             # Get default headers (from copilot-api configuration)
-            default_headers = self._get_default_headers(api_key)
+            default_headers = get_copilot_default_headers(api_key)
 
             # Merge with existing headers (user's extra_headers take priority)
             merged_headers = {**default_headers, **headers}
@@ -173,7 +168,7 @@ class GithubCopilotResponsesAPIConfig(OpenAIResponsesAPIConfig):
         api_base = (
             api_base
             or self.authenticator.get_api_base()
-            or self.GITHUB_COPILOT_API_BASE
+            or GITHUB_COPILOT_API_BASE
         )
 
         # Remove trailing slashes
@@ -184,25 +179,6 @@ class GithubCopilotResponsesAPIConfig(OpenAIResponsesAPIConfig):
 
     # ==================== Helper Methods ====================
 
-    def _get_default_headers(self, api_key: str) -> Dict[str, str]:
-        """
-        Get default headers for GitHub Copilot Responses API.
-
-        Based on copilot-api's header configuration.
-        """
-        return {
-            "Authorization": f"Bearer {api_key}",
-            "content-type": "application/json",
-            "copilot-integration-id": "vscode-chat",
-            "editor-version": "vscode/1.95.0",  # Fixed version for stability
-            "editor-plugin-version": EDITOR_PLUGIN_VERSION,
-            "user-agent": USER_AGENT,
-            "openai-intent": "conversation-panel",
-            "x-github-api-version": API_VERSION,
-            "x-request-id": str(uuid4()),
-            "x-vscode-user-agent-library-version": "electron-fetch",
-        }
-
     def _get_input_from_params(
         self, litellm_params: Optional[GenericLiteLLMParams]
     ) -> Optional[Union[str, ResponseInputParam]]:

litellm/llms/openai/chat/guardrail_translation/handler.py

@@ -14,8 +14,7 @@ Pattern Overview:
 This pattern can be replicated for other message formats (e.g., Anthropic).
 """
 
-import asyncio
-from typing import TYPE_CHECKING, Any, Coroutine, Dict, List, Optional, Tuple, cast
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, cast
 
 import litellm
 from litellm._logging import verbose_proxy_logger
@@ -51,31 +50,40 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
         if messages is None:
             return data
 
-        tasks: List[Coroutine[Any, Any, str]] = []
+        texts_to_check: List[str] = []
+        images_to_check: List[str] = []
         task_mappings: List[Tuple[int, Optional[int]]] = []
-        # Track (message_index, content_index) for each task
+        # Track (message_index, content_index) for each text
         # content_index is None for string content, int for list content
 
-        # Step 1: Extract all text content and create guardrail tasks
+        # Step 1: Extract all text content and images
         for msg_idx, message in enumerate(messages):
-            await self._extract_input_text_and_create_tasks(
+            self._extract_input_text_and_images(
                 message=message,
                 msg_idx=msg_idx,
-                tasks=tasks,
+                texts_to_check=texts_to_check,
+                images_to_check=images_to_check,
                 task_mappings=task_mappings,
-                guardrail_to_apply=guardrail_to_apply,
-                request_data=data,
             )
 
-        # Step 2: Run all guardrail tasks in parallel
-        responses = await asyncio.gather(*tasks)
+        # Step 2: Apply guardrail to all texts in batch
+        if texts_to_check:
+            guardrailed_texts, guardrailed_images = (
+                await guardrail_to_apply.apply_guardrail(
+                    texts=texts_to_check,
+                    request_data=data,
+                    input_type="request",
+                    images=images_to_check if images_to_check else None,
+                    logging_obj=litellm_logging_obj,
+                )
+            )
 
-        # Step 3: Map guardrail responses back to original message structure
-        await self._apply_guardrail_responses_to_input(
-            messages=messages,
-            responses=responses,
-            task_mappings=task_mappings,
-        )
+            # Step 3: Map guardrail responses back to original message structure
+            await self._apply_guardrail_responses_to_input(
+                messages=messages,
+                responses=guardrailed_texts,
+                task_mappings=task_mappings,
+            )
 
         verbose_proxy_logger.debug(
             "OpenAI Chat Completions: Processed input messages: %s", messages
@@ -83,19 +91,18 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
 
         return data
 
-    async def _extract_input_text_and_create_tasks(
+    def _extract_input_text_and_images(
         self,
         message: Dict[str, Any],
         msg_idx: int,
-        tasks: List,
+        texts_to_check: List[str],
+        images_to_check: List[str],
         task_mappings: List[Tuple[int, Optional[int]]],
-        guardrail_to_apply: "CustomGuardrail",
-        request_data: Optional[Dict[str, Any]] = None,
     ) -> None:
         """
-        Extract text content from a message and create guardrail tasks.
+        Extract text content and images from a message.
 
-        Override this method to customize text extraction logic.
+        Override this method to customize text/image extraction logic.
         """
         content = message.get("content", None)
         if content is None:
@@ -103,17 +110,25 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
 
         if isinstance(content, str):
             # Simple string content
-            tasks.append(guardrail_to_apply.apply_guardrail(text=content, request_data=request_data))
+            texts_to_check.append(content)
             task_mappings.append((msg_idx, None))
 
         elif isinstance(content, list):
             # List content (e.g., multimodal with text and images)
             for content_idx, content_item in enumerate(content):
+                # Extract text
                 text_str = content_item.get("text", None)
-                if text_str is None:
-                    continue
-                tasks.append(guardrail_to_apply.apply_guardrail(text=text_str, request_data=request_data))
-                task_mappings.append((msg_idx, int(content_idx)))
+                if text_str is not None:
+                    texts_to_check.append(text_str)
+                    task_mappings.append((msg_idx, int(content_idx)))
+
+                # Extract images (image_url)
+                if content_item.get("type") == "image_url":
+                    image_url = content_item.get("image_url", {})
+                    if isinstance(image_url, dict):
+                        url = image_url.get("url")
+                        if url:
+                            images_to_check.append(url)
 
     async def _apply_guardrail_responses_to_input(
         self,
@@ -150,6 +165,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
         response: "ModelResponse",
         guardrail_to_apply: "CustomGuardrail",
         litellm_logging_obj: Optional[Any] = None,
+        user_api_key_dict: Optional[Any] = None,
     ) -> Any:
         """
         Process output response by applying guardrails to text content.
@@ -157,6 +173,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
         Args:
             response: LiteLLM ModelResponse object
             guardrail_to_apply: The guardrail instance to apply
+            litellm_logging_obj: Optional logging object
+            user_api_key_dict: User API key metadata to pass to guardrails
 
         Returns:
             Modified response with guardrail applied to content
@@ -165,6 +183,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
             - String content: choice.message.content = "text here"
             - List content: choice.message.content = [{"type": "text", "text": "text here"}, ...]
         """
+
         # Step 0: Check if response has any text content to process
         if not self._has_text_content(response):
             verbose_proxy_logger.warning(
@@ -172,29 +191,49 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
             )
             return response
 
-        tasks: List[Coroutine[Any, Any, str]] = []
+        texts_to_check: List[str] = []
+        images_to_check: List[str] = []
         task_mappings: List[Tuple[int, Optional[int]]] = []
-        # Track (choice_index, content_index) for each task
+        # Track (choice_index, content_index) for each text
 
-        # Step 1: Extract all text content from response choices
+        # Step 1: Extract all text content and images from response choices
         for choice_idx, choice in enumerate(response.choices):
-            await self._extract_output_text_and_create_tasks(
+            self._extract_output_text_and_images(
                 choice=choice,
                 choice_idx=choice_idx,
-                tasks=tasks,
+                texts_to_check=texts_to_check,
+                images_to_check=images_to_check,
                 task_mappings=task_mappings,
-                guardrail_to_apply=guardrail_to_apply,
             )
 
-        # Step 2: Run all guardrail tasks in parallel
-        responses = await asyncio.gather(*tasks)
+        # Step 2: Apply guardrail to all texts in batch
+        if texts_to_check:
+            # Create a request_data dict with response info and user API key metadata
+            request_data: dict = {"response": response}
 
-        # Step 3: Map guardrail responses back to original response structure
-        await self._apply_guardrail_responses_to_output(
-            response=response,
-            responses=responses,
-            task_mappings=task_mappings,
-        )
+            # Add user API key metadata with prefixed keys
+            user_metadata = self.transform_user_api_key_dict_to_metadata(
+                user_api_key_dict
+            )
+            if user_metadata:
+                request_data["litellm_metadata"] = user_metadata
+
+            guardrailed_texts, guardrailed_images = (
+                await guardrail_to_apply.apply_guardrail(
+                    texts=texts_to_check,
+                    request_data=request_data,
+                    input_type="response",
+                    images=images_to_check if images_to_check else None,
+                    logging_obj=litellm_logging_obj,
+                )
+            )
+
+            # Step 3: Map guardrail responses back to original response structure
+            await self._apply_guardrail_responses_to_output(
+                response=response,
+                responses=guardrailed_texts,
+                task_mappings=task_mappings,
+            )
 
         verbose_proxy_logger.debug(
             "OpenAI Chat Completions: Processed output response: %s", response
@@ -214,19 +253,18 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
                     return True
         return False
 
-    async def _extract_output_text_and_create_tasks(
+    def _extract_output_text_and_images(
         self,
         choice: Any,
         choice_idx: int,
-        tasks: List,
+        texts_to_check: List[str],
+        images_to_check: List[str],
         task_mappings: List[Tuple[int, Optional[int]]],
-        guardrail_to_apply: "CustomGuardrail",
-        request_data: Optional[Dict[str, Any]] = None,
     ) -> None:
         """
-        Extract text content from a response choice and create guardrail tasks.
+        Extract text content and images from a response choice.
 
-        Override this method to customize text extraction logic.
+        Override this method to customize text/image extraction logic.
         """
         if not isinstance(choice, litellm.Choices):
             return
@@ -237,19 +275,26 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
 
         if choice.message.content and isinstance(choice.message.content, str):
             # Simple string content
-            tasks.append(
-                guardrail_to_apply.apply_guardrail(text=choice.message.content, request_data=request_data)
-            )
+            texts_to_check.append(choice.message.content)
             task_mappings.append((choice_idx, None))
 
         elif choice.message.content and isinstance(choice.message.content, list):
             # List content (e.g., multimodal response)
             for content_idx, content_item in enumerate(choice.message.content):
+                # Extract text
                 content_text = content_item.get("text")
                 if content_text:
-                    tasks.append(guardrail_to_apply.apply_guardrail(text=content_text, request_data=request_data))
+                    texts_to_check.append(content_text)
                     task_mappings.append((choice_idx, int(content_idx)))
 
+                # Extract images
+                if content_item.get("type") == "image_url":
+                    image_url = content_item.get("image_url", {})
+                    if isinstance(image_url, dict):
+                        url = image_url.get("url")
+                        if url:
+                            images_to_check.append(url)
+
     async def _apply_guardrail_responses_to_output(
         self,
         response: "ModelResponse",

litellm/llms/openai/completion/guardrail_translation/handler.py

@@ -53,41 +53,50 @@ class OpenAITextCompletionHandler(BaseTranslation):
 
         if isinstance(prompt, str):
             # Single string prompt
-            guardrailed_prompt = await guardrail_to_apply.apply_guardrail(text=prompt)
-            data["prompt"] = guardrailed_prompt
+            guardrailed_texts, _ = await guardrail_to_apply.apply_guardrail(
+                texts=[prompt],
+                request_data=data,
+                input_type="request",
+                logging_obj=litellm_logging_obj,
+            )
+            data["prompt"] = guardrailed_texts[0] if guardrailed_texts else prompt
 
             verbose_proxy_logger.debug(
                 "OpenAI Text Completion: Applied guardrail to string prompt. "
                 "Original length: %d, New length: %d",
                 len(prompt),
-                len(guardrailed_prompt),
+                len(data["prompt"]),
             )
 
         elif isinstance(prompt, list):
             # List of string prompts (batch completion)
-            guardrailed_prompts = []
+            texts_to_check = []
+            text_indices = []  # Track which prompts are strings
+
             for idx, p in enumerate(prompt):
                 if isinstance(p, str):
-                    guardrailed_p = await guardrail_to_apply.apply_guardrail(text=p)
-                    guardrailed_prompts.append(guardrailed_p)
-                    verbose_proxy_logger.debug(
-                        "OpenAI Text Completion: Applied guardrail to prompt[%d]. "
-                        "Original length: %d, New length: %d",
-                        idx,
-                        len(p),
-                        len(guardrailed_p),
-                    )
-                else:
-                    # For non-string items (e.g., token lists), keep unchanged
-                    guardrailed_prompts.append(p)
-                    verbose_proxy_logger.debug(
-                        "OpenAI Text Completion: Skipping guardrail for prompt[%d] "
-                        "(not a string, type: %s)",
-                        idx,
-                        type(p),
-                    )
+                    texts_to_check.append(p)
+                    text_indices.append(idx)
 
-            data["prompt"] = guardrailed_prompts
+            if texts_to_check:
+                guardrailed_texts, _ = await guardrail_to_apply.apply_guardrail(
+                    texts=texts_to_check,
+                    request_data=data,
+                    input_type="request",
+                    logging_obj=litellm_logging_obj,
+                )
+
+                # Replace guardrailed texts back
+                for guardrail_idx, prompt_idx in enumerate(text_indices):
+                    if guardrail_idx < len(guardrailed_texts):
+                        data["prompt"][prompt_idx] = guardrailed_texts[guardrail_idx]
+                        verbose_proxy_logger.debug(
+                            "OpenAI Text Completion: Applied guardrail to prompt[%d]. "
+                            "Original length: %d, New length: %d",
+                            prompt_idx,
+                            len(texts_to_check[guardrail_idx]),
+                            len(guardrailed_texts[guardrail_idx]),
+                        )
 
         else:
             verbose_proxy_logger.warning(
@@ -102,6 +111,7 @@ class OpenAITextCompletionHandler(BaseTranslation):
         response: "TextCompletionResponse",
         guardrail_to_apply: "CustomGuardrail",
         litellm_logging_obj: Optional[Any] = None,
+        user_api_key_dict: Optional[Any] = None,
     ) -> Any:
         """
         Process output response by applying guardrails to completion text.
@@ -109,6 +119,8 @@ class OpenAITextCompletionHandler(BaseTranslation):
         Args:
             response: Text completion response object
             guardrail_to_apply: The guardrail instance to apply
+            litellm_logging_obj: Optional logging object
+            user_api_key_dict: User API key metadata to pass to guardrails
 
         Returns:
             Modified response with guardrails applied to completion text
@@ -119,21 +131,46 @@ class OpenAITextCompletionHandler(BaseTranslation):
             )
             return response
 
-        # Apply guardrails to each choice's text
+        # Collect all texts to check
+        texts_to_check = []
+        choice_indices = []
+
         for idx, choice in enumerate(response.choices):
             if hasattr(choice, "text") and isinstance(choice.text, str):
-                original_text = choice.text
-                guardrailed_text = await guardrail_to_apply.apply_guardrail(
-                    text=original_text
-                )
-                choice.text = guardrailed_text
+                texts_to_check.append(choice.text)
+                choice_indices.append(idx)
 
-                verbose_proxy_logger.debug(
-                    "OpenAI Text Completion: Applied guardrail to choice[%d] text. "
-                    "Original length: %d, New length: %d",
-                    idx,
-                    len(original_text),
-                    len(guardrailed_text),
-                )
+        # Apply guardrails in batch
+        if texts_to_check:
+            # Create a request_data dict with response info and user API key metadata
+            request_data: dict = {"response": response}
+
+            # Add user API key metadata with prefixed keys
+            user_metadata = self.transform_user_api_key_dict_to_metadata(
+                user_api_key_dict
+            )
+            if user_metadata:
+                request_data["litellm_metadata"] = user_metadata
+
+            guardrailed_texts, _ = await guardrail_to_apply.apply_guardrail(
+                texts=texts_to_check,
+                request_data=request_data,
+                input_type="response",
+                logging_obj=litellm_logging_obj,
+            )
+
+            # Apply guardrailed texts back to choices
+            for guardrail_idx, choice_idx in enumerate(choice_indices):
+                if guardrail_idx < len(guardrailed_texts):
+                    original_text = response.choices[choice_idx].text
+                    response.choices[choice_idx].text = guardrailed_texts[guardrail_idx]
+
+                    verbose_proxy_logger.debug(
+                        "OpenAI Text Completion: Applied guardrail to choice[%d] text. "
+                        "Original length: %d, New length: %d",
+                        choice_idx,
+                        len(original_text),
+                        len(guardrailed_texts[guardrail_idx]),
+                    )
 
         return response

litellm/llms/openai/image_generation/guardrail_translation/handler.py

@@ -52,14 +52,19 @@ class OpenAIImageGenerationHandler(BaseTranslation):
 
         # Apply guardrail to the prompt
         if isinstance(prompt, str):
-            guardrailed_prompt = await guardrail_to_apply.apply_guardrail(text=prompt)
-            data["prompt"] = guardrailed_prompt
+            guardrailed_texts, _ = await guardrail_to_apply.apply_guardrail(
+                texts=[prompt],
+                request_data=data,
+                input_type="request",
+                logging_obj=litellm_logging_obj,
+            )
+            data["prompt"] = guardrailed_texts[0] if guardrailed_texts else prompt
 
             verbose_proxy_logger.debug(
                 "OpenAI Image Generation: Applied guardrail to prompt. "
                 "Original length: %d, New length: %d",
                 len(prompt),
-                len(guardrailed_prompt),
+                len(data["prompt"]),
             )
         else:
             verbose_proxy_logger.debug(
@@ -74,6 +79,7 @@ class OpenAIImageGenerationHandler(BaseTranslation):
         response: "ImageResponse",
         guardrail_to_apply: "CustomGuardrail",
         litellm_logging_obj: Optional[Any] = None,
+        user_api_key_dict: Optional[Any] = None,
     ) -> Any:
         """
         Process output response - typically not needed for image generation.
@@ -85,6 +91,8 @@ class OpenAIImageGenerationHandler(BaseTranslation):
         Args:
             response: Image generation response object
             guardrail_to_apply: The guardrail instance to apply
+            litellm_logging_obj: Optional logging object (unused)
+            user_api_key_dict: User API key metadata (unused)
 
         Returns:
             Unmodified response (images don't need text guardrails)

litellm/llms/openai/responses/guardrail_translation/handler.py

@@ -28,8 +28,7 @@ Output: response.output is List[GenericResponseOutputItem] where each has:
     - text: str
 """
 
-import asyncio
-from typing import TYPE_CHECKING, Any, Coroutine, List, Optional, Tuple, Union, cast
+from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Union, cast
 
 from litellm._logging import verbose_proxy_logger
 from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
@@ -69,10 +68,13 @@ class OpenAIResponsesHandler(BaseTranslation):
 
         # Handle simple string input
         if isinstance(input_data, str):
-            guardrail_response = await guardrail_to_apply.apply_guardrail(
-                text=input_data
+            guardrailed_texts, _ = await guardrail_to_apply.apply_guardrail(
+                texts=[input_data],
+                request_data=data,
+                input_type="request",
+                logging_obj=litellm_logging_obj,
             )
-            data["input"] = guardrail_response
+            data["input"] = guardrailed_texts[0] if guardrailed_texts else input_data
             verbose_proxy_logger.debug("OpenAI Responses API: Processed string input")
             return data
 
@@ -80,29 +82,38 @@ class OpenAIResponsesHandler(BaseTranslation):
         if not isinstance(input_data, list):
             return data
 
-        tasks: List[Coroutine[Any, Any, str]] = []
+        texts_to_check: List[str] = []
+        images_to_check: List[str] = []
         task_mappings: List[Tuple[int, Optional[int]]] = []
-        # Track (message_index, content_index) for each task
+        # Track (message_index, content_index) for each text
         # content_index is None for string content, int for list content
 
-        # Step 1: Extract all text content and create guardrail tasks
+        # Step 1: Extract all text content and images
         for msg_idx, message in enumerate(input_data):
-            await self._extract_input_text_and_create_tasks(
+            self._extract_input_text_and_images(
                 message=message,
                 msg_idx=msg_idx,
-                tasks=tasks,
+                texts_to_check=texts_to_check,
+                images_to_check=images_to_check,
                 task_mappings=task_mappings,
-                guardrail_to_apply=guardrail_to_apply,
             )
 
-        # Step 2: Run all guardrail tasks in parallel
-        if tasks:
-            responses = await asyncio.gather(*tasks)
+        # Step 2: Apply guardrail to all texts in batch
+        if texts_to_check:
+            guardrailed_texts, guardrailed_images = (
+                await guardrail_to_apply.apply_guardrail(
+                    texts=texts_to_check,
+                    request_data=data,
+                    input_type="request",
+                    images=images_to_check if images_to_check else None,
+                    logging_obj=litellm_logging_obj,
+                )
+            )
 
             # Step 3: Map guardrail responses back to original input structure
             await self._apply_guardrail_responses_to_input(
                 messages=input_data,
-                responses=responses,
+                responses=guardrailed_texts,
                 task_mappings=task_mappings,
             )
 
@@ -112,18 +123,18 @@ class OpenAIResponsesHandler(BaseTranslation):
 
         return data
 
-    async def _extract_input_text_and_create_tasks(
+    def _extract_input_text_and_images(
         self,
         message: Any,  # Can be Dict[str, Any] or ResponseInputParam
         msg_idx: int,
-        tasks: List[Coroutine[Any, Any, str]],
+        texts_to_check: List[str],
+        images_to_check: List[str],
         task_mappings: List[Tuple[int, Optional[int]]],
-        guardrail_to_apply: "CustomGuardrail",
     ) -> None:
         """
-        Extract text content from an input message and create guardrail tasks.
+        Extract text content and images from an input message.
 
-        Override this method to customize text extraction logic.
+        Override this method to customize text/image extraction logic.
         """
         content = message.get("content", None)
         if content is None:
@@ -131,18 +142,27 @@ class OpenAIResponsesHandler(BaseTranslation):
 
         if isinstance(content, str):
             # Simple string content
-            tasks.append(guardrail_to_apply.apply_guardrail(text=content))
+            texts_to_check.append(content)
             task_mappings.append((msg_idx, None))
 
         elif isinstance(content, list):
             # List content (e.g., multimodal with text and images)
             for content_idx, content_item in enumerate(content):
                 if isinstance(content_item, dict):
+                    # Extract text
                     text_str = content_item.get("text", None)
                     if text_str is not None:
-                        tasks.append(guardrail_to_apply.apply_guardrail(text=text_str))
+                        texts_to_check.append(text_str)
                         task_mappings.append((msg_idx, int(content_idx)))
 
+                    # Extract images
+                    if content_item.get("type") == "image_url":
+                        image_url = content_item.get("image_url", {})
+                        if isinstance(image_url, dict):
+                            url = image_url.get("url")
+                            if url:
+                                images_to_check.append(url)
+
     async def _apply_guardrail_responses_to_input(
         self,
         messages: Any,  # Can be List[Dict[str, Any]] or ResponseInputParam
@@ -179,6 +199,7 @@ class OpenAIResponsesHandler(BaseTranslation):
         response: "ResponsesAPIResponse",
         guardrail_to_apply: "CustomGuardrail",
         litellm_logging_obj: Optional[Any] = None,
+        user_api_key_dict: Optional[Any] = None,
     ) -> Any:
         """
         Process output response by applying guardrails to text content.
@@ -186,6 +207,8 @@ class OpenAIResponsesHandler(BaseTranslation):
         Args:
             response: LiteLLM ResponsesAPIResponse object
             guardrail_to_apply: The guardrail instance to apply
+            litellm_logging_obj: Optional logging object
+            user_api_key_dict: User API key metadata to pass to guardrails
 
         Returns:
             Modified response with guardrail applied to content
@@ -202,28 +225,47 @@ class OpenAIResponsesHandler(BaseTranslation):
             )
             return response
 
-        tasks: List[Coroutine[Any, Any, str]] = []
+        texts_to_check: List[str] = []
+        images_to_check: List[str] = []
         task_mappings: List[Tuple[int, int]] = []
-        # Track (output_item_index, content_index) for each task
+        # Track (output_item_index, content_index) for each text
 
         # Step 1: Extract all text content from response output
         for output_idx, output_item in enumerate(response.output):
-            await self._extract_output_text_and_create_tasks(
+            self._extract_output_text_and_images(
                 output_item=output_item,
                 output_idx=output_idx,
-                tasks=tasks,
+                texts_to_check=texts_to_check,
+                images_to_check=images_to_check,
                 task_mappings=task_mappings,
-                guardrail_to_apply=guardrail_to_apply,
             )
 
-        # Step 2: Run all guardrail tasks in parallel
-        if tasks:
-            responses = await asyncio.gather(*tasks)
+        # Step 2: Apply guardrail to all texts in batch
+        if texts_to_check:
+            # Create a request_data dict with response info and user API key metadata
+            request_data: dict = {"response": response}
+
+            # Add user API key metadata with prefixed keys
+            user_metadata = self.transform_user_api_key_dict_to_metadata(
+                user_api_key_dict
+            )
+            if user_metadata:
+                request_data["litellm_metadata"] = user_metadata
+
+            guardrailed_texts, guardrailed_images = (
+                await guardrail_to_apply.apply_guardrail(
+                    texts=texts_to_check,
+                    request_data=request_data,
+                    input_type="response",
+                    images=images_to_check if images_to_check else None,
+                    logging_obj=litellm_logging_obj,
+                )
+            )
 
             # Step 3: Map guardrail responses back to original response structure
             await self._apply_guardrail_responses_to_output(
                 response=response,
-                responses=responses,
+                responses=guardrailed_texts,
                 task_mappings=task_mappings,
             )
 
@@ -260,18 +302,18 @@ class OpenAIResponsesHandler(BaseTranslation):
                                 return True
         return False
 
-    async def _extract_output_text_and_create_tasks(
+    def _extract_output_text_and_images(
         self,
         output_item: Any,
         output_idx: int,
-        tasks: List,
+        texts_to_check: List[str],
+        images_to_check: List[str],
         task_mappings: List[Tuple[int, int]],
-        guardrail_to_apply: "CustomGuardrail",
     ) -> None:
         """
-        Extract text content from a response output item and create guardrail tasks.
+        Extract text content and images from a response output item.
 
-        Override this method to customize text extraction logic.
+        Override this method to customize text/image extraction logic.
         """
         # Handle both GenericResponseOutputItem and dict
         if isinstance(output_item, GenericResponseOutputItem):
@@ -299,7 +341,7 @@ class OpenAIResponsesHandler(BaseTranslation):
                 continue
 
             if text_content:
-                tasks.append(guardrail_to_apply.apply_guardrail(text=text_content))
+                texts_to_check.append(text_content)
                 task_mappings.append((output_idx, int(content_idx)))
 
     async def _apply_guardrail_responses_to_output(

litellm/llms/openai/speech/guardrail_translation/handler.py

@@ -50,16 +50,19 @@ class OpenAITextToSpeechHandler(BaseTranslation):
             return data
 
         if isinstance(input_text, str):
-            guardrailed_input = await guardrail_to_apply.apply_guardrail(
-                text=input_text
+            guardrailed_texts, _ = await guardrail_to_apply.apply_guardrail(
+                texts=[input_text],
+                request_data=data,
+                input_type="request",
+                logging_obj=litellm_logging_obj,
             )
-            data["input"] = guardrailed_input
+            data["input"] = guardrailed_texts[0] if guardrailed_texts else input_text
 
             verbose_proxy_logger.debug(
                 "OpenAI Text-to-Speech: Applied guardrail to input text. "
                 "Original length: %d, New length: %d",
                 len(input_text),
-                len(guardrailed_input),
+                len(data["input"]),
             )
         else:
             verbose_proxy_logger.debug(
@@ -74,6 +77,7 @@ class OpenAITextToSpeechHandler(BaseTranslation):
         response: "HttpxBinaryResponseContent",
         guardrail_to_apply: "CustomGuardrail",
         litellm_logging_obj: Optional[Any] = None,
+        user_api_key_dict: Optional[Any] = None,
     ) -> Any:
         """
         Process output - not applicable for text-to-speech.
@@ -84,6 +88,8 @@ class OpenAITextToSpeechHandler(BaseTranslation):
         Args:
             response: Binary audio response
             guardrail_to_apply: The guardrail instance (unused)
+            litellm_logging_obj: Optional logging object (unused)
+            user_api_key_dict: User API key metadata (unused)
 
         Returns:
             Unmodified response (audio data doesn't need text guardrails)

litellm/llms/openai/transcriptions/guardrail_translation/handler.py

@@ -56,6 +56,7 @@ class OpenAIAudioTranscriptionHandler(BaseTranslation):
         response: "TranscriptionResponse",
         guardrail_to_apply: "CustomGuardrail",
         litellm_logging_obj: Optional[Any] = None,
+        user_api_key_dict: Optional[Any] = None,
     ) -> Any:
         """
         Process output transcription by applying guardrails to transcribed text.
@@ -63,6 +64,8 @@ class OpenAIAudioTranscriptionHandler(BaseTranslation):
         Args:
             response: Transcription response object containing transcribed text
             guardrail_to_apply: The guardrail instance to apply
+            litellm_logging_obj: Optional logging object
+            user_api_key_dict: User API key metadata to pass to guardrails
 
         Returns:
             Modified response with guardrails applied to transcribed text
@@ -75,16 +78,29 @@ class OpenAIAudioTranscriptionHandler(BaseTranslation):
 
         if isinstance(response.text, str):
             original_text = response.text
-            guardrailed_text = await guardrail_to_apply.apply_guardrail(
-                text=original_text
+            # Create a request_data dict with response info and user API key metadata
+            request_data: dict = {"response": response}
+
+            # Add user API key metadata with prefixed keys
+            user_metadata = self.transform_user_api_key_dict_to_metadata(
+                user_api_key_dict
             )
-            response.text = guardrailed_text
+            if user_metadata:
+                request_data["litellm_metadata"] = user_metadata
+
+            guardrailed_texts, _ = await guardrail_to_apply.apply_guardrail(
+                texts=[original_text],
+                request_data=request_data,
+                input_type="response",
+                logging_obj=litellm_logging_obj,
+            )
+            response.text = guardrailed_texts[0] if guardrailed_texts else original_text
 
             verbose_proxy_logger.debug(
                 "OpenAI Audio Transcription: Applied guardrail to transcribed text. "
                 "Original length: %d, New length: %d",
                 len(original_text),
-                len(guardrailed_text),
+                len(response.text),
             )
         else:
             verbose_proxy_logger.debug(

litellm/llms/ovhcloud/audio_transcription/transformation.py

@@ -0,0 +1,156 @@
+"""
+Support for OVHCloud AI Endpoints `/v1/audio/transcriptions` endpoint.
+
+Our unified API follows the OpenAI standard.
+More information on our website: https://endpoints.ai.cloud.ovh.net
+"""
+
+from typing import List, Optional, Union
+
+import httpx
+
+from litellm.litellm_core_utils.audio_utils.utils import process_audio_file
+from litellm.llms.base_llm.audio_transcription.transformation import (
+    AudioTranscriptionRequestData,
+    BaseAudioTranscriptionConfig,
+)
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.secret_managers.main import get_secret_str
+from litellm.types.llms.openai import (
+    AllMessageValues,
+    OpenAIAudioTranscriptionOptionalParams,
+)
+from litellm.types.utils import FileTypes, TranscriptionResponse
+
+from ..utils import OVHCloudException
+
+
+class OVHCloudAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
+    def get_supported_openai_params(
+        self, model: str
+    ) -> List[OpenAIAudioTranscriptionOptionalParams]:
+        # OVHCloud implements the OpenAI-compatible Whisper interface.
+        # We pass through the same optional params as the OpenAI Whisper API.
+        return ["language", "prompt", "response_format", "timestamp_granularities", "temperature"]
+
+    def map_openai_params(
+        self,
+        non_default_params: dict,
+        optional_params: dict,
+        model: str,
+        drop_params: bool,
+    ) -> dict:
+        supported_params = self.get_supported_openai_params(model)
+        for k, v in non_default_params.items():
+            if k in supported_params:
+                optional_params[k] = v
+        return optional_params
+
+    def get_complete_url(
+        self,
+        api_base: Optional[str],
+        api_key: Optional[str],
+        model: str,
+        optional_params: dict,
+        litellm_params: dict,
+        stream: Optional[bool] = None,
+    ) -> str:
+        api_base = (
+            "https://oai.endpoints.kepler.ai.cloud.ovh.net/v1"
+            if api_base is None
+            else api_base.rstrip("/")
+        )
+        complete_url = f"{api_base}/audio/transcriptions"
+        return complete_url
+
+    def get_error_class(
+        self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
+    ) -> BaseLLMException:
+        return OVHCloudException(
+            message=error_message,
+            status_code=status_code,
+            headers=headers,
+        )
+
+    def validate_environment(
+        self,
+        headers: dict,
+        model: str,
+        messages: List[AllMessageValues],
+        optional_params: dict,
+        litellm_params: dict,
+        api_key: Optional[str] = None,
+        api_base: Optional[str] = None,
+    ) -> dict:
+        if api_key is None:
+            api_key = get_secret_str("OVHCLOUD_API_KEY")
+
+        default_headers = {
+            "Authorization": f"Bearer {api_key}",
+            "accept": "application/json",
+        }
+
+        # Caller can override / extend headers if needed
+        default_headers.update(headers or {})
+        return default_headers
+
+    def transform_audio_transcription_request(
+        self,
+        model: str,
+        audio_file: FileTypes,
+        optional_params: dict,
+        litellm_params: dict,
+    ) -> AudioTranscriptionRequestData:
+        """
+        Transform the audio transcription request into OpenAI-compatible form-data.
+
+        OVHCloud follows OpenAI's `/audio/transcriptions` format, so we:
+        - Build a multipart form-data body with `file`, `model`, and optional params
+        - Let the shared HTTP handler set the proper content-type boundary
+        """
+        processed_audio = process_audio_file(audio_file)
+
+        # Base form fields: model + OpenAI-compatible optional params
+        form_fields: dict = {
+            "model": model,
+        }
+
+        # Include OpenAI-compatible optional params
+        for key in self.get_supported_openai_params(model):
+            value = optional_params.get(key)
+            if value is not None:
+                form_fields[key] = value
+
+        files = {
+            "file": (
+                processed_audio.filename,
+                processed_audio.file_content,
+                processed_audio.content_type,
+            )
+        }
+
+        return AudioTranscriptionRequestData(data=form_fields, files=files)
+
+    def transform_audio_transcription_response(
+        self,
+        raw_response: httpx.Response,
+    ) -> TranscriptionResponse:
+        """
+        Transform OVHCloud audio transcription response to OpenAI-compatible TranscriptionResponse.
+        """
+        try:
+            response_json = raw_response.json()
+        except Exception:
+            raise OVHCloudException(
+                message=raw_response.text,
+                status_code=raw_response.status_code,
+                headers=raw_response.headers,
+            )
+
+        text = response_json.get("text") or response_json.get("transcript") or ""
+        response = TranscriptionResponse(text=text)
+
+        response._hidden_params = response_json
+        return response
+
+

litellm/llms/pass_through/guardrail_translation/handler.py

@@ -117,10 +117,12 @@ class PassThroughEndpointHandler(BaseTranslation):
             )
             return data
 
-        # Apply guardrail
+        # Apply guardrail (pass-through doesn't modify the text, just checks it)
         await guardrail_to_apply.apply_guardrail(
-            text=text_to_check,
+            texts=[text_to_check],
             request_data=data,
+            input_type="request",
+            logging_obj=litellm_logging_obj,
         )
 
         return data
@@ -130,9 +132,16 @@ class PassThroughEndpointHandler(BaseTranslation):
         response: Any,
         guardrail_to_apply: "CustomGuardrail",
         litellm_logging_obj: Optional["LiteLLMLoggingObj"] = None,
+        user_api_key_dict: Optional[Any] = None,
     ) -> Any:
         """
         Process output response by applying guardrails to targeted fields.
+
+        Args:
+            response: The response to process
+            guardrail_to_apply: The guardrail instance to apply
+            litellm_logging_obj: Optional logging object
+            user_api_key_dict: User API key metadata to pass to guardrails
         """
         if not isinstance(response, dict):
             verbose_proxy_logger.debug(
@@ -156,10 +165,24 @@ class PassThroughEndpointHandler(BaseTranslation):
         if not text_to_check:
             return response
 
-        # Apply guardrail
+        # Create a request_data dict with response info and user API key metadata
+        request_data: dict = (
+            {"response": response}
+            if not isinstance(response, dict)
+            else response.copy()
+        )
+
+        # Add user API key metadata with prefixed keys
+        user_metadata = self.transform_user_api_key_dict_to_metadata(user_api_key_dict)
+        if user_metadata:
+            request_data["litellm_metadata"] = user_metadata
+
+        # Apply guardrail (pass-through doesn't modify the text, just checks it)
         await guardrail_to_apply.apply_guardrail(
-            text=text_to_check,
-            request_data=response,
+            texts=[text_to_check],
+            request_data=request_data,
+            input_type="response",
+            logging_obj=litellm_logging_obj,
         )
 
         return response

litellm/llms/vertex_ai/text_to_speech/transformation.py

@@ -0,0 +1,472 @@
+"""
+Vertex AI Text-to-Speech transformation
+
+Maps OpenAI TTS spec to Google Cloud Text-to-Speech API
+Reference: https://cloud.google.com/text-to-speech/docs/reference/rest/v1/text/synthesize
+"""
+
+import base64
+from typing import TYPE_CHECKING, Any, Coroutine, Dict, Optional, Tuple, Union
+
+import httpx
+
+from litellm.llms.base_llm.text_to_speech.transformation import (
+    BaseTextToSpeechConfig,
+    TextToSpeechRequestData,
+)
+from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
+from litellm.types.llms.vertex_ai import VERTEX_CREDENTIALS_TYPES
+from litellm.types.llms.vertex_ai_text_to_speech import (
+    VertexTextToSpeechAudioConfig,
+    VertexTextToSpeechInput,
+    VertexTextToSpeechVoice,
+)
+
+if TYPE_CHECKING:
+    from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+    from litellm.types.llms.openai import HttpxBinaryResponseContent
+else:
+    LiteLLMLoggingObj = Any
+    HttpxBinaryResponseContent = Any
+
+
+class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase):
+    """
+    Configuration for Google Cloud/Vertex AI Text-to-Speech
+
+    Reference: https://cloud.google.com/text-to-speech/docs/reference/rest/v1/text/synthesize
+    """
+
+    # Default values
+    DEFAULT_LANGUAGE_CODE = "en-US"
+    DEFAULT_VOICE_NAME = "en-US-Studio-O"
+    DEFAULT_AUDIO_ENCODING = "LINEAR16"
+    DEFAULT_SPEAKING_RATE = "1"
+
+    # API endpoint
+    TTS_API_URL = "https://texttospeech.googleapis.com/v1/text:synthesize"
+
+    # Voice name mappings from OpenAI voices to Google Cloud voices
+    # Users can pass either:
+    # 1. OpenAI voice names (alloy, echo, fable, onyx, nova, shimmer) - will be mapped
+    # 2. Google Cloud/Vertex AI voice names (en-US-Studio-O, en-US-Wavenet-D, etc.) - used directly
+    VOICE_MAPPINGS = {
+        "alloy": "en-US-Studio-O",
+        "echo": "en-US-Studio-M",
+        "fable": "en-GB-Studio-B",
+        "onyx": "en-US-Wavenet-D",
+        "nova": "en-US-Studio-O",
+        "shimmer": "en-US-Wavenet-F",
+    }
+
+    # Response format mappings from OpenAI to Google Cloud audio encoding
+    FORMAT_MAPPINGS = {
+        "mp3": "MP3",
+        "opus": "OGG_OPUS",
+        "aac": "MP3",  # Google doesn't have AAC, use MP3
+        "flac": "FLAC",
+        "wav": "LINEAR16",
+        "pcm": "LINEAR16",
+    }
+
+    def __init__(self) -> None:
+        BaseTextToSpeechConfig.__init__(self)
+        VertexBase.__init__(self)
+
+    def _map_voice_to_vertex_format(
+        self,
+        voice: Optional[Union[str, Dict]],
+    ) -> Tuple[Optional[str], Optional[Dict]]:
+        """
+        Map voice to Vertex AI format.
+
+        Supports both:
+        1. OpenAI voice names (alloy, echo, fable, onyx, nova, shimmer) - will be mapped
+        2. Vertex AI voice names (en-US-Studio-O, en-US-Wavenet-D, etc.) - used directly
+        3. Dict with languageCode and name - used as-is
+
+        Returns:
+            Tuple of (voice_str, voice_dict) where:
+            - voice_str: Original string voice (for interface compatibility)
+            - voice_dict: Vertex AI format dict with languageCode and name
+        """
+        if voice is None:
+            return None, None
+
+        if isinstance(voice, dict):
+            # Already in Vertex AI format
+            return None, voice
+
+        # voice is a string
+        voice_str = voice
+
+        # Map OpenAI voice if it's a known OpenAI voice, otherwise use directly
+        if voice in self.VOICE_MAPPINGS:
+            mapped_voice_name = self.VOICE_MAPPINGS[voice]
+        else:
+            # Assume it's already a Vertex AI voice name
+            mapped_voice_name = voice
+
+        # Extract language code from voice name (e.g., "en-US-Studio-O" -> "en-US")
+        parts = mapped_voice_name.split("-")
+        if len(parts) >= 2:
+            language_code = f"{parts[0]}-{parts[1]}"
+        else:
+            language_code = self.DEFAULT_LANGUAGE_CODE
+
+        voice_dict = {
+            "languageCode": language_code,
+            "name": mapped_voice_name,
+        }
+
+        return voice_str, voice_dict
+
+    def dispatch_text_to_speech(
+        self,
+        model: str,
+        input: str,
+        voice: Optional[Union[str, Dict]],
+        optional_params: Dict,
+        litellm_params_dict: Dict,
+        logging_obj: "LiteLLMLoggingObj",
+        timeout: Union[float, httpx.Timeout],
+        extra_headers: Optional[Dict[str, Any]],
+        base_llm_http_handler: Any,
+        aspeech: bool,
+        api_base: Optional[str],
+        api_key: Optional[str],
+        **kwargs: Any,
+    ) -> Union[
+        "HttpxBinaryResponseContent",
+        Coroutine[Any, Any, "HttpxBinaryResponseContent"],
+    ]:
+        """
+        Dispatch method to handle Vertex AI TTS requests
+
+        This method encapsulates Vertex AI-specific credential resolution and parameter handling.
+        Voice mapping is handled in map_openai_params (similar to Azure AVA pattern).
+
+        Args:
+            base_llm_http_handler: The BaseLLMHTTPHandler instance from main.py
+        """
+        # Resolve Vertex AI credentials using VertexBase helpers
+        vertex_credentials = self.safe_get_vertex_ai_credentials(litellm_params_dict)
+        vertex_project = self.safe_get_vertex_ai_project(litellm_params_dict)
+        vertex_location = self.safe_get_vertex_ai_location(litellm_params_dict)
+
+        # Convert voice to string if it's a dict (extract name)
+        # Actual voice mapping happens in map_openai_params
+        voice_str: Optional[str] = None
+        if isinstance(voice, str):
+            voice_str = voice
+        elif isinstance(voice, dict):
+            # Extract voice name from dict if needed
+            voice_str = voice.get("name") if voice else None
+
+        # Store credentials in litellm_params for use in transform methods
+        litellm_params_dict.update({
+            "vertex_credentials": vertex_credentials,
+            "vertex_project": vertex_project,
+            "vertex_location": vertex_location,
+            "api_base": api_base,
+        })
+
+        # Call the text_to_speech_handler
+        response = base_llm_http_handler.text_to_speech_handler(
+            model=model,
+            input=input,
+            voice=voice_str,
+            text_to_speech_provider_config=self,
+            text_to_speech_optional_params=optional_params,
+            custom_llm_provider="vertex_ai",
+            litellm_params=litellm_params_dict,
+            logging_obj=logging_obj,
+            timeout=timeout,
+            extra_headers=extra_headers,
+            client=None,
+            _is_async=aspeech,
+        )
+
+        return response
+
+    def get_supported_openai_params(self, model: str) -> list:
+        """
+        Vertex AI TTS supports these OpenAI parameters
+
+        Note: Vertex AI also supports additional parameters like audioConfig
+        which can be passed but are not part of the OpenAI spec
+        """
+        return ["voice", "response_format", "speed"]
+
+    def map_openai_params(
+        self,
+        model: str,
+        optional_params: Dict,
+        voice: Optional[Union[str, Dict]] = None,
+        drop_params: bool = False,
+        kwargs: Dict = {},
+    ) -> Tuple[Optional[str], Dict]:
+        """
+        Map OpenAI parameters to Vertex AI TTS parameters
+
+        Voice handling (similar to Azure AVA):
+        - If voice is an OpenAI voice name (alloy, echo, etc.), it maps to a Vertex AI voice
+        - If voice is already a Vertex AI voice name (en-US-Studio-O, etc.), it's used directly
+        - If voice is a dict with languageCode and name, it's used as-is
+
+        Note: For Vertex AI, voice dict is stored in mapped_params["vertex_voice_dict"]
+        because the base class interface expects voice to be a string.
+
+        Returns:
+            Tuple of (mapped_voice_str, mapped_params)
+        """
+        mapped_params = {}
+
+        ##########################################################
+        # Map voice using helper
+        ##########################################################
+        mapped_voice_str, voice_dict = self._map_voice_to_vertex_format(voice)
+        if voice_dict is not None:
+            mapped_params["vertex_voice_dict"] = voice_dict
+
+        # Map response format
+        if "response_format" in optional_params:
+            format_name = optional_params["response_format"]
+            if format_name in self.FORMAT_MAPPINGS:
+                mapped_params["audioEncoding"] = self.FORMAT_MAPPINGS[format_name]
+            else:
+                # Try to use it directly as Google Cloud format
+                mapped_params["audioEncoding"] = format_name
+        else:
+            # Default to LINEAR16
+            mapped_params["audioEncoding"] = self.DEFAULT_AUDIO_ENCODING
+
+        # Map speed (OpenAI: 0.25-4.0, Vertex AI: speakingRate 0.25-4.0)
+        if "speed" in optional_params:
+            speed = optional_params["speed"]
+            if speed is not None:
+                mapped_params["speakingRate"] = str(speed)
+
+        # Pass through Vertex AI-specific parameters from kwargs
+        if "audioConfig" in kwargs:
+            mapped_params["audioConfig"] = kwargs["audioConfig"]
+
+        if "use_ssml" in kwargs:
+            mapped_params["use_ssml"] = kwargs["use_ssml"]
+
+        return mapped_voice_str, mapped_params
+
+    def validate_environment(
+        self,
+        headers: dict,
+        model: str,
+        api_key: Optional[str] = None,
+        api_base: Optional[str] = None,
+    ) -> dict:
+        """
+        Validate Vertex AI environment and set up authentication headers
+
+        Note: Actual authentication is handled in transform_text_to_speech_request
+        because Vertex AI requires OAuth2 token refresh
+        """
+        validated_headers = headers.copy()
+
+        # Content-Type for JSON
+        validated_headers["Content-Type"] = "application/json"
+        validated_headers["charset"] = "UTF-8"
+
+        return validated_headers
+
+    def get_complete_url(
+        self,
+        model: str,
+        api_base: Optional[str],
+        litellm_params: dict,
+    ) -> str:
+        """
+        Get the complete URL for Vertex AI TTS request
+
+        Google Cloud TTS endpoint: https://texttospeech.googleapis.com/v1/text:synthesize
+        """
+        if api_base:
+            return api_base
+
+        return self.TTS_API_URL
+
+    def _validate_vertex_input(
+        self,
+        input_data: VertexTextToSpeechInput,
+        optional_params: Dict,
+    ) -> VertexTextToSpeechInput:
+        """
+        Validate and transform input for Vertex AI TTS
+
+        Handles text vs SSML input detection and validation
+        """
+        # Remove None values
+        if input_data.get("text") is None:
+            input_data.pop("text", None)
+        if input_data.get("ssml") is None:
+            input_data.pop("ssml", None)
+
+        # Check if use_ssml is set
+        use_ssml = optional_params.get("use_ssml", False)
+
+        if use_ssml:
+            if "text" in input_data:
+                input_data["ssml"] = input_data.pop("text")
+            elif "ssml" not in input_data:
+                raise ValueError("SSML input is required when use_ssml is True.")
+        else:
+            # LiteLLM will auto-detect if text is in ssml format
+            # check if "text" is an ssml - in this case we should pass it as ssml instead of text
+            if input_data:
+                _text = input_data.get("text", None) or ""
+                if "<speak>" in _text:
+                    input_data["ssml"] = input_data.pop("text")
+
+        if not input_data:
+            raise ValueError("Either 'text' or 'ssml' must be provided.")
+        if "text" in input_data and "ssml" in input_data:
+            raise ValueError("Only one of 'text' or 'ssml' should be provided, not both.")
+
+        return input_data
+
+    def transform_text_to_speech_request(
+        self,
+        model: str,
+        input: str,
+        voice: Optional[str],
+        optional_params: Dict,
+        litellm_params: Dict,
+        headers: dict,
+    ) -> TextToSpeechRequestData:
+        """
+        Transform OpenAI TTS request to Vertex AI TTS format
+
+        This method handles:
+        1. Authentication with Vertex AI
+        2. Building the request body
+        3. Setting up headers
+
+        Returns:
+            TextToSpeechRequestData: Contains dict_body and headers
+        """
+        # Get Vertex AI credentials from litellm_params
+        vertex_credentials: Optional[VERTEX_CREDENTIALS_TYPES] = litellm_params.get(
+            "vertex_credentials"
+        )
+        vertex_project: Optional[str] = litellm_params.get("vertex_project")
+
+        ####### Authenticate with Vertex AI ########
+        _auth_header, vertex_project = self._ensure_access_token(
+            credentials=vertex_credentials,
+            project_id=vertex_project,
+            custom_llm_provider="vertex_ai_beta",
+        )
+
+        auth_header, _ = self._get_token_and_url(
+            model="",
+            auth_header=_auth_header,
+            gemini_api_key=None,
+            vertex_credentials=vertex_credentials,
+            vertex_project=vertex_project,
+            vertex_location=litellm_params.get("vertex_location"),
+            stream=False,
+            custom_llm_provider="vertex_ai_beta",
+            api_base=litellm_params.get("api_base"),
+        )
+
+        # Set authentication headers
+        headers["Authorization"] = f"Bearer {auth_header}"
+        headers["x-goog-user-project"] = vertex_project
+
+        ####### Build the request ################
+        vertex_input = VertexTextToSpeechInput(text=input)
+        vertex_input = self._validate_vertex_input(vertex_input, optional_params)
+
+        # Build voice configuration
+        # Check for voice dict stored in:
+        # 1. litellm_params by dispatch method
+        # 2. optional_params by map_openai_params
+        voice_dict = (
+            litellm_params.get("vertex_voice_dict")
+            or optional_params.get("vertex_voice_dict")
+        )
+        if voice_dict is not None and isinstance(voice_dict, dict):
+            vertex_voice = VertexTextToSpeechVoice(**voice_dict)
+        elif voice is not None and isinstance(voice, str):
+            # Handle string voice (shouldn't normally happen if dispatch was called)
+            parts = voice.split("-")
+            if len(parts) >= 2:
+                language_code = f"{parts[0]}-{parts[1]}"
+            else:
+                language_code = self.DEFAULT_LANGUAGE_CODE
+            vertex_voice = VertexTextToSpeechVoice(
+                languageCode=language_code,
+                name=voice,
+            )
+        else:
+            # Use defaults
+            vertex_voice = VertexTextToSpeechVoice(
+                languageCode=self.DEFAULT_LANGUAGE_CODE,
+                name=self.DEFAULT_VOICE_NAME,
+            )
+
+        # Build audio configuration
+        audio_encoding = optional_params.get("audioEncoding", self.DEFAULT_AUDIO_ENCODING)
+        speaking_rate = optional_params.get("speakingRate", self.DEFAULT_SPEAKING_RATE)
+
+        # Check for full audioConfig in optional_params
+        if "audioConfig" in optional_params:
+            vertex_audio_config = VertexTextToSpeechAudioConfig(**optional_params["audioConfig"])
+        else:
+            vertex_audio_config = VertexTextToSpeechAudioConfig(
+                audioEncoding=audio_encoding,
+                speakingRate=speaking_rate,
+            )
+
+        request_body: Dict[str, Any] = {
+            "input": dict(vertex_input),
+            "voice": dict(vertex_voice),
+            "audioConfig": dict(vertex_audio_config),
+        }
+
+        return TextToSpeechRequestData(
+            dict_body=request_body,
+            headers=headers,
+        )
+
+    def transform_text_to_speech_response(
+        self,
+        model: str,
+        raw_response: httpx.Response,
+        logging_obj: "LiteLLMLoggingObj",
+    ) -> "HttpxBinaryResponseContent":
+        """
+        Transform Vertex AI TTS response to standard format
+
+        Vertex AI returns JSON with base64-encoded audio content.
+        We decode it and return as HttpxBinaryResponseContent.
+        """
+        from litellm.types.llms.openai import HttpxBinaryResponseContent
+
+        # Parse JSON response
+        _json_response = raw_response.json()
+
+        # Get base64-encoded audio content
+        response_content = _json_response.get("audioContent")
+        if not response_content:
+            raise ValueError("No audioContent in Vertex AI TTS response")
+
+        # Decode base64 to get binary content
+        binary_data = base64.b64decode(response_content)
+
+        # Create an httpx.Response object with the binary data
+        response = httpx.Response(
+            status_code=200,
+            content=binary_data,
+        )
+
+        # Initialize the HttpxBinaryResponseContent instance
+        return HttpxBinaryResponseContent(response)

litellm/llms/watsonx/audio_transcription/transformation.py

@@ -4,11 +4,17 @@ Translates from OpenAI's `/v1/audio/transcriptions` to IBM WatsonX's `/ml/v1/aud
 WatsonX follows the OpenAI spec for audio transcription.
 """
 
-from typing import List, Optional
+from typing import Any, Dict, List, Optional
 
 import litellm
+from litellm.litellm_core_utils.audio_utils.utils import process_audio_file
 from litellm.types.llms.openai import OpenAIAudioTranscriptionOptionalParams
+from litellm.types.llms.watsonx import WatsonXAudioTranscriptionRequestBody
+from litellm.types.utils import FileTypes
 
+from ...base_llm.audio_transcription.transformation import (
+    AudioTranscriptionRequestData,
+)
 from ...openai.transcriptions.whisper_transformation import (
     OpenAIWhisperAudioTranscriptionConfig,
 )
@@ -40,6 +46,60 @@ class IBMWatsonXAudioTranscriptionConfig(
             "timestamp_granularities",
         ]
 
+    def transform_audio_transcription_request(
+        self,
+        model: str,
+        audio_file: FileTypes,
+        optional_params: dict,
+        litellm_params: dict,
+    ) -> AudioTranscriptionRequestData:
+        """
+        Transform the audio transcription request for WatsonX.
+        
+        WatsonX expects multipart/form-data with:
+        - file: the audio file
+        - model: the model name (without watsonx/ prefix)
+        - project_id: the project ID (as form field, not query param)
+        - other optional params
+        """
+        # Use common utility to process the audio file
+        processed_audio = process_audio_file(audio_file)
+        
+        # Get API params to extract project_id
+        api_params = _get_api_params(params=optional_params.copy())
+        
+        # Initialize form data with required fields
+        form_data: WatsonXAudioTranscriptionRequestBody = {
+            "model": model,
+            "project_id": api_params.get("project_id", ""),
+        }
+        
+        # Add supported OpenAI params to form data
+        supported_params = self.get_supported_openai_params(model)
+        for key, value in optional_params.items():
+            if key in supported_params and value is not None:
+                form_data[key] = value  # type: ignore
+        
+        # Set default response_format for cost calculation
+        if "response_format" not in form_data or (
+            form_data.get("response_format") in ["text", "json"]
+        ):
+            form_data["response_format"] = "verbose_json"
+        
+        # Prepare files dict with the audio file
+        files = {
+            "file": (
+                processed_audio.filename,
+                processed_audio.file_content,
+                processed_audio.content_type,
+            )
+        }
+        
+        # Convert TypedDict to regular dict for AudioTranscriptionRequestData
+        form_data_dict: Dict[str, Any] = dict(form_data)
+        
+        return AudioTranscriptionRequestData(data=form_data_dict, files=files)
+
     def get_complete_url(
         self,
         api_base: Optional[str],
@@ -52,7 +112,9 @@ class IBMWatsonXAudioTranscriptionConfig(
         """
         Construct the complete URL for WatsonX audio transcription.
 
-        URL format: {api_base}/ml/v1/audio/transcriptions?version={version}&project_id={project_id}
+        URL format: {api_base}/ml/v1/audio/transcriptions?version={version}
+        
+        Note: project_id is sent as form data, not as a query parameter
         """
         # Get base URL
         url = self._get_base_url(api_base=api_base)
@@ -61,18 +123,10 @@ class IBMWatsonXAudioTranscriptionConfig(
         # Add the audio transcription endpoint
         url = f"{url}/ml/v1/audio/transcriptions"
 
-        # Get API params for project_id
-        api_params = _get_api_params(params=optional_params.copy())
-
-        # Add version parameter
-        api_version = optional_params.pop(
+        # Add version parameter (only version in query string, not project_id)
+        api_version = optional_params.get(
             "api_version", None
         ) or litellm.WATSONX_DEFAULT_API_VERSION
         url = f"{url}?version={api_version}"
 
-        # Add project_id parameter
-        project_id = api_params.get("project_id")
-        if project_id:
-            url = f"{url}&project_id={project_id}"
-
         return url

litellm/llms/watsonx/common_utils.py

@@ -252,9 +252,13 @@ class IBMWatsonXMixin:
             Optional[str],
             optional_params.get("token") or get_secret_str("WATSONX_TOKEN"),
         )
+        zen_api_key = cast(
+            Optional[str],
+            optional_params.pop("zen_api_key", None) or get_secret_str("WATSONX_ZENAPIKEY"),
+        )
         if token:
             headers["Authorization"] = f"Bearer {token}"
-        elif zen_api_key := get_secret_str("WATSONX_ZENAPIKEY"):
+        elif zen_api_key:
             headers["Authorization"] = f"ZenApiKey {zen_api_key}"
         else:
             token = _generate_watsonx_token(api_key=api_key, token=token)

litellm/main.py

@@ -206,7 +206,6 @@ from .llms.vertex_ai.image_generation.image_generation_handler import (
 from .llms.vertex_ai.multimodal_embeddings.embedding_handler import (
     VertexMultimodalEmbedding,
 )
-from .llms.vertex_ai.text_to_speech.text_to_speech_handler import VertexTextToSpeechAPI
 from .llms.vertex_ai.vertex_ai_partner_models.main import VertexAIPartnerModels
 from .llms.vertex_ai.vertex_embeddings.embedding_handler import VertexEmbedding
 from .llms.vertex_ai.vertex_gemma_models.main import VertexAIGemmaModels
@@ -277,7 +276,7 @@ google_batch_embeddings = GoogleBatchEmbeddings()
 vertex_partner_models_chat_completion = VertexAIPartnerModels()
 vertex_gemma_chat_completion = VertexAIGemmaModels()
 vertex_model_garden_chat_completion = VertexAIModelGardenModels()
-vertex_text_to_speech = VertexTextToSpeechAPI()
+# vertex_text_to_speech is now replaced by VertexAITextToSpeechConfig
 sagemaker_llm = SagemakerLLM()
 watsonx_chat_completion = WatsonXChatHandler()
 openai_like_embedding = OpenAILikeEmbeddingHandler()
@@ -860,6 +859,7 @@ def mock_completion(
             raise mock_response
         # At this point, mock_response must be a string (all other types have been handled or returned early)
         mock_response = cast(str, mock_response)
+
         if n is None:
             model_response.choices[0].message.content = mock_response  # type: ignore
         else:
@@ -906,6 +906,7 @@ def mock_completion(
                 api_key="my-secret-key",
                 original_response="my-original-response",
             )
+
         return model_response
 
     except Exception as e:
@@ -942,10 +943,16 @@ def responses_api_bridge_check(
     return model_info, model
 
 
-def _should_allow_input_examples(custom_llm_provider: Optional[str], model: str) -> bool:
+def _should_allow_input_examples(
+    custom_llm_provider: Optional[str], model: str
+) -> bool:
     if custom_llm_provider == "anthropic":
         return True
-    if custom_llm_provider == "azure_ai" or custom_llm_provider == "bedrock" or custom_llm_provider == "vertex_ai":
+    if (
+        custom_llm_provider == "azure_ai"
+        or custom_llm_provider == "bedrock"
+        or custom_llm_provider == "vertex_ai"
+    ):
         return "claude" in model.lower()
     return False
 
@@ -961,7 +968,9 @@ def _drop_input_examples_from_tool(tool: dict) -> dict:
     return tool_copy
 
 
-def _drop_input_examples_from_tools(tools: Optional[List[dict]]) -> Optional[List[dict]]:
+def _drop_input_examples_from_tools(
+    tools: Optional[List[dict]],
+) -> Optional[List[dict]]:
     if tools is None:
         return None
     cleaned_tools: List[dict] = []
@@ -1735,7 +1744,7 @@ def completion(  # type: ignore # noqa: PLR0915
                         "Set `api_base` or the AZURE_AI_API_BASE env var."
                     )
                 api_key = AzureFoundryModelInfo.get_api_key(api_key)
-                
+
                 # Ensure the URL ends with /v1/messages for Anthropic
                 if api_base:
                     api_base = api_base.rstrip("/")
@@ -1746,7 +1755,7 @@ def completion(  # type: ignore # noqa: PLR0915
                         else:
                             api_base = api_base + "/anthropic"
                         api_base = api_base + "/v1/messages"
-                
+
                 response = azure_anthropic_chat_completions.completion(
                     model=model,
                     messages=messages,
@@ -4253,6 +4262,22 @@ def embedding(  # noqa: PLR0915
                 headers=headers or extra_headers,
                 litellm_params=litellm_params_dict,
             )
+        elif custom_llm_provider == "github_copilot":
+            api_key = (api_key or litellm.api_key)
+            response = base_llm_http_handler.embedding(
+                model=model,
+                input=input,
+                custom_llm_provider=custom_llm_provider,
+                api_base=api_base,
+                api_key=api_key,
+                logging_obj=logging,
+                timeout=timeout,
+                model_response=EmbeddingResponse(),
+                optional_params=optional_params,
+                client=client,
+                aembedding=aembedding,
+                litellm_params=litellm_params_dict,
+            )
         elif (
             model in litellm.open_ai_embedding_models
             or custom_llm_provider == "openai"
@@ -5860,9 +5885,7 @@ def speech(  # noqa: PLR0915
     custom_llm_provider: Optional[str] = None,
     aspeech: Optional[bool] = None,
     **kwargs,
-) -> Union[
-    HttpxBinaryResponseContent, Coroutine[Any, Any, HttpxBinaryResponseContent]
-]:
+) -> Union[HttpxBinaryResponseContent, Coroutine[Any, Any, HttpxBinaryResponseContent]]:
     user = kwargs.get("user", None)
     litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None)
     proxy_server_request = kwargs.get("proxy_server_request", None)
@@ -5907,7 +5930,9 @@ def speech(  # noqa: PLR0915
             kwargs=kwargs,
         )
 
-    logging_obj: LiteLLMLoggingObj = cast(LiteLLMLoggingObj, kwargs.get("litellm_logging_obj"))
+    logging_obj: LiteLLMLoggingObj = cast(
+        LiteLLMLoggingObj, kwargs.get("litellm_logging_obj")
+    )
     logging_obj.update_environment_variables(
         model=model,
         user=user,
@@ -6095,9 +6120,9 @@ def speech(  # noqa: PLR0915
                 ElevenLabsTextToSpeechConfig.ELEVENLABS_QUERY_PARAMS_KEY
             ] = query_params
 
-        litellm_params_dict[
-            ElevenLabsTextToSpeechConfig.ELEVENLABS_VOICE_ID_KEY
-        ] = voice_id
+        litellm_params_dict[ElevenLabsTextToSpeechConfig.ELEVENLABS_VOICE_ID_KEY] = (
+            voice_id
+        )
 
         if api_base is not None:
             litellm_params_dict["api_base"] = api_base
@@ -6119,30 +6144,13 @@ def speech(  # noqa: PLR0915
             _is_async=aspeech or False,
         )
     elif custom_llm_provider == "vertex_ai" or custom_llm_provider == "vertex_ai_beta":
+        from litellm.llms.vertex_ai.text_to_speech.transformation import (
+            VertexAITextToSpeechConfig,
+        )
+
         generic_optional_params = GenericLiteLLMParams(**kwargs)
 
-        api_base = generic_optional_params.api_base or ""
-        vertex_ai_project = (
-            generic_optional_params.vertex_project
-            or litellm.vertex_project
-            or get_secret_str("VERTEXAI_PROJECT")
-        )
-        vertex_ai_location = (
-            generic_optional_params.vertex_location
-            or litellm.vertex_location
-            or get_secret_str("VERTEXAI_LOCATION")
-        )
-        vertex_credentials = (
-            generic_optional_params.vertex_credentials
-            or get_secret_str("VERTEXAI_CREDENTIALS")
-        )
-
-        if voice is not None and not isinstance(voice, dict):
-            raise litellm.BadRequestError(
-                message=f"'voice' is required to be passed as a dict for Vertex AI TTS, passed in voice={voice}",
-                model=model,
-                llm_provider=custom_llm_provider,
-            )
+        # Handle Gemini models separately (they use speech_to_completion_bridge)
         if "gemini" in model:
             from .endpoints.speech.speech_to_completion_bridge.handler import (
                 speech_to_completion_bridge_handler,
@@ -6158,19 +6166,37 @@ def speech(  # noqa: PLR0915
                 logging_obj=logging_obj,
                 custom_llm_provider=custom_llm_provider,
             )
-        response = vertex_text_to_speech.audio_speech(
-            _is_async=aspeech,
-            vertex_credentials=vertex_credentials,
-            vertex_project=vertex_ai_project,
-            vertex_location=vertex_ai_location,
-            timeout=timeout,
-            api_base=api_base,
+
+        # Vertex AI Text-to-Speech (Google Cloud TTS)
+        if text_to_speech_provider_config is None:
+            text_to_speech_provider_config = VertexAITextToSpeechConfig()
+
+        # Cast to specific Vertex AI config type to access dispatch method
+        vertex_config = cast(
+            VertexAITextToSpeechConfig, text_to_speech_provider_config
+        )
+
+        # Store Vertex AI specific params in litellm_params_dict
+        litellm_params_dict.update({
+            "vertex_project": generic_optional_params.vertex_project,
+            "vertex_location": generic_optional_params.vertex_location,
+            "vertex_credentials": generic_optional_params.vertex_credentials,
+        })
+
+        response = vertex_config.dispatch_text_to_speech(
             model=model,
             input=input,
             voice=voice,
             optional_params=optional_params,
-            kwargs=kwargs,
+            litellm_params_dict=litellm_params_dict,
             logging_obj=logging_obj,
+            timeout=timeout,
+            extra_headers=headers,
+            base_llm_http_handler=base_llm_http_handler,
+            aspeech=aspeech or False,
+            api_base=generic_optional_params.api_base,
+            api_key=None,  # Vertex AI uses OAuth, not API key
+            **kwargs,
         )
     elif custom_llm_provider == "gemini":
         from .endpoints.speech.speech_to_completion_bridge.handler import (

litellm/model_prices_and_context_window_backup.json

@@ -6717,6 +6717,33 @@
         "supports_vision": true,
         "tool_use_system_prompt_tokens": 159
     },
+    "claude-opus-4-5": {
+        "cache_creation_input_token_cost": 6.25e-06,
+        "cache_creation_input_token_cost_above_1hr": 1e-05,
+        "cache_read_input_token_cost": 5e-07,
+        "input_cost_per_token": 5e-06,
+        "litellm_provider": "anthropic",
+        "max_input_tokens": 200000,
+        "max_output_tokens": 64000,
+        "max_tokens": 64000,
+        "mode": "chat",
+        "output_cost_per_token": 2.5e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_assistant_prefill": true,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "tool_use_system_prompt_tokens": 159
+    },
     "claude-sonnet-4-20250514": {
         "deprecation_date": "2026-05-14",
         "cache_creation_input_token_cost": 3.75e-06,
@@ -7824,26 +7851,298 @@
         "source": "https://www.databricks.com/product/pricing/foundation-model-serving"
     },
     "databricks/databricks-claude-3-7-sonnet": {
-        "input_cost_per_token": 2.5e-06,
-        "input_dbu_cost_per_token": 3.571e-05,
+        "input_cost_per_token": 2.9999900000000002e-06,
+        "input_dbu_cost_per_token": 4.2857e-05,
         "litellm_provider": "databricks",
         "max_input_tokens": 200000,
         "max_output_tokens": 128000,
         "max_tokens": 200000,
         "metadata": {
-            "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Claude 3.7 conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
         },
         "mode": "chat",
-        "output_cost_per_token": 1.7857e-05,
-        "output_db_cost_per_token": 0.000214286,
-        "source": "https://www.databricks.com/product/pricing/foundation-model-serving",
+        "output_cost_per_token": 1.5000020000000002e-05,
+        "output_dbu_cost_per_token": 0.000214286,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
         "supports_assistant_prefill": true,
         "supports_function_calling": true,
         "supports_reasoning": true,
         "supports_tool_choice": true
     },
+    "databricks/databricks-claude-haiku-4-5": {
+        "input_cost_per_token": 1.00002e-06,
+        "input_dbu_cost_per_token": 1.4286e-05,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 200000,
+        "max_output_tokens": 64000,
+        "max_tokens": 200000,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 5.00003e-06,
+        "output_dbu_cost_per_token": 7.1429e-05,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
+        "supports_assistant_prefill": true,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_tool_choice": true
+    },
+    "databricks/databricks-claude-opus-4": {
+        "input_cost_per_token": 1.5000020000000002e-05,
+        "input_dbu_cost_per_token": 0.000214286,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 200000,
+        "max_output_tokens": 32000,
+        "max_tokens": 200000,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 7.500003000000001e-05,
+        "output_dbu_cost_per_token": 0.001071429,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
+        "supports_assistant_prefill": true,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_tool_choice": true
+    },
+    "databricks/databricks-claude-opus-4-1": {
+        "input_cost_per_token": 1.5000020000000002e-05,
+        "input_dbu_cost_per_token": 0.000214286,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 200000,
+        "max_output_tokens": 32000,
+        "max_tokens": 200000,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 7.500003000000001e-05,
+        "output_dbu_cost_per_token": 0.001071429,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
+        "supports_assistant_prefill": true,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_tool_choice": true
+    },
+    "databricks/databricks-claude-opus-4-5": {
+        "input_cost_per_token": 5.00003e-06,
+        "input_dbu_cost_per_token": 7.1429e-05,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 200000,
+        "max_output_tokens": 64000,
+        "max_tokens": 200000,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 2.5000010000000002e-05,
+        "output_dbu_cost_per_token": 0.000357143,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
+        "supports_assistant_prefill": true,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_tool_choice": true
+    },
+    "databricks/databricks-claude-sonnet-4": {
+        "input_cost_per_token": 2.9999900000000002e-06,
+        "input_dbu_cost_per_token": 4.2857e-05,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 200000,
+        "max_output_tokens": 64000,
+        "max_tokens": 200000,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 1.5000020000000002e-05,
+        "output_dbu_cost_per_token": 0.000214286,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
+        "supports_assistant_prefill": true,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_tool_choice": true
+    },
+    "databricks/databricks-claude-sonnet-4-1": {
+        "input_cost_per_token": 2.9999900000000002e-06,
+        "input_dbu_cost_per_token": 4.2857e-05,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 200000,
+        "max_output_tokens": 64000,
+        "max_tokens": 200000,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 1.5000020000000002e-05,
+        "output_dbu_cost_per_token": 0.000214286,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
+        "supports_assistant_prefill": true,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_tool_choice": true
+    },
+    "databricks/databricks-claude-sonnet-4-5": {
+        "input_cost_per_token": 2.9999900000000002e-06,
+        "input_dbu_cost_per_token": 4.2857e-05,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 200000,
+        "max_output_tokens": 64000,
+        "max_tokens": 200000,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 1.5000020000000002e-05,
+        "output_dbu_cost_per_token": 0.000214286,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
+        "supports_assistant_prefill": true,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_tool_choice": true
+    },
+    "databricks/databricks-gemini-2-5-flash": {
+        "input_cost_per_token": 3.0001999999999996e-07,
+        "input_dbu_cost_per_token": 4.285999999999999e-06,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 65535,
+        "max_tokens": 1048576,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 2.49998e-06,
+        "output_dbu_cost_per_token": 3.5714e-05,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
+        "supports_function_calling": true,
+        "supports_tool_choice": true
+    },
+    "databricks/databricks-gemini-2-5-pro": {
+        "input_cost_per_token": 1.24999e-06,
+        "input_dbu_cost_per_token": 1.7857e-05,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 65536,
+        "max_tokens": 1048576,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 9.999990000000002e-06,
+        "output_dbu_cost_per_token": 0.000142857,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
+        "supports_function_calling": true,
+        "supports_tool_choice": true
+    },
+    "databricks/databricks-gemma-3-12b": {
+        "input_cost_per_token": 1.5000999999999998e-07,
+        "input_dbu_cost_per_token": 2.1429999999999996e-06,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 32000,
+        "max_tokens": 128000,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 5.0001e-07,
+        "output_dbu_cost_per_token": 7.143e-06,
+        "source": "https://www.databricks.com/product/pricing/foundation-model-serving"
+    },
+    "databricks/databricks-gpt-5": {
+        "input_cost_per_token": 1.24999e-06,
+        "input_dbu_cost_per_token": 1.7857e-05,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 400000,
+        "max_output_tokens": 128000,
+        "max_tokens": 400000,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 9.999990000000002e-06,
+        "output_dbu_cost_per_token": 0.000142857,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
+    },
+    "databricks/databricks-gpt-5-1": {
+        "input_cost_per_token": 1.24999e-06,
+        "input_dbu_cost_per_token": 1.7857e-05,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 400000,
+        "max_output_tokens": 128000,
+        "max_tokens": 400000,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 9.999990000000002e-06,
+        "output_dbu_cost_per_token": 0.000142857,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
+    },
+    "databricks/databricks-gpt-5-mini": {
+        "input_cost_per_token": 2.4997000000000006e-07,
+        "input_dbu_cost_per_token": 3.571e-06,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 400000,
+        "max_output_tokens": 128000,
+        "max_tokens": 400000,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 1.9999700000000004e-06,
+        "output_dbu_cost_per_token": 2.8571e-05,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
+    },
+    "databricks/databricks-gpt-5-nano": {
+        "input_cost_per_token": 4.998e-08,
+        "input_dbu_cost_per_token": 7.14e-07,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 400000,
+        "max_output_tokens": 128000,
+        "max_tokens": 400000,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 3.9998000000000007e-07,
+        "output_dbu_cost_per_token": 5.714000000000001e-06,
+        "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving"
+    },
+    "databricks/databricks-gpt-oss-120b": {
+        "input_cost_per_token": 1.5000999999999998e-07,
+        "input_dbu_cost_per_token": 2.1429999999999996e-06,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 131072,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 5.9997e-07,
+        "output_dbu_cost_per_token": 8.571e-06,
+        "source": "https://www.databricks.com/product/pricing/foundation-model-serving"
+    },
+    "databricks/databricks-gpt-oss-20b": {
+        "input_cost_per_token": 7e-08,
+        "input_dbu_cost_per_token": 1e-06,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 131072,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 3.0001999999999996e-07,
+        "output_dbu_cost_per_token": 4.285999999999999e-06,
+        "source": "https://www.databricks.com/product/pricing/foundation-model-serving"
+    },
     "databricks/databricks-gte-large-en": {
-        "input_cost_per_token": 1.2999e-07,
+        "input_cost_per_token": 1.2999000000000001e-07,
         "input_dbu_cost_per_token": 1.857e-06,
         "litellm_provider": "databricks",
         "max_input_tokens": 8192,
@@ -7868,14 +8167,14 @@
             "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
         },
         "mode": "chat",
-        "output_cost_per_token": 1.5e-06,
+        "output_cost_per_token": 1.5000300000000002e-06,
         "output_dbu_cost_per_token": 2.1429e-05,
         "source": "https://www.databricks.com/product/pricing/foundation-model-serving",
         "supports_tool_choice": true
     },
     "databricks/databricks-llama-4-maverick": {
-        "input_cost_per_token": 5e-06,
-        "input_dbu_cost_per_token": 7.143e-05,
+        "input_cost_per_token": 5.0001e-07,
+        "input_dbu_cost_per_token": 7.143e-06,
         "litellm_provider": "databricks",
         "max_input_tokens": 128000,
         "max_output_tokens": 128000,
@@ -7884,13 +8183,13 @@
             "notes": "Databricks documentation now provides both DBU costs (_dbu_cost_per_token) and dollar costs(_cost_per_token)."
         },
         "mode": "chat",
-        "output_cost_per_token": 1.5e-05,
-        "output_dbu_cost_per_token": 0.00021429,
+        "output_cost_per_token": 1.5000300000000002e-06,
+        "output_dbu_cost_per_token": 2.1429e-05,
         "source": "https://www.databricks.com/product/pricing/foundation-model-serving",
         "supports_tool_choice": true
     },
     "databricks/databricks-meta-llama-3-1-405b-instruct": {
-        "input_cost_per_token": 5e-06,
+        "input_cost_per_token": 5.00003e-06,
         "input_dbu_cost_per_token": 7.1429e-05,
         "litellm_provider": "databricks",
         "max_input_tokens": 128000,
@@ -7900,14 +8199,29 @@
             "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
         },
         "mode": "chat",
-        "output_cost_per_token": 1.500002e-05,
-        "output_db_cost_per_token": 0.000214286,
+        "output_cost_per_token": 1.5000020000000002e-05,
+        "output_dbu_cost_per_token": 0.000214286,
         "source": "https://www.databricks.com/product/pricing/foundation-model-serving",
         "supports_tool_choice": true
     },
+    "databricks/databricks-meta-llama-3-1-8b-instruct": {
+        "input_cost_per_token": 1.5000999999999998e-07,
+        "input_dbu_cost_per_token": 2.1429999999999996e-06,
+        "litellm_provider": "databricks",
+        "max_input_tokens": 200000,
+        "max_output_tokens": 128000,
+        "max_tokens": 200000,
+        "metadata": {
+            "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
+        },
+        "mode": "chat",
+        "output_cost_per_token": 4.5003000000000007e-07,
+        "output_dbu_cost_per_token": 6.429000000000001e-06,
+        "source": "https://www.databricks.com/product/pricing/foundation-model-serving"
+    },
     "databricks/databricks-meta-llama-3-3-70b-instruct": {
-        "input_cost_per_token": 1.00002e-06,
-        "input_dbu_cost_per_token": 1.4286e-05,
+        "input_cost_per_token": 5.0001e-07,
+        "input_dbu_cost_per_token": 7.143e-06,
         "litellm_provider": "databricks",
         "max_input_tokens": 128000,
         "max_output_tokens": 128000,
@@ -7916,8 +8230,8 @@
             "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
         },
         "mode": "chat",
-        "output_cost_per_token": 2.99999e-06,
-        "output_dbu_cost_per_token": 4.2857e-05,
+        "output_cost_per_token": 1.5000300000000002e-06,
+        "output_dbu_cost_per_token": 2.1429e-05,
         "source": "https://www.databricks.com/product/pricing/foundation-model-serving",
         "supports_tool_choice": true
     },
@@ -7932,7 +8246,7 @@
             "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
         },
         "mode": "chat",
-        "output_cost_per_token": 2.99999e-06,
+        "output_cost_per_token": 2.9999900000000002e-06,
         "output_dbu_cost_per_token": 4.2857e-05,
         "source": "https://www.databricks.com/product/pricing/foundation-model-serving",
         "supports_tool_choice": true
@@ -7948,13 +8262,13 @@
             "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
         },
         "mode": "chat",
-        "output_cost_per_token": 9.9902e-07,
+        "output_cost_per_token": 1.00002e-06,
         "output_dbu_cost_per_token": 1.4286e-05,
         "source": "https://www.databricks.com/product/pricing/foundation-model-serving",
         "supports_tool_choice": true
     },
     "databricks/databricks-mpt-30b-instruct": {
-        "input_cost_per_token": 9.9902e-07,
+        "input_cost_per_token": 1.00002e-06,
         "input_dbu_cost_per_token": 1.4286e-05,
         "litellm_provider": "databricks",
         "max_input_tokens": 8192,
@@ -7964,7 +8278,7 @@
             "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
         },
         "mode": "chat",
-        "output_cost_per_token": 9.9902e-07,
+        "output_cost_per_token": 1.00002e-06,
         "output_dbu_cost_per_token": 1.4286e-05,
         "source": "https://www.databricks.com/product/pricing/foundation-model-serving",
         "supports_tool_choice": true
@@ -10198,6 +10512,19 @@
         "supports_response_schema": true,
         "supports_tool_choice": true
     },
+    "fireworks_ai/accounts/fireworks/models/kimi-k2-instruct-0905": {
+        "input_cost_per_token": 6e-07,
+        "litellm_provider": "fireworks_ai",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 32768,
+        "max_tokens": 262144,
+        "mode": "chat",
+        "output_cost_per_token": 2.5e-06,
+        "source": "https://app.fireworks.ai/models/fireworks/kimi-k2-instruct-0905",
+        "supports_function_calling": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true
+    },
     "fireworks_ai/accounts/fireworks/models/kimi-k2-thinking": {
         "input_cost_per_token": 6e-07,
         "litellm_provider": "fireworks_ai",
@@ -24498,6 +24825,15 @@
         "supports_parallel_function_calling": true,
         "supports_tool_choice": true
     },
+    "vertex_ai/chirp": {
+        "input_cost_per_character": 30e-06,
+        "litellm_provider": "vertex_ai",
+        "mode": "audio_speech",
+        "source": "https://cloud.google.com/text-to-speech/pricing",
+        "supported_endpoints": [
+            "/v1/audio/speech"
+        ]
+    },
     "vertex_ai/claude-3-5-haiku": {
         "input_cost_per_token": 1e-06,
         "litellm_provider": "vertex_ai-anthropic_models",
@@ -26842,6 +27178,95 @@
         "supports_vision": true,
         "supports_web_search": true
     },
+    "zai/glm-4.6": {
+        "input_cost_per_token": 6e-07,
+        "output_cost_per_token": 2.2e-06,
+        "litellm_provider": "zai",
+        "max_input_tokens": 200000,
+        "max_output_tokens": 128000,
+        "mode": "chat",
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "source": "https://docs.z.ai/guides/overview/pricing"
+    },
+    "zai/glm-4.5": {
+        "input_cost_per_token": 6e-07,
+        "output_cost_per_token": 2.2e-06,
+        "litellm_provider": "zai",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 32000,
+        "mode": "chat",
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "source": "https://docs.z.ai/guides/overview/pricing"
+    },
+    "zai/glm-4.5v": {
+        "input_cost_per_token": 6e-07,
+        "output_cost_per_token": 1.8e-06,
+        "litellm_provider": "zai",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 32000,
+        "mode": "chat",
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "source": "https://docs.z.ai/guides/overview/pricing"
+    },
+    "zai/glm-4.5-x": {
+        "input_cost_per_token": 2.2e-06,
+        "output_cost_per_token": 8.9e-06,
+        "litellm_provider": "zai",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 32000,
+        "mode": "chat",
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "source": "https://docs.z.ai/guides/overview/pricing"
+    },
+    "zai/glm-4.5-air": {
+        "input_cost_per_token": 2e-07,
+        "output_cost_per_token": 1.1e-06,
+        "litellm_provider": "zai",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 32000,
+        "mode": "chat",
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "source": "https://docs.z.ai/guides/overview/pricing"
+    },
+    "zai/glm-4.5-airx": {
+        "input_cost_per_token": 1.1e-06,
+        "output_cost_per_token": 4.5e-06,
+        "litellm_provider": "zai",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 32000,
+        "mode": "chat",
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "source": "https://docs.z.ai/guides/overview/pricing"
+    },
+    "zai/glm-4-32b-0414-128k": {
+        "input_cost_per_token": 1e-07,
+        "output_cost_per_token": 1e-07,
+        "litellm_provider": "zai",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 32000,
+        "mode": "chat",
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "source": "https://docs.z.ai/guides/overview/pricing"
+    },
+    "zai/glm-4.5-flash": {
+        "input_cost_per_token": 0,
+        "output_cost_per_token": 0,
+        "litellm_provider": "zai",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 32000,
+        "mode": "chat",
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "source": "https://docs.z.ai/guides/overview/pricing"
+    },
     "vertex_ai/search_api": {
         "input_cost_per_query": 1.5e-03,
         "litellm_provider": "vertex_ai",

litellm/proxy/_experimental/mcp_server/mcp_server_manager.py

@@ -17,16 +17,15 @@ from urllib.parse import urlparse
 from fastapi import HTTPException
 from httpx import HTTPStatusError
 from mcp import ReadResourceResult, Resource
+from mcp.types import CallToolRequestParams as MCPCallToolRequestParams
 from mcp.types import (
-    CallToolRequestParams as MCPCallToolRequestParams,
+    CallToolResult,
     GetPromptRequestParams,
     GetPromptResult,
     Prompt,
     ResourceTemplate,
 )
-from mcp.types import CallToolResult
 from mcp.types import Tool as MCPTool
-
 from pydantic import AnyUrl
 
 import litellm
@@ -1949,7 +1948,12 @@ class MCPServerManager:
             ) = split_server_prefix_from_name(tool_name)
             if original_tool_name in self.tool_name_to_mcp_server_name_mapping:
                 for server in self.get_registry().values():
-                    if normalize_server_name(server.name) == normalize_server_name(
+                    if server.server_name is None:
+                        if normalize_server_name(server.name) == normalize_server_name(
+                            server_name_from_prefix
+                        ):
+                            return server
+                    elif normalize_server_name(server.server_name) == normalize_server_name(
                         server_name_from_prefix
                     ):
                         return server