Merge branch 'main' into litellm_dev_09_01_2025_p1

.circleci/config.yml

@@ -1477,6 +1477,7 @@ jobs:
             docker run -d \
               -p 4000:4000 \
               -e DATABASE_URL=$PROXY_DATABASE_URL \
+              -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \
               -e DISABLE_SCHEMA_UPDATE="True" \
               -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/schema.prisma \
               -v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/litellm/proxy/schema.prisma \
@@ -1912,6 +1913,7 @@ jobs:
               -e APORIA_API_BASE_1=$APORIA_API_BASE_1 \
               -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
               -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
+              -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \
               -e USE_DDTRACE=True \
               -e DD_API_KEY=$DD_API_KEY \
               -e DD_SITE=$DD_SITE \
@@ -2962,6 +2964,7 @@ jobs:
           command: |
             docker run --name my-app \
               -p 4000:4000 \
+              -e DEFAULT_NUM_WORKERS_LITELLM_PROXY=1 \
               -e DATABASE_URL="postgresql://wrong:wrong@wrong:5432/wrong" \
               myapp:latest \
               --port 4000 > docker_output.log 2>&1 || true

.github/workflows/test-litellm.yml

@@ -31,6 +31,7 @@ jobs:
         poetry run pip install "pytest-retry==1.6.3"
         poetry run pip install pytest-xdist
         poetry run pip install "google-genai==1.22.0"
+        poetry run pip install "google-cloud-aiplatform>=1.38"
         poetry run pip install "fastapi-offline==1.7.3"
     - name: Setup litellm-enterprise as local package
       run: |

cookbook/veo_video_generation.py

@@ -0,0 +1,311 @@
+#!/usr/bin/env python3
+"""
+Complete example for Veo video generation through LiteLLM proxy.
+
+This script demonstrates how to:
+1. Generate videos using Google's Veo model
+2. Poll for completion status
+3. Download the generated video file
+
+Requirements:
+- LiteLLM proxy running with Google AI Studio pass-through configured
+- Google AI Studio API key with Veo access
+"""
+
+import json
+import os
+import time
+import requests
+from typing import Optional
+
+
+class VeoVideoGenerator:
+    """Complete Veo video generation client using LiteLLM proxy."""
+    
+    def __init__(self, base_url: str = "http://localhost:4000/gemini/v1beta", 
+                 api_key: str = "sk-1234"):
+        """
+        Initialize the Veo video generator.
+        
+        Args:
+            base_url: Base URL for the LiteLLM proxy with Gemini pass-through
+            api_key: API key for LiteLLM proxy authentication
+        """
+        self.base_url = base_url
+        self.api_key = api_key
+        self.headers = {
+            "x-goog-api-key": api_key,
+            "Content-Type": "application/json"
+        }
+    
+    def generate_video(self, prompt: str) -> Optional[str]:
+        """
+        Initiate video generation with Veo.
+        
+        Args:
+            prompt: Text description of the video to generate
+            
+        Returns:
+            Operation name if successful, None otherwise
+        """
+        print(f"🎬 Generating video with prompt: '{prompt}'")
+        
+        url = f"{self.base_url}/models/veo-3.0-generate-preview:predictLongRunning"
+        payload = {
+            "instances": [{
+                "prompt": prompt
+            }]
+        }
+        
+        try:
+            response = requests.post(url, headers=self.headers, json=payload)
+            response.raise_for_status()
+            
+            data = response.json()
+            operation_name = data.get("name")
+            
+            if operation_name:
+                print(f"✅ Video generation started: {operation_name}")
+                return operation_name
+            else:
+                print("❌ No operation name returned")
+                print(f"Response: {json.dumps(data, indent=2)}")
+                return None
+                
+        except requests.RequestException as e:
+            print(f"❌ Failed to start video generation: {e}")
+            if hasattr(e, 'response') and e.response is not None:
+                try:
+                    error_data = e.response.json()
+                    print(f"Error details: {json.dumps(error_data, indent=2)}")
+                except:
+                    print(f"Error response: {e.response.text}")
+            return None
+    
+    def wait_for_completion(self, operation_name: str, max_wait_time: int = 600) -> Optional[str]:
+        """
+        Poll operation status until video generation is complete.
+        
+        Args:
+            operation_name: Name of the operation to monitor
+            max_wait_time: Maximum time to wait in seconds (default: 10 minutes)
+            
+        Returns:
+            Video URI if successful, None otherwise
+        """
+        print("⏳ Waiting for video generation to complete...")
+        
+        operation_url = f"{self.base_url}/{operation_name}"
+        start_time = time.time()
+        poll_interval = 10  # Start with 10 seconds
+        
+        while time.time() - start_time < max_wait_time:
+            try:
+                print(f"🔍 Polling status... ({int(time.time() - start_time)}s elapsed)")
+                
+                response = requests.get(operation_url, headers=self.headers)
+                response.raise_for_status()
+                
+                data = response.json()
+                
+                # Check for errors
+                if "error" in data:
+                    print("❌ Error in video generation:")
+                    print(json.dumps(data["error"], indent=2))
+                    return None
+                
+                # Check if operation is complete
+                is_done = data.get("done", False)
+                
+                if is_done:
+                    print("🎉 Video generation complete!")
+                    
+                    try:
+                        # Extract video URI from nested response
+                        video_uri = data["response"]["generateVideoResponse"]["generatedSamples"][0]["video"]["uri"]
+                        print(f"📹 Video URI: {video_uri}")
+                        return video_uri
+                    except KeyError as e:
+                        print(f"❌ Could not extract video URI: {e}")
+                        print("Full response:")
+                        print(json.dumps(data, indent=2))
+                        return None
+                
+                # Wait before next poll, with exponential backoff
+                time.sleep(poll_interval)
+                poll_interval = min(poll_interval * 1.2, 30)  # Cap at 30 seconds
+                
+            except requests.RequestException as e:
+                print(f"❌ Error polling operation status: {e}")
+                time.sleep(poll_interval)
+        
+        print(f"⏰ Timeout after {max_wait_time} seconds")
+        return None
+    
+    def download_video(self, video_uri: str, output_filename: str = "generated_video.mp4") -> bool:
+        """
+        Download the generated video file.
+        
+        Args:
+            video_uri: URI of the video to download (from Google's response)
+            output_filename: Local filename to save the video
+            
+        Returns:
+            True if download successful, False otherwise
+        """
+        print(f"⬇️  Downloading video...")
+        print(f"Original URI: {video_uri}")
+        
+        # Convert Google URI to LiteLLM proxy URI
+        # Example: files/abc123 -> /gemini/v1beta/files/abc123:download?alt=media
+        if video_uri.startswith("files/"):
+            download_path = f"{video_uri}:download?alt=media"
+        else:
+            download_path = video_uri
+            
+        litellm_download_url = f"{self.base_url}/{download_path}"
+        print(f"Download URL: {litellm_download_url}")
+        
+        try:
+            # Download with streaming and redirect handling
+            response = requests.get(
+                litellm_download_url, 
+                headers=self.headers, 
+                stream=True,
+                allow_redirects=True  # Handle redirects automatically
+            )
+            response.raise_for_status()
+            
+            # Save video file
+            with open(output_filename, 'wb') as f:
+                downloaded_size = 0
+                for chunk in response.iter_content(chunk_size=8192):
+                    if chunk:
+                        f.write(chunk)
+                        downloaded_size += len(chunk)
+                        
+                        # Progress indicator for large files
+                        if downloaded_size % (1024 * 1024) == 0:  # Every MB
+                            print(f"📦 Downloaded {downloaded_size / (1024*1024):.1f} MB...")
+            
+            # Verify file was created and has content
+            if os.path.exists(output_filename):
+                file_size = os.path.getsize(output_filename)
+                if file_size > 0:
+                    print(f"✅ Video downloaded successfully!")
+                    print(f"📁 Saved as: {output_filename}")
+                    print(f"📏 File size: {file_size / (1024*1024):.2f} MB")
+                    return True
+                else:
+                    print("❌ Downloaded file is empty")
+                    os.remove(output_filename)
+                    return False
+            else:
+                print("❌ File was not created")
+                return False
+                
+        except requests.RequestException as e:
+            print(f"❌ Download failed: {e}")
+            if hasattr(e, 'response') and e.response is not None:
+                print(f"Status code: {e.response.status_code}")
+                print(f"Response headers: {dict(e.response.headers)}")
+            return False
+    
+    def generate_and_download(self, prompt: str, output_filename: str = None) -> bool:
+        """
+        Complete workflow: generate video and download it.
+        
+        Args:
+            prompt: Text description for video generation
+            output_filename: Output filename (auto-generated if None)
+            
+        Returns:
+            True if successful, False otherwise
+        """
+        # Auto-generate filename if not provided
+        if output_filename is None:
+            timestamp = int(time.time())
+            safe_prompt = "".join(c for c in prompt[:30] if c.isalnum() or c in (' ', '-', '_')).rstrip()
+            output_filename = f"veo_video_{safe_prompt.replace(' ', '_')}_{timestamp}.mp4"
+        
+        print("=" * 60)
+        print("🎬 VEO VIDEO GENERATION WORKFLOW")
+        print("=" * 60)
+        
+        # Step 1: Generate video
+        operation_name = self.generate_video(prompt)
+        if not operation_name:
+            return False
+        
+        # Step 2: Wait for completion
+        video_uri = self.wait_for_completion(operation_name)
+        if not video_uri:
+            return False
+        
+        # Step 3: Download video
+        success = self.download_video(video_uri, output_filename)
+        
+        if success:
+            print("=" * 60)
+            print("🎉 SUCCESS! Video generation complete!")
+            print(f"📁 Video saved as: {output_filename}")
+            print("=" * 60)
+        else:
+            print("=" * 60)
+            print("❌ FAILED! Video generation or download failed")
+            print("=" * 60)
+        
+        return success
+
+
+def main():
+    """
+    Example usage of the VeoVideoGenerator.
+    
+    Configure these environment variables:
+    - LITELLM_BASE_URL: Your LiteLLM proxy URL (default: http://localhost:4000/gemini/v1beta)
+    - LITELLM_API_KEY: Your LiteLLM API key (default: sk-1234)
+    """
+    
+    # Configuration from environment or defaults
+    base_url = os.getenv("LITELLM_BASE_URL", "http://localhost:4000/gemini/v1beta")
+    api_key = os.getenv("LITELLM_API_KEY", "sk-1234")
+    
+    print("🚀 Starting Veo Video Generation Example")
+    print(f"📡 Using LiteLLM proxy at: {base_url}")
+    
+    # Initialize generator
+    generator = VeoVideoGenerator(base_url=base_url, api_key=api_key)
+    
+    # Example prompts - try different ones!
+    example_prompts = [
+        "A cat playing with a ball of yarn in a sunny garden",
+        "Ocean waves crashing against rocky cliffs at sunset",
+        "A bustling city street with people walking and cars passing by",
+        "A peaceful forest with sunlight filtering through the trees"
+    ]
+    
+    # Use first example or get from user
+    prompt = example_prompts[0]
+    print(f"🎬 Using prompt: '{prompt}'")
+    
+    # Generate and download video
+    success = generator.generate_and_download(prompt)
+    
+    if success:
+        print("\n✅ Example completed successfully!")
+        print("💡 Try modifying the prompt in the script for different videos!")
+    else:
+        print("\n❌ Example failed!")
+        print("🔧 Check your LiteLLM proxy configuration and Google AI Studio API key")
+        
+        # Troubleshooting tips
+        print("\n🔍 Troubleshooting:")
+        print("1. Ensure LiteLLM proxy is running with Google AI Studio pass-through")
+        print("2. Verify your Google AI Studio API key has Veo access")
+        print("3. Check that your prompt meets Veo's content guidelines")
+        print("4. Review the LiteLLM proxy logs for detailed error information")
+
+
+if __name__ == "__main__":
+    main()

docs/my-website/docs/completion/input.md

@@ -106,6 +106,7 @@ def completion(
     parallel_tool_calls: Optional[bool] = None,
     logprobs: Optional[bool] = None,
     top_logprobs: Optional[int] = None,
+    safety_identifier: Optional[str] = None,
     deployment_id=None,
     # soon to be deprecated params by OpenAI
     functions: Optional[List] = None,
@@ -196,6 +197,8 @@ def completion(
         
 - `top_logprobs`: *int (optional)* - An integer between 0 and 5 specifying the number of most likely tokens to return at each token position, each with an associated log probability. `logprobs` must be set to true if this parameter is used.
 
+- `safety_identifier`: *string (optional)* - A unique identifier for tracking and managing safety-related requests. This parameter helps with safety monitoring and compliance tracking.
+
 - `headers`: *dict (optional)* - A dictionary of headers to be sent with the request.
 
 - `extra_headers`: *dict (optional)* - Alternative to `headers`, used to send extra headers in LLM API request. 

docs/my-website/docs/completion/web_search.md

@@ -8,10 +8,25 @@ Use web search with litellm
 | Feature | Details |
 |---------|---------|
 | Supported Endpoints | - `/chat/completions` <br/> - `/responses` |
-| Supported Providers | `openai`, `xai`, `vertex_ai`, `gemini`, `perplexity` |
+| Supported Providers | `openai`, `xai`, `vertex_ai`, `anthropic`, `gemini`, `perplexity` |
 | LiteLLM Cost Tracking | ✅ Supported |
 | LiteLLM Version | `v1.71.0+` |
 
+## Which Search Engine is Used?
+
+Each provider uses their own search backend:
+
+| Provider | Search Engine | Notes |
+|----------|---------------|-------|
+| **OpenAI** (`gpt-4o-search-preview`) | OpenAI's internal search | Real-time web data |
+| **xAI** (`grok-3`) | xAI's search + X/Twitter | Real-time social media data |
+| **Google AI/Vertex** (`gemini-2.0-flash`) | **Google Search** | Uses actual Google search results |
+| **Anthropic** (`claude-3-5-sonnet`) | Anthropic's web search | Real-time web data |
+| **Perplexity** | Perplexity's search engine | AI-powered search and reasoning |
+
+:::info
+**Anthropic Web Search Models**: Claude models that support web search: `claude-3-5-sonnet-latest`, `claude-3-5-sonnet-20241022`, `claude-3-5-haiku-latest`, `claude-3-5-haiku-20241022`, `claude-3-7-sonnet-20250219`
+:::
 
 ## `/chat/completions` (litellm.completion)
 
@@ -56,6 +71,12 @@ model_list:
       model: xai/grok-3
       api_key: os.environ/XAI_API_KEY
   
+  # Anthropic
+  - model_name: claude-3-5-sonnet-latest
+    litellm_params:
+      model: anthropic/claude-3-5-sonnet-latest
+      api_key: os.environ/ANTHROPIC_API_KEY
+  
   # VertexAI
   - model_name: gemini-2-flash
     litellm_params:
@@ -143,6 +164,31 @@ response = completion(
 )
 ```
 
+**Anthropic (using web_search_options)**
+```python showLineNumbers
+from litellm import completion
+
+# Customize search context size for Anthropic
+response = completion(
+    model="anthropic/claude-3-5-sonnet-latest",
+    messages=[
+        {
+            "role": "user",
+            "content": "What was a positive news story from today?",
+        }
+    ],
+    web_search_options={
+        "search_context_size": "medium",  # Options: "low", "medium" (default), "high"
+        "user_location": {
+            "type": "approximate",
+            "approximate": {
+                "city": "San Francisco",
+            },
+        }
+    }
+)
+```
+
 **VertexAI/Gemini (using web_search_options)**
 ```python showLineNumbers
 from litellm import completion
@@ -375,6 +421,9 @@ assert litellm.supports_web_search(model="openai/gpt-4o-search-preview") == True
 # Check xAI models
 assert litellm.supports_web_search(model="xai/grok-3") == True
 
+# Check Anthropic models
+assert litellm.supports_web_search(model="anthropic/claude-3-5-sonnet-latest") == True
+
 # Check VertexAI models
 assert litellm.supports_web_search(model="gemini-2.0-flash") == True
 
@@ -405,6 +454,14 @@ model_list:
     model_info:
       supports_web_search: True
   
+  # Anthropic
+  - model_name: claude-3-5-sonnet-latest
+    litellm_params:
+      model: anthropic/claude-3-5-sonnet-latest
+      api_key: os.environ/ANTHROPIC_API_KEY
+    model_info:
+      supports_web_search: True
+  
   # VertexAI
   - model_name: gemini-2-flash
     litellm_params:

docs/my-website/docs/contributing.md

@@ -14,6 +14,11 @@ git clone https://github.com/BerriAI/litellm.git
 Tell the proxy where the UI is located
 ```bash
 export PROXY_BASE_URL="http://localhost:3000/"
+
+### ALSO ###  - set the basic env variables
+DATABASE_URL = "postgresql://<user>:<password>@<host>:<port>/<dbname>"
+LITELLM_MASTER_KEY = "sk-1234"
+STORE_MODEL_IN_DB = "True"
 ```
 
 ```bash

docs/my-website/docs/image_generation.md

@@ -124,8 +124,6 @@ Any non-openai params, will be treated as provider-specific params, and sent in
 
 - `size`: *string (optional)* The size of the generated images. Must be one of `1024x1024`, `1536x1024` (landscape), `1024x1536` (portrait), or `auto` (default value) for `gpt-image-1`, one of `256x256`, `512x512`, or `1024x1024` for `dall-e-2`, and one of `1024x1024`, `1792x1024`, or `1024x1792` for `dall-e-3`.
 
-- `input_fidelity`: *string (optional)* Controls how closely the model follows the input prompt. Supported for `gpt-image-1` model. Higher fidelity may improve prompt adherence but could affect generation speed.
-
 - `timeout`: *integer* - The maximum time, in seconds, to wait for the API to respond. Defaults to 600 seconds (10 minutes).
 
 - `user`: *string (optional)* A unique identifier representing your end-user, 

docs/my-website/docs/observability/callbacks.md

@@ -4,9 +4,14 @@
 
 liteLLM provides `input_callbacks`, `success_callbacks` and `failure_callbacks`, making it easy for you to send data to a particular provider depending on the status of your responses.
 
+:::tip
+**New to LiteLLM Callbacks?** Check out our comprehensive [Callback Management Guide](./callback_management.md) to understand when to use different callback hooks like `async_log_success_event` vs `async_post_call_success_hook`.
+:::
+
 liteLLM supports:
 
 - [Custom Callback Functions](https://docs.litellm.ai/docs/observability/custom_callback)
+- [Callback Management Guide](./callback_management.md) - **Comprehensive guide for choosing the right hooks**
 - [Lunary](https://lunary.ai/docs)
 - [Langfuse](https://langfuse.com/docs)
 - [LangSmith](https://www.langchain.com/langsmith)

docs/my-website/docs/observability/cloudzero.md

@@ -0,0 +1,209 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# CloudZero Integration
+
+LiteLLM provides an integration with CloudZero's AnyCost API, allowing you to export your LLM usage data to CloudZero for cost tracking analysis.
+
+## Overview
+
+| Property | Details |
+|----------|---------|
+| Description | Export LiteLLM usage data to CloudZero AnyCost API for cost tracking and analysis |
+| callback name | `cloudzero`|
+| Supported Operations | • Automatic hourly data export<br/>• Manual data export<br/>• Dry run testing<br/>• Cost and token usage tracking |
+| Data Format | CloudZero Billing Format (CBF) with proper resource tagging |
+| Export Frequency | Hourly (configurable via `CLOUDZERO_EXPORT_INTERVAL_MINUTES`) |
+
+## Environment Variables
+
+| Variable | Required | Description | Example |
+|----------|----------|-------------|---------|
+| `CLOUDZERO_API_KEY` | Yes | Your CloudZero API key | `cz_api_xxxxxxxxxx` |
+| `CLOUDZERO_CONNECTION_ID` | Yes | CloudZero connection ID for data submission | `conn_xxxxxxxxxx` |
+| `CLOUDZERO_TIMEZONE` | No | Timezone for date handling (default: UTC) | `America/New_York` |
+| `CLOUDZERO_EXPORT_INTERVAL_MINUTES` | No | Export frequency in minutes (default: 60) | `60` |
+
+## Setup
+
+### End to End Video Walkthrough
+This video walks through the entire process of setting up LiteLLM with CloudZero integration and viewing LiteLLM exported usage data in CloudZero.
+
+<iframe width="840" height="500" src="https://www.loom.com/embed/59b57593183f4cc3b1c05a2dd3277f92" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>
+
+### Step 1: Configure Environment Variables
+
+Set your CloudZero credentials in your environment:
+
+```bash
+export CLOUDZERO_API_KEY="cz_api_xxxxxxxxxx"
+export CLOUDZERO_CONNECTION_ID="conn_xxxxxxxxxx"
+export CLOUDZERO_TIMEZONE="UTC"  # Optional, defaults to UTC
+```
+
+### Step 2: Enable CloudZero Integration
+
+Add the CloudZero callback to your LiteLLM configuration YAML file:
+
+
+```yaml
+model_list:
+  - model_name: gpt-4o
+    litellm_params:
+      model: openai/gpt-4o
+      api_key: sk-xxxxxxx
+
+litellm_settings:
+  callbacks: ["cloudzero"]  # Enable CloudZero integration
+```
+
+### Step 3: Start LiteLLM Proxy
+
+Start your LiteLLM proxy with the configuration:
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+## Testing Your Setup
+
+### Dry Run Export
+
+Call the dry run endpoint to test your CloudZero configuration without sending data to CloudZero. This endpoint will not send any data to CloudZero, but will return the data that would be exported.
+
+```bash
+curl -X POST "http://localhost:4000/cloudzero/dry-run" \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer sk-1234" \
+  -d '{
+    "limit": 10
+  }' | jq
+```
+
+**Expected Response:**
+```json
+{
+  "message": "CloudZero dry run export completed successfully.",
+  "status": "success",
+  "dry_run_data": {
+    "usage_data": [...],
+    "cbf_data": [...],
+    "summary": {
+      "total_cost": 0.05,
+      "total_tokens": 1250,
+      "total_records": 10
+    }
+  }
+}
+```
+
+### Manual Export
+
+Call the export endpoint to send data immediately to CloudZero. We suggest setting a small `limit` to test the export. This will only export the last 10 records to CloudZero. Note: Cloudzero can take up to 15 minutes to process the exported data.
+
+```bash
+curl -X POST "http://localhost:4000/cloudzero/export" \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer sk-1234" \
+  -d '{
+    "limit": 10
+  }' | jq
+```
+
+**Expected Response:**
+```json
+{
+  "message": "CloudZero export completed successfully",
+  "status": "success"
+}
+```
+
+## Data Export Details
+
+### Automatic Export Schedule
+
+- **Frequency**: Every 60 minutes (configurable via `CLOUDZERO_EXPORT_INTERVAL_MINUTES`)
+- **Data Processing**: LiteLLM automatically processes and exports usage data hourly
+- **CloudZero Processing**: CloudZero typically takes 10-15 minutes to process data from LiteLLM
+
+### Data Format
+
+LiteLLM exports data in CloudZero Billing Format (CBF) with the following structure:
+
+```json
+{
+  "time/usage_start": "2024-01-15T14:00:00Z",
+  "cost/cost": 0.002,
+  "usage/amount": 150,
+  "usage/units": "tokens",
+  "resource/id": "czrn:litellm:openai:cross-region:team-123:llm-usage:gpt-4o",
+  "resource/service": "litellm",
+  "resource/account": "team-123",
+  "resource/region": "cross-region",
+  "resource/usage_family": "llm-usage",
+  "resource/tag:provider": "openai",
+  "resource/tag:model": "gpt-4o",
+  "resource/tag:prompt_tokens": "100",
+  "resource/tag:completion_tokens": "50"
+}
+```
+
+### Resource Tagging
+
+LiteLLM automatically creates comprehensive resource tags for cost attribution:
+
+- **Provider Tags**: `openai`, `anthropic`, `azure`, etc.
+- **Model Tags**: Specific model names like `gpt-4o`, `claude-3-sonnet`
+- **Team/User Tags**: Team IDs and user IDs for cost allocation
+- **Token Breakdown**: Separate tracking of prompt and completion tokens
+- **Usage Metrics**: Total tokens consumed per request
+
+## Advanced Configuration
+
+### Custom Export Frequency
+
+Change the export frequency (not recommended to go below 60 minutes):
+
+```bash
+export CLOUDZERO_EXPORT_INTERVAL_MINUTES=120  # Export every 2 hours
+```
+
+### Custom Time Range Export
+
+Export data for a specific time range:
+
+```bash
+curl -X POST "http://localhost:4000/cloudzero/export" \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer sk-1234" \
+  -d '{
+    "start_time_utc": "2024-01-15T00:00:00Z",
+    "end_time_utc": "2024-01-15T23:59:59Z",
+    "operation": "replace_hourly"
+  }' | jq
+```
+
+## Troubleshooting
+
+### Common Issues
+
+1. **Missing Credentials Error**
+   ```
+   CloudZero configuration missing. Please set CLOUDZERO_API_KEY and CLOUDZERO_CONNECTION_ID environment variables.
+   ```
+   **Solution**: Ensure both environment variables are set with valid values.
+
+2. **Connection Issues**
+   - Verify your CloudZero API key is valid
+   - Check that the connection ID exists in your CloudZero account
+   - Ensure your proxy has internet access to reach CloudZero's API
+
+3. **No Data in CloudZero**
+   - CloudZero can take 10-15 minutes to process data
+   - Check that your LiteLLM proxy is generating usage data
+   - Use the dry-run endpoint to verify data is being formatted correctly
+
+## Related Links
+
+- [CloudZero Documentation](https://docs.cloudzero.com/)
+- [CloudZero AnyCost API](https://docs.cloudzero.com/reference/anycost-api)

docs/my-website/docs/observability/custom_callback.md

@@ -4,7 +4,6 @@
 **For PROXY** [Go Here](../proxy/logging.md#custom-callback-class-async)
 ::: 
 
-
 ## Callback Class
 You can create a custom callback class to precisely log events as they occur in litellm. 
 
@@ -57,6 +56,17 @@ def async completion():
 asyncio.run(completion())
 ```
 
+## Common Hooks
+
+- `async_log_success_event` - Log successful API calls
+- `async_log_failure_event` - Log failed API calls  
+- `log_pre_api_call` - Log before API call
+- `log_post_api_call` - Log after API call
+
+**Proxy-only hooks** (only work with LiteLLM Proxy):
+- `async_post_call_success_hook` - Access user data + modify responses
+- `async_pre_call_hook` - Modify requests before sending
+
 ## Callback Functions
 If you just want to log on a specific event (e.g. on input) - you can use callback functions. 
 
@@ -174,260 +184,87 @@ async def test_chat_openai():
 asyncio.run(test_chat_openai())
 ```
 
-:::info
+## What's Available in kwargs?
 
-We're actively trying to expand this to other event types. [Tell us if you need this!](https://github.com/BerriAI/litellm/issues/1007)
-:::
-
-## What's in kwargs? 
-
-Notice we pass in a kwargs argument to custom callback. 
-```python
-def custom_callback(
-    kwargs,                 # kwargs to completion
-    completion_response,    # response from completion
-    start_time, end_time    # start/end time
-):
-    # Your custom code here
-    print("LITELLM: in custom callback function")
-    print("kwargs", kwargs)
-    print("completion_response", completion_response)
-    print("start_time", start_time)
-    print("end_time", end_time)
-```
-
-This is a dictionary containing all the model-call details (the params we receive, the values we send to the http endpoint, the response we receive, stacktrace in case of errors, etc.). 
-
-This is all logged in the [model_call_details via our Logger](https://github.com/BerriAI/litellm/blob/fc757dc1b47d2eb9d0ea47d6ad224955b705059d/litellm/utils.py#L246).
-
-Here's exactly what you can expect in the kwargs dictionary:
-```shell
-### DEFAULT PARAMS ### 
-"model": self.model,
-"messages": self.messages,
-"optional_params": self.optional_params, # model-specific params passed in
-"litellm_params": self.litellm_params, # litellm-specific params passed in (e.g. metadata passed to completion call)
-"start_time": self.start_time, # datetime object of when call was started
-
-### PRE-API CALL PARAMS ### (check via kwargs["log_event_type"]="pre_api_call")
-"input" = input # the exact prompt sent to the LLM API
-"api_key" = api_key # the api key used for that LLM API 
-"additional_args" = additional_args # any additional details for that API call (e.g. contains optional params sent)
-
-### POST-API CALL PARAMS ### (check via kwargs["log_event_type"]="post_api_call")
-"original_response" = original_response # the original http response received (saved via response.text)
-
-### ON-SUCCESS PARAMS ### (check via kwargs["log_event_type"]="successful_api_call")
-"complete_streaming_response" = complete_streaming_response # the complete streamed response (only set if `completion(..stream=True)`)
-"end_time" = end_time # datetime object of when call was completed
-
-### ON-FAILURE PARAMS ### (check via kwargs["log_event_type"]="failed_api_call")
-"exception" = exception # the Exception raised
-"traceback_exception" = traceback_exception # the traceback generated via `traceback.format_exc()`
-"end_time" = end_time # datetime object of when call was completed
-```
-
-
-### Cache hits
-
-Cache hits are logged in success events as `kwarg["cache_hit"]`. 
-
-Here's an example of accessing it: 
-
-  ```python
-  import litellm
-from litellm.integrations.custom_logger import CustomLogger
-from litellm import completion, acompletion, Cache
-
-class MyCustomHandler(CustomLogger):
-    async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): 
-        print(f"On Success")
-        print(f"Value of Cache hit: {kwargs['cache_hit']"})
-
-async def test_async_completion_azure_caching():
-    customHandler_caching = MyCustomHandler()
-    litellm.cache = Cache(type="redis", host=os.environ['REDIS_HOST'], port=os.environ['REDIS_PORT'], password=os.environ['REDIS_PASSWORD'])
-    litellm.callbacks = [customHandler_caching]
-    unique_time = time.time()
-    response1 = await litellm.acompletion(model="azure/chatgpt-v-2",
-                            messages=[{
-                                "role": "user",
-                                "content": f"Hi 👋 - i'm async azure {unique_time}"
-                            }],
-                            caching=True)
-    await asyncio.sleep(1)
-    print(f"customHandler_caching.states pre-cache hit: {customHandler_caching.states}")
-    response2 = await litellm.acompletion(model="azure/chatgpt-v-2",
-                            messages=[{
-                                "role": "user",
-                                "content": f"Hi 👋 - i'm async azure {unique_time}"
-                            }],
-                            caching=True)
-    await asyncio.sleep(1) # success callbacks are done in parallel
-    print(f"customHandler_caching.states post-cache hit: {customHandler_caching.states}")
-    assert len(customHandler_caching.errors) == 0
-    assert len(customHandler_caching.states) == 4 # pre, post, success, success
-  ```
-
-### Get complete streaming response
-
-LiteLLM will pass you the complete streaming response in the final streaming chunk as part of the kwargs for your custom callback function.
+The kwargs dictionary contains all the details about your API call:
 
 ```python
-# litellm.set_verbose = False
-        def custom_callback(
-            kwargs,                 # kwargs to completion
-            completion_response,    # response from completion
-            start_time, end_time    # start/end time
-        ):
-            # print(f"streaming response: {completion_response}")
-            if "complete_streaming_response" in kwargs: 
-                print(f"Complete Streaming Response: {kwargs['complete_streaming_response']}")
-        
-        # Assign the custom callback function
-        litellm.success_callback = [custom_callback]
-
-        response = completion(model="claude-instant-1", messages=messages, stream=True)
-        for idx, chunk in enumerate(response): 
-            pass
-```
-
-
-### Log additional metadata
-
-LiteLLM accepts a metadata dictionary in the completion call. You can pass additional metadata into your completion call via `completion(..., metadata={"key": "value"})`. 
-
-Since this is a [litellm-specific param](https://github.com/BerriAI/litellm/blob/b6a015404eed8a0fa701e98f4581604629300ee3/litellm/main.py#L235), it's accessible via kwargs["litellm_params"]
-
-```python
-from litellm import completion
-import os, litellm
-
-## set ENV variables
-os.environ["OPENAI_API_KEY"] = "your-api-key"
-
-messages = [{ "content": "Hello, how are you?","role": "user"}]
-
-def custom_callback(
-    kwargs,                 # kwargs to completion
-    completion_response,    # response from completion
-    start_time, end_time    # start/end time
-):
-    print(kwargs["litellm_params"]["metadata"])
+def custom_callback(kwargs, completion_response, start_time, end_time):
+    # Access common data
+    model = kwargs.get("model")
+    messages = kwargs.get("messages", [])
+    cost = kwargs.get("response_cost", 0)
+    cache_hit = kwargs.get("cache_hit", False)
     
-
-# Assign the custom callback function
-litellm.success_callback = [custom_callback]
-
-response = litellm.completion(model="gpt-3.5-turbo", messages=messages, metadata={"hello": "world"})
+    # Access metadata you passed in
+    metadata = kwargs.get("litellm_params", {}).get("metadata", {})
 ```
 
-## Examples
+**Key fields in kwargs:**
+- `model` - The model name
+- `messages` - Input messages  
+- `response_cost` - Calculated cost
+- `cache_hit` - Whether response was cached
+- `litellm_params.metadata` - Your custom metadata
 
-### Custom Callback to track costs for Streaming + Non-Streaming
-By default, the response cost is accessible in the logging object via `kwargs["response_cost"]` on success (sync + async)
+## Practical Examples
+
+### Track API Costs
 ```python
+def track_cost_callback(kwargs, completion_response, start_time, end_time):
+    cost = kwargs["response_cost"] # litellm calculates this for you
+    print(f"Request cost: ${cost}")
 
-# Step 1. Write your custom callback function
-def track_cost_callback(
-    kwargs,                 # kwargs to completion
-    completion_response,    # response from completion
-    start_time, end_time    # start/end time
-):
-    try:
-        response_cost = kwargs["response_cost"] # litellm calculates response cost for you
-        print("regular response_cost", response_cost)
-    except:
-        pass
-
-# Step 2. Assign the custom callback function
 litellm.success_callback = [track_cost_callback]
 
-# Step 3. Make litellm.completion call
-response = completion(
-    model="gpt-3.5-turbo",
-    messages=[
-        {
-            "role": "user",
-            "content": "Hi 👋 - i'm openai"
-        }
-    ]
-)
-
-print(response)
+response = completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hello"}])
 ```
 
-### Custom Callback to log transformed Input to LLMs
+### Log Inputs to LLMs
 ```python
-def get_transformed_inputs(
-    kwargs,
-):
+def get_transformed_inputs(kwargs):
     params_to_model = kwargs["additional_args"]["complete_input_dict"]
     print("params to model", params_to_model)
 
 litellm.input_callback = [get_transformed_inputs]
 
-def test_chat_openai():
-    try:
-        response = completion(model="claude-2",
-                              messages=[{
-                                  "role": "user",
-                                  "content": "Hi 👋 - i'm openai"
-                              }])
-
-        print(response)
-
-    except Exception as e:
-        print(e)
-        pass
+response = completion(model="claude-2", messages=[{"role": "user", "content": "Hello"}])
 ```
 
-#### Output
-```shell
-params to model {'model': 'claude-2', 'prompt': "\n\nHuman: Hi 👋 - i'm openai\n\nAssistant: ", 'max_tokens_to_sample': 256}
+### Send to External Service
+```python
+import requests
+
+def send_to_analytics(kwargs, completion_response, start_time, end_time):
+    data = {
+        "model": kwargs.get("model"),
+        "cost": kwargs.get("response_cost", 0),
+        "duration": (end_time - start_time).total_seconds()
+    }
+    requests.post("https://your-analytics.com/api", json=data)
+
+litellm.success_callback = [send_to_analytics]
 ```
 
-### Custom Callback to write to Mixpanel
+## Common Issues
+
+### Callback Not Called
+Make sure you:
+1. Register callbacks correctly: `litellm.callbacks = [MyHandler()]`
+2. Use the right hook names (check spelling)
+3. Don't use proxy-only hooks in library mode
+
+### Performance Issues  
+- Use async hooks for I/O operations
+- Don't block in callback functions
+- Handle exceptions properly:
 
 ```python
-import mixpanel
-import litellm
-from litellm import completion
-
-def custom_callback(
-    kwargs,                 # kwargs to completion
-    completion_response,    # response from completion
-    start_time, end_time    # start/end time
-):
-    # Your custom code here
-    mixpanel.track("LLM Response", {"llm_response": completion_response})
-
-
-# Assign the custom callback function
-litellm.success_callback = [custom_callback]
-
-response = completion(
-    model="gpt-3.5-turbo",
-    messages=[
-        {
-            "role": "user",
-            "content": "Hi 👋 - i'm openai"
-        }
-    ]
-)
-
-print(response)
-
+class SafeHandler(CustomLogger):
+    async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
+        try:
+            await external_service(response_obj)
+        except Exception as e:
+            print(f"Callback error: {e}")  # Log but don't break the flow
 ```
 
-
-
-
-
-
-
-
-
-
-
-

docs/my-website/docs/pass_through/google_ai_studio.md

@@ -230,6 +230,13 @@ curl -X POST "https://generativelanguage.googleapis.com/v1beta/models/gemini-1.5
 ```
 
 
+## **Example 4: Video Generation with Veo**
+
+Generate videos using Google's Veo model through LiteLLM pass-through routes.
+
+[**→ Complete Veo Video Generation Guide**](../proxy/veo_video_generation.md)
+
+
 ## Advanced 
 
 Pre-requisites

docs/my-website/docs/pass_through/intro.md

@@ -11,3 +11,43 @@ These endpoints are useful for 2 scenarios:
 ## How is your request handled? 
 
 The request is passed through to the provider's endpoint. The response is then passed back to the client. **No translation is done.**
+
+### Request Forwarding Process
+
+1. **Request Reception**: LiteLLM receives your request at `/provider/endpoint`
+2. **Authentication**: Your LiteLLM API key is validated and mapped to the provider's API key
+3. **Request Transformation**: Request is reformatted for the target provider's API
+4. **Forwarding**: Request is sent to the actual provider endpoint
+5. **Response Handling**: Provider response is returned directly to you
+
+### Authentication Flow
+
+```mermaid
+graph LR
+    A[Client Request] --> B[LiteLLM Proxy]
+    B --> C[Validate LiteLLM API Key]
+    C --> D[Map to Provider API Key]
+    D --> E[Forward to Provider]
+    E --> F[Return Response]
+```
+
+**Key Points:**
+- Use your **LiteLLM API key** in requests, not the provider's key
+- LiteLLM handles the provider authentication internally
+- Same authentication works across all passthrough endpoints
+
+### Error Handling
+
+**Provider Errors**: Forwarded directly to you with original error codes and messages
+
+**LiteLLM Errors**: 
+- `401`: Invalid LiteLLM API key
+- `404`: Provider or endpoint not supported
+- `500`: Internal routing/forwarding errors
+
+### Benefits
+
+- **Unified Authentication**: One API key for all providers
+- **Centralized Logging**: All requests logged through LiteLLM
+- **Cost Tracking**: Usage tracked across all endpoints
+- **Access Control**: Same permissions apply to passthrough endpoints

docs/my-website/docs/providers/bedrock.md

@@ -467,7 +467,7 @@ print(f"\nResponse: {resp}")
 
 ## Usage - 'thinking' / 'reasoning content'
 
-This is currently only supported for Anthropic's Claude 3.7 Sonnet + Deepseek R1.
+This is currently only supported for Anthropic's Claude 3.7 Sonnet + Deepseek R1 + GPT-OSS models.
 
 Works on v1.61.20+.
 

docs/my-website/docs/providers/databricks.md

@@ -282,6 +282,11 @@ ModelResponse(
 )
 ```
 
+### Citations
+
+Anthropic models served through Databricks can return citation metadata. LiteLLM
+exposes these via `response.choices[0].message.provider_specific_fields["citations"]`.
+
 ### Pass `thinking` to Anthropic models
 
 You can also pass the `thinking` parameter to Anthropic models.

docs/my-website/docs/providers/vertex_partner.md

@@ -15,6 +15,7 @@ import TabItem from '@theme/TabItem';
 | Mistral | `vertex_ai/mistral-*` | [Vertex AI - Mistral Models](https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/mistral) |
 | AI21 (Jamba) | `vertex_ai/jamba-*` | [Vertex AI - AI21 Models](https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/ai21) |
 | Qwen | `vertex_ai/qwen/*` | [Vertex AI - Qwen Models](https://cloud.google.com/vertex-ai/generative-ai/docs/maas/qwen) |
+| OpenAI (GPT-OSS) | `vertex_ai/openai/gpt-oss-*` | [Vertex AI - GPT-OSS Models](https://console.cloud.google.com/vertex-ai/publishers/openai/model-garden/) |
 | Model Garden | `vertex_ai/openai/{MODEL_ID}` or `vertex_ai/{MODEL_ID}` | [Vertex Model Garden](https://cloud.google.com/model-garden?hl=en) |
 
 ## Vertex AI - Anthropic (Claude)
@@ -658,6 +659,141 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
 </Tabs>
 
 
+## VertexAI GPT-OSS Models
+
+| Property | Details |
+|----------|---------|
+| Provider Route | `vertex_ai/openai/{MODEL}` |
+| Vertex Documentation | [Vertex AI - GPT-OSS Models](https://console.cloud.google.com/vertex-ai/publishers/openai/model-garden/) |
+
+**LiteLLM Supports all Vertex AI GPT-OSS Models.** Ensure you use the `vertex_ai/openai/` prefix for all Vertex AI GPT-OSS models.
+
+| Model Name       | Usage                        |
+|------------------|------------------------------|
+| vertex_ai/openai/gpt-oss-20b-maas | `completion('vertex_ai/openai/gpt-oss-20b-maas', messages)` |
+
+#### Usage
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+from litellm import completion
+import os
+
+os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = ""
+
+model = "openai/gpt-oss-20b-maas"
+
+vertex_ai_project = "your-vertex-project" # can also set this as os.environ["VERTEXAI_PROJECT"]
+vertex_ai_location = "your-vertex-location" # can also set this as os.environ["VERTEXAI_LOCATION"]
+
+response = completion(
+    model="vertex_ai/" + model,
+    messages=[{"role": "user", "content": "hi"}],
+    vertex_ai_project=vertex_ai_project,
+    vertex_ai_location=vertex_ai_location,
+)
+print("\nModel Response", response)
+```
+</TabItem>
+<TabItem value="proxy" label="Proxy">
+
+**1. Add to config**
+
+```yaml
+model_list:
+    - model_name: gpt-oss
+      litellm_params:
+        model: vertex_ai/openai/gpt-oss-20b-maas
+        vertex_ai_project: "my-test-project"
+        vertex_ai_location: "us-central1"
+```
+
+**2. Start proxy**
+
+```bash
+litellm --config /path/to/config.yaml
+
+# RUNNING at http://0.0.0.0:4000
+```
+
+**3. Test it!**
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+      --header 'Authorization: Bearer sk-1234' \
+      --header 'Content-Type: application/json' \
+      --data '{
+            "model": "gpt-oss", # 👈 the 'model_name' in config
+            "messages": [
+                {
+                "role": "user",
+                "content": "what llm are you"
+                }
+            ],
+        }'
+```
+
+</TabItem>
+</Tabs>
+
+#### Usage - `reasoning_effort`
+
+GPT-OSS models support the `reasoning_effort` parameter for enhanced reasoning capabilities.
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+from litellm import completion
+
+response = completion(
+    model="vertex_ai/openai/gpt-oss-20b-maas",
+    messages=[{"role": "user", "content": "Solve this complex problem step by step"}],
+    reasoning_effort="low",  # Options: "minimal", "low", "medium", "high"
+    vertex_ai_project="your-vertex-project",
+    vertex_ai_location="us-central1",
+)
+```
+
+</TabItem>
+
+<TabItem value="proxy" label="PROXY">
+
+1. Setup config.yaml
+
+```yaml
+model_list:
+- model_name: gpt-oss
+  litellm_params:
+    model: vertex_ai/openai/gpt-oss-20b-maas
+    vertex_ai_project: "my-test-project"
+    vertex_ai_location: "us-central1"
+```
+
+2. Start proxy
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+3. Test it! 
+
+```bash
+curl http://0.0.0.0:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer <YOUR-LITELLM-KEY>" \
+  -d '{
+    "model": "gpt-oss",
+    "messages": [{"role": "user", "content": "Solve this complex problem step by step"}],
+    "reasoning_effort": "low"
+  }'
+```
+
+</TabItem>
+</Tabs>
+
 ## Model Garden
 
 :::tip

docs/my-website/docs/providers/volcano.md

@@ -3,7 +3,7 @@ https://www.volcengine.com/docs/82379/1263482
 
 :::tip
 
-**We support ALL Volcengine NIM models, just set `model=volcengine/<any-model-on-volcengine>` as a prefix when sending litellm requests**
+**We support ALL Volcengine models including Chat and Embeddings, just set `model=volcengine/<any-model-on-volcengine>` as a prefix when sending litellm requests**
 
 :::
 
@@ -11,6 +11,8 @@ https://www.volcengine.com/docs/82379/1263482
 ```python
 # env variable
 os.environ['VOLCENGINE_API_KEY']
+# or
+os.environ['ARK_API_KEY']
 ```
 
 ## Sample Usage
@@ -64,9 +66,42 @@ for chunk in response:
     print(chunk)
 ```
 
+## Sample Usage - Embedding
+```python
+from litellm import embedding
+import os
 
-## Supported Models - 💥 ALL Volcengine NIM Models Supported!
-We support ALL `volcengine` models, just set `volcengine/<OUR_ENDPOINT_ID>` as a prefix when sending completion requests
+os.environ['VOLCENGINE_API_KEY'] = ""
+response = embedding(
+    model="volcengine/doubao-embedding-text-240715",
+    input=["hello world", "good morning"]
+)
+print(response)
+```
+
+### Supported Embedding Models
+- `doubao-embedding-large` (2048 dimensions)
+- `doubao-embedding-large-text-250515` (2048 dimensions)
+- `doubao-embedding-large-text-240915` (4096 dimensions)
+- `doubao-embedding` (2560 dimensions) 
+- `doubao-embedding-text-240715` (2560 dimensions)
+
+### Embedding Parameters
+```python
+from litellm import embedding
+
+response = embedding(
+    model="volcengine/doubao-embedding-text-240715",
+    input=["sample text"],
+    encoding_format="float",  # optional: "float" (default), "base64"
+    user="user-123",          # optional: user identifier for tracking
+)
+```
+
+## Supported Models - 💥 ALL Volcengine Models Supported!
+We support ALL `volcengine` models for both chat completions and embeddings:
+- **Chat Models**: Set `volcengine/<OUR_ENDPOINT_ID>` as a prefix when sending completion requests
+- **Embedding Models**: Use the specific model names listed above (e.g., `volcengine/doubao-embedding-text-240715`)
 
 ## Sample Usage - LiteLLM Proxy
 
@@ -74,14 +109,21 @@ We support ALL `volcengine` models, just set `volcengine/<OUR_ENDPOINT_ID>` as a
 
 ```yaml
 model_list:
+  # Chat model
   - model_name: volcengine-model
     litellm_params:
       model: volcengine/<OUR_ENDPOINT_ID>
       api_key: os.environ/VOLCENGINE_API_KEY
+  # Embedding model
+  - model_name: volcengine-embedding
+    litellm_params:
+      model: volcengine/doubao-embedding-text-240715
+      api_key: os.environ/VOLCENGINE_API_KEY
 ```
 
 ### Send Request
 
+#### Chat Completion
 ```shell
 curl --location 'http://localhost:4000/chat/completions' \
     --header 'Authorization: Bearer sk-1234' \
@@ -95,4 +137,15 @@ curl --location 'http://localhost:4000/chat/completions' \
         }
     ]
 }'
+```
+
+#### Embedding
+```shell
+curl --location 'http://localhost:4000/embeddings' \
+    --header 'Authorization: Bearer sk-1234' \
+    --header 'Content-Type: application/json' \
+    --data '{
+    "model": "volcengine-embedding",
+    "input": ["hello world", "good morning"]
+}'
 ```

docs/my-website/docs/proxy/access_control.md

@@ -4,7 +4,7 @@ Role-based access control (RBAC) is based on Organizations, Teams and Internal U
 
 - `Organizations` are the top-level entities that contain Teams.
 - `Team` - A Team is a collection of multiple `Internal Users`
-- `Internal Users` - users that can create keys, make LLM API calls, view usage on LiteLLM 
+- `Internal Users` - users that can create keys, make LLM API calls, view usage on LiteLLM. Users can be on multiple teams.
 - `Roles` define the permissions of an `Internal User`
 - `Virtual Keys` - Keys are used for authentication to the LiteLLM API. Keys are tied to a `Internal User` and `Team` 
 

docs/my-website/docs/proxy/admin_ui_sso.md

@@ -235,6 +235,13 @@ Example setting a local image (on your container)
 ```shell
 UI_LOGO_PATH="ui_images/logo.jpg"
 ```
+
+#### Or set your logo directly from Admin UI:
+<div style={{ display: 'flex', gap: '12px', alignItems: 'center' }}>
+  <Image img={require('../../img/admin_settings_ui_theme.png')} />
+  <Image img={require('../../img/admin_settings_ui_theme_logo.png')} />
+</div>
+
 #### Set Custom Color Theme
 - Navigate to [/enterprise/enterprise_ui](https://github.com/BerriAI/litellm/blob/main/enterprise/enterprise_ui/_enterprise_colors.json)
 - Inside the `enterprise_ui` directory, rename `_enterprise_colors.json` to `enterprise_colors.json`

docs/my-website/docs/proxy/call_hooks.md

@@ -6,6 +6,10 @@ import Image from '@theme/IdealImage';
 - Reject data before making llm api calls / before returning the response 
 - Enforce 'user' param for all openai endpoint calls
 
+:::tip
+**Understanding Callback Hooks?** Check out our [Callback Management Guide](../observability/callback_management.md) to understand the differences between proxy-specific hooks like `async_pre_call_hook` and general logging hooks like `async_log_success_event`.
+:::
+
 See a complete example with our [parallel request rate limiter](https://github.com/BerriAI/litellm/blob/main/litellm/proxy/hooks/parallel_request_limiter.py)
 
 ## Quick Start

docs/my-website/docs/proxy/config_settings.md

@@ -335,12 +335,15 @@ router_settings:
 | ANTHROPIC_API_KEY | API key for Anthropic service
 | ANTHROPIC_API_BASE | Base URL for Anthropic API. Default is https://api.anthropic.com
 | AWS_ACCESS_KEY_ID | Access Key ID for AWS services
+| AWS_BATCH_ROLE_ARN | ARN of the AWS IAM role for batch operations
 | AWS_DEFAULT_REGION | Default AWS region for service interactions when AWS_REGION is not set
 | AWS_PROFILE_NAME | AWS CLI profile name to be used
 | AWS_REGION | AWS region for service interactions (takes precedence over AWS_DEFAULT_REGION)
 | AWS_REGION_NAME | Default AWS region for service interactions
 | AWS_ROLE_ARN | ARN of the AWS IAM role to assume for authentication
 | AWS_ROLE_NAME | Role name for AWS IAM usage
+| AWS_S3_BUCKET_NAME | Name of the AWS S3 bucket for file operations
+| AWS_S3_OUTPUT_BUCKET_NAME | Name of the AWS S3 output bucket for batch operations
 | AWS_SECRET_ACCESS_KEY | Secret Access Key for AWS services
 | AWS_SESSION_NAME | Name for AWS session
 | AWS_WEB_IDENTITY_TOKEN | Web identity token for AWS
@@ -380,6 +383,8 @@ router_settings:
 | CIRCLE_OIDC_TOKEN_V2 | Version 2 of the OpenID Connect token for CircleCI
 | CLOUDZERO_API_KEY | CloudZero API key for authentication
 | CLOUDZERO_CONNECTION_ID | CloudZero connection ID for data submission
+| CLOUDZERO_EXPORT_INTERVAL_MINUTES | Interval in minutes for CloudZero data export operations
+| CLOUDZERO_MAX_FETCHED_DATA_RECORDS | Maximum number of data records to fetch from CloudZero
 | CLOUDZERO_TIMEZONE | Timezone for date handling (default: UTC)
 | CONFIG_FILE_PATH | File path for configuration file
 | CONFIDENT_API_KEY | API key for DeepEval integration
@@ -412,6 +417,7 @@ router_settings:
 | DEFAULT_ALLOWED_FAILS | Maximum failures allowed before cooling down a model. Default is 3
 | DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS | Default maximum tokens for Anthropic chat completions. Default is 4096
 | DEFAULT_BATCH_SIZE | Default batch size for operations. Default is 512
+| DEFAULT_CLIENT_DISCONNECT_CHECK_TIMEOUT_SECONDS | Timeout in seconds for checking client disconnection. Default is 1
 | DEFAULT_COOLDOWN_TIME_SECONDS | Duration in seconds to cooldown a model after failures. Default is 5
 | DEFAULT_CRON_JOB_LOCK_TTL_SECONDS | Time-to-live for cron job locks in seconds. Default is 60 (1 minute)
 | DEFAULT_FAILURE_THRESHOLD_PERCENT | Threshold percentage of failures to cool down a deployment. Default is 0.5 (50%)
@@ -438,6 +444,10 @@ router_settings:
 | DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET | Default high reasoning effort thinking budget. Default is 4096
 | DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET | Default low reasoning effort thinking budget. Default is 1024
 | DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET | Default medium reasoning effort thinking budget. Default is 2048
+| DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET | Default minimal reasoning effort thinking budget. Default is 512
+| DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH | Default minimal reasoning effort thinking budget for Gemini 2.5 Flash. Default is 512
+| DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE | Default minimal reasoning effort thinking budget for Gemini 2.5 Flash Lite. Default is 512
+| DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_PRO | Default minimal reasoning effort thinking budget for Gemini 2.5 Pro. Default is 512
 | DEFAULT_REDIS_SYNC_INTERVAL | Default Redis synchronization interval in seconds. Default is 1
 | DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND | Default price per second for Replicate GPU. Default is 0.001400
 | DEFAULT_REPLICATE_POLLING_DELAY_SECONDS | Default delay in seconds for Replicate polling. Default is 1
@@ -573,6 +583,10 @@ router_settings:
 | LITELLM_LOCAL_MODEL_COST_MAP | Local configuration for model cost mapping in LiteLLM
 | LITELLM_LOG | Enable detailed logging for LiteLLM
 | LITELLM_LOG_FILE | File path to write LiteLLM logs to. When set, logs will be written to both console and the specified file
+| LITELLM_LOGGER_NAME | Name for OTEL logger 
+| LITELLM_METER_NAME | Name for OTEL Meter 
+| LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS | Optionally enable semantic logs for OTEL
+| LITELLM_OTEL_INTEGRATION_ENABLE_METRICS | Optionally enable emantic metrics for OTEL
 | LITELLM_MASTER_KEY | Master key for proxy authentication
 | LITELLM_MODE | Operating mode for LiteLLM (e.g., production, development)
 | LITELLM_RATE_LIMIT_WINDOW_SIZE | Rate limit window size for LiteLLM. Default is 60

docs/my-website/docs/proxy/cost_tracking.md

@@ -505,11 +505,11 @@ litellm_settings:
 
 ### Disable user-agent tracking
 
-You can disable user-agent tracking by setting `litellm_settings.disable_user_agent_tracking` to `true`.
+You can disable user-agent tracking by setting `litellm_settings.disable_add_user_agent_to_request_tags` to `true`.
 
 ```yaml
 litellm_settings:
-  disable_user_agent_tracking: true
+  disable_add_user_agent_to_request_tags: true
 ```
 
 ## ✨ (Enterprise) Generate Spend Reports

docs/my-website/docs/proxy/enterprise.md

@@ -439,6 +439,33 @@ response = client.chat.completions.create(
 
 print(response)
 ```
+
+**Using Headers:**
+
+```python
+import openai
+client = openai.OpenAI(
+    api_key="sk-1234",
+    base_url="http://0.0.0.0:4000"
+)
+
+# Pass spend logs metadata via headers
+response = client.chat.completions.create(
+    model="gpt-3.5-turbo",
+    messages = [
+        {
+            "role": "user",
+            "content": "this is a test request, write a short poem"
+        }
+    ],
+    extra_headers={
+        "x-litellm-spend-logs-metadata": '{"user_id": "12345", "project_id": "proj_abc", "request_type": "chat_completion"}'
+    }
+)
+
+print(response)
+```
+
 </TabItem>
 
 
@@ -478,6 +505,43 @@ async function runOpenAI() {
 // Call the asynchronous function
 runOpenAI();
 ```
+
+**Using Headers:**
+
+```js
+const openai = require('openai');
+
+async function runOpenAI() {
+  const client = new openai.OpenAI({
+    apiKey: 'sk-1234',
+    baseURL: 'http://0.0.0.0:4000'
+  });
+
+  try {
+    const response = await client.chat.completions.create({
+      model: 'gpt-3.5-turbo',
+      messages: [
+        {
+          role: 'user',
+          content: "this is a test request, write a short poem"
+        },
+      ]
+    }, {
+      headers: {
+        'x-litellm-spend-logs-metadata': '{"user_id": "12345", "project_id": "proj_abc", "request_type": "chat_completion"}'
+      }
+    });
+    console.log(response);
+  } catch (error) {
+    console.log("got this exception from server");
+    console.error(error);
+  }
+}
+
+// Call the asynchronous function
+runOpenAI();
+```
+
 </TabItem>
 
 <TabItem value="Curl" label="Curl Request">
@@ -502,6 +566,29 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
     }
 }'
 ```
+
+</TabItem>
+
+<TabItem value="headers" label="Using Headers">
+
+Pass `x-litellm-spend-logs-metadata` as a request header with JSON string
+
+```shell
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+    --header 'Content-Type: application/json' \
+    --header 'Authorization: Bearer sk-1234' \
+    --header 'x-litellm-spend-logs-metadata: {"user_id": "12345", "project_id": "proj_abc", "request_type": "chat_completion"}' \
+    --data '{
+    "model": "gpt-3.5-turbo",
+    "messages": [
+        {
+        "role": "user",
+        "content": "what llm are you"
+        }
+    ]
+}'
+```
+
 </TabItem>
 <TabItem value="langchain" label="Langchain">
 

docs/my-website/docs/proxy/load_balancing.md

@@ -13,6 +13,23 @@ For more details on routing strategies / params, see [Routing](../routing.md)
 
 :::
 
+## How Load Balancing Works
+
+LiteLLM automatically distributes requests across multiple deployments of the same model using its built-in router. the proxy routes traffic to optimize performance and reliability.
+
+"simple-shuffle" routing strategy is used by default
+
+### Routing Strategies
+
+| Strategy | Description | When to Use |
+|----------|-------------|-------------|
+| **simple-shuffle** (recommended) | Randomly distributes requests | General purpose, good for even load distribution |
+| **least-busy** | Routes to deployment with fewest active requests | High concurrency scenarios |
+| **usage-based-routing** (bad for perf) | Routes to deployment with lowest current usage (RPM/TPM) | When you want to respect rate limits evenly |
+| **latency-based-routing** | Routes to fastest responding deployment | Latency-critical applications |
+| **cost-based-routing** | Routes to deployment with lowest cost | Cost-sensitive applications |
+
+
 ## Quick Start - Load Balancing
 #### Step 1 - Set deployments on config
 
@@ -106,49 +123,14 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
     ]
 }'
 ```
-</TabItem>
-<TabItem value="langchain" label="Langchain">
-
-```python
-from langchain.chat_models import ChatOpenAI
-from langchain.prompts.chat import (
-    ChatPromptTemplate,
-    HumanMessagePromptTemplate,
-    SystemMessagePromptTemplate,
-)
-from langchain.schema import HumanMessage, SystemMessage
-import os 
-
-os.environ["OPENAI_API_KEY"] = "anything"
-
-chat = ChatOpenAI(
-    openai_api_base="http://0.0.0.0:4000",
-    model="gpt-3.5-turbo",
-)
-
-messages = [
-    SystemMessage(
-        content="You are a helpful assistant that im using to make a test request to."
-    ),
-    HumanMessage(
-        content="test from litellm. tell me why it's amazing in 1 sentence"
-    ),
-]
-response = chat(messages)
-
-print(response)
-```
-
 </TabItem>
 
 </Tabs>
-
-
 ### Test - Loadbalancing
 
 In this request, the following will occur:
 1. A rate limit exception will be raised 
-2. LiteLLM proxy will retry the request on the model group (default is 3).
+2. LiteLLM proxy will retry the request on the model group (default retries are 3).
 
 ```bash
 curl -X POST 'http://0.0.0.0:4000/chat/completions' \
@@ -256,4 +238,16 @@ model_group_alias: Optional[Dict[str, Union[str, RouterModelGroupAliasItem]]] =
 class RouterModelGroupAliasItem(TypedDict):
     model: str
     hidden: bool  # if 'True', don't return on `/v1/models`, `/v1/model/info`, `/v1/model_group/info`
-```
+```
+
+### When You'll See Load Balancing in Action
+
+**Immediate Effects:**
+
+- Different deployments serve subsequent requests (visible in logs)
+- Better response times during high traffic
+
+**Observable Benefits:**
+- **Higher throughput**: More requests handled simultaneously across deployments
+- **Improved reliability**: If one deployment fails, traffic automatically routes to healthy ones
+- **Better resource utilization**: Load spread evenly across all available deployments

docs/my-website/docs/proxy/prometheus.md

@@ -63,7 +63,7 @@ Use this for for tracking per [user, key, team, etc.](virtual_keys)
 
 | Metric Name          | Description                          |
 |----------------------|--------------------------------------|
-| `litellm_spend_metric`                | Total Spend, per `"user", "key", "model", "team", "end-user"`                 |
+| `litellm_spend_metric`                | Total Spend, per `"end_user", "hashed_api_key", "api_key_alias", "model", "team", "team_alias", "user"`                 |
 | `litellm_total_tokens_metric`         | input + output tokens per `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "model"`     |
 | `litellm_input_tokens_metric`         | input tokens per `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "model"`     |
 | `litellm_output_tokens_metric`        | output tokens per `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "model"`             |
@@ -73,9 +73,9 @@ Use this for for tracking per [user, key, team, etc.](virtual_keys)
 
 | Metric Name          | Description                          |
 |----------------------|--------------------------------------|
-| `litellm_team_max_budget_metric`                    | Max Budget for Team Labels: `"team_id", "team_alias"`|
-| `litellm_remaining_team_budget_metric`             | Remaining Budget for Team (A team created on LiteLLM) Labels: `"team_id", "team_alias"`|
-| `litellm_team_budget_remaining_hours_metric`        | Hours before the team budget is reset Labels: `"team_id", "team_alias"`|
+| `litellm_team_max_budget_metric`                    | Max Budget for Team Labels: `"team", "team_alias"`|
+| `litellm_remaining_team_budget_metric`             | Remaining Budget for Team (A team created on LiteLLM) Labels: `"team", "team_alias"`|
+| `litellm_team_budget_remaining_hours_metric`        | Hours before the team budget is reset Labels: `"team", "team_alias"`|
 
 ### Virtual Key - Budget
 
@@ -119,8 +119,8 @@ Use this to track overall LiteLLM Proxy usage.
 
 | Metric Name          | Description                          |
 |----------------------|--------------------------------------|
-| `litellm_proxy_failed_requests_metric`             | Total number of failed responses from proxy - the client did not get a success response from litellm proxy. Labels: `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "exception_status", "exception_class"`          |
-| `litellm_proxy_total_requests_metric`             | Total number of requests made to the proxy server - track number of client side requests. Labels: `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "status_code"`          |
+| `litellm_proxy_failed_requests_metric`             | Total number of failed responses from proxy - the client did not get a success response from litellm proxy. Labels: `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "exception_status", "exception_class", "route"`          |
+| `litellm_proxy_total_requests_metric`             | Total number of requests made to the proxy server - track number of client side requests. Labels: `"end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "status_code", "user_email", "route"`          |
 
 ## LLM Provider Metrics
 
@@ -155,7 +155,7 @@ Use this for LLM API Error monitoring and tracking remaining rate limits and tok
 | Metric Name          | Description                          |
 |----------------------|--------------------------------------|
 | `litellm_remaining_requests_metric`             | Track `x-ratelimit-remaining-requests` returned from LLM API Deployment. Labels: `"model_group", "api_provider", "api_base", "litellm_model_name", "hashed_api_key", "api_key_alias"` |
-| `litellm_remaining_tokens`                | Track `x-ratelimit-remaining-tokens` return from LLM API Deployment. Labels: `"model_group", "api_provider", "api_base", "litellm_model_name", "hashed_api_key", "api_key_alias"` |
+| `litellm_remaining_tokens_metric`                | Track `x-ratelimit-remaining-tokens` return from LLM API Deployment. Labels: `"model_group", "api_provider", "api_base", "litellm_model_name", "hashed_api_key", "api_key_alias"` |
 
 ### Deployment State 
 | Metric Name          | Description                          |
@@ -167,16 +167,22 @@ Use this for LLM API Error monitoring and tracking remaining rate limits and tok
 
 | Metric Name          | Description                          |
 |----------------------|--------------------------------------|
-| `litellm_deployment_cooled_down`             | Number of times a deployment has been cooled down by LiteLLM load balancing logic. Labels: `"litellm_model_name", "model_id", "api_base", "api_provider", "exception_status"` |
+| `litellm_deployment_cooled_down`             | Number of times a deployment has been cooled down by LiteLLM load balancing logic. Labels: `"litellm_model_name", "model_id", "api_base", "api_provider"` |
 | `litellm_deployment_successful_fallbacks`           | Number of successful fallback requests from primary model -> fallback model. Labels: `"requested_model", "fallback_model", "hashed_api_key", "api_key_alias", "team", "team_alias", "exception_status", "exception_class"` |
 | `litellm_deployment_failed_fallbacks`               | Number of failed fallback requests from primary model -> fallback model. Labels: `"requested_model", "fallback_model", "hashed_api_key", "api_key_alias", "team", "team_alias", "exception_status", "exception_class"` |
 
+## Request Counting Metrics
+
+| Metric Name          | Description                          |
+|----------------------|--------------------------------------|
+| `litellm_requests_metric`             | Total number of requests tracked per endpoint. Labels: `"end_user", "hashed_api_key", "api_key_alias", "model", "team", "team_alias", "user", "user_email"` |
+
 ## Request Latency Metrics 
 
 | Metric Name          | Description                          |
 |----------------------|--------------------------------------|
 | `litellm_request_total_latency_metric`             | Total latency (seconds) for a request to LiteLLM Proxy Server - tracked for labels "end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "model" |
-| `litellm_overhead_latency_metric`             | Latency overhead (seconds) added by LiteLLM processing - tracked for labels "end_user", "hashed_api_key", "api_key_alias", "requested_model", "team", "team_alias", "user", "model" |
+| `litellm_overhead_latency_metric`             | Latency overhead (seconds) added by LiteLLM processing - tracked for labels "model_group", "api_provider", "api_base", "litellm_model_name", "hashed_api_key", "api_key_alias" |
 | `litellm_llm_api_latency_metric`  | Latency (seconds) for just the LLM API call - tracked for labels "model", "hashed_api_key", "api_key_alias", "team", "team_alias", "requested_model", "end_user", "user" |
 | `litellm_llm_api_time_to_first_token_metric`             | Time to first token for LLM API call - tracked for labels `model`, `hashed_api_key`, `api_key_alias`, `team`, `team_alias` [Note: only emitted for streaming requests] |
 
@@ -486,7 +492,6 @@ Here is a screenshot of the metrics you can monitor with the LiteLLM Grafana Das
 | Metric Name          | Description                          |
 |----------------------|--------------------------------------|
 | `litellm_llm_api_failed_requests_metric`             | **deprecated** use `litellm_proxy_failed_requests_metric` |
-| `litellm_requests_metric`             | **deprecated** use `litellm_proxy_total_requests_metric` |
 
 
 

docs/my-website/docs/proxy/request_headers.md

@@ -14,6 +14,8 @@ Special headers that are supported by LiteLLM.
 
 `x-litellm-num-retries`: Optional[int]: The number of retries for the request.
 
+`x-litellm-spend-logs-metadata`: Optional[str]: JSON string containing custom metadata to include in spend logs. Example: `{"user_id": "12345", "project_id": "proj_abc", "request_type": "chat_completion"}`. [Learn More](../proxy/enterprise#tracking-spend-with-custom-metadata)
+
 ## Anthropic Headers
 
 `anthropic-version` Optional[str]: The version of the Anthropic API to use.  

docs/my-website/docs/proxy/user_management_heirarchy.md

@@ -9,5 +9,5 @@ LiteLLM supports a hierarchy of users, teams, organizations, and budgets.
 
 - Organizations can have multiple teams. [API Reference](https://litellm-api.up.railway.app/#/organization%20management)
 - Teams can have multiple users. [API Reference](https://litellm-api.up.railway.app/#/team%20management)
-- Users can have multiple keys. [API Reference](https://litellm-api.up.railway.app/#/budget%20management)
+- Users can have multiple keys, and be on multiple teams. [API Reference](https://litellm-api.up.railway.app/#/budget%20management)
 - Keys can belong to either a team or a user. [API Reference](https://litellm-api.up.railway.app/#/end-user%20management)

docs/my-website/docs/proxy/veo_video_generation.md

@@ -0,0 +1,163 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Veo Video Generation with Google AI Studio
+
+Generate videos using Google's Veo model through LiteLLM's pass-through endpoints.
+
+## Quick Start
+
+LiteLLM allows you to use Google AI Studio's Veo video generation API through pass-through routes with zero configuration.
+
+### 1. Add Google AI Studio API Key to your environment 
+
+```bash
+export GEMINI_API_KEY="your_google_ai_studio_api_key"
+```
+
+### 2. Start LiteLLM Proxy 
+
+```bash
+litellm
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+### 3. Generate Video
+
+<Tabs>
+<TabItem value="python" label="Python">
+
+```python
+import requests
+import time
+import json
+
+# Configuration
+BASE_URL = "http://localhost:4000/gemini/v1beta"
+API_KEY = "anything"  # Use "anything" as the key
+
+headers = {
+    "x-goog-api-key": API_KEY,
+    "Content-Type": "application/json"
+}
+
+# Step 1: Initiate video generation
+def generate_video(prompt):
+    url = f"{BASE_URL}/models/veo-3.0-generate-preview:predictLongRunning"
+    payload = {
+        "instances": [{
+            "prompt": prompt
+        }]
+    }
+    
+    response = requests.post(url, headers=headers, json=payload)
+    response.raise_for_status()
+    
+    data = response.json()
+    return data.get("name")  # Operation name
+
+# Step 2: Poll for completion
+def wait_for_completion(operation_name):
+    operation_url = f"{BASE_URL}/{operation_name}"
+    
+    while True:
+        response = requests.get(operation_url, headers=headers)
+        response.raise_for_status()
+        
+        data = response.json()
+        
+        if data.get("done", False):
+            # Extract video URI
+            video_uri = data["response"]["generateVideoResponse"]["generatedSamples"][0]["video"]["uri"]
+            return video_uri
+        
+        time.sleep(10)  # Wait 10 seconds before next poll
+
+# Step 3: Download video
+def download_video(video_uri, filename="generated_video.mp4"):
+    # Replace Google URL with LiteLLM proxy URL
+    litellm_url = video_uri.replace(
+        "https://generativelanguage.googleapis.com/v1beta", 
+        BASE_URL
+    )
+    
+    response = requests.get(litellm_url, headers=headers, stream=True)
+    response.raise_for_status()
+    
+    with open(filename, 'wb') as f:
+        for chunk in response.iter_content(chunk_size=8192):
+            if chunk:
+                f.write(chunk)
+    
+    return filename
+
+# Complete workflow
+prompt = "A cat playing with a ball of yarn in a sunny garden"
+
+print("Generating video...")
+operation_name = generate_video(prompt)
+
+print("Waiting for completion...")
+video_uri = wait_for_completion(operation_name)
+
+print("Downloading video...")
+filename = download_video(video_uri)
+
+print(f"Video saved as: {filename}")
+```
+
+</TabItem>
+
+<TabItem value="curl" label="Curl">
+
+```bash
+# Step 1: Initiate video generation
+curl -X POST "http://localhost:4000/gemini/v1beta/models/veo-3.0-generate-preview:predictLongRunning" \
+  -H "x-goog-api-key: anything" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "instances": [{
+      "prompt": "A cat playing with a ball of yarn in a sunny garden"
+    }]
+  }'
+
+# Response will include operation name:
+# {"name": "operations/generate_12345"}
+
+# Step 2: Poll for completion
+curl -X GET "http://localhost:4000/gemini/v1beta/operations/generate_12345" \
+  -H "x-goog-api-key: anything"
+
+# Step 3: Download video (when done=true)
+curl -X GET "http://localhost:4000/gemini/v1beta/files/VIDEO_ID:download?alt=media" \
+  -H "x-goog-api-key: anything" \
+  --output generated_video.mp4
+```
+
+</TabItem>
+</Tabs>
+
+## Complete Example
+
+For a full working example with error handling and logging, see our [Veo Video Generation Cookbook](https://github.com/BerriAI/litellm/blob/main/cookbook/veo_video_generation.py).
+
+## How It Works
+
+1. **Video Generation Request**: Send a prompt to Veo's `predictLongRunning` endpoint
+2. **Operation Polling**: Monitor the long-running operation until completion
+3. **File Download**: Download the generated video through LiteLLM's pass-through with automatic redirect handling
+
+LiteLLM handles:
+- ✅ Authentication with Google AI Studio
+- ✅ Request routing and proxying
+- ✅ Automatic redirect handling for file downloads
+
+## Configuration Options
+
+### Environment Variables
+
+```bash
+export GEMINI_API_KEY="your_google_ai_studio_api_key"
+```
+

docs/my-website/docs/reasoning_content.md

@@ -12,7 +12,7 @@ Requires LiteLLM v1.63.0+
 Supported Providers:
 - Deepseek (`deepseek/`)
 - Anthropic API (`anthropic/`)
-- Bedrock (Anthropic + Deepseek) (`bedrock/`)
+- Bedrock (Anthropic + Deepseek + GPT-OSS) (`bedrock/`)
 - Vertex AI (Anthropic) (`vertexai/`)
 - OpenRouter (`openrouter/`)
 - XAI (`xai/`)
@@ -20,6 +20,7 @@ Supported Providers:
 - Vertex AI (`vertex_ai/`)
 - Perplexity (`perplexity/`)
 - Mistral AI (Magistral models) (`mistral/`)
+- Groq (`groq/`)
 
 LiteLLM will standardize the `reasoning_content` in the response and `thinking_blocks` in the assistant message.
 

docs/my-website/docs/tutorials/claude_responses_api.md

@@ -12,6 +12,12 @@ This tutorial is based on [Anthropic's official LiteLLM configuration documentat
 
 :::
 
+<br />
+
+### Video Walkthrough
+
+<iframe width="840" height="500" src="https://www.loom.com/embed/3c17d683cdb74d36a3698763cc558f56" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>
+
 ## Prerequisites
 
 - [Claude Code](https://docs.anthropic.com/en/docs/claude-code/overview) installed
@@ -83,11 +89,17 @@ curl -X POST http://0.0.0.0:4000/v1/messages \
 
 Configure Claude Code to use LiteLLM's unified endpoint:
 
+Either a virtual key / master key can be used here
+
 ```bash
 export ANTHROPIC_BASE_URL="http://0.0.0.0:4000"
 export ANTHROPIC_AUTH_TOKEN="$LITELLM_MASTER_KEY"
 ```
 
+:::tip
+LITELLM_MASTER_KEY gives claude access to all proxy models, whereas a virtual key would be limited to the models set in UI
+:::
+
 #### Method 2: Provider-specific Pass-through Endpoint
 
 Alternatively, use the Anthropic pass-through endpoint:

docs/my-website/release_notes/v1.74.7/index.md

@@ -148,7 +148,6 @@ Starting with this release, you can run health endpoints on an isolated process
     - New provider integration for v0.dev - [PR #12751](https://github.com/BerriAI/litellm/pull/12751), [Get Started](../../docs/providers/v0)
 - **[OpenAI](../../docs/providers/openai)**
     - Use OpenAI DeepResearch models with `litellm.completion` (`/chat/completions`) - [PR #12627](https://github.com/BerriAI/litellm/pull/12627) **DOC NEEDED**
-    - Add `input_fidelity` parameter for OpenAI image generation - [PR #12662](https://github.com/BerriAI/litellm/pull/12662), [Get Started](../../docs/image_generation)
 - **[Azure OpenAI](../../docs/providers/azure_openai)**
     - Use Azure OpenAI DeepResearch models with `litellm.completion` (`/chat/completions`) - [PR #12627](https://github.com/BerriAI/litellm/pull/12627) **DOC NEEDED**
     - Added `response_format` support for openai gpt-4.1 models - [PR #12745](https://github.com/BerriAI/litellm/pull/12745)

docs/my-website/release_notes/v1.76.0-stable/index.md

@@ -0,0 +1,189 @@
+---
+title: "[PRE-RELEASE]v1.76.0-stable - RPS Improvements"
+slug: "v1-76-0"
+date: 2025-08-23T10:00:00
+authors:
+  - name: Krrish Dholakia
+    title: CEO, LiteLLM
+    url: https://www.linkedin.com/in/krish-d/
+    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
+  - name: Ishaan Jaffer
+    title: CTO, LiteLLM
+    url: https://www.linkedin.com/in/reffajnaahsi/
+    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+
+hide_table_of_contents: false
+---
+
+import Image from '@theme/IdealImage';
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+:::info
+
+LiteLLM is hiring a **Founding Backend Engineer**, in San Francisco. 
+
+[Apply here](https://www.ycombinator.com/companies/litellm/jobs/6uvoBp3-founding-backend-engineer) if you're interested!
+:::
+
+
+
+
+
+## Deploy this version
+
+:::info
+
+This release is not live yet. 
+:::
+
+
+---
+
+## New Models / Updated Models
+
+#### Bugs
+- **[OpenAI](../../docs/providers/openai)**
+    - Gpt-5 chat: clarify does not support function calling [PR #13612](https://github.com/BerriAI/litellm/pull/13612), s/o  @[superpoussin22](https://github.com/superpoussin22)
+- **[VertexAI](../../docs/providers/vertex)**
+    - fix vertexai batch file format by @[thiagosalvatore](https://github.com/thiagosalvatore) in [PR #13576](https://github.com/BerriAI/litellm/pull/13576)
+- **[LiteLLM Proxy](../../docs/providers/litellm_proxy)**
+    - Add support for calling image_edits + image_generations via SDK to Proxy - [PR #13735](https://github.com/BerriAI/litellm/pull/13735)
+- **[OpenRouter](../../docs/providers/openrouter)**
+    - Fix max_output_tokens value for anthropic Claude 4 - [PR #13526](https://github.com/BerriAI/litellm/pull/13526)
+- **[Gemini](../../docs/providers/gemini)**
+    - Fix prompt caching cost calculation - [PR #13742](https://github.com/BerriAI/litellm/pull/13742)
+- **[Azure](../../docs/providers/azure)**
+    - Support `../openai/v1/respones` api base - [PR #13526](https://github.com/BerriAI/litellm/pull/13526)
+    - Fix azure/gpt-5-chat max_input_tokens - [PR #13660](https://github.com/BerriAI/litellm/pull/13660)
+- **[Groq](../../docs/providers/groq)**
+    - streaming ASCII encoding issue - [PR #13675](https://github.com/BerriAI/litellm/pull/13675)
+- **[Baseten](../../docs/providers/baseten)**
+    - Refactored integration to use new openai-compatible endpoints - [PR #13783](https://github.com/BerriAI/litellm/pull/13783)
+- **[Bedrock](../../docs/providers/bedrock)**
+    - fix application inference profile for pass-through endpoints for bedrock - [PR #13881](https://github.com/BerriAI/litellm/pull/13881)
+- **[DataRobot](../../docs/providers/datarobot)**
+    - Updated URL handling for DataRobot provider URL - [PR #13880](https://github.com/BerriAI/litellm/pull/13880)
+
+#### Features
+- **[Together AI](../../docs/providers/together)**
+    - Added Qwen3, Deepseek R1 0528 Throughput, GLM 4.5 and GPT-OSS models cost tracking - [PR #13637](https://github.com/BerriAI/litellm/pull/13637), s/o  @[Tasmay-Tibrewal](https://github.com/Tasmay-Tibrewal)
+- **[Fireworks AI](../../docs/providers/fireworks_ai)**
+    - add fireworks_ai/accounts/fireworks/models/deepseek-v3-0324 - [PR #13821](https://github.com/BerriAI/litellm/pull/13821)
+- **[VertexAI](../../docs/providers/vertex)**
+    - Add VertexAI qwen API Service - [PR #13828](https://github.com/BerriAI/litellm/pull/13828)
+    - Add new VertexAI image models vertex_ai/imagen-4.0-generate-001, vertex_ai/imagen-4.0-ultra-generate-001, vertex_ai/imagen-4.0-fast-generate-001  - [PR #13874](https://github.com/BerriAI/litellm/pull/13874)
+- **[Anthropic](../../docs/providers/anthropic)**
+    - Add long context support w/ cost tracking - [PR #13759](https://github.com/BerriAI/litellm/pull/13759)
+- **[DeepInfra](../../docs/providers/deepinfra)**
+    - Add rerank endpoint support for deepinfra - [PR #13820](https://github.com/BerriAI/litellm/pull/13820)
+    - Add new models for cost tracking - [PR #13883](https://github.com/BerriAI/litellm/pull/13883), s/o  @[Toy-97](https://github.com/Toy-97)
+- **[Bedrock](../../docs/providers/bedrock)**
+    - Add tool prompt caching on async calls - [PR #13803](https://github.com/BerriAI/litellm/pull/13803), s/o  @[UlookEE](https://github.com/UlookEE)
+    - role chaining and session name with webauthentication for aws bedrock - [PR #13753](https://github.com/BerriAI/litellm/pull/13753), s/o @[RichardoC](https://github.com/RichardoC)
+- **[Ollama](../../docs/providers/ollama)**
+    - Handle Ollama null response when using tool calling with non-tool trained models - [PR #13902](https://github.com/BerriAI/litellm/pull/13902)
+- **[OpenRouter](../../docs/providers/openrouter)**
+    - Add deepseek/deepseek-chat-v3.1 support - [PR #13897](https://github.com/BerriAI/litellm/pull/13897)
+- **[Mistral](../../docs/providers/mistral)**
+    - Add support for calling mistral files via chat completions - [PR #13866](https://github.com/BerriAI/litellm/pull/13866), s/o  @[jinskjoy](https://github.com/jinskjoy)
+    - Handle empty assistant content - [PR #13671](https://github.com/BerriAI/litellm/pull/13671)
+    - Support new ‘thinking’ response block - [PR #13671](https://github.com/BerriAI/litellm/pull/13671)
+- **[Databricks](../../docs/providers/databricks)**
+    - remove deprecated dbrx models (dbrx-instruct, llama 3.1) - [PR #13843](https://github.com/BerriAI/litellm/pull/13843)
+- **[AI/ML API](../../docs/providers/ai_ml_api)**
+    - Image gen api support - [PR #13893](https://github.com/BerriAI/litellm/pull/13893)
+
+
+## LLM API Endpoints
+#### Bugs
+- **[Responses API](../../docs/response_api)**
+    - add default api version for openai responses api calls - [PR #13526](https://github.com/BerriAI/litellm/pull/13526)
+    - support allowed_openai_params - [PR #13671](https://github.com/BerriAI/litellm/pull/13671)
+
+
+## MCP Gateway
+#### Bugs
+- fix StreamableHTTPSessionManager .run() error - [PR #13666](https://github.com/BerriAI/litellm/pull/13666)
+
+## Vector Stores 
+#### Bugs
+- **[Bedrock](../../docs/providers/bedrock)**
+    - Using LiteLLM Managed Credentials for Query - [PR #13787](https://github.com/BerriAI/litellm/pull/13787)
+
+## Management Endpoints / UI
+#### Bugs
+- **[Passthrough](../../docs/pass_through/intro)**
+    - Fix query passthrough deletion - [PR #13622](https://github.com/BerriAI/litellm/pull/13622)
+
+#### Features
+- **Models**
+    - Add Search Functionality for Public Model Names in Model Dashboard - [PR #13687](https://github.com/BerriAI/litellm/pull/13687)
+    - Auto-Add `azure/` to deployment Name in UI - [PR #13685](https://github.com/BerriAI/litellm/pull/13685)
+    - Models page row UI restructure - [PR #13771](https://github.com/BerriAI/litellm/pull/13771)
+- **Notifications**
+    - Add new notifications toast UI everywhere - [PR #13813](https://github.com/BerriAI/litellm/pull/13813)
+- **Keys**
+    - Fix key edit settings after regenerating a key - [PR #13815](https://github.com/BerriAI/litellm/pull/13815)
+    - Require team_id when creating service account keys - [PR #13873](https://github.com/BerriAI/litellm/pull/13873)
+    - Filter - show all options on filter option click - [PR #13858](https://github.com/BerriAI/litellm/pull/13858)
+- **Usage**
+    - Fix ‘Cannot read properties of undefined’ exception on user agent activity tab - [PR #13892](https://github.com/BerriAI/litellm/pull/13892)
+- **SSO**
+    - Free SSO usage for up to 5 users - [PR #13843](https://github.com/BerriAI/litellm/pull/13843)
+
+## Logging / Guardrail Integrations
+#### Bugs
+- **[Bedrock Guardrails](../../docs/proxy/guardrails/bedrock)**
+    - Add bedrock api key support - [PR #13835](https://github.com/BerriAI/litellm/pull/13835)
+#### Features
+- **[Datadog LLM Observability](../../docs/integrations/datadog)**
+    - Add support for Failure Logging [PR #13726](https://github.com/BerriAI/litellm/pull/13726)
+    - Add time to first token, litellm overhead, guardrail overhead latency metrics - [PR #13734](https://github.com/BerriAI/litellm/pull/13734)
+    - Add support for tracing guardrail input/output - [PR #13767](https://github.com/BerriAI/litellm/pull/13767)
+- **[Langfuse OTEL](../../docs/integrations/langfuse)**
+    - Allow using Key/Team Based Logging - [PR #13791](https://github.com/BerriAI/litellm/pull/13791)
+- **[AIM](../../docs/integrations/aim)**
+    - Migrate to new firewall API - [PR #13748](https://github.com/BerriAI/litellm/pull/13748)
+- **[OTEL](../../docs/observability/opentelemetry_integration)**
+    - Add OTEL tracing for actual LLM API call - [PR #13836](https://github.com/BerriAI/litellm/pull/13836)
+- **[MLFlow](../../docs/observability/mlflow_integration)**
+    - Include predicted output in MLflow tracing - [PR #13795](https://github.com/BerriAI/litellm/pull/13795), s/o @TomeHirata  
+
+
+## Performance / Loadbalancing / Reliability improvements
+#### Bugs
+- **[Cooldowns](../../docs/routing#how-cooldowns-work)**
+    - don't return raw Azure Exceptions to client (can contain prompt leakage) - [PR #13529](https://github.com/BerriAI/litellm/pull/13529)
+- **[Auto-router](../../docs/proxy/auto_routing)**
+    - Ensures the relevant dependencies for auto router existing on LiteLLM Docker - [PR #13788](https://github.com/BerriAI/litellm/pull/13788)
+- **Model Alias**
+    - Fix calling key with access to model alias - [PR #13830](https://github.com/BerriAI/litellm/pull/13830)
+
+#### Features
+- **[S3 Caching](../../docs/proxy/caching)**
+    - Use namespace as prefix for s3 cache - [PR #13704](https://github.com/BerriAI/litellm/pull/13704)
+    - Async S3 Caching support (4x RPS improvement) - [PR #13852](https://github.com/BerriAI/litellm/pull/13852), s/o @[michal-otmianowski](https://github.com/michal-otmianowski)
+- **Model Group header forwarding**
+    - reuse same logic as global header forwarding - [PR #13741](https://github.com/BerriAI/litellm/pull/13741)
+    - add support for hosted_vllm on UI - [PR #13885](https://github.com/BerriAI/litellm/pull/13885)
+- **Performance**
+    - Improve LiteLLM Python SDK RPS by +200 RPS (braintrust import + aiohttp transport fixes) - [PR #13839](https://github.com/BerriAI/litellm/pull/13839)
+    - Use O(1) Set lookups for model routing - [PR #13879](https://github.com/BerriAI/litellm/pull/13879)
+    - Reduce Significant CPU overhead from litellm_logging.py - [PR #13895](https://github.com/BerriAI/litellm/pull/13895)
+    - Improvements for Async Success Handler (Logging Callbacks) - Approx +130 RPS - [PR #13905](https://github.com/BerriAI/litellm/pull/13905)
+
+
+## General Proxy Improvements
+#### Bugs
+
+- **SDK**
+    - Fix litellm compatibility with newest release of openAI (>v1.100.0) - [PR #13728](https://github.com/BerriAI/litellm/pull/13728)
+- **Helm**
+    - Add possibility to configure resources for migrations-job - [PR #13617](https://github.com/BerriAI/litellm/pull/13617)
+    - Ensure Helm chart auto generated master keys follow sk-xxxx format - [PR #13871](https://github.com/BerriAI/litellm/pull/13871)
+    - Enhance database configuration: add support for optional endpointKey - [PR #13763](https://github.com/BerriAI/litellm/pull/13763)
+- **Rate Limits**
+    - fixing descriptor/response size mismatch on parallel_request_limiter_v3 - [PR #13863](https://github.com/BerriAI/litellm/pull/13863), s/o  @[luizrennocosta](https://github.com/luizrennocosta)
+- **Non-root**
+    - fix permission access on prisma migrate in non-root image - [PR #13848](https://github.com/BerriAI/litellm/pull/13848), s/o @[Ithanil](https://github.com/Ithanil)

docs/my-website/release_notes/v1.76.3-stable/index.md

@@ -0,0 +1,282 @@
+---
+title: "v1.76.3-stable - Performance, Video Generation & CloudZero Integration"
+slug: "v1-76-3"
+date: 2025-09-06T10:00:00
+authors:
+  - name: Krrish Dholakia
+    title: CEO, LiteLLM
+    url: https://www.linkedin.com/in/krish-d/
+    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
+  - name: Ishaan Jaffer
+    title: CTO, LiteLLM
+    url: https://www.linkedin.com/in/reffajnaahsi/
+    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+
+hide_table_of_contents: false
+---
+
+import Image from '@theme/IdealImage';
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+## Deploy this version
+
+<Tabs>
+<TabItem value="docker" label="Docker">
+
+``` showLineNumbers title="docker run litellm"
+docker run \
+-e STORE_MODEL_IN_DB=True \
+-p 4000:4000 \
+ghcr.io/berriai/litellm:v1.76.3
+```
+</TabItem>
+
+<TabItem value="pip" label="Pip">
+
+``` showLineNumbers title="pip install litellm"
+pip install litellm==1.76.3
+```
+
+</TabItem>
+</Tabs>
+
+---
+
+## Key Highlights
+
+- **Major Performance Improvements** +400 RPS when using correct amount of workers + CPU cores combination
+- **Video Generation Support** - Added Google AI Studio  and Vertex AI Veo Video Generation through LiteLLM Pass through routes
+- **CloudZero Integration** - New cost tracking integration for exporting LiteLLM Usage and Spend data to CloudZero. 
+
+## Major Changes 
+- **Performance Optimization**: LiteLLM Proxy now achieves +400 RPS when using correct amount of CPU cores - [PR #14153](https://github.com/BerriAI/litellm/pull/14153), [PR #14242](https://github.com/BerriAI/litellm/pull/14242)
+  
+  By default, LiteLLM will now use `num_workers = os.cpu_count()` to achieve optimal performance. 
+  
+  **Override Options:**
+  
+  Set environment variable:
+  ```bash
+  DEFAULT_NUM_WORKERS_LITELLM_PROXY=1
+  ```
+  
+  Or start LiteLLM Proxy with:
+  ```bash
+  litellm --num_workers 1
+  ```
+
+- **Security Fix**: Fixed memory_usage_in_mem_cache cache endpoint vulnerability - [PR #14229](https://github.com/BerriAI/litellm/pull/14229)
+
+---
+
+## Performance Improvements
+
+This release includes significant performance optimizations. On our internal benchmarks we saw 1 instance get +400 RPS when using correct amount of  workers + CPU cores combination.
+
+- **+400 RPS Performance Boost** - LiteLLM Proxy now uses correct amount of CPU cores for optimal performance - [PR #14153](https://github.com/BerriAI/litellm/pull/14153)
+- **Default CPU Workers** - Changed DEFAULT_NUM_WORKERS_LITELLM_PROXY default to number of CPUs - [PR #14242](https://github.com/BerriAI/litellm/pull/14242)
+
+
+---
+
+## New Models / Updated Models
+
+#### New Model Support
+
+| Provider    | Model                                  | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Features |
+| ----------- | -------------------------------------- | -------------- | ------------------- | -------------------- | -------- |
+| OpenRouter | `openrouter/openai/gpt-4.1` | 1M | $2.00 | $8.00 | Chat completions with vision |
+| OpenRouter | `openrouter/openai/gpt-4.1-mini` | 1M | $0.40 | $1.60 | Efficient chat completions |
+| OpenRouter | `openrouter/openai/gpt-4.1-nano` | 1M | $0.10 | $0.40 | Ultra-efficient chat |
+| Vertex AI | `vertex_ai/openai/gpt-oss-20b-maas` | 131K | $0.075 | $0.30 | Reasoning support |
+| Vertex AI | `vertex_ai/openai/gpt-oss-120b-maas` | 131K | $0.15 | $0.60 | Advanced reasoning |
+| Gemini | `gemini/veo-3.0-generate-preview` | 1K | - | $0.75/sec | Video generation |
+| Gemini | `gemini/veo-3.0-fast-generate-preview` | 1K | - | $0.40/sec | Fast video generation |
+| Gemini | `gemini/veo-2.0-generate-001` | 1K | - | $0.35/sec | Video generation |
+| Volcengine | `doubao-embedding-large` | 4K | Free | Free | 2048-dim embeddings |
+| Together AI | `together_ai/deepseek-ai/DeepSeek-V3.1` | 128K | $0.60 | $1.70 | Reasoning support |
+
+#### Features
+
+- **[Google Gemini](../../docs/providers/gemini)**
+    - Added 'thoughtSignature' support via 'thinking_blocks' - [PR #14122](https://github.com/BerriAI/litellm/pull/14122)
+    - Added support for reasoning_effort='minimal' for Gemini models - [PR #14262](https://github.com/BerriAI/litellm/pull/14262)
+- **[OpenRouter](../../docs/providers/openrouter)**
+    - Added GPT-4.1 model family - [PR #14101](https://github.com/BerriAI/litellm/pull/14101)
+- **[Groq](../../docs/providers/groq)**
+    - Added support for reasoning_effort parameter - [PR #14207](https://github.com/BerriAI/litellm/pull/14207)
+- **[X.AI](../../docs/providers/xai)**
+    - Fixed XAI cost calculation - [PR #14127](https://github.com/BerriAI/litellm/pull/14127)
+- **[Vertex AI](../../docs/providers/vertex)**
+    - Added support for GPT-OSS models on Vertex AI - [PR #14184](https://github.com/BerriAI/litellm/pull/14184)
+    - Added additionalProperties to Vertex AI Schema definition - [PR #14252](https://github.com/BerriAI/litellm/pull/14252)
+- **[VLLM](../../docs/providers/vllm)**
+    - Handle output parsing responses API output - [PR #14121](https://github.com/BerriAI/litellm/pull/14121)
+- **[Ollama](../../docs/providers/ollama)**
+    - Added unified 'thinking' param support via `reasoning_content` - [PR #14121](https://github.com/BerriAI/litellm/pull/14121)
+- **[Anthropic](../../docs/providers/anthropic)**
+    - Added supported text field to anthropic citation response - [PR #14126](https://github.com/BerriAI/litellm/pull/14126)
+- **[OCI Provider](../../docs/providers/oci)**
+    - Handle assistant messages with both content and tool_calls - [PR #14171](https://github.com/BerriAI/litellm/pull/14171)
+- **[Bedrock](../../docs/providers/bedrock)**
+    - Fixed structure output - [PR #14130](https://github.com/BerriAI/litellm/pull/14130)
+    - Added initial support for Bedrock Batches API - [PR #14190](https://github.com/BerriAI/litellm/pull/14190)
+- **[Databricks](../../docs/providers/databricks)**
+    - Added support for anthropic citation API in Databricks - [PR #14077](https://github.com/BerriAI/litellm/pull/14077)
+
+### Bug Fixes
+- **[Google Gemini (Google AI Studio + Vertex AI)](../../docs/providers/gemini)**
+    - Fixed Gemini 2.5 Pro schema validation with OpenAI-style type arrays in tools - [PR #14154](https://github.com/BerriAI/litellm/pull/14154)
+    - Fixed Gemini Tool Calling empty enum property - [PR #14155](https://github.com/BerriAI/litellm/pull/14155)
+
+#### New Provider Support
+
+- **[Volcengine](../../docs/providers/volcengine)**
+    - Added Volcengine embedding module with handler and transformation logic - [PR #14028](https://github.com/BerriAI/litellm/pull/14028)
+
+---
+
+## LLM API Endpoints
+
+#### Features
+
+- **[Images API](../../docs/image_generation)**
+    - Added pass through image generation and image editing on OpenAI - [PR #14292](https://github.com/BerriAI/litellm/pull/14292)
+    - Support extra_body parameter for image generation - [PR #14211](https://github.com/BerriAI/litellm/pull/14211)
+- **[Responses API](../../docs/response_api)**
+    - Fixed response API for reasoning item in input for litellm proxy - [PR #14200](https://github.com/BerriAI/litellm/pull/14200)
+    - Added structured output for SDK - [PR #14206](https://github.com/BerriAI/litellm/pull/14206)
+- **[Bedrock Passthrough](../../docs/pass_through/bedrock)**
+    - Support AWS_BEDROCK_RUNTIME_ENDPOINT on bedrock passthrough - [PR #14156](https://github.com/BerriAI/litellm/pull/14156)
+- **[Google AI Studio Passthrough](../../docs/pass_through/google_ai_studio)**
+    - Allow using Veo Video Generation through LiteLLM Pass through routes - [PR #14228](https://github.com/BerriAI/litellm/pull/14228)
+- **General**
+    - Added support for safety_identifier parameter in chat.completions.create - [PR #14174](https://github.com/BerriAI/litellm/pull/14174)
+    - Fixed misclassified 500 error on invalid image_url in /chat/completions request - [PR #14149](https://github.com/BerriAI/litellm/pull/14149)
+    - Fixed token count error for Gemini CLI - [PR #14133](https://github.com/BerriAI/litellm/pull/14133)
+
+#### Bugs
+
+- **General**
+    - Remove "/" or ":" from model name when being used as h11 header name - [PR #14191](https://github.com/BerriAI/litellm/pull/14191)
+    - Bug fix for openai.gpt-oss when using reasoning_effort parameter - [PR #14300](https://github.com/BerriAI/litellm/pull/14300)
+
+---
+
+## Spend Tracking, Budgets and Rate Limiting
+
+### Features
+    - Added header support for spend_logs_metadata - [PR #14186](https://github.com/BerriAI/litellm/pull/14186)
+    - Litellm passthrough cost tracking for chat completion - [PR #14256](https://github.com/BerriAI/litellm/pull/14256)
+
+### Bug Fixes
+    - Fixed TPM Rate Limit Bug - [PR #14237](https://github.com/BerriAI/litellm/pull/14237)
+    - Fixed Key Budget not resets at expectable times - [PR #14241](https://github.com/BerriAI/litellm/pull/14241)
+
+
+
+## Management Endpoints / UI
+
+#### Features
+
+- **UI Improvements**
+    - Logs page screen size fixed - [PR #14135](https://github.com/BerriAI/litellm/pull/14135)
+    - Create Organization Tooltip added on Success - [PR #14132](https://github.com/BerriAI/litellm/pull/14132)
+    - Back to Keys should say Back to Logs - [PR #14134](https://github.com/BerriAI/litellm/pull/14134)
+    - Add client side pagination on All Models table - [PR #14136](https://github.com/BerriAI/litellm/pull/14136)
+    - Model Filters UI improvement - [PR #14131](https://github.com/BerriAI/litellm/pull/14131)
+    - Remove table filter on user info page - [PR #14169](https://github.com/BerriAI/litellm/pull/14169)
+    - Team name badge added on the User Details - [PR #14003](https://github.com/BerriAI/litellm/pull/14003)
+    - Fix: Log page parameter passing error - [PR #14193](https://github.com/BerriAI/litellm/pull/14193)
+- **Authentication & Authorization**
+    - Support for ES256/ES384/ES512 and EdDSA JWT verification - [PR #14118](https://github.com/BerriAI/litellm/pull/14118)
+    - Ensure `team_id` is a required field for generating service account keys - [PR #14270](https://github.com/BerriAI/litellm/pull/14270)
+
+#### Bugs
+
+- **General**
+    - Validate store model in db setting - [PR #14269](https://github.com/BerriAI/litellm/pull/14269)
+
+---
+
+## Logging / Guardrail Integrations
+
+#### Features
+
+- **[Datadog](../../docs/proxy/logging#datadog)**
+    - Ensure `apm_id` is set on DD LLM Observability traces - [PR #14272](https://github.com/BerriAI/litellm/pull/14272)
+- **[Braintrust](../../docs/proxy/logging#braintrust)**
+    - Fix logging when OTEL is enabled - [PR #14122](https://github.com/BerriAI/litellm/pull/14122)
+- **[OTEL](../../docs/proxy/logging#otel)**
+    - Optional Metrics and Logs following semantic conventions - [PR #14179](https://github.com/BerriAI/litellm/pull/14179)
+- **[Slack Alerting](../../docs/proxy/alerting)**
+    - Added alert type to alert message to slack for easier handling - [PR #14176](https://github.com/BerriAI/litellm/pull/14176)
+
+#### Guardrails
+    - Added guardrail to the Anthropic API endpoint - [PR #14107](https://github.com/BerriAI/litellm/pull/14107)
+
+#### New Integration
+
+- **[CloudZero](../../docs/proxy/cost_tracking)**
+    - LiteLLM x CloudZero Integration for Cost Tracking - [PR #14296](https://github.com/BerriAI/litellm/pull/14296)
+
+---
+
+## Performance / Loadbalancing / Reliability improvements
+
+#### Features
+
+- **Performance**
+    - LiteLLM Proxy: +400 RPS when using correct amount of CPU cores - [PR #14153](https://github.com/BerriAI/litellm/pull/14153)
+    - Allow using `x-litellm-stream-timeout` header for stream timeout in requests - [PR #14147](https://github.com/BerriAI/litellm/pull/14147)
+    - Change DEFAULT_NUM_WORKERS_LITELLM_PROXY default to number CPUs - [PR #14242](https://github.com/BerriAI/litellm/pull/14242)
+- **Monitoring**
+    - Added Prometheus missing metrics - [PR #14139](https://github.com/BerriAI/litellm/pull/14139)
+- **Timeout**
+    - **Stream Timeout Control** - Allow using `x-litellm-stream-timeout` header for stream timeout in requests - [PR #14147](https://github.com/BerriAI/litellm/pull/14147)
+- **Routing**
+    - Fixed x-litellm-tags not routing with Responses API - [PR #14289](https://github.com/BerriAI/litellm/pull/14289)
+
+#### Bugs
+
+- **Security**
+    - Fixed memory_usage_in_mem_cache cache endpoint vulnerability - [PR #14229](https://github.com/BerriAI/litellm/pull/14229)
+
+---
+
+## General Proxy Improvements
+
+#### Features
+
+- **SCIM Support**
+    - Added better SCIM debugging - [PR #14221](https://github.com/BerriAI/litellm/pull/14221)
+    - Bug fixes for handling SCIM Group Memberships - [PR #14226](https://github.com/BerriAI/litellm/pull/14226)
+- **Kubernetes**
+    - Added optional PodDisruptionBudget for litellm proxy - [PR #14093](https://github.com/BerriAI/litellm/pull/14093)
+- **Error Handling**
+    - Add model to azure error message - [PR #14294](https://github.com/BerriAI/litellm/pull/14294)
+
+---
+
+## New Contributors
+* @iabhi4 made their first contribution in [PR #14093](https://github.com/BerriAI/litellm/pull/14093)
+* @zainhas made their first contribution in [PR #14087](https://github.com/BerriAI/litellm/pull/14087)
+* @LifeDJIK made their first contribution in [PR #14146](https://github.com/BerriAI/litellm/pull/14146)
+* @retanoj made their first contribution in [PR #14133](https://github.com/BerriAI/litellm/pull/14133)
+* @zhxlp made their first contribution in [PR #14193](https://github.com/BerriAI/litellm/pull/14193)
+* @kayoch1n made their first contribution in [PR #14191](https://github.com/BerriAI/litellm/pull/14191)
+* @kutsushitaneko made their first contribution in [PR #14171](https://github.com/BerriAI/litellm/pull/14171)
+* @mjmendo made their first contribution in [PR #14176](https://github.com/BerriAI/litellm/pull/14176)
+* @HarshavardhanK made their first contribution in [PR #14213](https://github.com/BerriAI/litellm/pull/14213)
+* @eycjur made their first contribution in [PR #14207](https://github.com/BerriAI/litellm/pull/14207)
+* @22mSqRi made their first contribution in [PR #14241](https://github.com/BerriAI/litellm/pull/14241)
+* @onlylhf made their first contribution in [PR #14028](https://github.com/BerriAI/litellm/pull/14028)
+* @btpemercier made their first contribution in [PR #11319](https://github.com/BerriAI/litellm/pull/11319)
+* @tremlin made their first contribution in [PR #14287](https://github.com/BerriAI/litellm/pull/14287)
+* @TobiMayr made their first contribution in [PR #14262](https://github.com/BerriAI/litellm/pull/14262)
+* @Eitan1112 made their first contribution in [PR #14252](https://github.com/BerriAI/litellm/pull/14252)
+
+---
+
+## **[Full Changelog](https://github.com/BerriAI/litellm/compare/v1.76.1-nightly...v1.76.3-nightly)**

enterprise/litellm_enterprise/integrations/prometheus.py

@@ -95,13 +95,14 @@ class PrometheusLogger(CustomLogger):
             self.litellm_llm_api_time_to_first_token_metric = self._histogram_factory(
                 "litellm_llm_api_time_to_first_token_metric",
                 "Time to first token for a models LLM API call",
-                labelnames=[
-                    "model",
-                    "hashed_api_key",
-                    "api_key_alias",
-                    "team",
-                    "team_alias",
-                ],
+                # labelnames=[
+                #     "model",
+                #     "hashed_api_key",
+                #     "api_key_alias",
+                #     "team",
+                #     "team_alias",
+                # ],
+                labelnames=self.get_labels_for_metric("litellm_llm_api_time_to_first_token_metric"),
                 buckets=LATENCY_BUCKETS,
             )
 
@@ -109,15 +110,7 @@ class PrometheusLogger(CustomLogger):
             self.litellm_spend_metric = self._counter_factory(
                 "litellm_spend_metric",
                 "Total spend on LLM requests",
-                labelnames=[
-                    "end_user",
-                    "hashed_api_key",
-                    "api_key_alias",
-                    "model",
-                    "team",
-                    "team_alias",
-                    "user",
-                ],
+                labelnames=self.get_labels_for_metric("litellm_spend_metric"),
             )
 
             # Counter for total_output_tokens
@@ -243,25 +236,18 @@ class PrometheusLogger(CustomLogger):
                 labelnames=["api_provider"],
             )
 
-            # Get all keys
-            _logged_llm_labels = [
-                UserAPIKeyLabelNames.v2_LITELLM_MODEL_NAME.value,
-                UserAPIKeyLabelNames.MODEL_ID.value,
-                UserAPIKeyLabelNames.API_BASE.value,
-                UserAPIKeyLabelNames.API_PROVIDER.value,
-            ]
-
             # Metric for deployment state
             self.litellm_deployment_state = self._gauge_factory(
                 "litellm_deployment_state",
                 "LLM Deployment Analytics - The state of the deployment: 0 = healthy, 1 = partial outage, 2 = complete outage",
-                labelnames=_logged_llm_labels,
+                labelnames=self.get_labels_for_metric("litellm_deployment_state")
             )
 
             self.litellm_deployment_cooled_down = self._counter_factory(
                 "litellm_deployment_cooled_down",
                 "LLM Deployment Analytics - Number of times a deployment has been cooled down by LiteLLM load balancing logic. exception_status is the status of the exception that caused the deployment to be cooled down",
-                labelnames=_logged_llm_labels + [EXCEPTION_STATUS],
+                # labelnames=_logged_llm_labels + [EXCEPTION_STATUS],
+                labelnames=self.get_labels_for_metric("litellm_deployment_cooled_down")
             )
 
             self.litellm_deployment_success_responses = self._counter_factory(
@@ -327,6 +313,7 @@ class PrometheusLogger(CustomLogger):
                 documentation="deprecated - use litellm_proxy_total_requests_metric. Total number of LLM calls to litellm - track total per API Key, team, user",
                 labelnames=self.get_labels_for_metric("litellm_requests_metric"),
             )
+
         except Exception as e:
             print_verbose(f"Got exception on init prometheus client {str(e)}")
             raise e

litellm/__init__.py

@@ -67,6 +67,7 @@ from litellm.constants import (
     bedrock_embedding_models,
     known_tokenizer_config,
     BEDROCK_INVOKE_PROVIDERS_LITERAL,
+    BEDROCK_CONVERSE_MODELS,
     DEFAULT_MAX_TOKENS,
     DEFAULT_SOFT_BUDGET,
     DEFAULT_ALLOWED_FAILS,
@@ -145,8 +146,11 @@ _custom_logger_compatible_callbacks_literal = Literal[
     "aws_sqs",
     "vector_store_pre_call_hook",
     "dotprompt",
+    "cloudzero",
 ]
-configured_cold_storage_logger: Optional[_custom_logger_compatible_callbacks_literal] = None
+configured_cold_storage_logger: Optional[
+    _custom_logger_compatible_callbacks_literal
+] = None
 logged_real_time_event_types: Optional[Union[List[str], Literal["*"]]] = None
 _known_custom_logger_compatible_callbacks: List = list(
     get_args(_custom_logger_compatible_callbacks_literal)
@@ -432,43 +436,10 @@ organization = None
 project = None
 config_path = None
 vertex_ai_safety_settings: Optional[dict] = None
-BEDROCK_CONVERSE_MODELS = [
-    "openai.gpt-oss-20b-1:0",
-    "openai.gpt-oss-120b-1:0",
-    "anthropic.claude-opus-4-1-20250805-v1:0",
-    "anthropic.claude-opus-4-20250514-v1:0",
-    "anthropic.claude-sonnet-4-20250514-v1:0",
-    "anthropic.claude-3-7-sonnet-20250219-v1:0",
-    "anthropic.claude-3-5-haiku-20241022-v1:0",
-    "anthropic.claude-3-5-sonnet-20241022-v2:0",
-    "anthropic.claude-3-5-sonnet-20240620-v1:0",
-    "anthropic.claude-3-opus-20240229-v1:0",
-    "anthropic.claude-3-sonnet-20240229-v1:0",
-    "anthropic.claude-3-haiku-20240307-v1:0",
-    "anthropic.claude-v2",
-    "anthropic.claude-v2:1",
-    "anthropic.claude-v1",
-    "anthropic.claude-instant-v1",
-    "ai21.jamba-instruct-v1:0",
-    "ai21.jamba-1-5-mini-v1:0",
-    "ai21.jamba-1-5-large-v1:0",
-    "meta.llama3-70b-instruct-v1:0",
-    "meta.llama3-8b-instruct-v1:0",
-    "meta.llama3-1-8b-instruct-v1:0",
-    "meta.llama3-1-70b-instruct-v1:0",
-    "meta.llama3-1-405b-instruct-v1:0",
-    "meta.llama3-70b-instruct-v1:0",
-    "mistral.mistral-large-2407-v1:0",
-    "mistral.mistral-large-2402-v1:0",
-    "mistral.mistral-small-2402-v1:0",
-    "meta.llama3-2-1b-instruct-v1:0",
-    "meta.llama3-2-3b-instruct-v1:0",
-    "meta.llama3-2-11b-instruct-v1:0",
-    "meta.llama3-2-90b-instruct-v1:0",
-]
 
 ####### COMPLETION MODELS ###################
-from typing import Set  
+from typing import Set
+
 open_ai_chat_completion_models: Set = set()
 open_ai_text_completion_models: Set = set()
 cohere_models: Set = set()
@@ -483,6 +454,7 @@ vertex_vision_models: Set = set()
 vertex_chat_models: Set = set()
 vertex_code_chat_models: Set = set()
 vertex_ai_image_models: Set = set()
+vertex_ai_video_models: Set = set()
 vertex_text_models: Set = set()
 vertex_code_text_models: Set = set()
 vertex_embedding_models: Set = set()
@@ -491,6 +463,7 @@ vertex_llama3_models: Set = set()
 vertex_deepseek_models: Set = set()
 vertex_ai_ai21_models: Set = set()
 vertex_mistral_models: Set = set()
+vertex_openai_models: Set = set()
 ai21_models: Set = set()
 ai21_chat_models: Set = set()
 nlp_cloud_models: Set = set()
@@ -544,6 +517,7 @@ recraft_models: Set = set()
 cometapi_models: Set = set()
 oci_models: Set = set()
 vercel_ai_gateway_models: Set = set()
+volcengine_models: Set = set()
 
 
 def is_bedrock_pricing_only_model(key: str) -> bool:
@@ -637,6 +611,12 @@ def add_known_models():
         elif value.get("litellm_provider") == "vertex_ai-image-models":
             key = key.replace("vertex_ai/", "")
             vertex_ai_image_models.add(key)
+        elif value.get("litellm_provider") == "vertex_ai-video-models":
+            key = key.replace("vertex_ai/", "")
+            vertex_ai_video_models.add(key)
+        elif value.get("litellm_provider") == "vertex_ai-openai_models":
+            key = key.replace("vertex_ai/", "")
+            vertex_openai_models.add(key)
         elif value.get("litellm_provider") == "ai21":
             if value.get("mode") == "chat":
                 ai21_chat_models.add(key)
@@ -748,6 +728,8 @@ def add_known_models():
             cometapi_models.add(key)
         elif value.get("litellm_provider") == "oci":
             oci_models.add(key)
+        elif value.get("litellm_provider") == "volcengine":
+            volcengine_models.add(key)
 
 
 add_known_models()
@@ -840,6 +822,7 @@ model_list = list(
     | cometapi_models
     | oci_models
     | vercel_ai_gateway_models
+    | volcengine_models
 )
 
 model_list_set = set(model_list)
@@ -860,7 +843,12 @@ models_by_provider: dict = {
     "openrouter": openrouter_models,
     "vercel_ai_gateway": vercel_ai_gateway_models,
     "datarobot": datarobot_models,
-    "vertex_ai": vertex_chat_models | vertex_text_models | vertex_anthropic_models | vertex_vision_models | vertex_language_models | vertex_deepseek_models,
+    "vertex_ai": vertex_chat_models
+    | vertex_text_models
+    | vertex_anthropic_models
+    | vertex_vision_models
+    | vertex_language_models
+    | vertex_deepseek_models,
     "ai21": ai21_models,
     "bedrock": bedrock_models | bedrock_converse_models,
     "petals": petals_models,
@@ -914,6 +902,7 @@ models_by_provider: dict = {
     "recraft": recraft_models,
     "cometapi": cometapi_models,
     "oci": oci_models,
+    "volcengine": volcengine_models,
 }
 
 # mapping for those models which have larger equivalents
@@ -1157,7 +1146,9 @@ from .llms.topaz.image_variations.transformation import TopazImageVariationConfi
 from litellm.llms.openai.completion.transformation import OpenAITextCompletionConfig
 from .llms.groq.chat.transformation import GroqChatConfig
 from .llms.voyage.embedding.transformation import VoyageEmbeddingConfig
-from .llms.voyage.embedding.transformation_contextual import VoyageContextualEmbeddingConfig
+from .llms.voyage.embedding.transformation_contextual import (
+    VoyageContextualEmbeddingConfig,
+)
 from .llms.infinity.embedding.transformation import InfinityEmbeddingConfig
 from .llms.azure_ai.chat.transformation import AzureAIStudioConfig
 from .llms.mistral.chat.transformation import MistralConfig
@@ -1221,7 +1212,9 @@ from .llms.jina_ai.embedding.transformation import JinaAIEmbeddingConfig
 from .llms.xai.chat.transformation import XAIChatConfig
 from .llms.xai.common_utils import XAIModelInfo
 from .llms.aiml.chat.transformation import AIMLChatConfig
-from .llms.volcengine import VolcEngineConfig
+from .llms.volcengine.chat.transformation import (
+    VolcEngineChatConfig as VolcEngineConfig,
+)
 from .llms.codestral.completion.transformation import CodestralTextCompletionConfig
 from .llms.azure.azure import (
     AzureOpenAIError,

litellm/batches/main.py

@@ -14,13 +14,15 @@ import asyncio
 import contextvars
 import os
 from functools import partial
-from typing import Any, Coroutine, Dict, Literal, Optional, Union
+from typing import Any, Coroutine, Dict, Literal, Optional, Union, cast
 
 import httpx
 
 import litellm
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.llms.azure.batches.handler import AzureBatchesAPI
+from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
+from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
 from litellm.llms.openai.openai import OpenAIBatchesAPI
 from litellm.llms.vertex_ai.batches.handler import VertexAIBatchPrediction
 from litellm.secret_managers.main import get_secret_str
@@ -31,13 +33,19 @@ from litellm.types.llms.openai import (
     RetrieveBatchRequest,
 )
 from litellm.types.router import GenericLiteLLMParams
-from litellm.types.utils import LiteLLMBatch
-from litellm.utils import client, get_litellm_params, supports_httpx_timeout
+from litellm.types.utils import LiteLLMBatch, LlmProviders
+from litellm.utils import (
+    ProviderConfigManager,
+    client,
+    get_litellm_params,
+    supports_httpx_timeout,
+)
 
 ####### ENVIRONMENT VARIABLES ###################
 openai_batches_instance = OpenAIBatchesAPI()
 azure_batches_instance = AzureBatchesAPI()
 vertex_ai_batches_instance = VertexAIBatchPrediction(gcs_bucket_name="")
+base_llm_http_handler = BaseLLMHTTPHandler()
 #################################################
 
 
@@ -46,7 +54,7 @@ async def acreate_batch(
     completion_window: Literal["24h"],
     endpoint: Literal["/v1/chat/completions", "/v1/embeddings", "/v1/completions"],
     input_file_id: str,
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock"] = "openai",
     metadata: Optional[Dict[str, str]] = None,
     extra_headers: Optional[Dict[str, str]] = None,
     extra_body: Optional[Dict[str, str]] = None,
@@ -94,7 +102,7 @@ def create_batch(
     completion_window: Literal["24h"],
     endpoint: Literal["/v1/chat/completions", "/v1/embeddings", "/v1/completions"],
     input_file_id: str,
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock"] = "openai",
     metadata: Optional[Dict[str, str]] = None,
     extra_headers: Optional[Dict[str, str]] = None,
     extra_body: Optional[Dict[str, str]] = None,
@@ -111,8 +119,8 @@ def create_batch(
         proxy_server_request = kwargs.get("proxy_server_request", None)
         model_info = kwargs.get("model_info", None)
         _is_async = kwargs.pop("acreate_batch", False) is True
-        litellm_params = get_litellm_params(**kwargs)
-        litellm_logging_obj: LiteLLMLoggingObj = kwargs.get("litellm_logging_obj", None)
+        litellm_params = dict(GenericLiteLLMParams(**kwargs))
+        litellm_logging_obj: LiteLLMLoggingObj = cast(LiteLLMLoggingObj, kwargs.get("litellm_logging_obj", None))
         ### TIMEOUT LOGIC ###
         timeout = optional_params.timeout or kwargs.get("request_timeout", 600) or 600
         litellm_logging_obj.update_environment_variables(
@@ -142,6 +150,7 @@ def create_batch(
             timeout = float(timeout)  # type: ignore
         elif timeout is None:
             timeout = 600.0
+        
 
         _create_batch_request = CreateBatchRequest(
             completion_window=completion_window,
@@ -151,6 +160,27 @@ def create_batch(
             extra_headers=extra_headers,
             extra_body=extra_body,
         )
+        provider_config = ProviderConfigManager.get_provider_batches_config(
+            model="",
+            provider=LlmProviders(custom_llm_provider),
+        )
+        if provider_config is not None:
+            response = base_llm_http_handler.create_batch(
+                provider_config=provider_config,
+                litellm_params=litellm_params,
+                create_batch_data=_create_batch_request,
+                headers=extra_headers or {},
+                api_base=optional_params.api_base,
+                api_key=optional_params.api_key,
+                logging_obj=litellm_logging_obj,
+                _is_async=_is_async,
+                client=client
+                if client is not None
+                and isinstance(client, (HTTPHandler, AsyncHTTPHandler))
+                else None,
+                timeout=timeout,
+            )
+            return response
         api_base: Optional[str] = None
         if custom_llm_provider == "openai":
             # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
@@ -322,20 +352,21 @@ def retrieve_batch(
     """
     try:
         optional_params = GenericLiteLLMParams(**kwargs)
-        litellm_logging_obj: LiteLLMLoggingObj = kwargs.get("litellm_logging_obj", None)
+        litellm_logging_obj: Optional[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj", None)
         ### TIMEOUT LOGIC ###
         timeout = optional_params.timeout or kwargs.get("request_timeout", 600) or 600
         litellm_params = get_litellm_params(
             custom_llm_provider=custom_llm_provider,
             **kwargs,
         )
-        litellm_logging_obj.update_environment_variables(
-            model=None,
-            user=None,
-            optional_params=optional_params.model_dump(),
-            litellm_params=litellm_params,
-            custom_llm_provider=custom_llm_provider,
-        )
+        if litellm_logging_obj is not None:
+            litellm_logging_obj.update_environment_variables(
+                model=None,
+                user=None,
+                optional_params=optional_params.model_dump(),
+                litellm_params=litellm_params,
+                custom_llm_provider=custom_llm_provider,
+            )
 
         if (
             timeout is not None

litellm/constants.py

@@ -14,7 +14,9 @@ DEFAULT_S3_BATCH_SIZE = int(os.getenv("DEFAULT_S3_BATCH_SIZE", 512))
 DEFAULT_SQS_FLUSH_INTERVAL_SECONDS = int(
     os.getenv("DEFAULT_SQS_FLUSH_INTERVAL_SECONDS", 10)
 )
-DEFAULT_NUM_WORKERS_LITELLM_PROXY = int(os.getenv("DEFAULT_NUM_WORKERS_LITELLM_PROXY", 4))
+DEFAULT_NUM_WORKERS_LITELLM_PROXY = int(
+    os.getenv("DEFAULT_NUM_WORKERS_LITELLM_PROXY", os.cpu_count() or 4)
+)
 DEFAULT_SQS_BATCH_SIZE = int(os.getenv("DEFAULT_SQS_BATCH_SIZE", 512))
 SQS_SEND_MESSAGE_ACTION = "SendMessage"
 SQS_API_VERSION = "2012-11-05"
@@ -49,6 +51,23 @@ SINGLE_DEPLOYMENT_TRAFFIC_FAILURE_THRESHOLD = int(
 DEFAULT_REASONING_EFFORT_DISABLE_THINKING_BUDGET = int(
     os.getenv("DEFAULT_REASONING_EFFORT_DISABLE_THINKING_BUDGET", 0)
 )
+
+# Gemini model-specific minimal thinking budget constants
+DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH = int(
+    os.getenv("DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH", 1)
+)
+DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_PRO = int(
+    os.getenv("DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_PRO", 128)
+)
+DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE = int(
+    os.getenv("DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE", 512)
+)
+
+# Generic fallback for unknown models
+DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET = int(
+    os.getenv("DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET", 128)
+)
+
 DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET = int(
     os.getenv("DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET", 1024)
 )
@@ -395,6 +414,7 @@ DEFAULT_CHAT_COMPLETION_PARAM_VALUES = {
     "reasoning_effort": None,
     "thinking": None,
     "web_search_options": None,
+    "safety_identifier": None,
 }
 
 openai_compatible_endpoints: List = [
@@ -745,6 +765,42 @@ BEDROCK_INVOKE_PROVIDERS_LITERAL = Literal[
     "deepseek_r1",
 ]
 
+BEDROCK_CONVERSE_MODELS = [
+    "openai.gpt-oss-20b-1:0",
+    "openai.gpt-oss-120b-1:0",
+    "anthropic.claude-opus-4-1-20250805-v1:0",
+    "anthropic.claude-opus-4-20250514-v1:0",
+    "anthropic.claude-sonnet-4-20250514-v1:0",
+    "anthropic.claude-3-7-sonnet-20250219-v1:0",
+    "anthropic.claude-3-5-haiku-20241022-v1:0",
+    "anthropic.claude-3-5-sonnet-20241022-v2:0",
+    "anthropic.claude-3-5-sonnet-20240620-v1:0",
+    "anthropic.claude-3-opus-20240229-v1:0",
+    "anthropic.claude-3-sonnet-20240229-v1:0",
+    "anthropic.claude-3-haiku-20240307-v1:0",
+    "anthropic.claude-v2",
+    "anthropic.claude-v2:1",
+    "anthropic.claude-v1",
+    "anthropic.claude-instant-v1",
+    "ai21.jamba-instruct-v1:0",
+    "ai21.jamba-1-5-mini-v1:0",
+    "ai21.jamba-1-5-large-v1:0",
+    "meta.llama3-70b-instruct-v1:0",
+    "meta.llama3-8b-instruct-v1:0",
+    "meta.llama3-1-8b-instruct-v1:0",
+    "meta.llama3-1-70b-instruct-v1:0",
+    "meta.llama3-1-405b-instruct-v1:0",
+    "meta.llama3-70b-instruct-v1:0",
+    "mistral.mistral-large-2407-v1:0",
+    "mistral.mistral-large-2402-v1:0",
+    "mistral.mistral-small-2402-v1:0",
+    "meta.llama3-2-1b-instruct-v1:0",
+    "meta.llama3-2-3b-instruct-v1:0",
+    "meta.llama3-2-11b-instruct-v1:0",
+    "meta.llama3-2-90b-instruct-v1:0",
+]
+
+
 open_ai_embedding_models: set = set(["text-embedding-ada-002"])
 cohere_embedding_models: set = set(
     [
@@ -834,6 +890,9 @@ AZURE_STORAGE_MSFT_VERSION = "2019-07-07"
 PROMETHEUS_BUDGET_METRICS_REFRESH_INTERVAL_MINUTES = int(
     os.getenv("PROMETHEUS_BUDGET_METRICS_REFRESH_INTERVAL_MINUTES", 5)
 )
+CLOUDZERO_EXPORT_INTERVAL_MINUTES = int(
+    os.getenv("CLOUDZERO_EXPORT_INTERVAL_MINUTES", 60)
+)
 MCP_TOOL_NAME_PREFIX = "mcp_tool"
 MAXIMUM_TRACEBACK_LINES_TO_LOG = int(os.getenv("MAXIMUM_TRACEBACK_LINES_TO_LOG", 100))
 
@@ -888,6 +947,8 @@ LITELLM_CLI_SESSION_TOKEN_PREFIX = "litellm-session-token"
 ########################### DB CRON JOB NAMES ###########################
 DB_SPEND_UPDATE_JOB_NAME = "db_spend_update_job"
 PROMETHEUS_EMIT_BUDGET_METRICS_JOB_NAME = "prometheus_emit_budget_metrics"
+CLOUDZERO_EXPORT_USAGE_DATA_JOB_NAME = "cloudzero_export_usage_data"
+CLOUDZERO_MAX_FETCHED_DATA_RECORDS = int(os.getenv("CLOUDZERO_MAX_FETCHED_DATA_RECORDS", 50000))
 SPEND_LOG_CLEANUP_JOB_NAME = "spend_log_cleanup"
 SPEND_LOG_RUN_LOOPS = int(os.getenv("SPEND_LOG_RUN_LOOPS", 500))
 SPEND_LOG_CLEANUP_BATCH_SIZE = int(os.getenv("SPEND_LOG_CLEANUP_BATCH_SIZE", 1000))

litellm/files/main.py

@@ -50,7 +50,7 @@ vertex_ai_files_instance = VertexAIFilesHandler()
 async def acreate_file(
     file: FileTypes,
     purpose: Literal["assistants", "batch", "fine-tune"],
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock"] = "openai",
     extra_headers: Optional[Dict[str, str]] = None,
     extra_body: Optional[Dict[str, str]] = None,
     **kwargs,
@@ -94,7 +94,7 @@ async def acreate_file(
 def create_file(
     file: FileTypes,
     purpose: Literal["assistants", "batch", "fine-tune"],
-    custom_llm_provider: Optional[Literal["openai", "azure", "vertex_ai"]] = None,
+    custom_llm_provider: Optional[Literal["openai", "azure", "vertex_ai", "bedrock"]] = None,
     extra_headers: Optional[Dict[str, str]] = None,
     extra_body: Optional[Dict[str, str]] = None,
     **kwargs,
@@ -109,7 +109,7 @@ def create_file(
     try:
         _is_async = kwargs.pop("acreate_file", False) is True
         optional_params = GenericLiteLLMParams(**kwargs)
-        litellm_params_dict = get_litellm_params(**kwargs)
+        litellm_params_dict = dict(**kwargs)
         logging_obj = cast(
             Optional[LiteLLMLoggingObj], kwargs.get("litellm_logging_obj")
         )

litellm/google_genai/adapters/handler.py

@@ -37,6 +37,10 @@ class GenerateContentToCompletionHandler:
 
         completion_kwargs: Dict[str, Any] = dict(completion_request)
 
+        # feed metadata for custom callback
+        if extra_kwargs is not None and "metadata" in extra_kwargs:
+            completion_kwargs["metadata"] = extra_kwargs["metadata"]
+
         if stream:
             completion_kwargs["stream"] = stream
 

litellm/images/main.py

@@ -90,12 +90,12 @@ async def aimage_generation(*args, **kwargs) -> ImageResponse:
             response = init_response
         elif asyncio.iscoroutine(init_response):
             response = await init_response  # type: ignore
-        
+
         if response is None:
             raise ValueError(
                 "Unable to get Image Response. Please pass a valid llm_provider."
             )
-        
+
         return response
     except Exception as e:
         custom_llm_provider = custom_llm_provider or "openai"
@@ -108,6 +108,8 @@ async def aimage_generation(*args, **kwargs) -> ImageResponse:
         )
 
 
+# fmt: off
+
 # Overload for when aimg_generation=True (returns Coroutine)
 @overload
 def image_generation(
@@ -119,7 +121,6 @@ def image_generation(
     size: Optional[str] = None,
     style: Optional[str] = None,
     user: Optional[str] = None,
-    input_fidelity: Optional[str] = None,
     timeout=600,  # default to 10 minutes
     api_key: Optional[str] = None,
     api_base: Optional[str] = None,
@@ -128,10 +129,11 @@ def image_generation(
     *,
     aimg_generation: Literal[True],
     **kwargs,
-) -> Coroutine[Any, Any, ImageResponse]:
+) -> Coroutine[Any, Any, ImageResponse]: 
     ...
 
 
+
 # Overload for when aimg_generation=False or not specified (returns ImageResponse)
 @overload
 def image_generation(
@@ -143,7 +145,6 @@ def image_generation(
     size: Optional[str] = None,
     style: Optional[str] = None,
     user: Optional[str] = None,
-    input_fidelity: Optional[str] = None,
     timeout=600,  # default to 10 minutes
     api_key: Optional[str] = None,
     api_base: Optional[str] = None,
@@ -152,9 +153,11 @@ def image_generation(
     *,
     aimg_generation: Literal[False] = False,
     **kwargs,
-) -> ImageResponse:
+) -> ImageResponse: 
     ...
 
+# fmt: on
+
 
 @client
 def image_generation(  # noqa: PLR0915
@@ -166,7 +169,6 @@ def image_generation(  # noqa: PLR0915
     size: Optional[str] = None,
     style: Optional[str] = None,
     user: Optional[str] = None,
-    input_fidelity: Optional[str] = None,
     timeout=600,  # default to 10 minutes
     api_key: Optional[str] = None,
     api_base: Optional[str] = None,
@@ -174,9 +176,9 @@ def image_generation(  # noqa: PLR0915
     custom_llm_provider=None,
     **kwargs,
 ) -> Union[
-        ImageResponse,
-        Coroutine[Any, Any, ImageResponse],
-    ]:
+    ImageResponse,
+    Coroutine[Any, Any, ImageResponse],
+]:
     """
     Maps the https://api.openai.com/v1/images/generations endpoint.
 
@@ -227,7 +229,6 @@ def image_generation(  # noqa: PLR0915
             "quality",
             "size",
             "style",
-            "input_fidelity",
         ]
         litellm_params = all_litellm_params
         default_params = openai_params + litellm_params
@@ -255,7 +256,6 @@ def image_generation(  # noqa: PLR0915
             size=size,
             style=style,
             user=user,
-            input_fidelity=input_fidelity,
             custom_llm_provider=custom_llm_provider,
             provider_config=image_generation_config,
             **non_default_params,
@@ -344,8 +344,10 @@ def image_generation(  # noqa: PLR0915
             litellm.LlmProviders.GEMINI,
         ):
             if image_generation_config is None:
-                raise ValueError(f"image generation config is not supported for {custom_llm_provider}")
-            
+                raise ValueError(
+                    f"image generation config is not supported for {custom_llm_provider}"
+                )
+
             return llm_http_handler.image_generation_handler(
                 api_key=api_key,
                 model=model,
@@ -360,6 +362,7 @@ def image_generation(  # noqa: PLR0915
             )
         elif custom_llm_provider == "azure_ai":
             from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
+
             api_base = AzureFoundryModelInfo.get_api_base(api_base)
             api_key = AzureFoundryModelInfo.get_api_key(api_key)
             if extra_headers is not None:
@@ -420,7 +423,7 @@ def image_generation(  # noqa: PLR0915
                 aimg_generation=aimg_generation,
                 client=client,
                 api_base=api_base,
-                api_key=api_key
+                api_key=api_key,
             )
         elif custom_llm_provider == "vertex_ai":
             vertex_ai_project = (
@@ -705,7 +708,7 @@ def image_edit(
         litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None)
         _is_async = kwargs.pop("async_call", False) is True
 
-        #add images / or return a single image
+        # add images / or return a single image
         images = image if isinstance(image, list) else [image]
 
         # get llm provider logic
@@ -716,11 +719,11 @@ def image_edit(
         )
 
         # get provider config
-        image_edit_provider_config: Optional[
-            BaseImageEditConfig
-        ] = ProviderConfigManager.get_provider_image_edit_config(
-            model=model,
-            provider=litellm.LlmProviders(custom_llm_provider),
+        image_edit_provider_config: Optional[BaseImageEditConfig] = (
+            ProviderConfigManager.get_provider_image_edit_config(
+                model=model,
+                provider=litellm.LlmProviders(custom_llm_provider),
+            )
         )
 
         if image_edit_provider_config is None:

litellm/integrations/SlackAlerting/slack_alerting.py

@@ -805,9 +805,9 @@ class SlackAlerting(CustomBatchLogger):
         ### UNIQUE CACHE KEY ###
         cache_key = provider + region_name
 
-        outage_value: Optional[ProviderRegionOutageModel] = (
-            await self.internal_usage_cache.async_get_cache(key=cache_key)
-        )
+        outage_value: Optional[
+            ProviderRegionOutageModel
+        ] = await self.internal_usage_cache.async_get_cache(key=cache_key)
 
         if (
             getattr(exception, "status_code", None) is None
@@ -1367,12 +1367,13 @@ Model Info:
         # Get the current timestamp
         current_time = datetime.now().strftime("%H:%M:%S")
         _proxy_base_url = os.getenv("PROXY_BASE_URL", None)
+        # Use .name if it's an enum, otherwise use as is
+        alert_type_name = getattr(alert_type, 'name', alert_type)
+        alert_type_formatted = f"Alert type: `{alert_type_name}`"
         if alert_type == "daily_reports" or alert_type == "new_model_added":
-            formatted_message = message
+            formatted_message = alert_type_formatted + message
         else:
-            formatted_message = (
-                f"Level: `{level}`\nTimestamp: `{current_time}`\n\nMessage: {message}"
-            )
+            formatted_message = f"{alert_type_formatted}\nLevel: `{level}`\nTimestamp: `{current_time}`\n\nMessage: {message}"
 
         if kwargs:
             for key, value in kwargs.items():
@@ -1388,9 +1389,9 @@ Model Info:
             self.alert_to_webhook_url is not None
             and alert_type in self.alert_to_webhook_url
         ):
-            slack_webhook_url: Optional[Union[str, List[str]]] = (
-                self.alert_to_webhook_url[alert_type]
-            )
+            slack_webhook_url: Optional[
+                Union[str, List[str]]
+            ] = self.alert_to_webhook_url[alert_type]
         elif self.default_webhook_url is not None:
             slack_webhook_url = self.default_webhook_url
         else:

litellm/integrations/braintrust_logging.py

@@ -1,13 +1,11 @@
 # What is this?
 ## Log success + failure events to Braintrust
 
-import copy
 import os
 from datetime import datetime
 from typing import Dict, Optional
 
 import httpx
-from pydantic import BaseModel
 
 import litellm
 from litellm import verbose_logger
@@ -24,7 +22,6 @@ API_BASE = "https://api.braintrustdata.com/v1"
 
 def get_utc_datetime():
     import datetime as dt
-    from datetime import datetime
 
     if hasattr(dt, "UTC"):
         return datetime.now(dt.UTC)  # type: ignore
@@ -45,9 +42,9 @@ class BraintrustLogger(CustomLogger):
             "Authorization": "Bearer " + self.api_key,
             "Content-Type": "application/json",
         }
-        self._project_id_cache: Dict[
-            str, str
-        ] = {}  # Cache mapping project names to IDs
+        self._project_id_cache: Dict[str, str] = (
+            {}
+        )  # Cache mapping project names to IDs
         self.global_braintrust_http_handler = get_async_httpx_client(
             llm_provider=httpxSpecialProvider.LoggingCallback
         )
@@ -108,43 +105,6 @@ class BraintrustLogger(CustomLogger):
         except httpx.HTTPStatusError as e:
             raise Exception(f"Failed to register project: {e.response.text}")
 
-    @staticmethod
-    def add_metadata_from_header(litellm_params: dict, metadata: dict) -> dict:
-        """
-        Adds metadata from proxy request headers to Braintrust logging if keys start with "braintrust_"
-        and overwrites litellm_params.metadata if already included.
-
-        For example if you want to append your trace to an existing `trace_id` via header, send
-        `headers: { ..., langfuse_existing_trace_id: your-existing-trace-id }` via proxy request.
-        """
-        if litellm_params is None:
-            return metadata
-
-        if litellm_params.get("proxy_server_request") is None:
-            return metadata
-
-        if metadata is None:
-            metadata = {}
-
-        proxy_headers = (
-            litellm_params.get("proxy_server_request", {}).get("headers", {}) or {}
-        )
-
-        for metadata_param_key in proxy_headers:
-            if metadata_param_key.startswith("braintrust"):
-                trace_param_key = metadata_param_key.replace("braintrust", "", 1)
-                if trace_param_key in metadata:
-                    verbose_logger.warning(
-                        f"Overwriting Braintrust `{trace_param_key}` from request header"
-                    )
-                else:
-                    verbose_logger.debug(
-                        f"Found Braintrust `{trace_param_key}` in request header"
-                    )
-                metadata[trace_param_key] = proxy_headers.get(metadata_param_key)
-
-        return metadata
-
     async def create_default_project_and_experiment(self):
         project = await self.global_braintrust_http_handler.post(
             f"{self.api_base}/project", headers=self.headers, json={"name": "litellm"}
@@ -169,7 +129,9 @@ class BraintrustLogger(CustomLogger):
         verbose_logger.debug("REACHES BRAINTRUST SUCCESS")
         try:
             litellm_call_id = kwargs.get("litellm_call_id")
+            standard_logging_object = kwargs.get("standard_logging_object", {})
             prompt = {"messages": kwargs.get("messages")}
+
             output = None
             choices = []
             if response_obj is not None and (
@@ -192,33 +154,13 @@ class BraintrustLogger(CustomLogger):
             ):
                 output = response_obj["data"]
 
-            litellm_params = kwargs.get("litellm_params", {})
-            metadata = (
-                litellm_params.get("metadata", {}) or {}
-            )  # if litellm_params['metadata'] == None
-            metadata = self.add_metadata_from_header(litellm_params, metadata)
-            clean_metadata = {}
-            try:
-                metadata = copy.deepcopy(
-                    metadata
-                )  # Avoid modifying the original metadata
-            except Exception:
-                new_metadata = {}
-                for key, value in metadata.items():
-                    if (
-                        isinstance(value, list)
-                        or isinstance(value, dict)
-                        or isinstance(value, str)
-                        or isinstance(value, int)
-                        or isinstance(value, float)
-                    ):
-                        new_metadata[key] = copy.deepcopy(value)
-                metadata = new_metadata
+            litellm_params = kwargs.get("litellm_params", {}) or {}
+            dynamic_metadata = litellm_params.get("metadata", {}) or {}
 
             # Get project_id from metadata or create default if needed
-            project_id = metadata.get("project_id")
+            project_id = dynamic_metadata.get("project_id")
             if project_id is None:
-                project_name = metadata.get("project_name")
+                project_name = dynamic_metadata.get("project_name")
                 project_id = (
                     self.get_project_id_sync(project_name) if project_name else None
                 )
@@ -229,8 +171,9 @@ class BraintrustLogger(CustomLogger):
                 project_id = self.default_project_id
 
             tags = []
-            if isinstance(metadata, dict):
-                for key, value in metadata.items():
+
+            if isinstance(dynamic_metadata, dict):
+                for key, value in dynamic_metadata.items():
                     # generate langfuse tags - Default Tags sent to Langfuse from LiteLLM Proxy
                     if (
                         litellm.langfuse_default_tags is not None
@@ -239,25 +182,12 @@ class BraintrustLogger(CustomLogger):
                     ):
                         tags.append(f"{key}:{value}")
 
-                    # clean litellm metadata before logging
-                    if key in [
-                        "headers",
-                        "endpoint",
-                        "caching_groups",
-                        "previous_models",
-                    ]:
-                        continue
-                    else:
-                        clean_metadata[key] = value
+                    if (
+                        isinstance(value, str) and key not in standard_logging_object
+                    ):  # support logging dynamic metadata to braintrust
+                        standard_logging_object[key] = value
 
             cost = kwargs.get("response_cost", None)
-            if cost is not None:
-                clean_metadata["litellm_response_cost"] = cost
-
-            # metadata.model is required for braintrust to calculate the "Estimated cost" metric
-            litellm_model = kwargs.get("model", None)
-            if litellm_model is not None:
-                clean_metadata["model"] = litellm_model
 
             metrics: Optional[dict] = None
             usage_obj = getattr(response_obj, "usage", None)
@@ -275,12 +205,12 @@ class BraintrustLogger(CustomLogger):
                 }
 
             # Allow metadata override for span name
-            span_name = metadata.get("span_name", "Chat Completion")
-            
+            span_name = dynamic_metadata.get("span_name", "Chat Completion")
+
             request_data = {
                 "id": litellm_call_id,
                 "input": prompt["messages"],
-                "metadata": clean_metadata,
+                "metadata": standard_logging_object,
                 "tags": tags,
                 "span_attributes": {"name": span_name, "type": "llm"},
             }
@@ -312,6 +242,7 @@ class BraintrustLogger(CustomLogger):
         verbose_logger.debug("REACHES BRAINTRUST SUCCESS")
         try:
             litellm_call_id = kwargs.get("litellm_call_id")
+            standard_logging_object = kwargs.get("standard_logging_object", {})
             prompt = {"messages": kwargs.get("messages")}
             output = None
             choices = []
@@ -336,32 +267,12 @@ class BraintrustLogger(CustomLogger):
                 output = response_obj["data"]
 
             litellm_params = kwargs.get("litellm_params", {})
-            metadata = (
-                litellm_params.get("metadata", {}) or {}
-            )  # if litellm_params['metadata'] == None
-            metadata = self.add_metadata_from_header(litellm_params, metadata)
-            clean_metadata = {}
-            new_metadata = {}
-            for key, value in metadata.items():
-                if (
-                    isinstance(value, list)
-                    or isinstance(value, str)
-                    or isinstance(value, int)
-                    or isinstance(value, float)
-                ):
-                    new_metadata[key] = value
-                elif isinstance(value, BaseModel):
-                    new_metadata[key] = value.model_dump_json()
-                elif isinstance(value, dict):
-                    for k, v in value.items():
-                        if isinstance(v, datetime):
-                            value[k] = v.isoformat()
-                    new_metadata[key] = value
+            dynamic_metadata = litellm_params.get("metadata", {}) or {}
 
             # Get project_id from metadata or create default if needed
-            project_id = metadata.get("project_id")
+            project_id = dynamic_metadata.get("project_id")
             if project_id is None:
-                project_name = metadata.get("project_name")
+                project_name = dynamic_metadata.get("project_name")
                 project_id = (
                     await self.get_project_id_async(project_name)
                     if project_name
@@ -374,8 +285,9 @@ class BraintrustLogger(CustomLogger):
                 project_id = self.default_project_id
 
             tags = []
-            if isinstance(metadata, dict):
-                for key, value in metadata.items():
+
+            if isinstance(dynamic_metadata, dict):
+                for key, value in dynamic_metadata.items():
                     # generate langfuse tags - Default Tags sent to Langfuse from LiteLLM Proxy
                     if (
                         litellm.langfuse_default_tags is not None
@@ -384,25 +296,12 @@ class BraintrustLogger(CustomLogger):
                     ):
                         tags.append(f"{key}:{value}")
 
-                    # clean litellm metadata before logging
-                    if key in [
-                        "headers",
-                        "endpoint",
-                        "caching_groups",
-                        "previous_models",
-                    ]:
-                        continue
-                    else:
-                        clean_metadata[key] = value
+                    if (
+                        isinstance(value, str) and key not in standard_logging_object
+                    ):  # support logging dynamic metadata to braintrust
+                        standard_logging_object[key] = value
 
             cost = kwargs.get("response_cost", None)
-            if cost is not None:
-                clean_metadata["litellm_response_cost"] = cost
-
-            # metadata.model is required for braintrust to calculate the "Estimated cost" metric
-            litellm_model = kwargs.get("model", None)
-            if litellm_model is not None:
-                clean_metadata["model"] = litellm_model
 
             metrics: Optional[dict] = None
             usage_obj = getattr(response_obj, "usage", None)
@@ -430,13 +329,13 @@ class BraintrustLogger(CustomLogger):
                     )
 
             # Allow metadata override for span name
-            span_name = metadata.get("span_name", "Chat Completion")
-            
+            span_name = dynamic_metadata.get("span_name", "Chat Completion")
+
             request_data = {
                 "id": litellm_call_id,
                 "input": prompt["messages"],
                 "output": output,
-                "metadata": clean_metadata,
+                "metadata": standard_logging_object,
                 "tags": tags,
                 "span_attributes": {"name": span_name, "type": "llm"},
             }

litellm/integrations/cloudzero/cloudzero.py

@@ -1,14 +1,15 @@
-import asyncio
 import os
-from datetime import datetime, timedelta
-from typing import Optional
+from datetime import datetime
+from typing import TYPE_CHECKING, Any, List, Optional, cast
 
+import litellm
 from litellm._logging import verbose_logger
 from litellm.integrations.custom_logger import CustomLogger
 
-from .cz_stream_api import CloudZeroStreamer
-from .database import LiteLLMDatabase
-from .transform import CBFTransformer
+if TYPE_CHECKING:
+    from apscheduler.schedulers.asyncio import AsyncIOScheduler
+else:
+    AsyncIOScheduler = Any
 
 
 class CloudZeroLogger(CustomLogger):
@@ -29,20 +30,80 @@ class CloudZeroLogger(CustomLogger):
         self.api_key = api_key or os.getenv("CLOUDZERO_API_KEY")
         self.connection_id = connection_id or os.getenv("CLOUDZERO_CONNECTION_ID") 
         self.timezone = timezone or os.getenv("CLOUDZERO_TIMEZONE", "UTC")
+        verbose_logger.debug(f"CloudZero Logger initialized with connection ID: {self.connection_id}, timezone: {self.timezone}")
 
-    async def export_usage_data(self, target_hour: datetime, limit: Optional[int] = 1000, operation: str = "replace_hourly"):
+    async def initialize_cloudzero_export_job(self):
         """
-        Exports the usage data for a specific hour to CloudZero.
+        Handler for initializing CloudZero export job.
 
-        - Reads spend logs from the DB for the specified hour
+        Runs when CloudZero logger starts up.
+
+        - If redis cache is available, we use the pod lock manager to acquire a lock and export the data.
+            - Ensures only one pod exports the data at a time.
+        - If redis cache is not available, we export the data directly.
+        """
+        from litellm.constants import (
+            CLOUDZERO_EXPORT_USAGE_DATA_JOB_NAME,
+        )
+        from litellm.proxy.proxy_server import proxy_logging_obj
+        pod_lock_manager = proxy_logging_obj.db_spend_update_writer.pod_lock_manager
+
+        # if using redis, ensure only one pod exports the data at a time
+        if pod_lock_manager and pod_lock_manager.redis_cache:
+            if await pod_lock_manager.acquire_lock(
+                cronjob_id=CLOUDZERO_EXPORT_USAGE_DATA_JOB_NAME
+            ):
+                try:
+                    await self._hourly_usage_data_export()
+                finally:
+                    await pod_lock_manager.release_lock(
+                        cronjob_id=CLOUDZERO_EXPORT_USAGE_DATA_JOB_NAME
+                    )
+        else:
+            # if not using redis, export the data directly
+            await self._hourly_usage_data_export()
+    
+    async def _hourly_usage_data_export(self):
+        """
+        Exports the hourly usage data to CloudZero.
+
+        Start time: 1 hour ago
+        End time: current time
+        """
+        from datetime import timedelta, timezone
+
+        from litellm.constants import CLOUDZERO_MAX_FETCHED_DATA_RECORDS
+        current_time_utc = datetime.now(timezone.utc)
+        one_hour_ago_utc = current_time_utc - timedelta(hours=1)
+        await self.export_usage_data(
+            limit=CLOUDZERO_MAX_FETCHED_DATA_RECORDS,
+            operation="replace_hourly",
+            start_time_utc=one_hour_ago_utc,
+            end_time_utc=current_time_utc
+        )
+
+
+    async def export_usage_data(
+        self, 
+        limit: Optional[int] = None, 
+        operation: str = "replace_hourly",
+        start_time_utc: Optional[datetime] = None,
+        end_time_utc: Optional[datetime] = None
+    ):
+        """
+        Exports the usage data to CloudZero.
+
+        - Reads data from the DB
         - Transforms the data to the CloudZero format
         - Sends the data to CloudZero
         
         Args:
-            target_hour: The specific hour to export data for
-            limit: Optional limit on number of records to export (default: 1000)
+            limit: Optional limit on number of records to export
             operation: CloudZero operation type ("replace_hourly" or "sum")
         """
+        from litellm.integrations.cloudzero.cz_stream_api import CloudZeroStreamer
+        from litellm.integrations.cloudzero.database import LiteLLMDatabase
+        from litellm.integrations.cloudzero.transform import CBFTransformer
         try:
             verbose_logger.debug("CloudZero Logger: Starting usage data export")
             
@@ -52,11 +113,27 @@ class CloudZeroLogger(CustomLogger):
                     "CloudZero configuration missing. Please set CLOUDZERO_API_KEY and CLOUDZERO_CONNECTION_ID environment variables."
                 )
 
-            # Fetch and transform data using helper
-            cbf_data = await self._fetch_cbf_data_for_hour(target_hour, limit)
+            # Initialize database connection and load data
+            database = LiteLLMDatabase()
+            verbose_logger.debug("CloudZero Logger: Loading usage data from database")
+            data = await database.get_usage_data(
+                limit=limit,
+                start_time_utc=start_time_utc,
+                end_time_utc=end_time_utc
+            )
+            
+            if data.is_empty():
+                verbose_logger.info("CloudZero Logger: No usage data found to export")
+                return
+
+            verbose_logger.debug(f"CloudZero Logger: Processing {len(data)} records")
+            
+            # Transform data to CloudZero CBF format
+            transformer = CBFTransformer()
+            cbf_data = transformer.transform(data)
             
             if cbf_data.is_empty():
-                verbose_logger.info("CloudZero Logger: No usage data found to export")
+                verbose_logger.warning("CloudZero Logger: No valid data after transformation")
                 return
 
             # Send data to CloudZero
@@ -75,60 +152,86 @@ class CloudZeroLogger(CustomLogger):
             verbose_logger.error(f"CloudZero Logger: Error exporting usage data: {str(e)}")
             raise
 
-    async def _fetch_cbf_data_for_hour(self, target_hour: datetime, limit: Optional[int] = 1000):
+    async def dry_run_export_usage_data(self, limit: Optional[int] = 10000):
         """
-        Helper method to fetch usage data for a specific hour and transform it to CloudZero CBF format.
+        Returns the data that would be exported to CloudZero without actually sending it.
         
         Args:
-            target_hour: The specific hour to fetch data for
-            limit: Optional limit on number of records to fetch (default: 1000)
+            limit: Limit number of records to display (default: 10000)
             
         Returns:
-            CBF formatted data ready for CloudZero ingestion
-        """
-        # Initialize database connection and load data
-        database = LiteLLMDatabase()
-        verbose_logger.debug(f"CloudZero Logger: Loading spend logs for hour {target_hour}")
-        data = await database.get_usage_data_for_hour(target_hour=target_hour, limit=limit)
-        
-        if data.is_empty():
-            verbose_logger.info("CloudZero Logger: No usage data found for the specified hour")
-            return data  # Return empty data
-
-        verbose_logger.debug(f"CloudZero Logger: Processing {len(data)} records")
-        
-        # Transform data to CloudZero CBF format
-        transformer = CBFTransformer()
-        cbf_data = transformer.transform(data)
-        
-        if cbf_data.is_empty():
-            verbose_logger.warning("CloudZero Logger: No valid data after transformation")
-            
-        return cbf_data
-
-    async def dry_run_export_usage_data(self, target_hour: datetime, limit: Optional[int] = 1000):
-        """
-        Only prints the spend logs data for a specific hour that would be exported to CloudZero.
-        
-        Args:
-            target_hour: The specific hour to export data for
-            limit: Limit number of records to display (default: 1000)
+            dict: Contains usage_data, cbf_data, and summary statistics
         """
+        from litellm.integrations.cloudzero.database import LiteLLMDatabase
+        from litellm.integrations.cloudzero.transform import CBFTransformer
         try:
             verbose_logger.debug("CloudZero Logger: Starting dry run export")
             
-            # Fetch and transform data using helper
-            cbf_data = await self._fetch_cbf_data_for_hour(target_hour, limit)
+            # Initialize database connection and load data
+            database = LiteLLMDatabase()
+            verbose_logger.debug("CloudZero Logger: Loading usage data for dry run")
+            data = await database.get_usage_data(limit=limit)
+            
+            if data.is_empty():
+                verbose_logger.warning("CloudZero Dry Run: No usage data found")
+                return {
+                    "usage_data": [],
+                    "cbf_data": [],
+                    "summary": {
+                        "total_records": 0,
+                        "total_cost": 0,
+                        "total_tokens": 0,
+                        "unique_accounts": 0,
+                        "unique_services": 0
+                    }
+                }
+
+            verbose_logger.debug(f"CloudZero Dry Run: Processing {len(data)} records...")
+            
+            # Convert usage data to dict format for response
+            usage_data_sample = data.head(50).to_dicts()  # Return first 50 rows
+
+            # Transform data to CloudZero CBF format
+            transformer = CBFTransformer()
+            cbf_data = transformer.transform(data)
             
             if cbf_data.is_empty():
-                verbose_logger.warning("CloudZero Dry Run: No usage data found")
-                return
+                verbose_logger.warning("CloudZero Dry Run: No valid data after transformation")
+                return {
+                    "usage_data": usage_data_sample,
+                    "cbf_data": [],
+                    "summary": {
+                        "total_records": len(usage_data_sample),
+                        "total_cost": sum(row.get('spend', 0) for row in usage_data_sample),
+                        "total_tokens": sum(row.get('prompt_tokens', 0) + row.get('completion_tokens', 0) for row in usage_data_sample),
+                        "unique_accounts": 0,
+                        "unique_services": 0
+                    }
+                }
 
-            # Display the transformed data on screen
-            self._display_cbf_data_on_screen(cbf_data)
+            # Convert CBF data to dict format for response
+            cbf_data_dict = cbf_data.to_dicts()
+            
+            # Calculate summary statistics
+            total_cost = sum(record.get('cost/cost', 0) for record in cbf_data_dict)
+            unique_accounts = len(set(record.get('resource/account', '') for record in cbf_data_dict if record.get('resource/account')))
+            unique_services = len(set(record.get('resource/service', '') for record in cbf_data_dict if record.get('resource/service')))
+            total_tokens = sum(record.get('usage/amount', 0) for record in cbf_data_dict)
             
             verbose_logger.info(f"CloudZero Logger: Dry run completed for {len(cbf_data)} records")
             
+            return {
+                "usage_data": usage_data_sample,
+                "cbf_data": cbf_data_dict,
+                "summary": {
+                    "total_records": len(cbf_data_dict),
+                    "total_cost": total_cost,
+                    "total_tokens": total_tokens,
+                    "unique_accounts": unique_accounts,
+                    "unique_services": unique_services
+                }
+            }
+            
         except Exception as e:
             verbose_logger.error(f"CloudZero Logger: Error in dry run export: {str(e)}")
             verbose_logger.error(f"CloudZero Dry Run Error: {str(e)}")
@@ -155,6 +258,11 @@ class CloudZeroLogger(CustomLogger):
         cbf_table = Table(show_header=True, header_style="bold cyan", box=SIMPLE, padding=(0, 1))
         cbf_table.add_column("time/usage_start", style="blue", no_wrap=False)
         cbf_table.add_column("cost/cost", style="green", justify="right", no_wrap=False)
+        cbf_table.add_column("entity_type", style="magenta", justify="right", no_wrap=False)
+        cbf_table.add_column("entity_id", style="magenta", justify="right", no_wrap=False)
+        cbf_table.add_column("team_id", style="cyan", no_wrap=False)
+        cbf_table.add_column("team_alias", style="cyan", no_wrap=False)
+        cbf_table.add_column("api_key_alias", style="yellow", no_wrap=False)
         cbf_table.add_column("usage/amount", style="yellow", justify="right", no_wrap=False)
         cbf_table.add_column("resource/id", style="magenta", no_wrap=False)
         cbf_table.add_column("resource/service", style="cyan", no_wrap=False)
@@ -170,10 +278,20 @@ class CloudZeroLogger(CustomLogger):
             resource_service = str(record.get('resource/service', 'N/A'))
             resource_account = str(record.get('resource/account', 'N/A'))
             resource_region = str(record.get('resource/region', 'N/A'))
+            entity_type = str(record.get('entity_type', 'N/A'))
+            entity_id = str(record.get('entity_id', 'N/A'))
+            team_id = str(record.get('resource/tag:team_id', 'N/A'))
+            team_alias = str(record.get('resource/tag:team_alias', 'N/A'))
+            api_key_alias = str(record.get('resource/tag:api_key_alias', 'N/A'))
 
             cbf_table.add_row(
                 time_usage_start,
                 cost_cost,
+                entity_type,
+                entity_id,
+                team_id,
+                team_alias,
+                api_key_alias,
                 usage_amount,
                 resource_id,
                 resource_service,
@@ -199,55 +317,33 @@ class CloudZeroLogger(CustomLogger):
         console.print(f"  Unique Services: {unique_services}")
 
         console.print("\n[dim]💡 This is the CloudZero CBF format ready for AnyCost ingestion[/dim]")
+    
+    @staticmethod
+    async def init_cloudzero_background_job(scheduler: AsyncIOScheduler):
+        """
+        Initialize the CloudZero background job.
 
-    async def init_background_job(self, redis_cache=None):
+        Starts the background job that exports the usage data to CloudZero every hour.
         """
-        Initialize a background job that exports usage data every hour.
-        Uses PodLockManager to ensure only one instance runs the export at a time.
+        from litellm.constants import CLOUDZERO_EXPORT_INTERVAL_MINUTES
+        from litellm.integrations.custom_logger import CustomLogger
         
-        Args:
-            redis_cache: Redis cache instance for pod locking
-        """
-        from litellm.proxy.db.db_transaction_queue.pod_lock_manager import (
-            PodLockManager,
+
+        prometheus_loggers: List[CustomLogger] = (
+            litellm.logging_callback_manager.get_custom_loggers_for_type(
+                callback_type=CloudZeroLogger
+            )
         )
-        
-        lock_manager = PodLockManager(redis_cache=redis_cache)
-        cronjob_id = "cloudzero_hourly_export"
-        
-        async def hourly_export_task():
-            while True:
-                try:
-                    # Calculate the previous completed hour
-                    now = datetime.utcnow()
-                    target_hour = now.replace(minute=0, second=0, microsecond=0)
-                    # Export data for the previous hour to ensure all data is available
-                    target_hour = target_hour - timedelta(hours=1)
-                    
-                    # Try to acquire lock
-                    lock_acquired = await lock_manager.acquire_lock(cronjob_id)
-                    
-                    if lock_acquired:
-                        try:
-                            verbose_logger.info(f"CloudZero Background Job: Starting export for hour {target_hour}")
-                            await self.export_usage_data(target_hour)
-                            verbose_logger.info(f"CloudZero Background Job: Completed export for hour {target_hour}")
-                        finally:
-                            # Always release the lock
-                            await lock_manager.release_lock(cronjob_id)
-                    else:
-                        verbose_logger.debug("CloudZero Background Job: Another instance is already running the export")
-                    
-                    # Wait until the next hour
-                    next_hour = (datetime.utcnow() + timedelta(hours=1)).replace(minute=0, second=0, microsecond=0)
-                    sleep_seconds = (next_hour - datetime.utcnow()).total_seconds()
-                    await asyncio.sleep(sleep_seconds)
-                    
-                except Exception as e:
-                    verbose_logger.error(f"CloudZero Background Job: Error in hourly export task: {str(e)}")
-                    # Sleep for 5 minutes before retrying on error
-                    await asyncio.sleep(300)
-        
-        # Start the background task
-        asyncio.create_task(hourly_export_task())
-        verbose_logger.debug("CloudZero Background Job: Initialized hourly export task")
+        # we need to get the initialized prometheus logger instance(s) and call logger.initialize_remaining_budget_metrics() on them
+        verbose_logger.debug("found %s cloudzero loggers", len(prometheus_loggers))
+        if len(prometheus_loggers) > 0:
+            cloudzero_logger = cast(CloudZeroLogger, prometheus_loggers[0])
+            verbose_logger.debug(
+                "Initializing remaining budget metrics as a cron job executing every %s minutes"
+                % CLOUDZERO_EXPORT_INTERVAL_MINUTES
+            )
+            scheduler.add_job(
+                cloudzero_logger.initialize_cloudzero_export_job,
+                "interval",
+                minutes=CLOUDZERO_EXPORT_INTERVAL_MINUTES
+            )

litellm/integrations/cloudzero/cz_resource_names.py

@@ -17,11 +17,16 @@
 """CloudZero Resource Names (CZRN) generation and validation for LiteLLM resources."""
 
 import re
+from enum import Enum
 from typing import Any, cast
 
 import litellm
 
 
+class CZEntityType(str, Enum):
+    TEAM = "team"
+
+
 class CZRNGenerator:
     """Generate CloudZero Resource Names (CZRNs) for LiteLLM resources."""
 
@@ -49,8 +54,8 @@ class CZRNGenerator:
         region = 'cross-region'
 
         # Use the actual entity_id (team_id or user_id) as the owner account
-        entity_id = row.get('entity_id', 'unknown')
-        owner_account_id = self._normalize_component(entity_id)
+        team_id = row.get('team_id', 'unknown')
+        owner_account_id = self._normalize_component(team_id)
 
         resource_type = 'llm-usage'
 

litellm/integrations/cloudzero/database.py

@@ -12,14 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-# CHANGELOG: 2025-07-23 - Added support for using LiteLLM_SpendLogs table for CBF mapping (ishaan-jaff)
 # CHANGELOG: 2025-01-19 - Refactored to use daily spend tables for proper CBF mapping (erik.peterson)
 # CHANGELOG: 2025-01-19 - Migrated from pandas to polars for database operations (erik.peterson)
 # CHANGELOG: 2025-01-19 - Initial database module for LiteLLM data extraction (erik.peterson)
 
 """Database connection and data extraction for LiteLLM."""
 
-from datetime import datetime, timedelta
+from datetime import datetime
 from typing import Any, Dict, Optional
 
 import polars as pl
@@ -37,61 +36,88 @@ class LiteLLMDatabase:
             )
         return prisma_client
 
-    async def get_usage_data_for_hour(self, target_hour: datetime, limit: Optional[int] = 1000) -> pl.DataFrame:
-        """Retrieve spend logs for a specific hour from LiteLLM_SpendLogs table with batching."""
+    async def get_usage_data(
+        self, 
+        limit: Optional[int] = None,
+        start_time_utc: Optional[datetime] = None,
+        end_time_utc: Optional[datetime] = None
+    ) -> pl.DataFrame:
+        """Retrieve usage data from LiteLLM daily user spend table."""
         client = self._ensure_prisma_client()
         
-        # Calculate hour range
-        hour_start = target_hour.replace(minute=0, second=0, microsecond=0)
-        hour_end = hour_start + timedelta(hours=1)
+        # Build WHERE clause for time filtering
+        where_conditions = []
+        if start_time_utc:
+            where_conditions.append(f"dus.created_at >= '{start_time_utc.isoformat()}'")
+        if end_time_utc:
+            where_conditions.append(f"dus.created_at <= '{end_time_utc.isoformat()}'")
         
-        # Convert datetime objects to ISO format strings for PostgreSQL compatibility
-        hour_start_str = hour_start.isoformat()
-        hour_end_str = hour_end.isoformat()
+        where_clause = ""
+        if where_conditions:
+            where_clause = "WHERE " + " AND ".join(where_conditions)
         
-        # Query to get spend logs for the specific hour
-        query = """
-        SELECT *
-        FROM "LiteLLM_SpendLogs"
-        WHERE "startTime" >= $1::timestamp 
-          AND "startTime" < $2::timestamp
-        ORDER BY "startTime" ASC
+        # Query to get user spend data with team information
+        query = f"""
+        SELECT
+            dus.id,
+            dus.date,
+            dus.user_id,
+            dus.api_key,
+            dus.model,
+            dus.model_group,
+            dus.custom_llm_provider,
+            dus.prompt_tokens,
+            dus.completion_tokens,
+            dus.spend,
+            dus.api_requests,
+            dus.successful_requests,
+            dus.failed_requests,
+            dus.cache_creation_input_tokens,
+            dus.cache_read_input_tokens,
+            dus.created_at,
+            dus.updated_at,
+            vt.team_id,
+            vt.key_alias as api_key_alias,
+            tt.team_alias
+        FROM "LiteLLM_DailyUserSpend" dus
+        LEFT JOIN "LiteLLM_VerificationToken" vt ON dus.api_key = vt.token
+        LEFT JOIN "LiteLLM_TeamTable" tt ON vt.team_id = tt.team_id
+        {where_clause}
+        ORDER BY dus.date DESC, dus.created_at DESC
         """
 
         if limit:
             query += f" LIMIT {limit}"
 
         try:
-            db_response = await client.db.query_raw(query, hour_start_str, hour_end_str)
-            # Convert the response to polars DataFrame
-            return pl.DataFrame(db_response) if db_response else pl.DataFrame()
+            db_response = await client.db.query_raw(query)
+            # Convert the response to polars DataFrame with full schema inference
+            # This prevents schema mismatch errors when data types vary across rows
+            return pl.DataFrame(db_response, infer_schema_length=None)
         except Exception as e:
-            raise Exception(f"Error retrieving spend logs for hour {target_hour}: {str(e)}")
-
+            raise Exception(f"Error retrieving usage data: {str(e)}")
 
     async def get_table_info(self) -> Dict[str, Any]:
-        """Get information about the LiteLLM_SpendLogs table."""
+        """Get information about the daily user spend table."""
         client = self._ensure_prisma_client()
         
         try:
-            # Get row count from SpendLogs table
-            spend_logs_count = await self._get_table_row_count('LiteLLM_SpendLogs')
+            # Get row count from user spend table
+            user_count = await self._get_table_row_count('LiteLLM_DailyUserSpend')
 
-            # Get column structure from spend logs table
+            # Get column structure from user spend table
             query = """
             SELECT column_name, data_type, is_nullable
             FROM information_schema.columns
-            WHERE table_name = 'LiteLLM_SpendLogs'
+            WHERE table_name = 'LiteLLM_DailyUserSpend'
             ORDER BY ordinal_position;
             """
             columns_response = await client.db.query_raw(query)
 
             return {
                 'columns': columns_response,
-                'row_count': spend_logs_count,
-                'table_breakdown': {
-                    'spend_logs': spend_logs_count
-                }
+                'row_count': user_count,
+                'table_name': 'LiteLLM_DailyUserSpend'
             }
         except Exception as e:
             raise Exception(f"Error getting table info: {str(e)}")

litellm/integrations/cloudzero/transform.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-# CHANGELOG: 2025-01-19 - Updated CBF transformation for LiteLLM_SpendLogs with hourly aggregation and team_id focus (ishaan-jaff)
+# CHANGELOG: 2025-01-19 - Updated CBF transformation for daily spend tables and proper CloudZero mapping (erik.peterson)
 # CHANGELOG: 2025-01-19 - Migrated from pandas to polars for data transformation (erik.peterson)
 # CHANGELOG: 2025-01-19 - Initial CBF transformation module (erik.peterson)
 
@@ -24,7 +24,7 @@ from typing import Any, Optional
 import polars as pl
 
 from ...types.integrations.cloudzero import CBFRecord
-from .cz_resource_names import CZRNGenerator
+from .cz_resource_names import CZEntityType, CZRNGenerator
 
 
 class CBFTransformer:
@@ -35,160 +35,99 @@ class CBFTransformer:
         self.czrn_generator = CZRNGenerator()
 
     def transform(self, data: pl.DataFrame) -> pl.DataFrame:
-        """Transform LiteLLM SpendLogs data to hourly aggregated CBF format."""
+        """Transform LiteLLM data to CBF format, dropping records with zero successful_requests or invalid CZRNs."""
         if data.is_empty():
             return pl.DataFrame()
 
-        # Filter out records with zero spend or invalid team_id
+        # Filter out records with zero successful_requests first
         original_count = len(data)
-        filtered_data = data.filter(
-            (pl.col('spend') > 0) & 
-            (pl.col('team_id').is_not_null()) &
-            (pl.col('team_id') != "")
-        )
-        filtered_count = len(filtered_data)
-        zero_spend_dropped = original_count - filtered_count
+        if 'successful_requests' in data.columns:
+            filtered_data = data.filter(pl.col('successful_requests') > 0)
+            zero_requests_dropped = original_count - len(filtered_data)
+        else:
+            filtered_data = data
+            zero_requests_dropped = 0
 
-        if filtered_data.is_empty():
-            from rich.console import Console
-            console = Console()
-            console.print(f"[yellow]⚠️  Dropped all {original_count:,} records due to zero spend or missing team_id[/yellow]")
-            return pl.DataFrame()
-
-        # Aggregate data to hourly level
-        hourly_aggregated = self._aggregate_to_hourly(filtered_data)
-        
-        # Transform aggregated data to CBF format
         cbf_data = []
         czrn_dropped_count = 0
-        
-        for row in hourly_aggregated.iter_rows(named=True):
+        filtered_count = len(filtered_data)
+
+        for row in filtered_data.iter_rows(named=True):
             try:
                 cbf_record = self._create_cbf_record(row)
+                # Only include the record if CZRN generation was successful
                 cbf_data.append(cbf_record)
             except Exception:
                 # Skip records that fail CZRN generation
                 czrn_dropped_count += 1
                 continue
 
-        # Print summary of transformations
+        # Print summary of dropped records if any
         from rich.console import Console
         console = Console()
 
-        if zero_spend_dropped > 0:
-            console.print(f"[yellow]⚠️  Dropped {zero_spend_dropped:,} of {original_count:,} records with zero spend or missing team_id[/yellow]")
+        if zero_requests_dropped > 0:
+            console.print(f"[yellow]⚠️  Dropped {zero_requests_dropped:,} of {original_count:,} records with zero successful_requests[/yellow]")
 
         if czrn_dropped_count > 0:
-            console.print(f"[yellow]⚠️  Dropped {czrn_dropped_count:,} of {len(hourly_aggregated):,} aggregated records due to invalid CZRNs[/yellow]")
+            console.print(f"[yellow]⚠️  Dropped {czrn_dropped_count:,} of {filtered_count:,} filtered records due to invalid CZRNs[/yellow]")
 
         if len(cbf_data) > 0:
-            console.print(f"[green]✓ Successfully transformed {len(cbf_data):,} hourly aggregated records[/green]")
+            console.print(f"[green]✓ Successfully transformed {len(cbf_data):,} records[/green]")
 
         return pl.DataFrame(cbf_data)
 
-    def _aggregate_to_hourly(self, data: pl.DataFrame) -> pl.DataFrame:
-        """Aggregate spend logs to hourly level by team_id, key_name, model, and tags."""
-        
-        # Extract hour from startTime, skip tags and metadata for now
-        data_with_hour = data.with_columns([
-            pl.col('startTime').str.to_datetime().dt.truncate('1h').alias('usage_hour'),
-            pl.lit([]).cast(pl.List(pl.String)).alias('parsed_tags'),  # Empty tags list for now
-            pl.lit("").alias('key_name')  # Empty key name for now
-        ])
-        
-        # Skip tag explosion for now - just add a null tag column
-        all_data = data_with_hour.with_columns([
-            pl.lit(None, dtype=pl.String).alias('tag')
-        ])
-        
-        # Group by hour, team_id, key_name, model, provider, and tag
-        aggregated = all_data.group_by([
-            'usage_hour',
-            'team_id', 
-            'key_name',
-            'model',
-            'model_group',
-            'custom_llm_provider',
-            'tag'
-        ]).agg([
-            pl.col('spend').sum().alias('total_spend'),
-            pl.col('total_tokens').sum().alias('total_tokens'),
-            pl.col('prompt_tokens').sum().alias('total_prompt_tokens'),
-            pl.col('completion_tokens').sum().alias('total_completion_tokens'),
-            pl.col('request_id').count().alias('request_count'),
-            pl.col('api_key').first().alias('api_key_sample'),  # Keep one for reference
-            pl.col('status').filter(pl.col('status') == 'success').count().alias('successful_requests'),
-            pl.col('status').filter(pl.col('status') != 'success').count().alias('failed_requests')
-        ])
-        return aggregated
-
-
     def _create_cbf_record(self, row: dict[str, Any]) -> CBFRecord:
-        """Create a single CBF record from aggregated hourly spend data."""
+        """Create a single CBF record from LiteLLM daily spend row."""
 
-        # Helper function to extract scalar values from polars data
-        def extract_scalar(value):
-            if hasattr(value, 'item') and not isinstance(value, (str, int, float, bool)):
-                return value.item() if value is not None else None
-            return value
+        # Parse date (daily spend tables use date strings like '2025-04-19')
+        usage_date = self._parse_date(row.get('date'))
 
-        # Use the aggregated hour as usage time
-        usage_time = self._parse_datetime(extract_scalar(row.get('usage_hour')))
-        
-        # Use team_id as the primary entity_id
-        entity_id = str(extract_scalar(row.get('team_id', '')))
-        key_name = str(extract_scalar(row.get('key_name', '')))
-        model = str(extract_scalar(row.get('model', '')))
-        model_group = str(extract_scalar(row.get('model_group', '')))
-        provider = str(extract_scalar(row.get('custom_llm_provider', '')))
-        tag = extract_scalar(row.get('tag'))
-        
-        # Calculate aggregated metrics
-        total_spend = float(extract_scalar(row.get('total_spend', 0.0)) or 0.0)
-        total_tokens = int(extract_scalar(row.get('total_tokens', 0)) or 0)
-        total_prompt_tokens = int(extract_scalar(row.get('total_prompt_tokens', 0)) or 0)
-        total_completion_tokens = int(extract_scalar(row.get('total_completion_tokens', 0)) or 0)
-        request_count = int(extract_scalar(row.get('request_count', 0)) or 0)
-        successful_requests = int(extract_scalar(row.get('successful_requests', 0)) or 0)
-        failed_requests = int(extract_scalar(row.get('failed_requests', 0)) or 0)
+        # Calculate total tokens
+        prompt_tokens = int(row.get('prompt_tokens', 0))
+        completion_tokens = int(row.get('completion_tokens', 0))
+        total_tokens = prompt_tokens + completion_tokens
 
         # Create CloudZero Resource Name (CZRN) as resource_id
-        # Create a mock row for CZRN generation with team_id as entity_id
-        czrn_row = {
-            'entity_id': entity_id,
-            'entity_type': 'team',
-            'model': model,
-            'custom_llm_provider': provider,
-            'api_key': str(extract_scalar(row.get('api_key_sample', '')))
-        }
-        resource_id = self.czrn_generator.create_from_litellm_data(czrn_row)
+        resource_id = self.czrn_generator.create_from_litellm_data(row)
 
-        # Build dimensions for CloudZero tracking
-        dimensions = {
-            'entity_type': 'team',
-            'entity_id': entity_id,
-            'key_name': key_name,
-            'model': model,
-            'model_group': model_group,
-            'provider': provider,
-            'request_count': str(request_count),
-            'successful_requests': str(successful_requests),
-            'failed_requests': str(failed_requests),
-        }
+        # Build dimensions for CloudZero
+        model = str(row.get('model', ''))
+        api_key_hash = str(row.get('api_key', ''))[:8]  # First 8 chars for identification
         
-        # Add tag if present
-        if tag is not None and str(tag) not in ['', 'null', 'None']:
-            dimensions['tag'] = str(tag)
+        # Handle team information with fallbacks
+        team_id = row.get('team_id')
+        team_alias = row.get('team_alias')
+        
+        # Use team_alias if available, otherwise team_id, otherwise fallback to 'unknown'
+        entity_id = str(team_alias) if team_alias else (str(team_id) if team_id else 'unknown')
+        
+        dimensions = {
+            'entity_type': CZEntityType.TEAM.value,
+            'entity_id': entity_id,
+            'team_id': str(team_id) if team_id else 'unknown',
+            'team_alias': str(team_alias) if team_alias else 'unknown',
+            'model': model,
+            'model_group': str(row.get('model_group', '')),
+            'provider': str(row.get('custom_llm_provider', '')),
+            'api_key_prefix': api_key_hash,
+            'api_key_alias': str(row.get('api_key_alias', '')),
+            'api_requests': str(row.get('api_requests', 0)),
+            'successful_requests': str(row.get('successful_requests', 0)),
+            'failed_requests': str(row.get('failed_requests', 0)),
+            'cache_creation_tokens': str(row.get('cache_creation_input_tokens', 0)),
+            'cache_read_tokens': str(row.get('cache_read_input_tokens', 0)),
+        }
 
         # Extract CZRN components to populate corresponding CBF columns
         czrn_components = self.czrn_generator.extract_components(resource_id)
-        service_type, provider_czrn, region, owner_account_id, resource_type, cloud_local_id = czrn_components
+        service_type, provider, region, owner_account_id, resource_type, cloud_local_id = czrn_components
 
         # CloudZero CBF format with proper column names
         cbf_record = {
             # Required CBF fields
-            'time/usage_start': usage_time.isoformat() if usage_time else None,  # Required: ISO-formatted UTC datetime
-            'cost/cost': total_spend,  # Required: billed cost
+            'time/usage_start': usage_date.isoformat() if usage_date else None,  # Required: ISO-formatted UTC datetime
+            'cost/cost': float(row.get('spend', 0.0)),  # Required: billed cost
             'resource/id': resource_id,  # Required when resource tags are present
 
             # Usage metrics for token consumption
@@ -206,41 +145,42 @@ class CBFTransformer:
         }
 
         # Add CZRN components that don't have direct CBF column mappings as resource tags
-        cbf_record['resource/tag:provider'] = provider_czrn  # CZRN provider component
+        cbf_record['resource/tag:provider'] = provider  # CZRN provider component
         cbf_record['resource/tag:model'] = cloud_local_id  # CZRN cloud-local-id component (model)
-
+        
         # Add resource tags for all dimensions (using resource/tag:<key> format)
         for key, value in dimensions.items():
-            # Ensure value is a scalar and not empty
-            if hasattr(value, 'item') and not isinstance(value, str):
-                value = value.item() if value is not None else None
-            if value is not None and str(value) not in ['', 'N/A', 'None', 'null']:  # Only add non-empty tags
+            if value and value != 'N/A' and value != 'unknown':  # Only add meaningful tags
                 cbf_record[f'resource/tag:{key}'] = str(value)
 
         # Add token breakdown as resource tags for analysis
-        if total_prompt_tokens > 0:
-            cbf_record['resource/tag:prompt_tokens'] = str(total_prompt_tokens)
-        if total_completion_tokens > 0:
-            cbf_record['resource/tag:completion_tokens'] = str(total_completion_tokens)
+        if prompt_tokens > 0:
+            cbf_record['resource/tag:prompt_tokens'] = str(prompt_tokens)
+        if completion_tokens > 0:
+            cbf_record['resource/tag:completion_tokens'] = str(completion_tokens)
         if total_tokens > 0:
             cbf_record['resource/tag:total_tokens'] = str(total_tokens)
 
         return CBFRecord(cbf_record)
 
-    def _parse_datetime(self, datetime_obj) -> Optional[datetime]:
-        """Parse datetime object to ensure proper format."""
-        if datetime_obj is None:
+    def _parse_date(self, date_str) -> Optional[datetime]:
+        """Parse date string from daily spend tables (e.g., '2025-04-19')."""
+        if date_str is None:
             return None
 
-        if isinstance(datetime_obj, datetime):
-            return datetime_obj
+        if isinstance(date_str, datetime):
+            return date_str
 
-        if isinstance(datetime_obj, str):
+        if isinstance(date_str, str):
             try:
-                # Try to parse ISO format
-                return pl.Series([datetime_obj]).str.to_datetime().item()
+                # Parse date string and set to midnight UTC for daily aggregation
+                return pl.Series([date_str]).str.to_datetime("%Y-%m-%d").item()
             except Exception:
-                return None
+                try:
+                    # Fallback: try ISO format parsing
+                    return pl.Series([date_str]).str.to_datetime().item()
+                except Exception:
+                    return None
 
         return None
 

litellm/integrations/custom_guardrail.py

@@ -119,11 +119,8 @@ class CustomGuardrail(CustomLogger):
         """
         if "guardrails" in data:
             return data["guardrails"]
-        metadata = data.get("metadata") or {}
-        requested_guardrails = metadata.get("guardrails") or []
-        if requested_guardrails:
-            return requested_guardrails
-        return requested_guardrails
+        metadata = data.get("litellm_metadata") or data.get("metadata", {})
+        return metadata.get("guardrails") or []
 
     def _guardrail_is_in_requested_guardrails(
         self,

litellm/integrations/datadog/datadog_llm_obs.py

@@ -19,6 +19,7 @@ import litellm
 from litellm._logging import verbose_logger
 from litellm.integrations.custom_batch_logger import CustomBatchLogger
 from litellm.integrations.datadog.datadog import DataDogLogger
+from litellm.litellm_core_utils.dd_tracing import tracer
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     handle_any_messages_to_chat_completion_str_messages_conversion,
 )
@@ -216,7 +217,7 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
             time_to_first_token=self._get_time_to_first_token_seconds(standard_logging_payload),
         )
 
-        return LLMObsPayload(
+        payload: LLMObsPayload = LLMObsPayload(
             parent_id=metadata.get("parent_id", "undefined"),
             trace_id=standard_logging_payload.get("trace_id", str(uuid.uuid4())),
             span_id=metadata.get("span_id", str(uuid.uuid4())),
@@ -230,6 +231,26 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
                 self._get_datadog_tags(standard_logging_object=standard_logging_payload)
             ],
         )
+
+        apm_trace_id = self._get_apm_trace_id()
+        if apm_trace_id is not None:
+            payload["apm_id"] = apm_trace_id
+
+        return payload
+
+    def _get_apm_trace_id(self) -> Optional[str]:
+        """Retrieve the current APM trace ID if available."""
+        try:
+            current_span_fn = getattr(tracer, "current_span", None)
+            if callable(current_span_fn):
+                current_span = current_span_fn()
+                if current_span is not None:
+                    trace_id = getattr(current_span, "trace_id", None)
+                    if trace_id is not None:
+                        return str(trace_id)
+        except Exception:
+            pass
+        return None
     
     def _assemble_error_info(self, standard_logging_payload: StandardLoggingPayload) -> Optional[DDLLMObsError]:
         """

litellm/integrations/opentelemetry.py

@@ -15,6 +15,8 @@ from litellm.types.utils import (
     StandardLoggingPayload,
 )
 
+# OpenTelemetry imports moved to individual functions to avoid import errors when not installed
+
 if TYPE_CHECKING:
     from opentelemetry.sdk.trace.export import SpanExporter as _SpanExporter
     from opentelemetry.trace import Context as _Context
@@ -41,6 +43,8 @@ else:
     Context = Any
 
 LITELLM_TRACER_NAME = os.getenv("OTEL_TRACER_NAME", "litellm")
+LITELLM_METER_NAME = os.getenv("LITELLM_METER_NAME", "litellm")
+LITELLM_LOGGER_NAME = os.getenv("LITELLM_LOGGER_NAME", "litellm")
 # Remove the hardcoded LITELLM_RESOURCE dictionary - we'll create it properly later
 RAW_REQUEST_SPAN_NAME = "raw_gen_ai_request"
 LITELLM_REQUEST_SPAN_NAME = "litellm_request"
@@ -83,6 +87,8 @@ class OpenTelemetryConfig:
     exporter: Union[str, SpanExporter] = "console"
     endpoint: Optional[str] = None
     headers: Optional[str] = None
+    enable_metrics: bool = False
+    enable_events: bool = False
 
     @classmethod
     def from_env(cls):
@@ -104,6 +110,14 @@ class OpenTelemetryConfig:
         headers = os.getenv(
             "OTEL_EXPORTER_OTLP_HEADERS", os.getenv("OTEL_HEADERS")
         )  # example: OTEL_HEADERS=x-honeycomb-team=B85YgLm96***"
+        enable_metrics: bool = (
+            os.getenv("LITELLM_OTEL_INTEGRATION_ENABLE_METRICS", "false").lower()
+            == "true"
+        )
+        enable_events: bool = (
+            os.getenv("LITELLM_OTEL_INTEGRATION_ENABLE_EVENTS", "false").lower()
+            == "true"
+        )
 
         if exporter == "in_memory":
             return cls(exporter=InMemorySpanExporter())
@@ -111,6 +125,8 @@ class OpenTelemetryConfig:
             exporter=exporter,
             endpoint=endpoint,
             headers=headers,  # example: OTEL_HEADERS=x-honeycomb-team=B85YgLm96***"
+            enable_metrics=enable_metrics,
+            enable_events=enable_events,
         )
 
 
@@ -119,27 +135,22 @@ class OpenTelemetry(CustomLogger):
         self,
         config: Optional[OpenTelemetryConfig] = None,
         callback_name: Optional[str] = None,
+        # injection points for testing
+        tracer_provider: Optional[Any] = None,
+        logger_provider: Optional[Any] = None,
+        meter_provider: Optional[Any] = None,
         **kwargs,
     ):
-        from opentelemetry import trace
-        from opentelemetry.sdk.trace import TracerProvider
-        from opentelemetry.trace import SpanKind
 
         if config is None:
             config = OpenTelemetryConfig.from_env()
 
         self.config = config
+        self.callback_name = callback_name
         self.OTEL_EXPORTER = self.config.exporter
         self.OTEL_ENDPOINT = self.config.endpoint
         self.OTEL_HEADERS = self.config.headers
-        provider = TracerProvider(resource=_get_litellm_resource())
-        provider.add_span_processor(self._get_span_processor())
-        self.callback_name = callback_name
-
-        trace.set_tracer_provider(provider)
-        self.tracer = trace.get_tracer(LITELLM_TRACER_NAME)
-
-        self.span_kind = SpanKind
+        self._init_tracing(tracer_provider)
 
         _debug_otel = str(os.getenv("DEBUG_OTEL", "False")).lower()
 
@@ -156,6 +167,8 @@ class OpenTelemetry(CustomLogger):
 
         # init CustomLogger params
         super().__init__(**kwargs)
+        self._init_metrics(meter_provider)
+        self._init_logs(logger_provider)
         self._init_otel_logger_on_litellm_proxy()
 
     def _init_otel_logger_on_litellm_proxy(self):
@@ -178,14 +191,109 @@ class OpenTelemetry(CustomLogger):
             litellm.service_callback.append("otel")
         setattr(proxy_server, "open_telemetry_logger", self)
 
+    def _init_tracing(self, tracer_provider):
+        from opentelemetry import trace
+        from opentelemetry.sdk.trace import TracerProvider
+        from opentelemetry.trace import SpanKind
+
+        # use provided tracer or create a new one
+        if tracer_provider is None:
+            tracer_provider = TracerProvider(resource=_get_litellm_resource())
+            # Only add OTLP span processor if we created the tracer provider ourselves
+            tracer_provider.add_span_processor(self._get_span_processor())
+
+        # register global provider and grab our tracer
+        trace.set_tracer_provider(tracer_provider)
+        self.tracer = trace.get_tracer(LITELLM_TRACER_NAME)
+        self.span_kind = SpanKind
+
+    def _init_metrics(self, meter_provider):
+        if not self.config.enable_metrics:
+            self._operation_duration_histogram = None
+            self._token_usage_histogram = None
+            self._cost_histogram = None
+            return
+
+        from opentelemetry import metrics
+        from opentelemetry.sdk.metrics import Histogram, MeterProvider
+
+        # Only create OTLP infrastructure if no custom meter provider is provided
+        if meter_provider is None:
+            from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import (
+                OTLPMetricExporter,
+            )
+            from opentelemetry.sdk.metrics.export import (
+                AggregationTemporality,
+                PeriodicExportingMetricReader,
+            )
+
+            _metric_exporter = OTLPMetricExporter(
+                endpoint=self.config.endpoint,
+                headers=OpenTelemetry._get_headers_dictionary(self.config.headers),
+                preferred_temporality={Histogram: AggregationTemporality.DELTA},
+            )
+            _metric_reader = PeriodicExportingMetricReader(
+                _metric_exporter, export_interval_millis=10000
+            )
+
+            meter_provider = MeterProvider(
+                metric_readers=[_metric_reader], resource=_get_litellm_resource()
+            )
+            meter = meter_provider.get_meter(__name__)
+        else:
+            # Use the provided meter provider as-is, without creating additional OTLP infrastructure
+            meter = meter_provider.get_meter(__name__)
+
+        metrics.set_meter_provider(meter_provider)
+
+        self._operation_duration_histogram = meter.create_histogram(
+            name="gen_ai.client.operation.duration", # Replace with semconv constant in otel 1.38
+            description="GenAI operation duration",
+            unit="s",
+        )
+        self._token_usage_histogram = meter.create_histogram(
+            name="gen_ai.client.token.usage", # Replace with semconv constant in otel 1.38
+            description="GenAI token usage",
+            unit="{token}",
+        )
+        self._cost_histogram = meter.create_histogram(
+            name="gen_ai.client.token.cost",
+            description="GenAI request cost",
+            unit="USD",
+        )
+
+    def _init_logs(self, logger_provider):
+        # nothing to do if events disabled
+        if not self.config.enable_events:
+            return
+
+        from opentelemetry._logs import set_logger_provider
+        from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter
+        from opentelemetry.sdk._logs import LoggerProvider as OTLoggerProvider
+        from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
+
+        # set up log pipeline
+        if logger_provider is None:
+            logger_provider = OTLoggerProvider()
+            # Only add OTLP exporter if we created the logger provider ourselves
+            logger_provider.add_log_record_processor(
+                BatchLogRecordProcessor(
+                    OTLPLogExporter(
+                        endpoint=self.config.endpoint,
+                        headers=self._get_headers_dictionary(self.config.headers),
+                    )
+                )
+            )
+        set_logger_provider(logger_provider)
+
     def log_success_event(self, kwargs, response_obj, start_time, end_time):
-        self._handle_sucess(kwargs, response_obj, start_time, end_time)
+        self._handle_success(kwargs, response_obj, start_time, end_time)
 
     def log_failure_event(self, kwargs, response_obj, start_time, end_time):
         self._handle_failure(kwargs, response_obj, start_time, end_time)
 
     async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
-        self._handle_sucess(kwargs, response_obj, start_time, end_time)
+        self._handle_success(kwargs, response_obj, start_time, end_time)
 
     async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
         self._handle_failure(kwargs, response_obj, start_time, end_time)
@@ -372,9 +480,9 @@ class OpenTelemetry(CustomLogger):
 
     def _get_dynamic_otel_headers_from_kwargs(self, kwargs) -> Optional[dict]:
         """Extract dynamic headers from kwargs if available."""
-        standard_callback_dynamic_params: Optional[StandardCallbackDynamicParams] = (
-            kwargs.get("standard_callback_dynamic_params")
-        )
+        standard_callback_dynamic_params: Optional[
+            StandardCallbackDynamicParams
+        ] = kwargs.get("standard_callback_dynamic_params")
 
         if not standard_callback_dynamic_params:
             return None
@@ -414,50 +522,185 @@ class OpenTelemetry(CustomLogger):
     # End of Team/Key Based Logging Control Flow
     #########################################################
 
-    def _handle_sucess(self, kwargs, response_obj, start_time, end_time):
-        from opentelemetry import trace
-        from opentelemetry.trace import Status, StatusCode
+    def _handle_success(self, kwargs, response_obj, start_time, end_time):
 
         verbose_logger.debug(
             "OpenTelemetry Logger: Logging kwargs: %s, OTEL config settings=%s",
             kwargs,
             self.config,
         )
+        ctx, parent_span = self._get_span_context(kwargs)
+
+        # 1. Primary span
+        span = self._start_primary_span(kwargs, response_obj, start_time, end_time, ctx)
+
+        # 2. Raw‐request sub-span (if enabled)
+        self._maybe_log_raw_request(kwargs, response_obj, start_time, end_time, span)
+
+        # 3. Guardrail span
+        self._create_guardrail_span(kwargs=kwargs, context=ctx)
+
+        # 4. Metrics & cost recording
+        self._record_metrics(kwargs, response_obj, start_time, end_time)
+
+        # 5. Semantic logs. 
+        if self.config.enable_events:
+            self._emit_semantic_logs(kwargs, response_obj, span)
+
+        # 6. End parent span
+        if parent_span is not None:
+            parent_span.end(end_time=self._to_ns(datetime.now()))
+
+    def _start_primary_span(self, kwargs, response_obj, start_time, end_time, context):
+        from opentelemetry.trace import Status, StatusCode
 
-        _parent_context, parent_otel_span = self._get_span_context(kwargs)
-        # Span 1: Request sent to litellm SDK
         otel_tracer: Tracer = self.get_tracer_to_use_for_request(kwargs)
         span = otel_tracer.start_span(
             name=self._get_span_name(kwargs),
             start_time=self._to_ns(start_time),
-            context=_parent_context,
+            context=context,
         )
         span.set_status(Status(StatusCode.OK))
         self.set_attributes(span, kwargs, response_obj)
+        span.end(end_time=self._to_ns(end_time))
+        return span
 
-        if litellm.turn_off_message_logging is True:
-            pass
-        elif self.message_logging is not True:
-            pass
-        else:
-            # Span 2: Raw Request / Response to LLM
-            raw_request_span = otel_tracer.start_span(
-                name=RAW_REQUEST_SPAN_NAME,
-                start_time=self._to_ns(start_time),
-                context=trace.set_span_in_context(span),
+    def _maybe_log_raw_request(
+        self, kwargs, response_obj, start_time, end_time, parent_span
+    ):
+        from opentelemetry import trace
+        from opentelemetry.trace import Status, StatusCode
+
+        # only log raw LLM request/response if message_logging is on and not globally turned off
+        if litellm.turn_off_message_logging or not self.message_logging:
+            return
+
+        otel_tracer: Tracer = self.get_tracer_to_use_for_request(kwargs)
+        raw_span = otel_tracer.start_span(
+            name=RAW_REQUEST_SPAN_NAME,
+            start_time=self._to_ns(start_time),
+            context=trace.set_span_in_context(parent_span),
+        )
+        raw_span.set_status(Status(StatusCode.OK))
+        self.set_raw_request_attributes(raw_span, kwargs, response_obj)
+        raw_span.end(end_time=self._to_ns(end_time))
+
+    def _record_metrics(self, kwargs, response_obj, start_time, end_time):
+        duration_s = (end_time - start_time).total_seconds()
+        params = kwargs.get("litellm_params") or {}
+        provider = params.get("custom_llm_provider", "Unknown")
+
+        common_attrs = {
+            "gen_ai.operation.name": "chat",
+            "gen_ai.system": provider,
+            "gen_ai.request.model": kwargs.get("model"),
+            "gen_ai.framework": "litellm",
+        }
+
+        std_log = kwargs.get("standard_logging_object")
+        md = getattr(std_log, "metadata", None) or (std_log or {}).get("metadata", {})
+        for key in [
+            "user_api_key_hash",
+            "user_api_key_alias",
+            "user_api_key_team_id",
+            "user_api_key_org_id",
+            "user_api_key_user_id",
+            "user_api_key_team_alias",
+            "user_api_key_user_email",
+            "spend_logs_metadata",
+            "requester_ip_address",
+            "requester_metadata",
+            "user_api_key_end_user_id",
+            "prompt_management_metadata",
+            "applied_guardrails",
+            "mcp_tool_call_metadata",
+            "vector_store_request_metadata",
+        ]:
+            if md.get(key) is not None:
+                common_attrs[f"metadata.{key}"] = str(md[key])
+
+        if self._operation_duration_histogram:
+            self._operation_duration_histogram.record(
+                duration_s, attributes=common_attrs
+            )
+            if (
+                response_obj
+                and (usage := response_obj.get("usage"))
+                and self._token_usage_histogram
+            ):
+                in_attrs = {**common_attrs, "gen_ai.token.type": "input"}
+                out_attrs = {**common_attrs, "gen_ai.token.type": "completion"}
+                self._token_usage_histogram.record(
+                    usage.get("prompt_tokens", 0), attributes=in_attrs
+                )
+                self._token_usage_histogram.record(
+                    usage.get("completion_tokens", 0), attributes=out_attrs
+                )
+
+        cost = kwargs.get("response_cost")
+        if self._cost_histogram and cost:
+            self._cost_histogram.record(cost, attributes=common_attrs)
+
+    def _emit_semantic_logs(self, kwargs, response_obj, span: Span):
+        if not self.config.enable_events:
+            return
+
+        from opentelemetry._logs import get_logger, LogRecord
+        otel_logger = get_logger(LITELLM_LOGGER_NAME)
+
+        parent_ctx = span.get_span_context()
+        provider = (kwargs.get("litellm_params") or {}).get(
+            "custom_llm_provider", "Unknown"
+        )
+
+        # per-message events
+        for msg in kwargs.get("messages", []):
+            role = msg.get("role", "user")
+            attrs = {"event_name": "gen_ai.content.prompt", "gen_ai.system": provider}
+            if role == "tool" and msg.get("id"):
+                attrs["id"] = msg["id"]
+            if self.message_logging and msg.get("content"):
+                attrs["gen_ai.prompt"] = msg["content"]
+
+            otel_logger.emit(
+                LogRecord(
+                    attributes=attrs,
+                    body=msg.copy(),
+                    trace_id=parent_ctx.trace_id,
+                    span_id=parent_ctx.span_id,
+                    trace_flags=parent_ctx.trace_flags,
+                )
             )
 
-            raw_request_span.set_status(Status(StatusCode.OK))
-            self.set_raw_request_attributes(raw_request_span, kwargs, response_obj)
-            raw_request_span.end(end_time=self._to_ns(end_time))
+        # per-choice events
+        for idx, choice in enumerate(response_obj.get("choices", [])):
+            attrs = {
+                "event_name": "gen_ai.content.completion",
+                "gen_ai.system": provider,
+                "index": idx,
+                "finish_reason": choice.get("finish_reason"),
+            }
+            body_msg = choice.get("message", {})
+            if self.message_logging and body_msg.get("content"):
+                attrs["message.content"] = body_msg["content"]
+            body = {
+                "index": idx,
+                "finish_reason": choice.get("finish_reason"),
+                "message": {"role": body_msg.get("role", "assistant")},
+            }
+            if self.message_logging and body_msg.get("content"):
+                body["message"]["content"] = body_msg["content"]
 
-        span.end(end_time=self._to_ns(end_time))
+            otel_logger.emit(
+                LogRecord(
+                    attributes=attrs,
+                    body=body,
+                    trace_id=parent_ctx.trace_id,
+                    span_id=parent_ctx.span_id,
+                    trace_flags=parent_ctx.trace_flags,
+                )
+            )
 
-        # Create span for guardrail information
-        self._create_guardrail_span(kwargs=kwargs, context=_parent_context)
-
-        if parent_otel_span is not None:
-            parent_otel_span.end(end_time=self._to_ns(datetime.now()))
 
     def _create_guardrail_span(
         self, kwargs: Optional[dict], context: Optional[Context]

litellm/litellm_core_utils/custom_logger_registry.py

@@ -38,6 +38,7 @@ try:
     from litellm_enterprise.integrations.prometheus import PrometheusLogger
 except Exception:
     PrometheusLogger = None
+from litellm.integrations.cloudzero.cloudzero import CloudZeroLogger
 from litellm.integrations.dotprompt import DotpromptManager
 from litellm.integrations.s3_v2 import S3Logger
 from litellm.integrations.sqs import SQSLogger
@@ -86,6 +87,7 @@ class CustomLoggerRegistry:
         "dynamic_rate_limiter": _PROXY_DynamicRateLimitHandler,
         "vector_store_pre_call_hook": VectorStorePreCallHook,
         "dotprompt": DotpromptManager,
+        "cloudzero": CloudZeroLogger,
     }
 
     try:

litellm/litellm_core_utils/duration_parser.py

@@ -1,7 +1,7 @@
 """
 Helper utilities for parsing durations - 1s, 1d, 10d, 30d, 1mo, 2mo
 
-duration_in_seconds is used in diff parts of the code base, example 
+duration_in_seconds is used in diff parts of the code base, example
 - Router - Provider budget routing
 - Proxy - Key, Team Generation
 """
@@ -192,6 +192,10 @@ def _handle_day_reset(
     current_time: datetime, base_midnight: datetime, value: int, timezone: timezone
 ) -> datetime:
     """Handle day-based reset times."""
+    # Handle zero value - immediate expiration
+    if value == 0:
+        return current_time
+
     if value == 1:  # Daily reset at midnight
         return base_midnight + timedelta(days=1)
     elif value == 7:  # Weekly reset on Monday at midnight
@@ -234,6 +238,10 @@ def _handle_hour_reset(
     current_time: datetime, base_midnight: datetime, value: int
 ) -> datetime:
     """Handle hour-based reset times."""
+    # Handle zero value - immediate expiration
+    if value == 0:
+        return current_time
+
     current_hour = current_time.hour
     current_minute = current_time.minute
     current_second = current_time.second
@@ -266,6 +274,10 @@ def _handle_minute_reset(
     current_time: datetime, base_midnight: datetime, value: int
 ) -> datetime:
     """Handle minute-based reset times."""
+    # Handle zero value - immediate expiration
+    if value == 0:
+        return current_time
+
     current_hour = current_time.hour
     current_minute = current_time.minute
     current_second = current_time.second
@@ -306,6 +318,10 @@ def _handle_second_reset(
     current_time: datetime, base_midnight: datetime, value: int
 ) -> datetime:
     """Handle second-based reset times."""
+    # Handle zero value - immediate expiration
+    if value == 0:
+        return current_time
+
     current_hour = current_time.hour
     current_minute = current_time.minute
     current_second = current_time.second

litellm/litellm_core_utils/get_llm_provider_logic.py

@@ -320,6 +320,7 @@ def get_llm_provider(  # noqa: PLR0915
             or model in litellm.vertex_embedding_models
             or model in litellm.vertex_vision_models
             or model in litellm.vertex_ai_image_models
+            or model in litellm.vertex_ai_video_models
         ):
             custom_llm_provider = "vertex_ai"
         ## ai21

litellm/litellm_core_utils/litellm_logging.py

@@ -1164,7 +1164,6 @@ class Logging(LiteLLMLoggingBaseClass):
 
         used for consistent cost calculation across response headers + logging integrations.
         """
-
         if isinstance(result, BaseModel) and hasattr(result, "_hidden_params"):
             hidden_params = getattr(result, "_hidden_params", {})
             if (
@@ -3361,7 +3360,14 @@ def _init_custom_logger_compatible_class(  # noqa: PLR0915
             galileo_logger = GalileoObserve()
             _in_memory_loggers.append(galileo_logger)
             return galileo_logger  # type: ignore
-
+        elif logging_integration == "cloudzero":
+            from litellm.integrations.cloudzero.cloudzero import CloudZeroLogger
+            for callback in _in_memory_loggers:
+                if isinstance(callback, CloudZeroLogger):
+                    return callback  # type: ignore
+            cloudzero_logger = CloudZeroLogger()
+            _in_memory_loggers.append(cloudzero_logger)
+            return cloudzero_logger  # type: ignore
         elif logging_integration == "deepeval":
             for callback in _in_memory_loggers:
                 if isinstance(callback, DeepEvalLogger):
@@ -3581,6 +3587,11 @@ def get_custom_logger_compatible_class(  # noqa: PLR0915
             for callback in _in_memory_loggers:
                 if isinstance(callback, GalileoObserve):
                     return callback
+        elif logging_integration == "cloudzero":
+            from litellm.integrations.cloudzero.cloudzero import CloudZeroLogger
+            for callback in _in_memory_loggers:
+                if isinstance(callback, CloudZeroLogger):
+                    return callback
         elif logging_integration == "deepeval":
             for callback in _in_memory_loggers:
                 if isinstance(callback, DeepEvalLogger):

litellm/litellm_core_utils/safe_json_dumps.py

@@ -1,5 +1,6 @@
 import json
 from typing import Any, Union
+
 from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
 
 

litellm/llms/base_llm/__init__.py

@@ -1,5 +1,6 @@
 from .anthropic_messages.transformation import BaseAnthropicMessagesConfig
 from .audio_transcription.transformation import BaseAudioTranscriptionConfig
+from .batches.transformation import BaseBatchesConfig
 from .chat.transformation import BaseConfig
 from .embedding.transformation import BaseEmbeddingConfig
 from .image_edit.transformation import BaseImageEditConfig
@@ -12,4 +13,5 @@ __all__ = [
     "BaseAnthropicMessagesConfig",
     "BaseEmbeddingConfig",
     "BaseImageEditConfig",
+    "BaseBatchesConfig",
 ]

litellm/llms/base_llm/batches/transformation.py

@@ -0,0 +1,176 @@
+import types
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
+
+import httpx
+from httpx import Headers
+
+from litellm.types.llms.openai import (
+    AllMessageValues,
+    CreateBatchRequest,
+)
+from litellm.types.utils import LiteLLMBatch, LlmProviders
+
+if TYPE_CHECKING:
+    from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
+
+    from ..chat.transformation import BaseLLMException as _BaseLLMException
+
+    LiteLLMLoggingObj = _LiteLLMLoggingObj
+    BaseLLMException = _BaseLLMException
+else:
+    LiteLLMLoggingObj = Any
+    BaseLLMException = Any
+
+
+class BaseBatchesConfig(ABC):
+    """
+    Abstract base class for batch processing configurations across different LLM providers.
+    
+    This class defines the interface that all provider-specific batch configurations
+    must implement to work with LiteLLM's unified batch processing system.
+    """
+
+    def __init__(self):
+        pass
+
+    @property
+    @abstractmethod
+    def custom_llm_provider(self) -> LlmProviders:
+        """Return the LLM provider type for this configuration."""
+        pass
+
+    @classmethod
+    def get_config(cls):
+        """Get configuration dictionary for this class."""
+        return {
+            k: v
+            for k, v in cls.__dict__.items()
+            if not k.startswith("__")
+            and not k.startswith("_abc")
+            and not isinstance(
+                v,
+                (
+                    types.FunctionType,
+                    types.BuiltinFunctionType,
+                    classmethod,
+                    staticmethod,
+                ),
+            )
+            and v is not None
+        }
+
+    @abstractmethod
+    def validate_environment(
+        self,
+        headers: dict,
+        model: str,
+        messages: List[AllMessageValues],
+        optional_params: dict,
+        litellm_params: dict,
+        api_key: Optional[str] = None,
+        api_base: Optional[str] = None,
+    ) -> dict:
+        """
+        Validate and prepare environment-specific headers and parameters.
+        
+        Args:
+            headers: HTTP headers dictionary
+            model: Model name
+            messages: List of messages
+            optional_params: Optional parameters
+            litellm_params: LiteLLM parameters
+            api_key: API key
+            api_base: API base URL
+            
+        Returns:
+            Updated headers dictionary
+        """
+        pass
+
+    @abstractmethod
+    def get_complete_batch_url(
+        self,
+        api_base: Optional[str],
+        api_key: Optional[str],
+        model: str,
+        optional_params: Dict,
+        litellm_params: Dict,
+        data: CreateBatchRequest,
+    ) -> str:
+        """
+        Get the complete URL for batch creation request.
+        
+        Args:
+            api_base: Base API URL
+            api_key: API key
+            model: Model name
+            optional_params: Optional parameters
+            litellm_params: LiteLLM parameters
+            data: Batch creation request data
+            
+        Returns:
+            Complete URL for the batch request
+        """
+        pass
+
+    @abstractmethod
+    def transform_create_batch_request(
+        self,
+        model: str,
+        create_batch_data: CreateBatchRequest,
+        optional_params: dict,
+        litellm_params: dict,
+    ) -> Union[bytes, str, Dict[str, Any]]:
+        """
+        Transform the batch creation request to provider-specific format.
+        
+        Args:
+            model: Model name
+            create_batch_data: Batch creation request data
+            optional_params: Optional parameters
+            litellm_params: LiteLLM parameters
+            
+        Returns:
+            Transformed request data
+        """
+        pass
+
+    @abstractmethod
+    def transform_create_batch_response(
+        self,
+        model: Optional[str],
+        raw_response: httpx.Response,
+        logging_obj: LiteLLMLoggingObj,
+        litellm_params: dict,
+    ) -> LiteLLMBatch:
+        """
+        Transform provider-specific batch response to LiteLLM format.
+        
+        Args:
+            model: Model name
+            raw_response: Raw HTTP response
+            logging_obj: Logging object
+            litellm_params: LiteLLM parameters
+            
+        Returns:
+            LiteLLM batch object
+        """
+        pass
+
+    @abstractmethod
+    def get_error_class(
+        self, error_message: str, status_code: int, headers: Union[Dict, Headers]
+    ) -> "BaseLLMException":
+        """
+        Get the appropriate error class for this provider.
+        
+        Args:
+            error_message: Error message
+            status_code: HTTP status code
+            headers: Response headers
+            
+        Returns:
+            Provider-specific exception class
+        """
+        pass

litellm/llms/base_llm/files/transformation.py

@@ -35,6 +35,16 @@ class BaseFilesConfig(BaseConfig):
     def custom_llm_provider(self) -> LlmProviders:
         pass
 
+    @property
+    def file_upload_http_method(self) -> str:
+        """
+        HTTP method to use for file uploads.
+        Override this in provider configs if they need different methods.
+        Default is POST (used by most providers like OpenAI, Anthropic).
+        S3-based providers like Bedrock should return "PUT".
+        """
+        return "POST"
+
     @abstractmethod
     def get_supported_openai_params(
         self, model: str

litellm/llms/base_llm/passthrough/transformation.py

@@ -31,30 +31,26 @@ class BasePassthroughConfig(BaseLLMModelInfo):
         Args:
             endpoint: str - the endpoint to add to the url
             base_target_url: str - the base url to add the endpoint to
-            request_query_params: dict - the query params to add to the url
+            request_query_params: Optional[dict] - the query params to add to the url
         Returns:
-            str - the formatted url
+            httpx.URL - the formatted url
         """
         from urllib.parse import urlencode
 
         import httpx
 
-        encoded_endpoint = httpx.URL(endpoint).path
+        base = base_target_url.rstrip('/')
+        endpoint = endpoint.lstrip('/')
+        full_url = f"{base}/{endpoint}"
 
-        # Ensure endpoint starts with '/' for proper URL construction
-        if not encoded_endpoint.startswith("/"):
-            encoded_endpoint = "/" + encoded_endpoint
-
-        # Construct the full target URL using httpx
-        base_url = httpx.URL(base_target_url)
-        updated_url = base_url.copy_with(path=encoded_endpoint)
+        url = httpx.URL(full_url)
 
         if request_query_params:
-            # Create a new URL with the merged query params
-            updated_url = updated_url.copy_with(
+            url = url.copy_with(
                 query=urlencode(request_query_params).encode("ascii")
             )
-        return updated_url
+
+        return url
 
     @abstractmethod
     def get_complete_url(

litellm/llms/bedrock/batches/transformation.py

@@ -0,0 +1,254 @@
+import os
+import time
+from typing import Any, Dict, List, Literal, Optional, Union, cast
+
+from httpx import Headers, Response
+
+from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.types.llms.bedrock import (
+    BedrockBatchJobStatus,
+    BedrockCreateBatchRequest,
+    BedrockCreateBatchResponse,
+    BedrockInputDataConfig,
+    BedrockOutputDataConfig,
+    BedrockS3InputDataConfig,
+    BedrockS3OutputDataConfig,
+)
+from litellm.types.llms.openai import (
+    AllMessageValues,
+    CreateBatchRequest,
+)
+from litellm.types.utils import LiteLLMBatch, LlmProviders
+
+from ..base_aws_llm import BaseAWSLLM
+from ..common_utils import CommonBatchFilesUtils
+
+
+class BedrockBatchesConfig(BaseAWSLLM, BaseBatchesConfig):
+    """
+    Config for Bedrock Batches - handles batch job creation and management for Bedrock
+    """
+    
+    def __init__(self):
+        super().__init__()
+        self.common_utils = CommonBatchFilesUtils()
+
+    @property
+    def custom_llm_provider(self) -> LlmProviders:
+        return LlmProviders.BEDROCK
+
+    def validate_environment(
+        self,
+        headers: dict,
+        model: str,
+        messages: List[AllMessageValues],
+        optional_params: dict,
+        litellm_params: dict,
+        api_key: Optional[str] = None,
+        api_base: Optional[str] = None,
+    ) -> dict:
+        """
+        Validate and prepare environment for Bedrock batch requests.
+        AWS credentials are handled by BaseAWSLLM.
+        """
+        # Add any Bedrock-specific headers if needed
+        return headers
+
+    def get_complete_batch_url(
+        self,
+        api_base: Optional[str],
+        api_key: Optional[str],
+        model: str,
+        optional_params: Dict,
+        litellm_params: Dict,
+        data: CreateBatchRequest,
+    ) -> str:
+        """
+        Get the complete URL for Bedrock batch creation.
+        Bedrock batch jobs are created via the model invocation job API.
+        """
+        aws_region_name = self._get_aws_region_name(optional_params, model)
+        
+        # Bedrock model invocation job endpoint
+        # Format: https://bedrock.{region}.amazonaws.com/model-invocation-job
+        bedrock_endpoint = f"https://bedrock.{aws_region_name}.amazonaws.com/model-invocation-job"
+        
+        return bedrock_endpoint
+
+
+
+
+
+
+
+    def transform_create_batch_request(
+        self,
+        model: str,
+        create_batch_data: CreateBatchRequest,
+        optional_params: dict,
+        litellm_params: dict,
+    ) -> Dict[str, Any]:
+        """
+        Transform the batch creation request to Bedrock format.
+        
+        Bedrock batch inference requires:
+        - modelId: The Bedrock model ID
+        - jobName: Unique name for the batch job
+        - inputDataConfig: Configuration for input data (S3 location)
+        - outputDataConfig: Configuration for output data (S3 location)
+        - roleArn: IAM role ARN for the batch job
+        """
+        # Get required parameters
+        input_file_id = create_batch_data.get("input_file_id")
+        if not input_file_id:
+            raise ValueError("input_file_id is required for Bedrock batch creation")
+        
+        # Extract S3 information from file ID using common utility
+        input_bucket, input_key = self.common_utils.parse_s3_uri(input_file_id)
+        
+        # Get output S3 configuration
+        output_bucket = litellm_params.get("s3_output_bucket_name") or os.getenv("AWS_S3_OUTPUT_BUCKET_NAME")
+        if not output_bucket:
+            # Use same bucket as input if no output bucket specified
+            output_bucket = input_bucket
+        
+        # Get IAM role ARN
+        role_arn = (
+            litellm_params.get("aws_batch_role_arn") 
+            or optional_params.get("aws_batch_role_arn")
+            or os.getenv("AWS_BATCH_ROLE_ARN")
+        )
+        if not role_arn:
+            raise ValueError(
+                "AWS IAM role ARN is required for Bedrock batch jobs. "
+                "Set 'aws_batch_role_arn' in litellm_params or AWS_BATCH_ROLE_ARN env var"
+            )
+        
+        # Get the actual Bedrock model ID using common utility
+        bedrock_model_id = self.common_utils.extract_model_from_s3_file_path(input_file_id, optional_params)
+        
+        if not bedrock_model_id:
+            raise ValueError("Could not determine Bedrock model ID. Ensure the model is specified in the input file or passed as a parameter.")
+        
+        # Generate job name with the correct model ID using common utility
+        job_name = self.common_utils.generate_unique_job_name(bedrock_model_id, prefix="litellm")
+        output_key = f"litellm-batch-outputs/{job_name}/"
+        
+        # Build input data config
+        input_data_config: BedrockInputDataConfig = {
+            "s3InputDataConfig": BedrockS3InputDataConfig(
+                s3Uri=f"s3://{input_bucket}/{input_key}"
+            )
+        }
+        
+        # Build output data config
+        output_data_config: BedrockOutputDataConfig = {
+            "s3OutputDataConfig": BedrockS3OutputDataConfig(
+                s3Uri=f"s3://{output_bucket}/{output_key}"
+            )
+        }
+        
+        # Create Bedrock batch request with proper typing
+        bedrock_request: BedrockCreateBatchRequest = {
+            "modelId": bedrock_model_id,
+            "jobName": job_name,
+            "inputDataConfig": input_data_config,
+            "outputDataConfig": output_data_config,
+            "roleArn": role_arn
+        }
+        
+        # Add optional parameters if provided
+        completion_window = create_batch_data.get("completion_window")
+        if completion_window:
+            # Map OpenAI completion window to Bedrock timeout
+            # OpenAI uses "24h", Bedrock expects timeout in hours
+            if completion_window == "24h":
+                bedrock_request["timeoutDurationInHours"] = 24
+
+        # For Bedrock, we need to return a pre-signed request with AWS auth headers
+        # Use common utility for AWS signing
+        endpoint_url = f"https://bedrock.{self._get_aws_region_name(optional_params, model)}.amazonaws.com/model-invocation-job"
+        signed_headers, signed_data = self.common_utils.sign_aws_request(
+            service_name="bedrock",
+            data=bedrock_request,
+            endpoint_url=endpoint_url,
+            optional_params=optional_params,
+            method="POST"
+        )
+        
+        # Return a pre-signed request format that the HTTP handler can use
+        return {
+            "method": "POST",
+            "url": endpoint_url,
+            "headers": signed_headers,
+            "data": signed_data.decode('utf-8')
+        }
+
+    def transform_create_batch_response(
+        self,
+        model: Optional[str],
+        raw_response: Response,
+        logging_obj: Any,
+        litellm_params: dict,
+    ) -> LiteLLMBatch:
+        """
+        Transform Bedrock batch creation response to LiteLLM format.
+        """
+        try:
+            response_data: BedrockCreateBatchResponse = raw_response.json()
+        except Exception as e:
+            raise ValueError(f"Failed to parse Bedrock batch response: {e}")
+        
+        # Extract information from typed Bedrock response
+        job_arn = response_data.get("jobArn", "")
+        status: BedrockBatchJobStatus = response_data.get("status", "Submitted")
+        
+        # Map Bedrock status to OpenAI-compatible status
+        status_mapping: Dict[BedrockBatchJobStatus, str] = {
+            "Submitted": "validating",
+            "InProgress": "in_progress", 
+            "Completed": "completed",
+            "Failed": "failed",
+            "Stopping": "cancelling",
+            "Stopped": "cancelled"
+        }
+        
+        openai_status = cast(Literal["validating", "failed", "in_progress", "finalizing", "completed", "expired", "cancelling", "cancelled"], status_mapping.get(status, "validating"))
+        
+        # Get original request data from litellm_params if available
+        original_request = litellm_params.get("original_batch_request", {})
+        
+        # Create LiteLLM batch object
+        return LiteLLMBatch(
+            id=job_arn,  # Use ARN as the batch ID
+            object="batch",
+            endpoint=original_request.get("endpoint", "/v1/chat/completions"),
+            errors=None,
+            input_file_id=original_request.get("input_file_id", ""),
+            completion_window=original_request.get("completion_window", "24h"),
+            status=openai_status,
+            output_file_id=None,  # Will be populated when job completes
+            error_file_id=None,
+            created_at=int(time.time()),
+            in_progress_at=int(time.time()) if status == "InProgress" else None,
+            expires_at=None,
+            finalizing_at=None,
+            completed_at=None,
+            failed_at=None,
+            expired_at=None,
+            cancelling_at=None,
+            cancelled_at=None,
+            request_counts=None,
+            metadata=original_request.get("metadata", {}),
+        )
+
+    def get_error_class(
+        self, error_message: str, status_code: int, headers: Union[Dict, Headers]
+    ) -> BaseLLMException:
+        """
+        Get Bedrock-specific error class using common utility.
+        """
+        return self.common_utils.get_error_class(error_message, status_code, headers)
+
+

litellm/llms/bedrock/chat/converse_transformation.py

@@ -10,6 +10,8 @@ from typing import List, Literal, Optional, Tuple, Union, cast, overload
 import httpx
 
 import litellm
+from litellm._logging import verbose_logger
+from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
 from litellm.litellm_core_utils.core_helpers import map_finish_reason
 from litellm.litellm_core_utils.litellm_logging import Logging
 from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
@@ -48,14 +50,19 @@ from litellm.types.utils import (
 )
 from litellm.utils import add_dummy_tool, has_tool_call_blocks, supports_reasoning
 
-from ..common_utils import BedrockError, BedrockModelInfo, get_bedrock_tool_name, get_anthropic_beta_from_headers
+from ..common_utils import (
+    BedrockError,
+    BedrockModelInfo,
+    get_anthropic_beta_from_headers,
+    get_bedrock_tool_name,
+)
 
 # Computer use tool prefixes supported by Bedrock
 BEDROCK_COMPUTER_USE_TOOLS = [
     "computer_use_preview",
     "computer_",
     "bash_",
-    "text_editor_"
+    "text_editor_",
 ]
 
 
@@ -163,7 +170,9 @@ class AmazonConverseConfig(BaseConfig):
             # only anthropic and mistral support tool choice config. otherwise (E.g. cohere) will fail the call - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ToolChoice.html
             supported_params.append("tool_choice")
 
-        if (
+        if "gpt-oss" in model:
+            supported_params.append("reasoning_effort")
+        elif (
             "claude-3-7" in model
             or "claude-sonnet-4" in model
             or "claude-opus-4" in model
@@ -233,7 +242,7 @@ class AmazonConverseConfig(BaseConfig):
         """Check if computer use tools are being used in the request."""
         if tools is None:
             return False
-        
+
         for tool in tools:
             if "type" in tool:
                 tool_type = tool["type"]
@@ -247,17 +256,17 @@ class AmazonConverseConfig(BaseConfig):
     ) -> List[dict]:
         """Transform computer use tools to Bedrock format."""
         transformed_tools: List[dict] = []
-        
+
         for tool in computer_use_tools:
             tool_type = tool.get("type", "")
-            
+
             # Check if this is a computer use tool with the startswith method
             is_computer_use_tool = False
             for computer_use_prefix in BEDROCK_COMPUTER_USE_TOOLS:
                 if tool_type.startswith(computer_use_prefix):
                     is_computer_use_tool = True
                     break
-            
+
             transformed_tool: dict = {}
             if is_computer_use_tool:
                 if tool_type.startswith("computer_") and "function" in tool:
@@ -266,7 +275,7 @@ class AmazonConverseConfig(BaseConfig):
                     transformed_tool = {
                         "type": tool_type,
                         "name": func.get("name", "computer"),
-                        **func.get("parameters", {})
+                        **func.get("parameters", {}),
                     }
                 else:
                     # Direct tools - just need to ensure name is present
@@ -279,27 +288,29 @@ class AmazonConverseConfig(BaseConfig):
             else:
                 # Pass through other tools as-is
                 transformed_tool = dict(tool)
-                
+
             transformed_tools.append(transformed_tool)
-            
+
         return transformed_tools
 
     def _separate_computer_use_tools(
         self, tools: List[OpenAIChatCompletionToolParam], model: str
-    ) -> Tuple[List[OpenAIChatCompletionToolParam], List[OpenAIChatCompletionToolParam]]:
+    ) -> Tuple[
+        List[OpenAIChatCompletionToolParam], List[OpenAIChatCompletionToolParam]
+    ]:
         """
         Separate computer use tools from regular function tools.
-        
+
         Args:
             tools: List of tools to separate
             model: The model name to check if it supports computer use
-            
+
         Returns:
             Tuple of (computer_use_tools, regular_tools)
         """
         computer_use_tools = []
         regular_tools = []
-        
+
         for tool in tools:
             if "type" in tool:
                 tool_type = tool["type"]
@@ -314,15 +325,12 @@ class AmazonConverseConfig(BaseConfig):
                     regular_tools.append(tool)
             else:
                 regular_tools.append(tool)
-            
+
         return computer_use_tools, regular_tools
 
-
-
     def _create_json_tool_call_for_response_format(
         self,
         json_schema: Optional[dict] = None,
-        schema_name: str = "json_tool_call",
         description: Optional[str] = None,
     ) -> ChatCompletionToolParam:
         """
@@ -344,10 +352,12 @@ class AmazonConverseConfig(BaseConfig):
                 "properties": {},
             }
         else:
+            # Use the schema as-is for Bedrock
+            # Bedrock requires the tool schema to be of type "object" and doesn't need unwrapping
             _input_schema = json_schema
 
         tool_param_function_chunk = ChatCompletionToolParamFunctionChunk(
-            name=schema_name, parameters=_input_schema
+            name=RESPONSE_FORMAT_TOOL_NAME, parameters=_input_schema
         )
         if description:
             tool_param_function_chunk["description"] = description
@@ -386,56 +396,9 @@ class AmazonConverseConfig(BaseConfig):
 
         for param, value in non_default_params.items():
             if param == "response_format" and isinstance(value, dict):
-                ignore_response_format_types = ["text"]
-                if value["type"] in ignore_response_format_types:  # value is a no-op
-                    continue
-
-                json_schema: Optional[dict] = None
-                schema_name: str = ""
-                description: Optional[str] = None
-                if "response_schema" in value:
-                    json_schema = value["response_schema"]
-                    schema_name = "json_tool_call"
-                elif "json_schema" in value:
-                    json_schema = value["json_schema"]["schema"]
-                    schema_name = value["json_schema"]["name"]
-                    description = value["json_schema"].get("description")
-
-                if "type" in value and value["type"] == "text":
-                    continue
-
-                """
-                Follow similar approach to anthropic - translate to a single tool call. 
-
-                When using tools in this way: - https://docs.anthropic.com/en/docs/build-with-claude/tool-use#json-mode
-                - You usually want to provide a single tool
-                - You should set tool_choice (see Forcing tool use) to instruct the model to explicitly use that tool
-                - Remember that the model will pass the input to the tool, so the name of the tool and description should be from the model’s perspective.
-                """
-                _tool = self._create_json_tool_call_for_response_format(
-                    json_schema=json_schema,
-                    schema_name=schema_name if schema_name != "" else "json_tool_call",
-                    description=description,
+                optional_params = self._translate_response_format_param(
+                    value=value, model=model, optional_params=optional_params, non_default_params=non_default_params, is_thinking_enabled=is_thinking_enabled
                 )
-                optional_params = self._add_tools_to_optional_params(
-                    optional_params=optional_params, tools=[_tool]
-                )
-
-                if (
-                    litellm.utils.supports_tool_choice(
-                        model=model, custom_llm_provider=self.custom_llm_provider
-                    )
-                    and not is_thinking_enabled
-                ):
-
-                    optional_params["tool_choice"] = ToolChoiceValuesBlock(
-                        tool=SpecificToolChoiceBlock(
-                            name=schema_name if schema_name != "" else "json_tool_call"
-                        )
-                    )
-                optional_params["json_mode"] = True
-                if non_default_params.get("stream", False) is True:
-                    optional_params["fake_stream"] = True
             if param == "max_tokens" or param == "max_completion_tokens":
                 optional_params["maxTokens"] = value
             if param == "stream":
@@ -466,14 +429,82 @@ class AmazonConverseConfig(BaseConfig):
             if param == "thinking":
                 optional_params["thinking"] = value
             elif param == "reasoning_effort" and isinstance(value, str):
-                optional_params["thinking"] = AnthropicConfig._map_reasoning_effort(
-                    value
-                )
+                if "gpt-oss" in model:
+                    # GPT-OSS models: keep reasoning_effort as-is
+                    # It will be passed through to additionalModelRequestFields
+                    optional_params["reasoning_effort"] = value
+                else:
+                    # Anthropic and other models: convert to thinking parameter
+                    optional_params["thinking"] = AnthropicConfig._map_reasoning_effort(
+                        value
+                    )
 
-        self.update_optional_params_with_thinking_tokens(
-            non_default_params=non_default_params, optional_params=optional_params
+        # Only update thinking tokens for non-GPT-OSS models
+        if "gpt-oss" not in model:
+            self.update_optional_params_with_thinking_tokens(
+                non_default_params=non_default_params, optional_params=optional_params
+            )
+
+        return optional_params
+    
+    def _translate_response_format_param(
+        self, 
+        value: dict, 
+        model: str, 
+        optional_params: dict,
+        non_default_params: dict,
+        is_thinking_enabled: bool,
+    ) -> dict:
+        """
+        Handles translation of response_format parameter to Bedrock format.
+
+        Returns `optional_params` with the translated response_format parameter.
+        """
+        ignore_response_format_types = ["text"]
+        if value["type"] in ignore_response_format_types:  # value is a no-op
+            return optional_params
+
+        json_schema: Optional[dict] = None
+        description: Optional[str] = None
+        if "response_schema" in value:
+            json_schema = value["response_schema"]
+        elif "json_schema" in value:
+            json_schema = value["json_schema"]["schema"]
+            description = value["json_schema"].get("description")
+
+        if "type" in value and value["type"] == "text":
+            return optional_params
+
+        """
+        Follow similar approach to anthropic - translate to a single tool call. 
+
+        When using tools in this way: - https://docs.anthropic.com/en/docs/build-with-claude/tool-use#json-mode
+        - You usually want to provide a single tool
+        - You should set tool_choice (see Forcing tool use) to instruct the model to explicitly use that tool
+        - Remember that the model will pass the input to the tool, so the name of the tool and description should be from the model’s perspective.
+        """
+        _tool = self._create_json_tool_call_for_response_format(
+            json_schema=json_schema,
+            description=description,
+        )
+        optional_params = self._add_tools_to_optional_params(
+            optional_params=optional_params, tools=[_tool]
         )
 
+        if (
+            litellm.utils.supports_tool_choice(
+                model=model, custom_llm_provider=self.custom_llm_provider
+            )
+            and not is_thinking_enabled
+        ):
+
+            optional_params["tool_choice"] = ToolChoiceValuesBlock(
+                tool=SpecificToolChoiceBlock(name=RESPONSE_FORMAT_TOOL_NAME)
+            )
+        optional_params["json_mode"] = True
+        if non_default_params.get("stream", False) is True:
+            optional_params["fake_stream"] = True
+        
         return optional_params
 
     def update_optional_params_with_thinking_tokens(
@@ -597,7 +628,6 @@ class AmazonConverseConfig(BaseConfig):
 
         return {}
 
-
     def _transform_request_helper(
         self,
         model: str,
@@ -653,36 +683,38 @@ class AmazonConverseConfig(BaseConfig):
         )
 
         original_tools = inference_params.pop("tools", [])
-        
+
         # Initialize bedrock_tools
         bedrock_tools: List[ToolBlock] = []
-        
+
         # Collect anthropic_beta values from user headers
         anthropic_beta_list = []
         if headers:
             user_betas = get_anthropic_beta_from_headers(headers)
             anthropic_beta_list.extend(user_betas)
-        
+
         # Only separate tools if computer use tools are actually present
         if original_tools and self.is_computer_use_tool_used(original_tools, model):
             # Separate computer use tools from regular function tools
             computer_use_tools, regular_tools = self._separate_computer_use_tools(
                 original_tools, model
             )
-            
+
             # Process regular function tools using existing logic
             bedrock_tools = _bedrock_tools_pt(regular_tools)
-            
+
             # Add computer use tools and anthropic_beta if needed (only when computer use tools are present)
             if computer_use_tools:
                 anthropic_beta_list.append("computer-use-2024-10-22")
                 # Transform computer use tools to proper Bedrock format
-                transformed_computer_tools = self._transform_computer_use_tools(computer_use_tools)
+                transformed_computer_tools = self._transform_computer_use_tools(
+                    computer_use_tools
+                )
                 additional_request_params["tools"] = transformed_computer_tools
         else:
             # No computer use tools, process all tools as regular tools
             bedrock_tools = _bedrock_tools_pt(original_tools)
-        
+
         # Set anthropic_beta in additional_request_params if we have any beta features
         if anthropic_beta_list:
             # Remove duplicates while preserving order
@@ -693,7 +725,7 @@ class AmazonConverseConfig(BaseConfig):
                     unique_betas.append(beta)
                     seen.add(beta)
             additional_request_params["anthropic_beta"] = unique_betas
-        
+
         bedrock_tool_config: Optional[ToolConfigBlock] = None
         if len(bedrock_tools) > 0:
             tool_choice_values: ToolChoiceValuesBlock = inference_params.pop(
@@ -1119,10 +1151,37 @@ class AmazonConverseConfig(BaseConfig):
                 self._transform_thinking_blocks(reasoningContentBlocks)
             )
         chat_completion_message["content"] = content_str
-        if json_mode is True and tools is not None and len(tools) == 1:
-            # to support 'json_schema' logic on bedrock models
+        if (
+            json_mode is True
+            and tools is not None
+            and len(tools) == 1
+            and tools[0]["function"].get("name") == RESPONSE_FORMAT_TOOL_NAME
+        ):
+            verbose_logger.debug(
+                "Processing JSON tool call response for response_format"
+            )
             json_mode_content_str: Optional[str] = tools[0]["function"].get("arguments")
             if json_mode_content_str is not None:
+                import json
+
+                # Bedrock returns the response wrapped in a "properties" object
+                # We need to extract the actual content from this wrapper
+                try:
+
+                    response_data = json.loads(json_mode_content_str)
+
+                    # If Bedrock wrapped the response in "properties", extract the content
+                    if (
+                        isinstance(response_data, dict)
+                        and "properties" in response_data
+                        and len(response_data) == 1
+                    ):
+                        response_data = response_data["properties"]
+                        json_mode_content_str = json.dumps(response_data)
+                except json.JSONDecodeError:
+                    # If parsing fails, use the original response
+                    pass
+
                 chat_completion_message["content"] = json_mode_content_str
         else:
             chat_completion_message["tool_calls"] = tools
@@ -1182,7 +1241,6 @@ class AmazonConverseConfig(BaseConfig):
         if api_key:
             headers["Authorization"] = f"Bearer {api_key}"
         return headers
-    
 
     def should_fake_stream(
         self,

litellm/llms/bedrock/common_utils.py

@@ -6,6 +6,9 @@ import json
 import os
 from typing import TYPE_CHECKING, Dict, List, Literal, Optional, Union
 
+if TYPE_CHECKING:
+    from litellm.types.llms.bedrock import BedrockCreateBatchRequest
+
 import httpx
 
 import litellm
@@ -608,3 +611,218 @@ def get_anthropic_beta_from_headers(headers: dict) -> List[str]:
     
     # Split comma-separated values and strip whitespace
     return [beta.strip() for beta in anthropic_beta_header.split(",")]
+
+
+class CommonBatchFilesUtils:
+    """
+    Common utilities for Bedrock batch and file operations.
+    Provides shared functionality to reduce code duplication between batches and files.
+    """
+    
+    def __init__(self):
+        # Import here to avoid circular imports
+        from .base_aws_llm import BaseAWSLLM
+        self._base_aws = BaseAWSLLM()
+
+    def get_bedrock_model_id_from_litellm_model(self, model: str) -> str:
+        """
+        Extract the actual Bedrock model ID from LiteLLM model name.
+        
+        Args:
+            model: LiteLLM model name (e.g., "bedrock/anthropic.claude-3-sonnet-20240229-v1:0")
+            
+        Returns:
+            Bedrock model ID (e.g., "anthropic.claude-3-sonnet-20240229-v1:0")
+        """
+        if model.startswith("bedrock/"):
+            return model[8:]  # Remove "bedrock/" prefix
+        return model
+
+    def parse_s3_uri(self, s3_uri: str) -> tuple:
+        """
+        Parse S3 URI into bucket and key components.
+        
+        Args:
+            s3_uri: S3 URI (e.g., "s3://bucket/key/path")
+            
+        Returns:
+            Tuple of (bucket, key)
+            
+        Raises:
+            ValueError: If URI format is invalid
+        """
+        if not s3_uri.startswith("s3://"):
+            raise ValueError(f"Invalid S3 URI format: {s3_uri}")
+        
+        s3_parts = s3_uri[5:].split("/", 1)  # Remove "s3://" and split on first "/"
+        if len(s3_parts) != 2:
+            raise ValueError(f"Invalid S3 URI format: {s3_uri}")
+        
+        return s3_parts[0], s3_parts[1]  # bucket, key
+
+    def extract_model_from_s3_file_path(self, s3_uri: str, optional_params: dict) -> str:
+        """
+        Extract model ID from S3 file path.
+        
+        The Bedrock file transformation creates S3 objects with the model name embedded:
+        Format: s3://bucket/litellm-bedrock-files-{model}-{uuid}.jsonl
+        """
+        # Check if model is provided in optional_params first
+        if "model" in optional_params and optional_params["model"]:
+            return self.get_bedrock_model_id_from_litellm_model(optional_params["model"])
+        
+        # Extract model from S3 URI path
+        # Expected format: s3://bucket/litellm-bedrock-files-{model}-{uuid}.jsonl
+        try:
+            bucket, object_key = self.parse_s3_uri(s3_uri)
+            
+            # Extract model from object key if it follows our naming pattern
+            if object_key.startswith("litellm-bedrock-files-"):
+                # Remove prefix and suffix to get model part
+                model_part = object_key[22:]  # Remove "litellm-bedrock-files-"
+                # Find the last dash before the UUID
+                parts = model_part.split("-")
+                if len(parts) > 1:
+                    # Reconstruct model name (everything except the last UUID part and .jsonl)
+                    model_name = "-".join(parts[:-1])
+                    if model_name.endswith(".jsonl"):
+                        model_name = model_name[:-6]  # Remove .jsonl
+                    return model_name
+        except Exception:
+            pass
+        
+        # Fallback to default model
+        return "anthropic.claude-3-5-sonnet-20240620-v1:0"
+
+    def sign_aws_request(
+        self,
+        service_name: str,
+        data: Union[str, dict, "BedrockCreateBatchRequest"],
+        endpoint_url: str,
+        optional_params: dict,
+        method: str = "POST",
+    ) -> tuple:
+        """
+        Sign AWS request using Signature Version 4.
+        
+        Args:
+            service_name: AWS service name ("bedrock" or "s3")
+            data: Request data (string or dict)
+            endpoint_url: Full endpoint URL
+            optional_params: Optional parameters containing AWS credentials
+            method: HTTP method (default: POST)
+            
+        Returns:
+            Tuple of (signed_headers, signed_data)
+        """
+        try:
+            from botocore.auth import SigV4Auth
+            from botocore.awsrequest import AWSRequest
+        except ImportError:
+            raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.")
+
+        # Get AWS credentials using existing methods
+        aws_region_name = self._base_aws._get_aws_region_name(
+            optional_params=optional_params, model=""
+        )
+        credentials = self._base_aws.get_credentials(
+            aws_access_key_id=optional_params.get("aws_access_key_id"),
+            aws_secret_access_key=optional_params.get("aws_secret_access_key"),
+            aws_session_token=optional_params.get("aws_session_token"),
+            aws_region_name=aws_region_name,
+            aws_session_name=optional_params.get("aws_session_name"),
+            aws_profile_name=optional_params.get("aws_profile_name"),
+            aws_role_name=optional_params.get("aws_role_name"),
+            aws_web_identity_token=optional_params.get("aws_web_identity_token"),
+            aws_sts_endpoint=optional_params.get("aws_sts_endpoint"),
+        )
+        
+        # Prepare the request data
+        if isinstance(data, dict):
+            import json
+            request_data = json.dumps(data)
+        else:
+            request_data = data
+        
+        # Prepare headers
+        headers = {"Content-Type": "application/json"}
+        
+        # Create AWS request and sign it
+        sigv4 = SigV4Auth(credentials, service_name, aws_region_name)
+        request = AWSRequest(
+            method=method.upper(), url=endpoint_url, data=request_data, headers=headers
+        )
+        sigv4.add_auth(request)
+        prepped = request.prepare()
+        
+        return dict(prepped.headers), request_data.encode('utf-8') if isinstance(request_data, str) else request_data
+
+    def generate_unique_job_name(self, model: str, prefix: str = "litellm") -> str:
+        """
+        Generate a unique job name for AWS services.
+        AWS services often have length limits, so this creates a concise name.
+        
+        Args:
+            model: Model name to include in the job name
+            prefix: Prefix for the job name
+            
+        Returns:
+            Unique job name (≤ 63 characters for Bedrock compatibility)
+        """
+        import fastuuid as uuid
+        unique_id = str(uuid.uuid4())[:8]
+        # Format: {prefix}-batch-{model}-{uuid}
+        # Example: litellm-batch-claude-266c398e
+        job_name = f"{prefix}-batch-{unique_id}"
+        
+        return job_name
+
+    def get_s3_bucket_and_key_from_config(
+        self, 
+        litellm_params: dict, 
+        optional_params: dict,
+        bucket_env_var: str = "AWS_S3_BUCKET_NAME",
+        key_prefix: str = "litellm"
+    ) -> tuple:
+        """
+        Get S3 bucket and generate a unique key from configuration.
+        
+        Args:
+            litellm_params: LiteLLM parameters
+            optional_params: Optional parameters
+            bucket_env_var: Environment variable name for bucket
+            key_prefix: Prefix for the S3 key
+            
+        Returns:
+            Tuple of (bucket_name, object_key)
+        """
+        import time
+        import uuid
+
+        # Get bucket name
+        bucket_name = (
+            litellm_params.get("s3_bucket_name") 
+            or optional_params.get("s3_bucket_name")
+            or os.getenv(bucket_env_var)
+        )
+        if not bucket_name:
+            raise ValueError(f"S3 bucket name is required. Set 's3_bucket_name' parameter or {bucket_env_var} env var")
+        
+        # Generate unique object key
+        timestamp = int(time.time())
+        unique_id = str(uuid.uuid4())[:8]
+        object_key = f"{key_prefix}-{timestamp}-{unique_id}"
+        
+        return bucket_name, object_key
+
+    def get_error_class(
+        self, error_message: str, status_code: int, headers: Union[Dict, httpx.Headers]
+    ) -> BaseLLMException:
+        """
+        Get Bedrock-specific error class.
+        """
+        return BedrockError(
+            status_code=status_code, 
+            message=error_message, 
+            headers=headers
+        )

litellm/llms/bedrock/files/transformation.py

@@ -0,0 +1,607 @@
+import json
+import os
+import time
+import uuid
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+from httpx import Headers, Response
+
+from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.base_llm.files.transformation import (
+    BaseFilesConfig,
+    LiteLLMLoggingObj,
+)
+from litellm.types.llms.openai import (
+    AllMessageValues,
+    CreateFileRequest,
+    FileTypes,
+    OpenAICreateFileRequestOptionalParams,
+    OpenAIFileObject,
+    PathLike,
+)
+from litellm.types.utils import ExtractedFileData, LlmProviders
+
+from ..base_aws_llm import BaseAWSLLM
+from ..common_utils import BedrockError
+
+
+class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
+    """
+    Config for Bedrock Files - handles S3 uploads for Bedrock batch processing
+    """
+    
+    def __init__(self):
+        self.jsonl_transformation = BedrockJsonlFilesTransformation()
+        super().__init__()
+
+    @property
+    def custom_llm_provider(self) -> LlmProviders:
+        return LlmProviders.BEDROCK
+
+    @property
+    def file_upload_http_method(self) -> str:
+        """
+        Bedrock files are uploaded to S3, which requires PUT requests
+        """
+        return "PUT"
+
+    def validate_environment(
+        self,
+        headers: dict,
+        model: str,
+        messages: List[AllMessageValues],
+        optional_params: dict,
+        litellm_params: dict,
+        api_key: Optional[str] = None,
+        api_base: Optional[str] = None,
+    ) -> dict:
+        # No additional headers needed for S3 uploads - AWS credentials handled by BaseAWSLLM
+        return headers
+
+
+
+    def _get_content_from_openai_file(self, openai_file_content: FileTypes) -> str:
+        """
+        Helper to extract content from various OpenAI file types and return as string.
+
+        Handles:
+        - Direct content (str, bytes, IO[bytes])
+        - Tuple formats: (filename, content, [content_type], [headers])
+        - PathLike objects
+        """
+        content: Union[str, bytes] = b""
+        # Extract file content from tuple if necessary
+        if isinstance(openai_file_content, tuple):
+            # Take the second element which is always the file content
+            file_content = openai_file_content[1]
+        else:
+            file_content = openai_file_content
+
+        # Handle different file content types
+        if isinstance(file_content, str):
+            # String content can be used directly
+            content = file_content
+        elif isinstance(file_content, bytes):
+            # Bytes content can be decoded
+            content = file_content
+        elif isinstance(file_content, PathLike):  # PathLike
+            with open(str(file_content), "rb") as f:
+                content = f.read()
+        elif hasattr(file_content, "read"):  # IO[bytes]
+            # File-like objects need to be read
+            content = file_content.read()
+
+        # Ensure content is string
+        if isinstance(content, bytes):
+            content = content.decode("utf-8")
+
+        return content
+
+    def _get_s3_object_name_from_batch_jsonl(
+        self,
+        openai_jsonl_content: List[Dict[str, Any]],
+    ) -> str:
+        """
+        Gets a unique S3 object name for the Bedrock batch processing job
+
+        named as: litellm-bedrock-files/{model}/{uuid}
+        """
+        _model = openai_jsonl_content[0].get("body", {}).get("model", "")
+        # Remove bedrock/ prefix if present
+        if _model.startswith("bedrock/"):
+            _model = _model[8:]
+        object_name = f"litellm-bedrock-files-{_model}-{uuid.uuid4()}.jsonl"
+        return object_name
+
+    def get_object_name(
+        self, extracted_file_data: ExtractedFileData, purpose: str
+    ) -> str:
+        """
+        Get the object name for the request
+        """
+        extracted_file_data_content = extracted_file_data.get("content")
+
+        if extracted_file_data_content is None:
+            raise ValueError("file content is required")
+
+        if purpose == "batch":
+            ## 1. If jsonl, check if there's a model name
+            file_content = self._get_content_from_openai_file(
+                extracted_file_data_content
+            )
+
+            # Split into lines and parse each line as JSON
+            openai_jsonl_content = [
+                json.loads(line) for line in file_content.splitlines() if line.strip()
+            ]
+            if len(openai_jsonl_content) > 0:
+                return self._get_s3_object_name_from_batch_jsonl(openai_jsonl_content)
+
+        ## 2. If not jsonl, return the filename
+        filename = extracted_file_data.get("filename")
+        if filename:
+            return filename
+        ## 3. If no file name, return timestamp
+        return str(int(time.time()))
+
+    def get_complete_file_url(
+        self,
+        api_base: Optional[str],
+        api_key: Optional[str],
+        model: str,
+        optional_params: Dict,
+        litellm_params: Dict,
+        data: CreateFileRequest,
+    ) -> str:
+        """
+        Get the complete S3 URL for the file upload request
+        """
+        bucket_name = litellm_params.get("s3_bucket_name") or os.getenv("AWS_S3_BUCKET_NAME")
+        if not bucket_name:
+            raise ValueError("S3 bucket_name is required. Set 's3_bucket_name' in litellm_params or AWS_S3_BUCKET_NAME env var")
+        
+        aws_region_name = self._get_aws_region_name(optional_params, model)
+        
+        file_data = data.get("file")
+        purpose = data.get("purpose")
+        if file_data is None:
+            raise ValueError("file is required")
+        if purpose is None:
+            raise ValueError("purpose is required")
+        extracted_file_data = extract_file_data(file_data)
+        object_name = self.get_object_name(extracted_file_data, purpose)
+        
+        # S3 endpoint URL format
+        s3_endpoint_url = optional_params.get("s3_endpoint_url") or f"https://s3.{aws_region_name}.amazonaws.com"
+        
+        return f"{s3_endpoint_url}/{bucket_name}/{object_name}"
+
+    def get_supported_openai_params(
+        self, model: str
+    ) -> List[OpenAICreateFileRequestOptionalParams]:
+        return []
+
+    def map_openai_params(
+        self,
+        non_default_params: dict,
+        optional_params: dict,
+        model: str,
+        drop_params: bool,
+    ) -> dict:
+        return optional_params
+
+    def _get_bedrock_provider_from_model(self, model: str) -> Optional[str]:
+        """
+        Extract provider from Bedrock model name
+        """
+        if model.startswith("anthropic."):
+            return "anthropic"
+        elif model.startswith("cohere."):
+            return "cohere"
+        elif model.startswith("meta.") or model.startswith("llama"):
+            return "meta"
+        elif model.startswith("mistral."):
+            return "mistral"
+        elif model.startswith("ai21."):
+            return "ai21"
+        elif model.startswith("amazon."):
+            return "amazon"
+        else:
+            return None
+
+    def _map_openai_to_bedrock_params(
+        self,
+        openai_request_body: Dict[str, Any],
+        provider: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        """
+        Transform OpenAI request body to Bedrock-compatible modelInput parameters using existing transformation logic
+        """
+        _model = openai_request_body.get("model", "")
+        messages = openai_request_body.get("messages", [])
+        
+        # Use existing Anthropic transformation logic for Anthropic models
+        if provider == "anthropic":
+            from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import (
+                AmazonAnthropicClaudeConfig,
+            )
+            
+            anthropic_config = AmazonAnthropicClaudeConfig()
+            
+            # Extract optional params (everything except model and messages)
+            optional_params = {k: v for k, v in openai_request_body.items() if k not in ["model", "messages"]}
+            
+            # Transform using existing Anthropic logic
+            bedrock_params = anthropic_config.transform_request(
+                model=_model,
+                messages=messages,
+                optional_params=optional_params,
+                litellm_params={},
+                headers={}
+            )
+            
+            return bedrock_params
+        else:
+            # For other providers, use basic mapping
+            bedrock_params = {
+                "messages": messages,
+                **{k: v for k, v in openai_request_body.items() if k not in ["model", "messages"]}
+            }
+            return bedrock_params
+
+    def _transform_openai_jsonl_content_to_bedrock_jsonl_content(
+        self, openai_jsonl_content: List[Dict[str, Any]]
+    ) -> List[Dict[str, Any]]:
+        """
+        Transforms OpenAI JSONL content to Bedrock batch format
+        
+        Bedrock batch format: { "recordId": "alphanumeric string", "modelInput": {JSON body} }
+        Example:
+        {
+            "recordId": "CALL0000001", 
+            "modelInput": {
+                "anthropic_version": "bedrock-2023-05-31", 
+                "max_tokens": 1024,
+                "messages": [ 
+                    { 
+                        "role": "user", 
+                        "content": [{"type": "text", "text": "Hello"}]
+                    }
+                ]
+            }
+        }
+        """
+        
+        bedrock_jsonl_content = []
+        for idx, _openai_jsonl_content in enumerate(openai_jsonl_content):
+            # Extract the request body from OpenAI format
+            openai_body = _openai_jsonl_content.get("body", {})
+            model = openai_body.get("model", "")
+            
+            # Determine provider from model name
+            provider = self._get_bedrock_provider_from_model(model)
+            
+            # Transform to Bedrock modelInput format
+            model_input = self._map_openai_to_bedrock_params(
+                openai_request_body=openai_body,
+                provider=provider
+            )
+            
+            # Create Bedrock batch record
+            record_id = _openai_jsonl_content.get("custom_id", f"CALL{str(idx).zfill(7)}")
+            bedrock_record = {
+                "recordId": record_id,
+                "modelInput": model_input
+            }
+                    
+            bedrock_jsonl_content.append(bedrock_record)
+        return bedrock_jsonl_content
+
+    def transform_create_file_request(
+        self,
+        model: str,
+        create_file_data: CreateFileRequest,
+        optional_params: dict,
+        litellm_params: dict,
+    ) -> Union[bytes, str, dict]:
+        """
+        Transform file request and return a pre-signed request for S3.
+        This keeps the HTTP handler clean by doing all the signing here.
+        """
+        file_data = create_file_data.get("file")
+        if file_data is None:
+            raise ValueError("file is required")
+        extracted_file_data = extract_file_data(file_data)
+        extracted_file_data_content = extracted_file_data.get("content")
+        
+        # Get and transform the file content
+        if (
+            create_file_data.get("purpose") == "batch"
+            and extracted_file_data.get("content_type") == "application/jsonl"
+            and extracted_file_data_content is not None
+        ):
+            ## Transform JSONL content to Bedrock format
+            original_file_content = self._get_content_from_openai_file(
+                extracted_file_data_content
+            )
+            openai_jsonl_content = [
+                json.loads(line) for line in original_file_content.splitlines() if line.strip()
+            ]
+            bedrock_jsonl_content = (
+                self._transform_openai_jsonl_content_to_bedrock_jsonl_content(
+                    openai_jsonl_content
+                )
+            )
+            file_content = "\n".join(json.dumps(item) for item in bedrock_jsonl_content)
+        elif isinstance(extracted_file_data_content, bytes):
+            file_content = extracted_file_data_content.decode('utf-8')
+        elif isinstance(extracted_file_data_content, str):
+            file_content = extracted_file_data_content
+        else:
+            raise ValueError("Unsupported file content type")
+        
+        # Get the S3 URL for upload
+        api_base = self.get_complete_file_url(
+            api_base=None,
+            api_key=None,
+            model=model,
+            optional_params=optional_params,
+            litellm_params=litellm_params,
+            data=create_file_data,
+        )
+        
+        # Sign the request and return a pre-signed request object
+        signed_headers, signed_body = self._sign_s3_request(
+            content=file_content,
+            api_base=api_base,
+            optional_params=optional_params,
+        )
+        
+        # Return a dict that tells the HTTP handler exactly what to do
+        return {
+            "method": "PUT",
+            "url": api_base,
+            "headers": signed_headers,
+            "data": signed_body or file_content,
+        }
+
+    def _sign_s3_request(
+        self,
+        content: str,
+        api_base: str,
+        optional_params: dict,
+    ) -> Tuple[dict, str]:
+        """
+        Sign S3 PUT request using the same proven logic as S3Logger.
+        Reuses the exact pattern from litellm/integrations/s3_v2.py
+        """
+        try:
+            import hashlib
+
+            import requests
+            from botocore.auth import SigV4Auth
+            from botocore.awsrequest import AWSRequest
+        except ImportError:
+            raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.")
+
+        # Get AWS credentials using existing methods
+        aws_region_name = self._get_aws_region_name(
+            optional_params=optional_params, model=""
+        )
+        credentials = self.get_credentials(
+            aws_access_key_id=optional_params.get("aws_access_key_id"),
+            aws_secret_access_key=optional_params.get("aws_secret_access_key"),
+            aws_session_token=optional_params.get("aws_session_token"),
+            aws_region_name=aws_region_name,
+            aws_session_name=optional_params.get("aws_session_name"),
+            aws_profile_name=optional_params.get("aws_profile_name"),
+            aws_role_name=optional_params.get("aws_role_name"),
+            aws_web_identity_token=optional_params.get("aws_web_identity_token"),
+            aws_sts_endpoint=optional_params.get("aws_sts_endpoint"),
+        )
+        
+        # Calculate SHA256 hash of the content (REQUIRED for S3)
+        content_hash = hashlib.sha256(content.encode("utf-8")).hexdigest()
+
+        # Prepare headers with required S3 headers (same as s3_v2.py)
+        request_headers = {
+            "Content-Type": "application/json",  # JSONL files are JSON content
+            "x-amz-content-sha256": content_hash,  # REQUIRED by S3
+            "Content-Language": "en",
+            "Cache-Control": "private, immutable, max-age=31536000, s-maxage=0",
+        }
+
+        # Use requests.Request to prepare the request (same pattern as s3_v2.py)
+        req = requests.Request("PUT", api_base, data=content, headers=request_headers)
+        prepped = req.prepare()
+
+        # Sign the request with S3 service
+        aws_request = AWSRequest(
+            method=prepped.method,
+            url=prepped.url,
+            data=prepped.body,
+            headers=prepped.headers,
+        )
+        
+        # Get region name for non-LLM API calls (same as s3_v2.py)
+        signing_region = self.get_aws_region_name_for_non_llm_api_calls(
+            aws_region_name=aws_region_name
+        )
+        
+        SigV4Auth(credentials, "s3", signing_region).add_auth(aws_request)
+
+        # Return signed headers and body
+        signed_body = aws_request.body
+        if isinstance(signed_body, bytes):
+            signed_body = signed_body.decode('utf-8')
+        elif signed_body is None:
+            signed_body = content  # Fallback to original content
+        
+        return dict(aws_request.headers), signed_body
+
+    def transform_create_file_response(
+        self,
+        model: Optional[str],
+        raw_response: Response,
+        logging_obj: LiteLLMLoggingObj,
+        litellm_params: dict,
+    ) -> OpenAIFileObject:
+        """
+        Transform S3 File upload response into OpenAI-style FileObject
+        """
+        # For S3 uploads, we typically get an ETag and other metadata
+        response_headers = raw_response.headers
+        
+        # Extract S3 object information from the response
+        # S3 PUT object returns ETag and other metadata in headers
+        content_length = response_headers.get("Content-Length", "0")
+        
+        # Extract bucket and key from the request URL or litellm_params
+        bucket_name = litellm_params.get("s3_bucket_name") or os.getenv("AWS_S3_BUCKET_NAME")
+        
+        # Generate file ID in S3 format
+        object_key = getattr(logging_obj, 'object_key', None) or f"file-{int(time.time())}"
+        file_id = f"s3://{bucket_name}/{object_key}"
+        
+        # Extract filename from object key
+        filename = object_key.split("/")[-1] if "/" in object_key else object_key
+        
+        return OpenAIFileObject(
+            purpose="batch",  # Default purpose for Bedrock files
+            id=file_id,
+            filename=filename,
+            created_at=int(time.time()),  # Current timestamp
+            status="uploaded",
+            bytes=int(content_length) if content_length.isdigit() else 0,
+            object="file",
+        )
+
+    def get_error_class(
+        self, error_message: str, status_code: int, headers: Union[Dict, Headers]
+    ) -> BaseLLMException:
+        return BedrockError(
+            status_code=status_code, message=error_message, headers=headers
+        )
+
+
+class BedrockJsonlFilesTransformation:
+    """
+    Transforms OpenAI /v1/files/* requests to Bedrock S3 file uploads for batch processing
+    """
+
+    def transform_openai_file_content_to_bedrock_file_content(
+        self, openai_file_content: Optional[FileTypes] = None
+    ) -> Tuple[str, str]:
+        """
+        Transforms OpenAI FileContentRequest to Bedrock S3 file format
+        """
+
+        if openai_file_content is None:
+            raise ValueError("contents of file are None")
+        # Read the content of the file
+        file_content = self._get_content_from_openai_file(openai_file_content)
+
+        # Split into lines and parse each line as JSON
+        openai_jsonl_content = [
+            json.loads(line) for line in file_content.splitlines() if line.strip()
+        ]
+        bedrock_jsonl_content = (
+            self._transform_openai_jsonl_content_to_bedrock_jsonl_content(
+                openai_jsonl_content
+            )
+        )
+        bedrock_jsonl_string = "\n".join(
+            json.dumps(item) for item in bedrock_jsonl_content
+        )
+        object_name = self._get_s3_object_name(
+            openai_jsonl_content=openai_jsonl_content
+        )
+        return bedrock_jsonl_string, object_name
+
+    def _transform_openai_jsonl_content_to_bedrock_jsonl_content(
+        self, openai_jsonl_content: List[Dict[str, Any]]
+    ):
+        """
+        Delegate to the main BedrockFilesConfig transformation method
+        """
+        config = BedrockFilesConfig()
+        return config._transform_openai_jsonl_content_to_bedrock_jsonl_content(openai_jsonl_content)
+
+    def _get_s3_object_name(
+        self,
+        openai_jsonl_content: List[Dict[str, Any]],
+    ) -> str:
+        """
+        Gets a unique S3 object name for the Bedrock batch processing job
+
+        named as: litellm-bedrock-files-{model}-{uuid}
+        """
+        _model = openai_jsonl_content[0].get("body", {}).get("model", "")
+        # Remove bedrock/ prefix if present
+        if _model.startswith("bedrock/"):
+            _model = _model[8:]
+        object_name = f"litellm-bedrock-files-{_model}-{uuid.uuid4()}.jsonl"
+        return object_name
+
+
+
+    def _get_content_from_openai_file(self, openai_file_content: FileTypes) -> str:
+        """
+        Helper to extract content from various OpenAI file types and return as string.
+
+        Handles:
+        - Direct content (str, bytes, IO[bytes])
+        - Tuple formats: (filename, content, [content_type], [headers])
+        - PathLike objects
+        """
+        content: Union[str, bytes] = b""
+        # Extract file content from tuple if necessary
+        if isinstance(openai_file_content, tuple):
+            # Take the second element which is always the file content
+            file_content = openai_file_content[1]
+        else:
+            file_content = openai_file_content
+
+        # Handle different file content types
+        if isinstance(file_content, str):
+            # String content can be used directly
+            content = file_content
+        elif isinstance(file_content, bytes):
+            # Bytes content can be decoded
+            content = file_content
+        elif isinstance(file_content, PathLike):  # PathLike
+            with open(str(file_content), "rb") as f:
+                content = f.read()
+        elif hasattr(file_content, "read"):  # IO[bytes]
+            # File-like objects need to be read
+            content = file_content.read()
+
+        # Ensure content is string
+        if isinstance(content, bytes):
+            content = content.decode("utf-8")
+
+        return content
+
+    def transform_s3_bucket_response_to_openai_file_object(
+        self, create_file_data: CreateFileRequest, s3_upload_response: Dict[str, Any]
+    ) -> OpenAIFileObject:
+        """
+        Transforms S3 Bucket upload file response to OpenAI FileObject
+        """
+        # S3 response typically contains ETag, key, etc.
+        object_key = s3_upload_response.get("Key", "")
+        bucket_name = s3_upload_response.get("Bucket", "")
+        
+        # Extract filename from object key
+        filename = object_key.split("/")[-1] if "/" in object_key else object_key
+        
+        return OpenAIFileObject(
+            purpose=create_file_data.get("purpose", "batch"),
+            id=f"s3://{bucket_name}/{object_key}",
+            filename=filename,
+            created_at=int(time.time()),  # Current timestamp
+            status="uploaded",
+            bytes=s3_upload_response.get("ContentLength", 0),
+            object="file",
+        )

litellm/llms/bedrock/passthrough/transformation.py

@@ -41,9 +41,15 @@ class BedrockPassthroughConfig(
             model_id=None,
         )
 
-        api_base = f"https://bedrock-runtime.{aws_region_name}.amazonaws.com"
+        aws_bedrock_runtime_endpoint = optional_params.get("aws_bedrock_runtime_endpoint")
+        endpoint_url, _ = self.get_runtime_endpoint(
+            api_base=api_base,
+            aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint,
+            aws_region_name=aws_region_name,
+            endpoint_type="runtime",
+        )
 
-        return self.format_url(endpoint, api_base, request_query_params or {}), api_base
+        return self.format_url(endpoint, endpoint_url, request_query_params or {}), endpoint_url
 
     def sign_request(
         self,

litellm/llms/custom_httpx/http_handler.py

@@ -212,6 +212,7 @@ class AsyncHTTPHandler:
             verify=ssl_config,
             cert=cert,
             headers=headers,
+            follow_redirects=True,
         )
 
     async def close(self):
@@ -687,6 +688,7 @@ class HTTPHandler:
                 verify=ssl_config,
                 cert=cert,
                 headers=headers,
+                follow_redirects=True,
             )
         else:
             self.client = client

litellm/llms/custom_httpx/llm_http_handler.py

@@ -28,6 +28,7 @@ from litellm.llms.base_llm.audio_transcription.transformation import (
     BaseAudioTranscriptionConfig,
 )
 from litellm.llms.base_llm.base_model_iterator import MockResponseIterator
+from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig
 from litellm.llms.base_llm.chat.transformation import BaseConfig
 from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig
 from litellm.llms.base_llm.files.transformation import BaseFilesConfig
@@ -58,6 +59,7 @@ from litellm.types.llms.anthropic_messages.anthropic_response import (
     AnthropicMessagesResponse,
 )
 from litellm.types.llms.openai import (
+    CreateBatchRequest,
     CreateFileRequest,
     OpenAIFileObject,
     ResponseInputParam,
@@ -66,7 +68,12 @@ from litellm.types.llms.openai import (
 from litellm.types.rerank import OptionalRerankParams, RerankResponse
 from litellm.types.responses.main import DeleteResponseResult
 from litellm.types.router import GenericLiteLLMParams
-from litellm.types.utils import EmbeddingResponse, FileTypes, TranscriptionResponse
+from litellm.types.utils import (
+    EmbeddingResponse,
+    FileTypes,
+    LiteLLMBatch,
+    TranscriptionResponse,
+)
 from litellm.types.vector_stores import (
     VectorStoreCreateOptionalRequestParams,
     VectorStoreCreateResponse,
@@ -2212,15 +2219,38 @@ class BaseLLMHTTPHandler:
         else:
             sync_httpx_client = client
 
-        if isinstance(transformed_request, str) or isinstance(
-            transformed_request, bytes
-        ):
-            upload_response = sync_httpx_client.post(
-                url=api_base,
-                headers=headers,
-                data=transformed_request,
+        if isinstance(transformed_request, dict) and "method" in transformed_request:
+            # Handle pre-signed requests (e.g., from Bedrock S3 uploads)
+            upload_response = getattr(sync_httpx_client, transformed_request["method"].lower())(
+                url=transformed_request["url"],
+                headers=transformed_request["headers"],
+                data=transformed_request["data"],
                 timeout=timeout,
             )
+        elif isinstance(transformed_request, str) or isinstance(
+            transformed_request, bytes
+        ):
+            # Handle traditional file uploads
+            # Ensure transformed_request is a string for httpx compatibility
+            if isinstance(transformed_request, bytes):
+                transformed_request = transformed_request.decode('utf-8')
+            
+            # Use the HTTP method specified by the provider config
+            http_method = provider_config.file_upload_http_method.upper()
+            if http_method == "PUT":
+                upload_response = sync_httpx_client.put(
+                    url=api_base,
+                    headers=headers,
+                    data=transformed_request,
+                    timeout=timeout,
+                )
+            else:  # Default to POST
+                upload_response = sync_httpx_client.post(
+                    url=api_base,
+                    headers=headers,
+                    data=transformed_request,
+                    timeout=timeout,
+                )
         else:
             try:
                 # Step 1: Initial request to get upload URL
@@ -2280,16 +2310,52 @@ class BaseLLMHTTPHandler:
             )
         else:
             async_httpx_client = client
+        
+        #########################################################
+        # Debug Logging
+        #########################################################
+        logging_obj.pre_call(
+            input="",
+            api_key="",
+            additional_args={
+                "complete_input_dict": transformed_request,
+                "api_base": api_base,
+                "headers": headers,
+            },
+        )
 
-        if isinstance(transformed_request, str) or isinstance(
-            transformed_request, bytes
-        ):
-            upload_response = await async_httpx_client.post(
-                url=api_base,
-                headers=headers,
-                data=transformed_request,
+        if isinstance(transformed_request, dict) and "method" in transformed_request:
+            # Handle pre-signed requests (e.g., from Bedrock S3 uploads)
+            upload_response = await getattr(async_httpx_client, transformed_request["method"].lower())(
+                url=transformed_request["url"],
+                headers=transformed_request["headers"],
+                data=transformed_request["data"],
                 timeout=timeout,
             )
+        elif isinstance(transformed_request, str) or isinstance(
+            transformed_request, bytes
+        ):
+            # Handle traditional file uploads
+            # Ensure transformed_request is a string for httpx compatibility
+            if isinstance(transformed_request, bytes):
+                transformed_request = transformed_request.decode('utf-8')
+            
+            # Use the HTTP method specified by the provider config
+            http_method = provider_config.file_upload_http_method.upper()
+            if http_method == "PUT":
+                upload_response = await async_httpx_client.put(
+                    url=api_base,
+                    headers=headers,
+                    data=transformed_request,
+                    timeout=timeout,
+                )
+            else:  # Default to POST
+                upload_response = await async_httpx_client.post(
+                    url=api_base,
+                    headers=headers,
+                    data=transformed_request,
+                    timeout=timeout,
+                )
         else:
             try:
                 # Step 1: Initial request to get upload URL
@@ -2330,6 +2396,188 @@ class BaseLLMHTTPHandler:
             litellm_params=litellm_params,
         )
 
+    def create_batch(
+        self,
+        create_batch_data: "CreateBatchRequest",
+        litellm_params: dict,
+        provider_config: "BaseBatchesConfig",
+        headers: dict,
+        api_base: Optional[str],
+        api_key: Optional[str],
+        logging_obj: "LiteLLMLoggingObj",
+        _is_async: bool = False,
+        client: Optional[Union["HTTPHandler", "AsyncHTTPHandler"]] = None,
+        timeout: Optional[Union[float, httpx.Timeout]] = None,
+    ) -> Union["LiteLLMBatch", Coroutine[Any, Any, "LiteLLMBatch"]]:
+        """
+        Creates a batch using provider-specific batch creation process
+        """
+        # get config from model, custom llm provider
+        headers = provider_config.validate_environment(
+            api_key=api_key,
+            headers=headers,
+            model="",
+            messages=[],
+            optional_params={},
+            litellm_params=litellm_params,
+        )
+
+        api_base = provider_config.get_complete_batch_url(
+            api_base=api_base,
+            api_key=api_key,
+            model="",
+            optional_params={},
+            litellm_params=litellm_params,
+            data=create_batch_data,
+        )
+        if api_base is None:
+            raise ValueError("api_base is required for create_batch")
+
+        # Get the transformed request data
+        transformed_request = provider_config.transform_create_batch_request(
+            model="",
+            create_batch_data=create_batch_data,
+            litellm_params=litellm_params,
+            optional_params={},
+        )
+
+        if _is_async:
+            return self.async_create_batch(
+                transformed_request=transformed_request,
+                litellm_params=litellm_params,
+                provider_config=provider_config,
+                headers=headers,
+                api_base=api_base,
+                logging_obj=logging_obj,
+                client=client,
+                timeout=timeout,
+                create_batch_data=create_batch_data,
+            )
+
+        if client is None or not isinstance(client, HTTPHandler):
+            sync_httpx_client = _get_httpx_client()
+        else:
+            sync_httpx_client = client
+
+        try:
+            if isinstance(transformed_request, dict) and "method" in transformed_request:
+                # Handle pre-signed requests (e.g., from Bedrock with AWS auth)
+                batch_response = getattr(sync_httpx_client, transformed_request["method"].lower())(
+                    url=transformed_request["url"],
+                    headers=transformed_request["headers"],
+                    data=transformed_request["data"],
+                    timeout=timeout,
+                )
+            elif isinstance(transformed_request, dict):
+                # For other providers that use JSON requests
+                batch_response = sync_httpx_client.post(
+                    url=api_base,
+                    headers={**headers, "Content-Type": "application/json"},
+                    json=transformed_request,
+                    timeout=timeout,
+                )
+            else:
+                # Handle other request types if needed
+                batch_response = sync_httpx_client.post(
+                    url=api_base,
+                    headers=headers,
+                    data=transformed_request,
+                    timeout=timeout,
+                )
+        except Exception as e:
+            verbose_logger.exception(f"Error creating batch: {e}")
+            raise self._handle_error(
+                e=e,
+                provider_config=provider_config,
+            )
+
+        # Store original request for response transformation
+        litellm_params_with_request = {**litellm_params, "original_batch_request": create_batch_data}
+        
+        return provider_config.transform_create_batch_response(
+            model=None,
+            raw_response=batch_response,
+            logging_obj=logging_obj,
+            litellm_params=litellm_params_with_request,
+        )
+
+    async def async_create_batch(
+        self,
+        transformed_request: Union[bytes, str, dict],
+        litellm_params: dict,
+        provider_config: "BaseBatchesConfig",
+        headers: dict,
+        api_base: str,
+        logging_obj: "LiteLLMLoggingObj",
+        client: Optional[Union["HTTPHandler", "AsyncHTTPHandler"]] = None,
+        timeout: Optional[Union[float, httpx.Timeout]] = None,
+        create_batch_data: Optional["CreateBatchRequest"] = None,
+    ):
+        """
+        Async version of create_batch
+        """
+        if client is None or not isinstance(client, AsyncHTTPHandler):
+            async_httpx_client = get_async_httpx_client(
+                llm_provider=provider_config.custom_llm_provider
+            )
+        else:
+            async_httpx_client = client
+        
+        #########################################################
+        # Debug Logging
+        #########################################################
+        logging_obj.pre_call(
+            input="",
+            api_key="",
+            additional_args={
+                "complete_input_dict": transformed_request,
+                "api_base": api_base,
+                "headers": headers,
+            },
+        )
+
+        try:
+            if isinstance(transformed_request, dict) and "method" in transformed_request:
+                # Handle pre-signed requests (e.g., from Bedrock with AWS auth)
+                batch_response = await getattr(async_httpx_client, transformed_request["method"].lower())(
+                    url=transformed_request["url"],
+                    headers=transformed_request["headers"],
+                    data=transformed_request["data"],
+                    timeout=timeout,
+                )
+            elif isinstance(transformed_request, dict):
+                # For other providers that use JSON requests
+                batch_response = await async_httpx_client.post(
+                    url=api_base,
+                    headers={**headers, "Content-Type": "application/json"},
+                    json=transformed_request,
+                    timeout=timeout,
+                )
+            else:
+                # Handle other request types if needed
+                batch_response = await async_httpx_client.post(
+                    url=api_base,
+                    headers=headers,
+                    data=transformed_request,
+                    timeout=timeout,
+                )
+        except Exception as e:
+            verbose_logger.exception(f"Error creating batch: {e}")
+            raise self._handle_error(
+                e=e,
+                provider_config=provider_config,
+            )
+
+        # Store original request for response transformation (for async version)
+        litellm_params_with_request = {**litellm_params, "original_batch_request": create_batch_data or {}}
+        
+        return provider_config.transform_create_batch_response(
+            model=None,
+            raw_response=batch_response,
+            logging_obj=logging_obj,
+            litellm_params=litellm_params_with_request,
+        )
+
     def list_files(self):
         """
         Lists all files
@@ -2381,6 +2629,7 @@ class BaseLLMHTTPHandler:
             BaseVectorStoreConfig,
             BaseGoogleGenAIGenerateContentConfig,
             BaseAnthropicMessagesConfig,
+            BaseBatchesConfig,
             "BasePassthroughConfig",
         ],
     ):

litellm/llms/databricks/chat/transformation.py

@@ -26,7 +26,6 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo
     _should_convert_tool_call_to_json_mode,
 )
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
-    handle_messages_with_content_list_to_str_conversion,
     strip_name_from_messages,
 )
 from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
@@ -301,7 +300,6 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
     ) -> Union[List[AllMessageValues], Coroutine[Any, Any, List[AllMessageValues]]]:
         """
         Databricks does not support:
-        - content in list format.
         - 'name' in user message.
         """
         new_messages = []
@@ -311,7 +309,6 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
             else:
                 _message = message
             new_messages.append(_message)
-        new_messages = handle_messages_with_content_list_to_str_conversion(new_messages)
         new_messages = strip_name_from_messages(new_messages)
 
         if is_async:
@@ -379,6 +376,25 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
                         thinking_blocks.append(thinking_block)
         return reasoning_content, thinking_blocks
 
+    @staticmethod
+    def extract_citations(
+        content: Optional[AllDatabricksContentValues],
+    ) -> Optional[List[Any]]:
+        if content is None:
+            return None
+        citations = []
+        if isinstance(content, list):
+            for item in content:
+                text = item.get("text", None)
+                if citations_item := item.get("citations"):
+                    citations.append(
+                        [
+                            {**citation, "supported_text": text}
+                            for citation in citations_item
+                        ]
+                    )
+        return citations or None
+
     def _transform_dbrx_choices(
         self, choices: List[DatabricksChoice], json_mode: Optional[bool] = None
     ) -> List[Choices]:
@@ -427,12 +443,19 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
                     choice["message"].get("content")
                 )
 
+                citations = DatabricksConfig.extract_citations(
+                    choice["message"].get("content")
+                )
+
                 translated_message = Message(
                     role="assistant",
                     content=content_str,
                     reasoning_content=reasoning_content,
                     thinking_blocks=thinking_blocks,
                     tool_calls=choice["message"].get("tool_calls"),
+                    provider_specific_fields={"citations": citations}
+                    if citations is not None
+                    else None,
                 )
 
             if finish_reason is None:
@@ -561,6 +584,17 @@ class DatabricksChatResponseIterator(BaseModelResponseIterator):
                     for _tc in tool_calls:
                         if _tc.get("function", {}).get("arguments") == "{}":
                             _tc["function"]["arguments"] = ""  # avoid invalid json
+                if isinstance(choice["delta"]["content"], list) and (
+                    content := choice["delta"]["content"]
+                ):
+                    if citations := content[0].get("citations"):
+                        # TODO: Databricks delta does not include supported text or chunk type.
+                        # Add either here once Databricks supports it to enable citation linkage.
+                        choice["delta"].setdefault("provider_specific_fields", {})[
+                            "citation"
+                        ] = citations[
+                            0
+                        ]  # Databricks Content item always has citation as a list of list
                 # extract the content str
                 content_str = DatabricksConfig.extract_content_str(
                     choice["delta"].get("content")

litellm/llms/groq/chat/transformation.py

@@ -6,6 +6,8 @@ from typing import Any, Coroutine, List, Literal, Optional, Tuple, Union, cast,
 import httpx
 from pydantic import BaseModel
 
+import litellm
+from litellm._logging import verbose_logger
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.llms.openai import (
@@ -55,6 +57,10 @@ class GroqChatConfig(OpenAILikeChatConfig):
             if key != "self" and value is not None:
                 setattr(self.__class__, key, value)
 
+    @property
+    def custom_llm_provider(self) -> Optional[str]:
+        return "groq"
+
     @classmethod
     def get_config(cls):
         return super().get_config()
@@ -65,6 +71,15 @@ class GroqChatConfig(OpenAILikeChatConfig):
             base_params.remove("max_retries")
         except ValueError:
             pass
+
+        try:
+            if litellm.supports_reasoning(
+                model=model, custom_llm_provider=self.custom_llm_provider
+            ):
+                base_params.append("reasoning_effort")
+        except Exception as e:
+            verbose_logger.debug(f"Error checking if model supports reasoning: {e}")
+
         return base_params
 
     @overload

litellm/llms/oci/chat/transformation.py

@@ -772,7 +772,14 @@ def adapt_messages_to_generic_oci_standard(
         tool_calls = message.get("tool_calls")
         tool_call_id = message.get("tool_call_id")
 
-        if role in ["system", "user", "assistant"] and content is not None:
+        if role == "assistant" and tool_calls is not None:
+            if not isinstance(tool_calls, list):
+                raise Exception("Prop `tool_calls` must be a list of tool calls")
+            new_messages.append(
+                adapt_messages_to_generic_oci_standard_tool_call(role, tool_calls)
+            )
+
+        elif role in ["system", "user", "assistant"] and content is not None:
             if not isinstance(content, (str, list)):
                 raise Exception(
                     "Prop `content` must be a string or a list of content items"
@@ -781,13 +788,6 @@ def adapt_messages_to_generic_oci_standard(
                 adapt_messages_to_generic_oci_standard_content_message(role, content)
             )
 
-        elif role == "assistant" and tool_calls is not None:
-            if not isinstance(tool_calls, list):
-                raise Exception("Prop `tool_calls` must be a list of tool calls")
-            new_messages.append(
-                adapt_messages_to_generic_oci_standard_tool_call(role, tool_calls)
-            )
-
         elif role == "tool":
             if not isinstance(tool_call_id, str):
                 raise Exception("Prop `tool_call_id` is required and must be a string")

litellm/llms/ollama/chat/transformation.py

@@ -137,6 +137,7 @@ class OllamaChatConfig(BaseConfig):
             "tool_choice",
             "functions",
             "response_format",
+            "reasoning_effort",
         ]
 
     def map_openai_params(
@@ -175,6 +176,8 @@ class OllamaChatConfig(BaseConfig):
                 if value.get("json_schema") and value["json_schema"].get("schema"):
                     optional_params["format"] = value["json_schema"]["schema"]
             ### FUNCTION CALLING LOGIC ###
+            if param == "reasoning_effort" and value is not None:
+                optional_params["think"] = True
             if param == "tools":
                 ## CHECK IF MODEL SUPPORTS TOOL CALLING ##
                 try:
@@ -212,9 +215,9 @@ class OllamaChatConfig(BaseConfig):
                     litellm.add_function_to_prompt = (
                         True  # so that main.py adds the function call to the prompt
                     )
-                    optional_params[
-                        "functions_unsupported_model"
-                    ] = non_default_params.get("functions")
+                    optional_params["functions_unsupported_model"] = (
+                        non_default_params.get("functions")
+                    )
         non_default_params.pop("tool_choice", None)  # causes ollama requests to hang
         non_default_params.pop("functions", None)  # causes ollama requests to hang
         return optional_params
@@ -346,11 +349,31 @@ class OllamaChatConfig(BaseConfig):
 
         ## RESPONSE OBJECT
         model_response.choices[0].finish_reason = "stop"
+        response_json_message = response_json.get("message")
+        if response_json_message is not None:
+            if "thinking" in response_json_message:
+                # remap 'thinking' to 'reasoning_content'
+                response_json_message["reasoning_content"] = response_json_message[
+                    "thinking"
+                ]
+                del response_json_message["thinking"]
+            elif response_json_message.get("content") is not None:
+                # parse reasoning content from content
+                from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
+                    _parse_content_for_reasoning,
+                )
+
+                reasoning_content, content = _parse_content_for_reasoning(
+                    response_json_message["content"]
+                )
+                response_json_message["reasoning_content"] = reasoning_content
+                response_json_message["content"] = content
+
         if (
             request_data.get("format", "") == "json"
             and litellm_params.get("function_name") is not None
         ):
-            function_call = json.loads(response_json["message"]["content"])
+            function_call = json.loads(response_json_message["content"])
             message = litellm.Message(
                 content=None,
                 tool_calls=[
@@ -367,11 +390,13 @@ class OllamaChatConfig(BaseConfig):
                         "type": "function",
                     }
                 ],
+                reasoning_content=response_json_message.get("reasoning_content"),
             )
             model_response.choices[0].message = message  # type: ignore
             model_response.choices[0].finish_reason = "tool_calls"
         else:
-            _message = litellm.Message(**response_json["message"])
+
+            _message = litellm.Message(**response_json_message)
             model_response.choices[0].message = _message  # type: ignore
         model_response.created = int(time.time())
         model_response.model = "ollama_chat/" + model
@@ -412,6 +437,9 @@ class OllamaChatConfig(BaseConfig):
 
 
 class OllamaChatCompletionResponseIterator(BaseModelResponseIterator):
+    started_reasoning_content: bool = False
+    finished_reasoning_content: bool = False
+
     def _is_function_call_complete(self, function_args: Union[str, dict]) -> bool:
         if isinstance(function_args, dict):
             return True
@@ -465,8 +493,38 @@ class OllamaChatCompletionResponseIterator(BaseModelResponseIterator):
                         if is_function_call_complete:
                             tool_call["id"] = str(uuid.uuid4())
 
+            # PROCESS REASONING CONTENT
+            reasoning_content: Optional[str] = None
+            content: Optional[str] = None
+            if chunk["message"].get("thinking") is not None:
+                if self.started_reasoning_content is False:
+                    reasoning_content = chunk["message"].get("thinking")
+                    self.started_reasoning_content = True
+                elif self.finished_reasoning_content is False:
+                    reasoning_content = chunk["message"].get("thinking")
+                    self.finished_reasoning_content = True
+            elif chunk["message"].get("content") is not None:
+                message_content = chunk["message"].get("content")
+                if "<think>" in message_content:
+                    message_content = message_content.replace("<think>", "")
+
+                    self.started_reasoning_content = True
+
+                if "</think>" in message_content and self.started_reasoning_content:
+                    message_content = message_content.replace("</think>", "")
+                    self.finished_reasoning_content = True
+
+                if (
+                    self.started_reasoning_content
+                    and not self.finished_reasoning_content
+                ):
+                    reasoning_content = message_content
+                else:
+                    content = message_content
+
             delta = Delta(
-                content=chunk["message"].get("content", ""),
+                content=content,
+                reasoning_content=reasoning_content,
                 tool_calls=tool_calls,
             )
 

litellm/llms/ollama/completion/transformation.py

@@ -19,13 +19,13 @@ from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMExcepti
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.llms.openai import AllMessageValues, ChatCompletionUsageBlock
 from litellm.types.utils import (
+    Delta,
     GenericStreamingChunk,
     ModelInfoBase,
     ModelResponse,
     ModelResponseStream,
     ProviderField,
     StreamingChoices,
-    Delta,
 )
 
 from ..common_utils import OllamaError, _convert_image
@@ -92,9 +92,9 @@ class OllamaConfig(BaseConfig):
     repeat_penalty: Optional[float] = None
     temperature: Optional[float] = None
     seed: Optional[int] = None
-    stop: Optional[
-        list
-    ] = None  # stop is a list based on this - https://github.com/ollama/ollama/pull/442
+    stop: Optional[list] = (
+        None  # stop is a list based on this - https://github.com/ollama/ollama/pull/442
+    )
     tfs_z: Optional[float] = None
     num_predict: Optional[int] = None
     top_k: Optional[int] = None
@@ -154,6 +154,7 @@ class OllamaConfig(BaseConfig):
             "stop",
             "response_format",
             "max_completion_tokens",
+            "reasoning_effort",
         ]
 
     def map_openai_params(
@@ -166,19 +167,21 @@ class OllamaConfig(BaseConfig):
         for param, value in non_default_params.items():
             if param == "max_tokens" or param == "max_completion_tokens":
                 optional_params["num_predict"] = value
-            if param == "stream":
+            elif param == "stream":
                 optional_params["stream"] = value
-            if param == "temperature":
+            elif param == "temperature":
                 optional_params["temperature"] = value
-            if param == "seed":
+            elif param == "seed":
                 optional_params["seed"] = value
-            if param == "top_p":
+            elif param == "top_p":
                 optional_params["top_p"] = value
-            if param == "frequency_penalty":
+            elif param == "frequency_penalty":
                 optional_params["frequency_penalty"] = value
-            if param == "stop":
+            elif param == "stop":
                 optional_params["stop"] = value
-            if param == "response_format" and isinstance(value, dict):
+            elif param == "reasoning_effort" and value is not None:
+                optional_params["think"] = True
+            elif param == "response_format" and isinstance(value, dict):
                 if value["type"] == "json_object":
                     optional_params["format"] = "json"
                 elif value["type"] == "json_schema":
@@ -258,12 +261,17 @@ class OllamaConfig(BaseConfig):
         api_key: Optional[str] = None,
         json_mode: Optional[bool] = None,
     ) -> ModelResponse:
+        from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
+            _parse_content_for_reasoning,
+        )
+
         response_json = raw_response.json()
         ## RESPONSE OBJECT
         model_response.choices[0].finish_reason = "stop"
         if request_data.get("format", "") == "json":
             # Check if response field exists and is not empty before parsing JSON
             response_text = response_json.get("response", "")
+
             if not response_text or not response_text.strip():
                 # Handle empty response gracefully - set empty content
                 message = litellm.Message(content="")
@@ -288,7 +296,9 @@ class OllamaConfig(BaseConfig):
                                     "id": f"call_{str(uuid.uuid4())}",
                                     "function": {
                                         "name": function_call["name"],
-                                        "arguments": json.dumps(function_call["arguments"]),
+                                        "arguments": json.dumps(
+                                            function_call["arguments"]
+                                        ),
                                     },
                                     "type": "function",
                                 }
@@ -305,11 +315,28 @@ class OllamaConfig(BaseConfig):
                         model_response.choices[0].finish_reason = "stop"
                 except json.JSONDecodeError:
                     # If JSON parsing fails, treat as regular text response
-                    message = litellm.Message(content=response_text)
+                    ## output parse reasoning content from response_text
+                    reasoning_content: Optional[str] = None
+                    content: Optional[str] = None
+                    if response_text is not None:
+                        reasoning_content, content = _parse_content_for_reasoning(
+                            response_text
+                        )
+                    message = litellm.Message(
+                        content=content, reasoning_content=reasoning_content
+                    )
                     model_response.choices[0].message = message  # type: ignore
                     model_response.choices[0].finish_reason = "stop"
         else:
-            model_response.choices[0].message.content = response_json["response"]  # type: ignore
+            response_text = response_json.get("response", "")
+            content = None
+            reasoning_content = None
+            if response_text is not None and isinstance(response_text, str):
+                reasoning_content, content = _parse_content_for_reasoning(response_text)
+            else:
+                content = response_text  # type: ignore
+            model_response.choices[0].message.content = content  # type: ignore
+            model_response.choices[0].message.reasoning_content = reasoning_content  # type: ignore
         model_response.created = int(time.time())
         model_response.model = "ollama/" + model
         _prompt = request_data.get("prompt", "")
@@ -434,12 +461,21 @@ class OllamaConfig(BaseConfig):
 
 
 class OllamaTextCompletionResponseIterator(BaseModelResponseIterator):
+    def __init__(
+        self, streaming_response, sync_stream: bool, json_mode: Optional[bool] = False
+    ):
+        super().__init__(streaming_response, sync_stream, json_mode)
+        self.started_reasoning_content: bool = False
+        self.finished_reasoning_content: bool = False
+
     def _handle_string_chunk(
         self, str_line: str
     ) -> Union[GenericStreamingChunk, ModelResponseStream]:
         return self.chunk_parser(json.loads(str_line))
 
-    def chunk_parser(self, chunk: dict) -> Union[GenericStreamingChunk, ModelResponseStream]:
+    def chunk_parser(
+        self, chunk: dict
+    ) -> Union[GenericStreamingChunk, ModelResponseStream]:
         try:
             if "error" in chunk:
                 raise Exception(f"Ollama Error - {chunk}")
@@ -469,12 +505,42 @@ class OllamaTextCompletionResponseIterator(BaseModelResponseIterator):
                 )
             elif chunk["response"]:
                 text = chunk["response"]
-                return GenericStreamingChunk(
-                    text=text,
-                    is_finished=is_finished,
-                    finish_reason="stop",
+                reasoning_content: Optional[str] = None
+                content: Optional[str] = None
+                if text is not None:
+                    if "<think>" in text:
+                        text = text.replace("<think>", "")
+                        self.started_reasoning_content = True
+                    elif "</think>" in text:
+                        text = text.replace("</think>", "")
+                        self.finished_reasoning_content = True
+
+                    if (
+                        self.started_reasoning_content
+                        and not self.finished_reasoning_content
+                    ):
+                        reasoning_content = text
+                    else:
+                        content = text
+
+                return ModelResponseStream(
+                    choices=[
+                        StreamingChoices(
+                            index=0,
+                            delta=Delta(
+                                reasoning_content=reasoning_content, content=content
+                            ),
+                        )
+                    ],
+                    finish_reason=finish_reason,
                     usage=None,
                 )
+                # return GenericStreamingChunk(
+                #     text=text,
+                #     is_finished=is_finished,
+                #     finish_reason="stop",
+                #     usage=None,
+                # )
             elif "thinking" in chunk and not chunk["response"]:
                 # Return reasoning content as ModelResponseStream so UIs can render it
                 thinking_content = chunk.get("thinking") or ""

litellm/llms/openai/chat/gpt_5_transformation.py

@@ -28,7 +28,18 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
         base_gpt_series_params.extend(gpt_5_only_params)
         if not supports_tool_choice(model=model):
             base_gpt_series_params.remove("tool_choice")
-        return base_gpt_series_params
+
+        non_supported_params = [
+            "logprobs",
+            "top_p",
+            "presence_penalty",
+            "frequency_penalty",
+            "top_logprobs",
+        ]
+
+        return [
+            param for param in base_gpt_series_params if param not in non_supported_params
+        ]
 
     def map_openai_params(
         self,

litellm/llms/openai/chat/gpt_transformation.py

@@ -158,6 +158,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
             "parallel_tool_calls",
             "audio",
             "web_search_options",
+            "safety_identifier",
         ]  # works across all models
 
         model_specific_params = []

litellm/llms/openai/image_generation/gpt_transformation.py

@@ -16,7 +16,6 @@ class GPTImageGenerationConfig(BaseImageGenerationConfig):
     ) -> List[OpenAIImageGenerationOptionalParams]:
         return [
             "background",
-            "input_fidelity",
             "moderation",
             "n",
             "output_compression",

litellm/llms/openai/responses/transformation.py

@@ -1,6 +1,15 @@
-from typing import TYPE_CHECKING, Any, Dict, Optional, Union, cast, get_type_hints
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Dict,
+    Optional,
+    Union,
+    cast,
+    get_type_hints,
+)
 
 import httpx
+from openai.types.responses import ResponseReasoningItem
 from pydantic import BaseModel
 
 import litellm
@@ -92,12 +101,67 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
                 # if it's pydantic, convert to dict
                 if isinstance(item, BaseModel):
                     validated_input.append(item.model_dump(exclude_none=True))
+                elif isinstance(item, dict):
+                    # Handle reasoning items specifically to filter out status=None
+                    verbose_logger.debug(f"Handling reasoning item: {item}")
+                    if item.get("type") == "reasoning":
+                        # Type assertion since we know it's a dict at this point
+                        dict_item = cast(Dict[str, Any], item)
+                        filtered_item = self._handle_reasoning_item(dict_item)
+                    else:
+                        # For other dict items, just pass through
+                        filtered_item = cast(Dict[str, Any], item)
+                    validated_input.append(filtered_item)
                 else:
                     validated_input.append(item)
-            return validated_input
+            return validated_input  # type: ignore
         # Input is expected to be either str or List, no single BaseModel expected
         return input
 
+    def _handle_reasoning_item(self, item: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Handle reasoning items specifically to filter out status=None using OpenAI's model.
+        Issue: https://github.com/BerriAI/litellm/issues/13484
+        OpenAI API does not accept ReasoningItem(status=None), so we need to:
+        1. Check if the item is a reasoning type
+        2. Create a ResponseReasoningItem object with the item data
+        3. Convert it back to dict with exclude_none=True to filter None values
+        """
+        verbose_logger.debug(f"Handling reasoning item: {item}")
+        if item.get("type") == "reasoning":
+            try:
+                # Ensure required fields are present for ResponseReasoningItem
+                item_data = dict(item)
+                if "id" not in item_data:
+                    item_data["id"] = f"reasoning_{hash(str(item_data))}"
+                if "summary" not in item_data:
+                    item_data["summary"] = (
+                        item_data.get("reasoning_content", "")[:100] + "..."
+                        if len(item_data.get("reasoning_content", "")) > 100
+                        else item_data.get("reasoning_content", "")
+                    )
+
+                # Create ResponseReasoningItem object from the item data
+                reasoning_item = ResponseReasoningItem(**item_data)
+
+                # Convert back to dict with exclude_none=True to exclude None fields
+                dict_reasoning_item = reasoning_item.model_dump(exclude_none=True)
+
+                return dict_reasoning_item
+            except Exception as e:
+                verbose_logger.debug(
+                    f"Failed to create ResponseReasoningItem, falling back to manual filtering: {e}"
+                )
+                # Fallback: manually filter out known None fields
+                filtered_item = {
+                    k: v
+                    for k, v in item.items()
+                    if v is not None
+                    or k not in {"status", "content", "encrypted_content"}
+                }
+                return filtered_item
+        return item
+
     def transform_response_api_response(
         self,
         model: str,

litellm/llms/vertex_ai/gemini/transformation.py

@@ -105,6 +105,64 @@ def _process_gemini_image(image_url: str, format: Optional[str] = None) -> PartT
         raise e
 
 
+def _snake_to_camel(snake_str: str) -> str:
+    """Convert snake_case to camelCase"""
+    components = snake_str.split("_")
+    return components[0] + "".join(x.capitalize() for x in components[1:])
+
+
+def _camel_to_snake(camel_str: str) -> str:
+    """Convert camelCase to snake_case"""
+    import re
+
+    return re.sub(r"(?<!^)(?=[A-Z])", "_", camel_str).lower()
+
+
+def _get_equivalent_key(key: str, available_keys: set) -> Optional[str]:
+    """
+    Get the equivalent key from available keys, checking both camelCase and snake_case variants
+    """
+    if key in available_keys:
+        return key
+
+    # Try camelCase version
+    camel_key = _snake_to_camel(key)
+    if camel_key in available_keys:
+        return camel_key
+
+    # Try snake_case version
+    snake_key = _camel_to_snake(key)
+    if snake_key in available_keys:
+        return snake_key
+
+    return None
+
+
+def check_if_part_exists_in_parts(
+    parts: List[PartType], part: PartType, excluded_keys: List[str] = []
+) -> bool:
+    """
+    Check if a part exists in a list of parts
+    Handles both camelCase and snake_case key variations (e.g., function_call vs functionCall)
+    """
+    keys_to_compare = set(part.keys()) - set(excluded_keys)
+    for p in parts:
+        p_keys = set(p.keys())
+        # Check if all keys in part have equivalent values in p
+        match_found = True
+        for key in keys_to_compare:
+            equivalent_key = _get_equivalent_key(key, p_keys)
+            if equivalent_key is None or p.get(equivalent_key, None) != part.get(
+                key, None
+            ):
+                match_found = False
+                break
+
+        if match_found:
+            return True
+    return False
+
+
 def _gemini_convert_messages_with_history(  # noqa: PLR0915
     messages: List[AllMessageValues],
 ) -> List[ContentType]:
@@ -236,10 +294,33 @@ def _gemini_convert_messages_with_history(  # noqa: PLR0915
                 assistant_msg = ChatCompletionAssistantMessage(**msg_dict)  # type: ignore
                 _message_content = assistant_msg.get("content", None)
                 reasoning_content = assistant_msg.get("reasoning_content", None)
+                thinking_blocks = assistant_msg.get("thinking_blocks")
                 if reasoning_content is not None:
                     assistant_content.append(
                         PartType(thought=True, text=reasoning_content)
                     )
+                if thinking_blocks is not None:
+                    for block in thinking_blocks:
+                        block_thinking_str = block.get("thinking")
+                        block_signature = block.get("signature")
+                        if (
+                            block_thinking_str is not None
+                            and block_signature is not None
+                        ):
+                            try:
+                                assistant_content.append(
+                                    PartType(
+                                        thoughtSignature=block_signature,
+                                        **json.loads(block_thinking_str),
+                                    )
+                                )
+                            except Exception:
+                                assistant_content.append(
+                                    PartType(
+                                        thoughtSignature=block_signature,
+                                        text=block_thinking_str,
+                                    )
+                                )
                 if _message_content is not None and isinstance(_message_content, list):
                     _parts = []
                     for element in _message_content:
@@ -262,9 +343,17 @@ def _gemini_convert_messages_with_history(  # noqa: PLR0915
                     assistant_msg.get("tool_calls", []) is not None
                     or assistant_msg.get("function_call") is not None
                 ):  # support assistant tool invoke conversion
-                    assistant_content.extend(
-                        convert_to_gemini_tool_call_invoke(assistant_msg)
+                    gemini_tool_call_parts = convert_to_gemini_tool_call_invoke(
+                        assistant_msg
                     )
+                    ## check if gemini_tool_call already exists in assistant_content
+                    for gemini_tool_call_part in gemini_tool_call_parts:
+                        if not check_if_part_exists_in_parts(
+                            assistant_content,
+                            gemini_tool_call_part,
+                            excluded_keys=["thoughtSignature"],
+                        ):
+                            assistant_content.append(gemini_tool_call_part)
                     last_message_with_tool_calls = assistant_msg
 
                 msg_i += 1
@@ -476,6 +565,7 @@ async def async_transform_request_body(
         optional_params=optional_params,
     )
 
+
 def _default_user_message_when_system_message_passed() -> ChatCompletionUserMessage:
     """
     Returns a default user message when a "system" message is passed in gemini fails.
@@ -484,6 +574,7 @@ def _default_user_message_when_system_message_passed() -> ChatCompletionUserMess
     """
     return ChatCompletionUserMessage(content=".", role="user")
 
+
 def _transform_system_message(
     supports_system_message: bool, messages: List[AllMessageValues]
 ) -> Tuple[Optional[SystemInstructions], List[AllMessageValues]]:

litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py

@@ -30,6 +30,10 @@ from litellm.constants import (
     DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET,
     DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET,
     DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET,
+    DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET,
+    DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH,
+    DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_PRO,
+    DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE,
 )
 from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
 from litellm.llms.custom_httpx.http_handler import (
@@ -43,6 +47,7 @@ from litellm.types.llms.gemini import BidiGenerateContentServerMessage
 from litellm.types.llms.openai import (
     AllMessageValues,
     ChatCompletionResponseMessage,
+    ChatCompletionThinkingBlock,
     ChatCompletionToolCallChunk,
     ChatCompletionToolCallFunctionChunk,
     ChatCompletionToolParamFunctionChunk,
@@ -422,8 +427,25 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
     @staticmethod
     def _map_reasoning_effort_to_thinking_budget(
         reasoning_effort: str,
+        model: Optional[str] = None,
     ) -> GeminiThinkingConfig:
-        if reasoning_effort == "low":
+        if reasoning_effort == "minimal":
+            # Use model-specific minimum thinking budget or fallback
+            # Check for exact matches first, then partial matches
+            if model and "gemini-2.5-flash-lite" in model.lower():
+                budget = DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE
+            elif model and "gemini-2.5-pro" in model.lower():
+                budget = DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_PRO
+            elif model and "gemini-2.5-flash" in model.lower():
+                budget = DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH
+            else:
+                budget = DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET
+
+            return {
+                "thinkingBudget": budget,
+                "includeThoughts": True,
+            }
+        elif reasoning_effort == "low":
             return {
                 "thinkingBudget": DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET,
                 "includeThoughts": True,
@@ -600,7 +622,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
                 optional_params["seed"] = value
             elif param == "reasoning_effort" and isinstance(value, str):
                 optional_params["thinkingConfig"] = (
-                    VertexGeminiConfig._map_reasoning_effort_to_thinking_budget(value)
+                    VertexGeminiConfig._map_reasoning_effort_to_thinking_budget(
+                        value, model
+                    )
                 )
             elif param == "thinking":
                 optional_params["thinkingConfig"] = (
@@ -794,6 +818,24 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
 
         return content_str, reasoning_content_str
 
+    def _extract_thinking_blocks_from_parts(
+        self, parts: List[HttpxPartType]
+    ) -> List[ChatCompletionThinkingBlock]:
+        """Extract thinking blocks from parts if present"""
+        thinking_blocks: List[ChatCompletionThinkingBlock] = []
+        for part in parts:
+            if "thoughtSignature" in part:
+                part_copy = part.copy()
+                part_copy.pop("thoughtSignature")
+                thinking_blocks.append(
+                    ChatCompletionThinkingBlock(
+                        type="thinking",
+                        thinking=json.dumps(part_copy),
+                        signature=part["thoughtSignature"],
+                    )
+                )
+        return thinking_blocks
+
     def _extract_image_response_from_parts(
         self, parts: List[HttpxPartType]
     ) -> Optional[List[ImageURLListItem]]:
@@ -1237,6 +1279,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
         tools: Optional[List[ChatCompletionToolCallChunk]] = []
         functions: Optional[ChatCompletionToolCallFunctionChunk] = None
         cumulative_tool_call_index: int = 0
+        thinking_blocks: Optional[List[ChatCompletionThinkingBlock]] = None
 
         for idx, candidate in enumerate(_candidates):
             if "content" not in candidate:
@@ -1274,6 +1317,12 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
                     )
                 )
 
+                thinking_blocks = (
+                    VertexGeminiConfig()._extract_thinking_blocks_from_parts(
+                        parts=candidate["content"]["parts"]
+                    )
+                )
+
                 if audio_response is not None:
                     cast(Dict[str, Any], chat_completion_message)[
                         "audio"
@@ -1310,6 +1359,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
             if functions is not None:
                 chat_completion_message["function_call"] = functions
 
+            if thinking_blocks is not None:
+                chat_completion_message["thinking_blocks"] = thinking_blocks  # type: ignore
+
             if isinstance(model_response, ModelResponseStream):
                 choice = VertexGeminiConfig._create_streaming_choice(
                     chat_completion_message=chat_completion_message,

litellm/llms/vertex_ai/vertex_ai_partner_models/gpt_oss/transformation.py

@@ -0,0 +1,27 @@
+import litellm
+from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
+
+
+class VertexAIGPTOSSTransformation(OpenAIGPTConfig):
+    """
+    Transformation for GPT-OSS model from VertexAI
+
+    https://console.cloud.google.com/vertex-ai/publishers/openai/model-garden/gpt-oss-120b-maas?hl=id
+    """
+    def __init__(self):
+        super().__init__()
+    
+    def get_supported_openai_params(self, model: str) -> list:
+        base_gpt_series_params = super().get_supported_openai_params(model=model)
+        gpt_oss_only_params = ["reasoning_effort"]
+        base_gpt_series_params.extend(gpt_oss_only_params)
+
+        #########################################################
+        # VertexAI - GPT-OSS does not support tool calls
+        #########################################################
+        if litellm.supports_function_calling(model=model) is False:
+            TOOL_CALLING_PARAMS_TO_REMOVE = ["tool", "tool_choice", "function_call", "functions"]
+            base_gpt_series_params = [param for param in base_gpt_series_params if param not in TOOL_CALLING_PARAMS_TO_REMOVE]
+
+        return base_gpt_series_params
+

litellm/llms/vertex_ai/vertex_ai_partner_models/main.py

@@ -1,5 +1,6 @@
 # What is this?
 ## API Handler for calling Vertex AI Partner Models
+from enum import Enum
 from typing import Callable, Optional, Union
 
 import httpx  # type: ignore
@@ -27,6 +28,16 @@ class VertexAIError(Exception):
             self.message
         )  # Call the base class constructor with the parameters it needs
 
+class PartnerModelPrefixes(str, Enum):
+    META_PREFIX = "meta/"
+    DEEPSEEK_PREFIX = "deepseek-ai"
+    MISTRAL_PREFIX = "mistral"
+    CODERESTAL_PREFIX = "codestral"
+    JAMBA_PREFIX = "jamba"
+    CLAUDE_PREFIX = "claude"
+    QWEN_PREFIX = "qwen"
+    GPT_OSS_PREFIX = "openai/gpt-oss-"
+
 
 class VertexAIPartnerModels(VertexBase):
     def __init__(self) -> None:
@@ -42,13 +53,14 @@ class VertexAIPartnerModels(VertexBase):
             bool: True if the model string is a Vertex AI Partner Model, False otherwise
         """
         if (
-            model.startswith("meta/")
-            or model.startswith("deepseek-ai")
-            or model.startswith("mistral")
-            or model.startswith("codestral")
-            or model.startswith("jamba")
-            or model.startswith("claude")
-            or model.startswith("qwen")
+            model.startswith(PartnerModelPrefixes.META_PREFIX)
+            or model.startswith(PartnerModelPrefixes.DEEPSEEK_PREFIX)
+            or model.startswith(PartnerModelPrefixes.MISTRAL_PREFIX)
+            or model.startswith(PartnerModelPrefixes.CODERESTAL_PREFIX)
+            or model.startswith(PartnerModelPrefixes.JAMBA_PREFIX)
+            or model.startswith(PartnerModelPrefixes.CLAUDE_PREFIX)
+            or model.startswith(PartnerModelPrefixes.QWEN_PREFIX)
+            or model.startswith(PartnerModelPrefixes.GPT_OSS_PREFIX)
         ):
             return True
         return False
@@ -57,8 +69,9 @@ class VertexAIPartnerModels(VertexBase):
     def should_use_openai_handler(model: str):
         OPENAI_LIKE_VERTEX_PROVIDERS = [
             "llama",
-            "deepseek-ai",
-            "qwen",
+            PartnerModelPrefixes.DEEPSEEK_PREFIX,
+            PartnerModelPrefixes.QWEN_PREFIX,
+            PartnerModelPrefixes.GPT_OSS_PREFIX,
         ]
         if any(provider in model for provider in OPENAI_LIKE_VERTEX_PROVIDERS):
             return True

litellm/llms/volcengine/__init__.py

@@ -0,0 +1,24 @@
+"""
+Volcengine LLM Provider
+Support for Volcengine (ByteDance) chat and embedding models
+"""
+
+from .chat.transformation import VolcEngineChatConfig
+from .common_utils import (
+    VolcEngineError,
+    get_volcengine_base_url,
+    get_volcengine_headers,
+)
+from .embedding import VolcEngineEmbeddingConfig
+
+# For backward compatibility, keep the old class name
+VolcEngineConfig = VolcEngineChatConfig
+
+__all__ = [
+    "VolcEngineChatConfig",
+    "VolcEngineConfig",  # backward compatibility
+    "VolcEngineEmbeddingConfig",
+    "VolcEngineError",
+    "get_volcengine_base_url",
+    "get_volcengine_headers",
+]

litellm/llms/volcengine/chat/transformation.py

@@ -3,7 +3,7 @@ from typing import Optional, Union
 from litellm.llms.openai_like.chat.transformation import OpenAILikeChatConfig
 
 
-class VolcEngineConfig(OpenAILikeChatConfig):
+class VolcEngineChatConfig(OpenAILikeChatConfig):
     frequency_penalty: Optional[int] = None
     function_call: Optional[Union[str, dict]] = None
     functions: Optional[list] = None
@@ -82,17 +82,19 @@ class VolcEngineConfig(OpenAILikeChatConfig):
 
         if "thinking" in optional_params:
             thinking_value = optional_params.pop("thinking")
-            
+
             # Handle disabled thinking case - don't add to extra_body if disabled
             if (
-                thinking_value is not None 
-                and isinstance(thinking_value, dict) 
+                thinking_value is not None
+                and isinstance(thinking_value, dict)
                 and thinking_value.get("type") == "disabled"
             ):
                 # Skip adding thinking parameter when it's disabled
                 pass
             else:
                 # Add thinking parameter to extra_body for all other cases
-                optional_params.setdefault("extra_body", {})["thinking"] = thinking_value
+                optional_params.setdefault("extra_body", {})[
+                    "thinking"
+                ] = thinking_value
 
         return optional_params

litellm/llms/volcengine/common_utils.py

@@ -0,0 +1,62 @@
+"""
+Common utilities for Volcengine LLM provider
+"""
+
+from typing import Optional
+
+import httpx
+
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+
+
+class VolcEngineError(BaseLLMException):
+    """
+    Custom exception class for Volcengine provider errors.
+    """
+
+    def __init__(
+        self, status_code: int, message: str, headers: Optional[httpx.Headers] = None
+    ):
+        self.status_code = status_code
+        self.message = message
+        self.headers = headers or httpx.Headers()
+        super().__init__(
+            status_code=status_code, message=message, headers=dict(self.headers)
+        )
+
+
+def get_volcengine_base_url(api_base: Optional[str] = None) -> str:
+    """
+    Get the base URL for Volcengine API calls.
+
+    Args:
+        api_base: Optional custom API base URL
+
+    Returns:
+        The base URL to use for API calls
+    """
+    if api_base:
+        return api_base
+    return "https://ark.cn-beijing.volces.com"
+
+
+def get_volcengine_headers(api_key: str, extra_headers: Optional[dict] = None) -> dict:
+    """
+    Get headers for Volcengine API calls.
+
+    Args:
+        api_key: The API key for authentication
+        extra_headers: Optional additional headers
+
+    Returns:
+        Dictionary of headers
+    """
+    headers = {
+        "Content-Type": "application/json",
+        "Authorization": f"Bearer {api_key}",
+    }
+
+    if extra_headers:
+        headers.update(extra_headers)
+
+    return headers

litellm/llms/volcengine/embedding/__init__.py

@@ -0,0 +1,7 @@
+"""
+Volcengine Embedding Module
+"""
+
+from .transformation import VolcEngineEmbeddingConfig
+
+__all__ = ["VolcEngineEmbeddingConfig"]

litellm/llms/volcengine/embedding/transformation.py

@@ -0,0 +1,211 @@
+"""
+Volcengine Embedding Transformation
+Transforms OpenAI embedding requests to Volcengine format
+"""
+
+from typing import List, Optional, Union, Dict, Any
+import httpx
+from litellm.types.llms.openai import AllEmbeddingInputValues, AllMessageValues
+from litellm.types.utils import EmbeddingResponse
+from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from ..common_utils import get_volcengine_base_url, get_volcengine_headers
+
+
+class VolcEngineEmbeddingConfig(BaseEmbeddingConfig):
+    """
+    Configuration class for Volcengine embedding models.
+    Reference: https://ark.cn-beijing.volces.com/api/v3/embeddings
+    """
+
+    def __init__(
+        self,
+        encoding_format: Optional[str] = None,
+    ) -> None:
+        locals_ = locals().copy()
+        for key, value in locals_.items():
+            if key != "self" and value is not None:
+                setattr(self.__class__, key, value)
+
+    @classmethod
+    def get_config(cls):
+        return super().get_config()
+
+    def get_supported_openai_params(self, model: str) -> List[str]:
+        """
+        Get the list of OpenAI parameters supported by Volcengine embedding models.
+
+        Args:
+            model: The model name
+
+        Returns:
+            List of supported parameter names
+        """
+        return [
+            "encoding_format",
+            "user",
+            "extra_headers",
+        ]
+
+    def get_complete_url(
+        self,
+        api_base: Optional[str],
+        api_key: Optional[str],
+        model: str,
+        optional_params: dict,
+        litellm_params: dict,
+        stream: Optional[bool] = None,
+    ) -> str:
+        """
+        Get the complete URL for volcengine embedding API calls.
+        
+        Args:
+            api_base: Optional custom API base URL
+            api_key: API key (not used for URL construction)
+            model: Model name (not used for URL construction)
+            optional_params: Optional parameters (not used for URL construction)
+            litellm_params: LiteLLM parameters (not used for URL construction)
+            stream: Stream parameter (not used for URL construction)
+            
+        Returns:
+            Complete URL for the embedding API endpoint
+        """
+        base_url = get_volcengine_base_url(api_base)
+        # Construct the complete URL with /embeddings endpoint
+        if base_url.endswith("/api/v3"):
+            return f"{base_url}/embeddings"
+        else:
+            return f"{base_url}/api/v3/embeddings"
+
+    def map_openai_params(
+        self,
+        non_default_params: Dict[str, Any],
+        optional_params: Dict[str, Any],
+        model: str,
+        drop_params: bool,
+    ) -> Dict[str, Any]:
+        """
+        Map OpenAI embedding parameters to Volcengine format.
+
+        Args:
+            non_default_params: Parameters that are not default values
+            optional_params: Optional parameters dict to update
+            model: The model name
+            drop_params: Whether to drop unsupported parameters
+
+        Returns:
+            Updated optional_params dict
+        """
+        for param, value in non_default_params.items():
+            if param == "encoding_format":
+                # Volcengine supports: float, base64, null
+                if value in ["float", "base64", None]:
+                    optional_params["encoding_format"] = value
+                else:
+                    if not drop_params:
+                        raise ValueError(
+                            f"Unsupported encoding_format: {value}. Volcengine supports: float, base64, null"
+                        )
+            elif param == "user":
+                # Keep user parameter as-is
+                optional_params["user"] = value
+            elif param in self.get_supported_openai_params(model):
+                optional_params[param] = value
+            elif not drop_params:
+                raise ValueError(f"Unsupported parameter for Volcengine: {param}")
+
+        return optional_params
+
+
+
+    def transform_embedding_request(
+        self,
+        model: str,
+        input: AllEmbeddingInputValues,
+        optional_params: dict,
+        headers: dict,
+    ) -> dict:
+        """Transform embedding request to Volcengine format"""
+        # Prepare request data (only the JSON body, not the full request)
+        data = {
+            "model": model,
+            "input": input if isinstance(input, list) else [input],
+        }
+
+        # Add optional parameters from optional_params
+        if "encoding_format" in optional_params:
+            encoding_format = optional_params["encoding_format"]
+            if encoding_format is not None:
+                data["encoding_format"] = encoding_format
+
+        if "user" in optional_params:
+            user = optional_params["user"]
+            if user is not None:
+                data["user"] = user
+
+        return data
+
+    def transform_embedding_response(
+        self,
+        model: str,
+        raw_response: httpx.Response,
+        model_response: EmbeddingResponse,
+        logging_obj: LiteLLMLoggingObj,
+        api_key: Optional[str],
+        request_data: dict,
+        optional_params: dict,
+        litellm_params: dict,
+    ) -> EmbeddingResponse:
+        """Transform Volcengine response to EmbeddingResponse"""
+        try:
+            response_json = raw_response.json()
+        except Exception as e:
+            raise ValueError(f"Failed to parse Volcengine response as JSON: {str(e)}")
+
+        # Volcengine response format matches OpenAI format closely
+        # Just need to ensure all required fields are present
+        transformed_response = {
+            "object": "list",
+            "data": response_json.get("data", []),
+            "model": response_json.get("model", model),
+            "usage": response_json.get("usage", {}),
+        }
+
+        # Add id if present
+        if "id" in response_json:
+            transformed_response["id"] = response_json["id"]
+        
+        # Create EmbeddingResponse from transformed data
+        return EmbeddingResponse(**transformed_response)
+
+    def validate_environment(
+        self,
+        headers: dict,
+        model: str,
+        messages: List[AllMessageValues],
+        optional_params: dict,
+        litellm_params: dict,
+        api_key: Optional[str] = None,
+        api_base: Optional[str] = None,
+    ) -> dict:
+        """Validate environment and return headers"""
+        # Get Volcengine headers
+        if api_key is None:
+            raise ValueError("api_key is required for Volcengine authentication")
+        volcengine_headers = get_volcengine_headers(api_key)
+        return {**headers, **volcengine_headers}
+
+    def get_error_class(
+        self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
+    ) -> BaseLLMException:
+        """Get error class for Volcengine errors"""
+        from ..common_utils import VolcEngineError
+        # Convert dict to httpx.Headers if needed
+        if isinstance(headers, dict):
+            headers = httpx.Headers(headers)
+        return VolcEngineError(
+            status_code=status_code,
+            message=error_message,
+            headers=headers,
+        )

litellm/main.py

@@ -356,7 +356,8 @@ async def acompletion(
     logprobs: Optional[bool] = None,
     top_logprobs: Optional[int] = None,
     deployment_id=None,
-    reasoning_effort: Optional[Literal["minimal", "low", "medium", "high"]] = None,
+    reasoning_effort: Optional[Literal["none", "minimal", "low", "medium", "high", "default"]] = None,
+    safety_identifier: Optional[str] = None,
     # set api_base, api_version, api_key
     base_url: Optional[str] = None,
     api_version: Optional[str] = None,
@@ -493,6 +494,7 @@ async def acompletion(
         "api_key": api_key,
         "model_list": model_list,
         "reasoning_effort": reasoning_effort,
+        "safety_identifier": safety_identifier,
         "extra_headers": extra_headers,
         "acompletion": True,  # assuming this is a required parameter
         "thinking": thinking,
@@ -500,7 +502,7 @@ async def acompletion(
     }
     if custom_llm_provider is None:
         _, custom_llm_provider, _, _ = get_llm_provider(
-            model=model, api_base=completion_kwargs.get("base_url", None)
+            model=model, custom_llm_provider=custom_llm_provider, api_base=completion_kwargs.get("base_url", None)
         )
 
     fallbacks = fallbacks or litellm.model_fallbacks
@@ -895,7 +897,7 @@ def completion(  # type: ignore # noqa: PLR0915
     logit_bias: Optional[dict] = None,
     user: Optional[str] = None,
     # openai v1.0+ new params
-    reasoning_effort: Optional[Literal["minimal", "low", "medium", "high"]] = None,
+    reasoning_effort: Optional[Literal["none", "minimal", "low", "medium", "high", "default"]] = None,
     response_format: Optional[Union[dict, Type[BaseModel]]] = None,
     seed: Optional[int] = None,
     tools: Optional[List] = None,
@@ -906,6 +908,7 @@ def completion(  # type: ignore # noqa: PLR0915
     web_search_options: Optional[OpenAIWebSearchOptions] = None,
     deployment_id=None,
     extra_headers: Optional[dict] = None,
+    safety_identifier: Optional[str] = None,
     # soon to be deprecated params by OpenAI
     functions: Optional[List] = None,
     function_call: Optional[str] = None,
@@ -1243,6 +1246,7 @@ def completion(  # type: ignore # noqa: PLR0915
             "reasoning_effort": reasoning_effort,
             "thinking": thinking,
             "web_search_options": web_search_options,
+            "safety_identifier": safety_identifier,
             "allowed_openai_params": kwargs.get("allowed_openai_params"),
         }
         optional_params = get_optional_params(
@@ -3667,7 +3671,7 @@ async def aembedding(*args, **kwargs) -> EmbeddingResponse:
     model = args[0] if len(args) > 0 else kwargs["model"]
     ### PASS ARGS TO Embedding ###
     kwargs["aembedding"] = True
-    custom_llm_provider = None
+    custom_llm_provider = kwargs.get("custom_llm_provider", None)
     try:
         # Use a partial function to pass your keyword arguments
         func = partial(embedding, *args, **kwargs)
@@ -3677,7 +3681,7 @@ async def aembedding(*args, **kwargs) -> EmbeddingResponse:
         func_with_context = partial(ctx.run, func)
 
         _, custom_llm_provider, _, _ = get_llm_provider(
-            model=model, api_base=kwargs.get("api_base", None)
+            model=model, custom_llm_provider=custom_llm_provider, api_base=kwargs.get("api_base", None)
         )
 
         # Await normally
@@ -4499,6 +4503,36 @@ def embedding(  # noqa: PLR0915
                 client=client,
                 aembedding=aembedding,
             )
+        elif custom_llm_provider == "volcengine":
+            volcengine_key = (
+                api_key
+                or litellm.api_key
+                or get_secret_str("ARK_API_KEY")
+                or get_secret_str("VOLCENGINE_API_KEY")
+            )
+            if volcengine_key is None:
+                raise ValueError(
+                    "Missing API key for Volcengine. Set ARK_API_KEY or VOLCENGINE_API_KEY environment variable or pass api_key parameter."
+                )
+            if extra_headers is not None and isinstance(extra_headers, dict):
+                headers = extra_headers
+            else:
+                headers = {}
+            response = base_llm_http_handler.embedding(
+                model=model,
+                input=input,
+                timeout=timeout,
+                custom_llm_provider=custom_llm_provider,
+                logging_obj=logging,
+                api_base=api_base,
+                optional_params=optional_params,
+                litellm_params={},
+                model_response=EmbeddingResponse(),
+                api_key=volcengine_key,
+                client=client,
+                aembedding=aembedding,
+                headers=headers,
+            )
         elif custom_llm_provider in litellm._custom_providers:
             custom_handler: Optional[CustomLLM] = None
             for item in litellm.custom_provider_map:

litellm/model_prices_and_context_window_backup.json

@@ -5817,16 +5817,6 @@
         "supports_response_schema": true,
         "supports_tool_choice": true
     },
-    "groq/llama3-8b-8192": {
-        "max_tokens": 8192,
-        "max_input_tokens": 8192,
-        "max_output_tokens": 8192,
-        "input_cost_per_token": 5e-08,
-        "output_cost_per_token": 8e-08,
-        "litellm_provider": "groq",
-        "mode": "chat",
-        "supports_tool_choice": true
-    },
     "groq/llama-3.2-1b-preview": {
         "max_tokens": 8192,
         "max_input_tokens": 8192,
@@ -5907,17 +5897,6 @@
         "supports_tool_choice": true,
         "deprecation_date": "2025-04-14"
     },
-    "groq/llama3-70b-8192": {
-        "max_tokens": 8192,
-        "max_input_tokens": 8192,
-        "max_output_tokens": 8192,
-        "input_cost_per_token": 5.9e-07,
-        "output_cost_per_token": 7.9e-07,
-        "litellm_provider": "groq",
-        "mode": "chat",
-        "supports_response_schema": true,
-        "supports_tool_choice": true
-    },
     "groq/llama-3.1-8b-instant": {
         "max_tokens": 8192,
         "max_input_tokens": 128000,
@@ -6178,21 +6157,7 @@
         "supports_tool_choice": true,
         "source": "https://inference-docs.cerebras.ai/support/pricing"
     },
-    "cerebras/openai/gpt-oss-20b": {
-        "max_tokens": 32768,
-        "max_input_tokens": 131072,
-        "max_output_tokens": 32768,
-        "input_cost_per_token": 7e-08,
-        "output_cost_per_token": 3e-07,
-        "litellm_provider": "cerebras",
-        "mode": "chat",
-        "supports_function_calling": true,
-        "supports_parallel_function_calling": true,
-        "supports_response_schema": true,
-        "supports_reasoning": true,
-        "supports_tool_choice": true,
-        "source": "https://inference-docs.cerebras.ai/support/pricing"
-    },
+
     "cerebras/openai/gpt-oss-120b": {
         "max_tokens": 32768,
         "max_input_tokens": 131072,
@@ -8027,8 +7992,8 @@
         "max_pdf_size_mb": 30,
         "input_cost_per_audio_token": 1e-06,
         "input_cost_per_token": 3e-07,
-        "output_cost_per_token": 2.5e-06,
-        "output_cost_per_reasoning_token": 2.5e-06,
+        "output_cost_per_token": 3e-05,
+        "output_cost_per_reasoning_token": 3e-05,
         "output_cost_per_image": 0.039,
         "litellm_provider": "gemini",
         "mode": "chat",
@@ -8391,8 +8356,8 @@
         "max_pdf_size_mb": 30,
         "input_cost_per_audio_token": 1e-06,
         "input_cost_per_token": 3e-07,
-        "output_cost_per_token": 2.5e-06,
-        "output_cost_per_reasoning_token": 2.5e-06,
+        "output_cost_per_token": 3e-05,
+        "output_cost_per_reasoning_token": 3e-05,
         "output_cost_per_image": 0.039,
         "litellm_provider": "vertex_ai-language-models",
         "mode": "chat",
@@ -9519,6 +9484,48 @@
         "source": "https://aistudio.google.com",
         "supports_tool_choice": true
     },
+    "gemini/veo-3.0-generate-preview": {
+        "max_tokens": 1024,
+        "max_input_tokens": 1024,
+        "output_cost_per_second": 0.75,
+        "litellm_provider": "gemini",
+        "mode": "video_generation",
+        "supported_modalities": [
+            "text"
+        ],
+        "supported_output_modalities": [
+            "video"
+        ],
+        "source": "https://ai.google.dev/gemini-api/docs/video"
+    },
+    "gemini/veo-3.0-fast-generate-preview": {
+        "max_tokens": 1024,
+        "max_input_tokens": 1024,
+        "output_cost_per_second": 0.40,
+        "litellm_provider": "gemini",
+        "mode": "video_generation",
+        "supported_modalities": [
+            "text"
+        ],
+        "supported_output_modalities": [
+            "video"
+        ],
+        "source": "https://ai.google.dev/gemini-api/docs/video"
+    },
+    "gemini/veo-2.0-generate-001": {
+        "max_tokens": 1024,
+        "max_input_tokens": 1024,
+        "output_cost_per_second": 0.35,
+        "litellm_provider": "gemini",
+        "mode": "video_generation",
+        "supported_modalities": [
+            "text"
+        ],
+        "supported_output_modalities": [
+            "video"
+        ],
+        "source": "https://ai.google.dev/gemini-api/docs/video"
+    },
     "vertex_ai/claude-opus-4-1": {
         "max_tokens": 4096,
         "max_input_tokens": 200000,
@@ -9905,6 +9912,28 @@
         "supports_tool_choice": true,
         "supports_prompt_caching": true
     },
+    "vertex_ai/openai/gpt-oss-20b-maas": {
+        "max_tokens": 32768,
+        "max_input_tokens": 131072,
+        "max_output_tokens": 32768,
+        "input_cost_per_token": 0.075e-06,
+        "output_cost_per_token": 0.30e-06,
+        "litellm_provider": "vertex_ai-openai_models",
+        "mode": "chat",
+        "supports_reasoning": true,
+        "source": "https://console.cloud.google.com/vertex-ai/publishers/openai/model-garden/gpt-oss-120b-maas"
+    },
+    "vertex_ai/openai/gpt-oss-120b-maas": {
+        "max_tokens": 32768,
+        "max_input_tokens": 131072,
+        "max_output_tokens": 32768,
+        "input_cost_per_token": 0.15e-06,
+        "output_cost_per_token": 0.60e-06,
+        "litellm_provider": "vertex_ai-openai_models",
+        "mode": "chat",
+        "supports_reasoning": true,
+        "source": "https://console.cloud.google.com/vertex-ai/publishers/openai/model-garden/gpt-oss-120b-maas"
+    },
     "vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas": {
         "max_tokens": 32768,
         "max_input_tokens": 262144,
@@ -10314,6 +10343,48 @@
         "mode": "image_generation",
         "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing"
     },
+    "vertex_ai/veo-3.0-generate-preview": {
+        "max_tokens": 1024,
+        "max_input_tokens": 1024,
+        "output_cost_per_second": 0.75,
+        "litellm_provider": "vertex_ai-video-models",
+        "mode": "video_generation",
+        "supported_modalities": [
+            "text"
+        ],
+        "supported_output_modalities": [
+            "video"
+        ],
+        "source": "https://ai.google.dev/gemini-api/docs/video"
+    },
+    "vertex_ai/veo-3.0-fast-generate-preview": {
+        "max_tokens": 1024,
+        "max_input_tokens": 1024,
+        "output_cost_per_second": 0.40,
+        "litellm_provider": "vertex_ai-video-models",
+        "mode": "video_generation",
+        "supported_modalities": [
+            "text"
+        ],
+        "supported_output_modalities": [
+            "video"
+        ],
+        "source": "https://ai.google.dev/gemini-api/docs/video"
+    },
+    "vertex_ai/veo-2.0-generate-001": {
+        "max_tokens": 1024,
+        "max_input_tokens": 1024,
+        "output_cost_per_second": 0.35,
+        "litellm_provider": "vertex_ai-video-models",
+        "mode": "video_generation",
+        "supported_modalities": [
+            "text"
+        ],
+        "supported_output_modalities": [
+            "video"
+        ],
+        "source": "https://ai.google.dev/gemini-api/docs/video"
+    },
     "text-embedding-004": {
         "max_tokens": 2048,
         "max_input_tokens": 2048,
@@ -20962,5 +21033,65 @@
         "metadata": {
             "notes": "DALL-E 2 via AI/ML API - Reliable text-to-image generation"
         }
+    },
+    "doubao-embedding-large": {
+        "max_tokens": 4096,
+        "max_input_tokens": 4096,
+        "output_vector_size": 2048,
+        "input_cost_per_token": 0.0,
+        "output_cost_per_token": 0.0,
+        "litellm_provider": "volcengine",
+        "mode": "embedding",
+        "metadata": {
+            "notes": "Volcengine Doubao embedding model - large version with 2048 dimensions"
+        }
+    },
+    "doubao-embedding-large-text-250515": {
+        "max_tokens": 4096,
+        "max_input_tokens": 4096,
+        "output_vector_size": 2048,
+        "input_cost_per_token": 0.0,
+        "output_cost_per_token": 0.0,
+        "litellm_provider": "volcengine",
+        "mode": "embedding",
+        "metadata": {
+            "notes": "Volcengine Doubao embedding model - text-250515 version with 2048 dimensions"
+        }
+    },
+    "doubao-embedding-large-text-240915": {
+        "max_tokens": 4096,
+        "max_input_tokens": 4096,
+        "output_vector_size": 4096,
+        "input_cost_per_token": 0.0,
+        "output_cost_per_token": 0.0,
+        "litellm_provider": "volcengine",
+        "mode": "embedding",
+        "metadata": {
+            "notes": "Volcengine Doubao embedding model - text-240915 version with 4096 dimensions"
+        }
+    },
+    "doubao-embedding": {
+        "max_tokens": 4096,
+        "max_input_tokens": 4096,
+        "output_vector_size": 2560,
+        "input_cost_per_token": 0.0,
+        "output_cost_per_token": 0.0,
+        "litellm_provider": "volcengine",
+        "mode": "embedding",
+        "metadata": {
+            "notes": "Volcengine Doubao embedding model - standard version with 2560 dimensions"
+        }
+    },
+    "doubao-embedding-text-240715": {
+        "max_tokens": 4096,
+        "max_input_tokens": 4096,
+        "output_vector_size": 2560,
+        "input_cost_per_token": 0.0,
+        "output_cost_per_token": 0.0,
+        "litellm_provider": "volcengine",
+        "mode": "embedding",
+        "metadata": {
+            "notes": "Volcengine Doubao embedding model - text-240715 version with 2560 dimensions"
+        }
     }
 }

litellm/proxy/_new_secret_config.yaml

@@ -1,30 +1,27 @@
 model_list:
- - model_name: fake-openai-endpoint
-   litellm_params:
-    model: openai/fake
-    api_key: fake-key
-    api_base: https://exampleopenaiendpoint-production.up.railway.app/
- - model_name: gpt-5-mini
-   litellm_params:
-    model: azure/gpt-5-mini
-    api_base: os.environ/AZURE_GPT_5_MINI_API_BASE # runs os.getenv("AZURE_API_BASE")
-    api_key: os.environ/AZURE_GPT_5_MINI_API_KEY # runs os.getenv("AZURE_API_KEY")
-    stream_timeout: 60
-    merge_reasoning_content_in_choices: true
-   model_info:
-    mode: chat
+  - model_name: fake-openai-endpoint
+    litellm_params:
+      model: openai/fake
+      api_key: fake-key
+      api_base: https://exampleopenaiendpoint-production.up.railway.app/
+  - model_name: gpt-5-mini
+    litellm_params:
+      model: azure/gpt-5-mini
+      api_base: os.environ/AZURE_GPT_5_MINI_API_BASE # runs os.getenv("AZURE_API_BASE")
+      api_key: os.environ/AZURE_GPT_5_MINI_API_KEY # runs os.getenv("AZURE_API_KEY")
+      stream_timeout: 60
+      merge_reasoning_content_in_choices: true
+    model_info:
+      mode: chat
+  - model_name: ollama-deepseek-r1
+    litellm_params:
+      model: ollama/deepseek-r1:1.5b
+    model_info:
+      mode: chat
 
 router_settings:
   model_group_alias: {"my-fake-gpt-4": "fake-openai-endpoint"}
-
+  
 litellm_settings:
   callbacks: ["otel"]
-  cache: true
-  cache_params:
-    type: redis
-    ttl: 600
-    supported_call_types: ["acompletion", "completion"]
-
-  model_group_settings:
-    forward_client_headers_to_llm_api:
-      - fake-openai-endpoint
+  success_callback: ["braintrust"]

litellm/proxy/_types.py

@@ -2,7 +2,16 @@ import enum
 import json
 import uuid
 from datetime import datetime
-from typing import TYPE_CHECKING, Any, Callable, Dict, List, Literal, Optional, Union
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    Dict,
+    List,
+    Literal,
+    Optional,
+    Union,
+)
 
 import httpx
 from pydantic import (
@@ -388,7 +397,11 @@ class LiteLLMRoutes(enum.Enum):
     ]
 
     # NOTE: ROUTES ONLY FOR MASTER KEY - only the Master Key should be able to Reset Spend
-    master_key_only_routes = ["/global/spend/reset"]
+    master_key_only_routes = [
+        "/global/spend/reset",
+        "/memory-usage-in-mem-cache",
+        "/memory-usage-in-mem-cache-items",
+    ]
 
     key_management_routes = [
         KeyManagementRoutes.KEY_GENERATE,
@@ -774,7 +787,6 @@ class GenerateKeyRequest(KeyRequestBase):
         description="Type of key that determines default allowed routes.",
     )
 
-
 class GenerateKeyResponse(KeyRequestBase):
     key: str  # type: ignore
     key_name: Optional[str] = None
@@ -2908,6 +2920,12 @@ class LitellmDataForBackendLLMCall(TypedDict, total=False):
     user: Optional[str]
     num_retries: Optional[int]
 
+class LitellmMetadataFromRequestHeaders(TypedDict, total=False):
+    """
+    Headers a user can pass that will get added to litellm metadata for the request
+    """
+    spend_logs_metadata: Optional[dict]
+
 
 class JWTKeyItem(TypedDict, total=False):
     kid: str

litellm/proxy/anthropic_endpoints/endpoints.py

@@ -90,6 +90,17 @@ async def anthropic_response(  # noqa: PLR0915
             user_api_key_dict=user_api_key_dict, data=data, call_type="text_completion"
         )
 
+        tasks = []
+        tasks.append(
+            proxy_logging_obj.during_call_hook(
+                data=data,
+                user_api_key_dict=user_api_key_dict,
+                call_type=ProxyBaseLLMRequestProcessing._get_pre_call_type(
+                    route_type="anthropic_messages"  # type: ignore
+                ),
+            )
+        )
+
         ### ROUTE THE REQUESTs ###
         router_model_names = llm_router.model_names if llm_router is not None else []
 
@@ -97,23 +108,21 @@ async def anthropic_response(  # noqa: PLR0915
         if (
             llm_router is not None and data["model"] in router_model_names
         ):  # model in router model list
-            llm_response = asyncio.create_task(llm_router.aanthropic_messages(**data))
+            llm_coro = llm_router.aanthropic_messages(**data)
         elif (
             llm_router is not None
             and llm_router.model_group_alias is not None
             and data["model"] in llm_router.model_group_alias
         ):  # model set in model_group_alias
-            llm_response = asyncio.create_task(llm_router.aanthropic_messages(**data))
+            llm_coro = llm_router.aanthropic_messages(**data)
         elif (
             llm_router is not None and data["model"] in llm_router.deployment_names
         ):  # model in router deployments, calling a specific deployment on the router
-            llm_response = asyncio.create_task(
-                llm_router.aanthropic_messages(**data, specific_deployment=True)
-            )
+            llm_coro = llm_router.aanthropic_messages(**data, specific_deployment=True)
         elif (
             llm_router is not None and data["model"] in llm_router.get_model_ids()
         ):  # model in router model list
-            llm_response = asyncio.create_task(llm_router.aanthropic_messages(**data))
+            llm_coro = llm_router.aanthropic_messages(**data)
         elif (
             llm_router is not None
             and data["model"] not in router_model_names
@@ -122,9 +131,9 @@ async def anthropic_response(  # noqa: PLR0915
                 or len(llm_router.pattern_router.patterns) > 0
             )
         ):  # model in router deployments, calling a specific deployment on the router
-            llm_response = asyncio.create_task(llm_router.aanthropic_messages(**data))
+            llm_coro = llm_router.aanthropic_messages(**data)
         elif user_model is not None:  # `litellm --model <your-model-name>`
-            llm_response = asyncio.create_task(litellm.anthropic_messages(**data))
+            llm_coro = litellm.anthropic_messages(**data)
         else:
             raise HTTPException(
                 status_code=status.HTTP_400_BAD_REQUEST,
@@ -134,8 +143,16 @@ async def anthropic_response(  # noqa: PLR0915
                 },
             )
 
-        # Await the llm_response task
-        response = await llm_response
+        tasks.append(llm_coro)
+
+        # wait for call to end
+        llm_responses = asyncio.gather(
+            *tasks
+        )  # run the moderation check in parallel to the actual llm api call
+
+        responses = await llm_responses
+
+        response = responses[1]
 
         hidden_params = getattr(response, "_hidden_params", {}) or {}
         model_id = hidden_params.get("model_id", None) or ""
@@ -183,6 +200,11 @@ async def anthropic_response(  # noqa: PLR0915
                 headers=dict(fastapi_response.headers),
             )
 
+        ### CALL HOOKS ### - modify outgoing data
+        response = await proxy_logging_obj.post_call_success_hook(
+            data=data, user_api_key_dict=user_api_key_dict, response=response # type: ignore
+        )
+
         verbose_proxy_logger.info("\nResponse from Litellm:\n{}".format(response))
         return response
     except Exception as e:

litellm/proxy/auth/handle_jwt.py

@@ -7,7 +7,6 @@ JWT token must have 'litellm_proxy_admin' in scope.
 """
 
 import fnmatch
-import json
 import os
 from typing import Any, List, Literal, Optional, Set, Tuple, cast
 
@@ -484,7 +483,7 @@ class JWTHandler:
         # Supported algos: https://pyjwt.readthedocs.io/en/stable/algorithms.html
         # "Warning: Make sure not to mix symmetric and asymmetric algorithms that interpret
         #   the key in different ways (e.g. HS* and RS*)."
-        algorithms = ["RS256", "RS384", "RS512", "PS256", "PS384", "PS512"]
+        algorithms = ["RS256", "RS384", "RS512", "PS256", "PS384", "PS512", "ES256", "ES384", "ES512", "EdDSA"]
 
         audience = os.getenv("JWT_AUDIENCE")
         decode_options = None
@@ -492,7 +491,7 @@ class JWTHandler:
             decode_options = {"verify_aud": False}
 
         import jwt
-        from jwt.algorithms import RSAAlgorithm
+        from jwt.api_jwk import PyJWK
 
         header = jwt.get_unverified_header(token)
 
@@ -512,14 +511,21 @@ class JWTHandler:
                 jwk["n"] = public_key["n"]
             if "e" in public_key:
                 jwk["e"] = public_key["e"]
+            if "x" in public_key:
+                jwk["x"] = public_key["x"]
+            if "y" in public_key:
+                jwk["y"] = public_key["y"]
+            if "crv" in public_key:
+                jwk["crv"] = public_key["crv"]
 
-            public_key_rsa = RSAAlgorithm.from_jwk(json.dumps(jwk))
+            # parse RSA/EC/OKP keys
+            public_key_obj = PyJWK.from_dict(jwk).key
 
             try:
                 # decode the token using the public key
                 payload = jwt.decode(
                     token,
-                    public_key_rsa,  # type: ignore
+                    public_key_obj,  # type: ignore
                     algorithms=algorithms,
                     options=decode_options,
                     audience=audience,
@@ -534,9 +540,7 @@ class JWTHandler:
                 raise Exception(f"Validation fails: {str(e)}")
         elif public_key is not None and isinstance(public_key, str):
             try:
-                cert = x509.load_pem_x509_certificate(
-                    public_key.encode(), default_backend()
-                )
+                cert = x509.load_pem_x509_certificate(public_key.encode(), default_backend())
 
                 # Extract public key
                 key = cert.public_key().public_bytes(
@@ -561,7 +565,7 @@ class JWTHandler:
                 raise Exception(f"Validation fails: {str(e)}")
 
         raise Exception("Invalid JWT Submitted")
-
+    
     async def close(self):
         await self.http_handler.close()
 
@@ -1210,4 +1214,4 @@ class JWTAuthManager:
             end_user_object=end_user_object,
             token=api_key,
             team_membership=team_membership_object,
-        )
+        )

litellm/proxy/common_request_processing.py

@@ -109,7 +109,6 @@ async def create_streaming_response(
     final_status_code = default_status_code
 
     try:
-
         # Handle coroutine that returns a generator
         if asyncio.iscoroutine(generator):
             generator = await generator
@@ -118,7 +117,6 @@ async def create_streaming_response(
         first_chunk_value = await generator.__anext__()
 
         if first_chunk_value is not None:
-
             try:
                 error_code_from_chunk = await _parse_event_data_for_error(
                     first_chunk_value
@@ -132,7 +130,6 @@ async def create_streaming_response(
                 verbose_proxy_logger.debug(f"Error parsing first chunk value: {e}")
 
     except StopAsyncIteration:
-
         # Generator was empty. Default status
         async def empty_gen() -> AsyncGenerator[str, None]:
             if False:
@@ -145,7 +142,6 @@ async def create_streaming_response(
             status_code=default_status_code,
         )
     except Exception as e:
-
         # Unexpected error consuming first chunk.
         verbose_proxy_logger.exception(
             f"Error consuming first chunk from generator: {e}"
@@ -168,7 +164,6 @@ async def create_streaming_response(
             with tracer.trace(DD_TRACER_STREAMING_CHUNK_YIELD_RESOURCE):
                 yield first_chunk_value
         async for chunk in generator:
-
             with tracer.trace(DD_TRACER_STREAMING_CHUNK_YIELD_RESOURCE):
                 yield chunk
 
@@ -462,7 +457,6 @@ class ProxyBaseLLMRequestProcessing:
         ) or self._is_streaming_response(
             response
         ):  # use generate_responses to stream responses
-
             custom_headers = ProxyBaseLLMRequestProcessing.get_custom_headers(
                 user_api_key_dict=user_api_key_dict,
                 call_id=logging_obj.litellm_call_id,
@@ -480,7 +474,6 @@ class ProxyBaseLLMRequestProcessing:
             if route_type == "allm_passthrough_route":
                 # Check if response is an async generator
                 if self._is_streaming_response(response):
-
                     if asyncio.iscoroutine(response):
                         generator = await response
                     else:
@@ -501,7 +494,6 @@ class ProxyBaseLLMRequestProcessing:
                         headers=custom_headers,
                     )
             else:
-
                 selected_data_generator = select_data_generator(
                     response=response,
                     user_api_key_dict=user_api_key_dict,
@@ -740,7 +732,11 @@ class ProxyBaseLLMRequestProcessing:
         verbose_proxy_logger.debug("inside generator")
         try:
             str_so_far = ""
-            async for chunk in response:
+            async for chunk in proxy_logging_obj.async_post_call_streaming_iterator_hook(
+                user_api_key_dict=user_api_key_dict,
+                response=response,
+                request_data=request_data,
+            ):
                 verbose_proxy_logger.debug(
                     "async_data_generator: received streaming chunk - {}".format(chunk)
                 )

litellm/proxy/common_utils/callback_utils.py

@@ -317,17 +317,26 @@ def get_remaining_tokens_and_requests_from_request_data(data: Dict) -> Dict[str,
     _metadata = data.get("metadata", None) or {}
     model_group = get_model_group_from_request_data(data)
 
+    # The h11 package considers "/" or ":" invalid and raise a LocalProtocolError
+    h11_model_group_name = (
+        model_group.replace("/", "-").replace(":", "-") if model_group else None
+    )
+
     # Remaining Requests
     remaining_requests_variable_name = f"litellm-key-remaining-requests-{model_group}"
     remaining_requests = _metadata.get(remaining_requests_variable_name, None)
     if remaining_requests:
-        headers[f"x-litellm-key-remaining-requests-{model_group}"] = remaining_requests
+        headers[f"x-litellm-key-remaining-requests-{h11_model_group_name}"] = (
+            remaining_requests
+        )
 
     # Remaining Tokens
     remaining_tokens_variable_name = f"litellm-key-remaining-tokens-{model_group}"
     remaining_tokens = _metadata.get(remaining_tokens_variable_name, None)
     if remaining_tokens:
-        headers[f"x-litellm-key-remaining-tokens-{model_group}"] = remaining_tokens
+        headers[f"x-litellm-key-remaining-tokens-{h11_model_group_name}"] = (
+            remaining_tokens
+        )
 
     return headers
 

litellm/proxy/common_utils/debug_utils.py

@@ -5,10 +5,12 @@ import os
 import tracemalloc
 from collections import Counter
 
-from fastapi import APIRouter
+from fastapi import APIRouter, Depends
 
 from litellm import get_secret_str
 from litellm._logging import verbose_proxy_logger
+from litellm.proxy._types import UserAPIKeyAuth
+from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
 
 router = APIRouter()
 
@@ -84,7 +86,9 @@ if os.environ.get("LITELLM_PROFILE", "false").lower() == "true":
 
 
 @router.get("/memory-usage-in-mem-cache", include_in_schema=False)
-async def memory_usage_in_mem_cache():
+async def memory_usage_in_mem_cache(
+    _: UserAPIKeyAuth = Depends(user_api_key_auth),
+):
     # returns the size of all in-memory caches on the proxy server
     """
     1. user_api_key_cache
@@ -121,7 +125,9 @@ async def memory_usage_in_mem_cache():
 
 
 @router.get("/memory-usage-in-mem-cache-items", include_in_schema=False)
-async def memory_usage_in_mem_cache_items():
+async def memory_usage_in_mem_cache_items(
+    _: UserAPIKeyAuth = Depends(user_api_key_auth),
+):
     # returns the size of all in-memory caches on the proxy server
     """
     1. user_api_key_cache

litellm/proxy/google_endpoints/endpoints.py

@@ -173,15 +173,24 @@ async def google_count_tokens(request: Request, model_name: str):
     """
     from litellm.proxy.common_utils.http_parsing_utils import _read_request_body
     from litellm.proxy.proxy_server import token_counter as internal_token_counter
+    from litellm.google_genai.adapters.transformation import GoogleGenAIAdapter
 
     data = await _read_request_body(request=request)
     contents = data.get("contents", [])
     #Create TokenCountRequest for the internal endpoint
     from litellm.proxy._types import TokenCountRequest
 
+    # Translate contents to openai format messages using the adapter
+    messages = (
+        GoogleGenAIAdapter()
+        .translate_generate_content_to_completion(model_name, contents)
+        .get("messages", [])
+    )
+
     token_request = TokenCountRequest(
         model=model_name,
-        contents=contents
+        contents=contents,
+        messages=messages,  # compatibility when use openai-like endpoint
     )
 
     # Call the internal token counter function with direct request flag set to False
@@ -192,11 +201,17 @@ async def google_count_tokens(request: Request, model_name: str):
     if token_response is not None:
         # cast the response to the well known format
         original_response: dict = token_response.original_response or {}
-        return TokenCountDetailsResponse(
-            totalTokens=original_response.get("totalTokens", 0),
-            promptTokensDetails=original_response.get("promptTokensDetails", []),
-        )
-    
+        if original_response:
+            return TokenCountDetailsResponse(
+                totalTokens=original_response.get("totalTokens", 0),
+                promptTokensDetails=original_response.get("promptTokensDetails", []),
+            )
+        else:
+            return TokenCountDetailsResponse(
+                totalTokens=token_response.total_tokens or 0,
+                promptTokensDetails=[],
+            )
+
     #########################################################
     # Return the response in the well known format
     #########################################################

litellm/proxy/hooks/parallel_request_limiter_v3.py

@@ -68,6 +68,32 @@ end
 return results
 """
 
+TOKEN_INCREMENT_SCRIPT = """
+local results = {}
+
+-- Process each key/increment_value/ttl triplet
+for i = 1, #KEYS do
+    local key = KEYS[i]
+    local increment_value = tonumber(ARGV[i * 2 - 1])
+    local ttl_seconds = tonumber(ARGV[i * 2])
+
+    -- Increment the value
+    local new_value = redis.call('INCRBYFLOAT', key, increment_value)
+
+    -- Handle TTL: only set expire if ttl_seconds > 0 and key has no current TTL
+    -- ttl_seconds can be 0 (no TTL) or positive (set TTL)
+    if ttl_seconds and ttl_seconds > 0 then
+        local current_ttl = redis.call('TTL', key)
+        if current_ttl == -1 then
+            redis.call('EXPIRE', key, ttl_seconds)
+        end
+    end
+
+    table.insert(results, new_value)
+end
+
+return results
+"""
 
 class RateLimitDescriptorRateLimitObject(TypedDict, total=False):
     requests_per_unit: Optional[int]
@@ -109,8 +135,14 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
                     BATCH_RATE_LIMITER_SCRIPT
                 )
             )
+            self.token_increment_script = (
+                self.internal_usage_cache.dual_cache.redis_cache.async_register_script(
+                    TOKEN_INCREMENT_SCRIPT
+                )
+            )
         else:
             self.batch_rate_limiter_script = None
+            self.token_increment_script = None
 
         self.window_size = int(os.getenv("LITELLM_RATE_LIMIT_WINDOW_SIZE", 60))
 
@@ -567,6 +599,62 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
 
         return pipeline_operations
 
+    async def async_increment_tokens_with_ttl_preservation(
+        self,
+        pipeline_operations: List["RedisPipelineIncrementOperation"],
+        parent_otel_span: Optional[Span] = None,
+    ) -> None:
+        """
+        Increment token counters using Lua script to preserve existing TTL.
+        This prevents TTL reset on every token increment.
+        """
+        if not pipeline_operations:
+            return
+
+        # Check if script is available
+        if self.token_increment_script is None:
+            verbose_proxy_logger.debug("TTL preservation script not available, using regular pipeline")
+            await self.internal_usage_cache.dual_cache.async_increment_cache_pipeline(
+                increment_list=pipeline_operations,
+                litellm_parent_otel_span=parent_otel_span,
+            )
+            return
+
+        try:
+            # Use Lua script for all operations
+            keys = []
+            args = []
+
+            for op in pipeline_operations:
+                # Convert None TTL to 0 for Lua script
+                ttl_value = op["ttl"] if op["ttl"] is not None else 0
+                
+                verbose_proxy_logger.debug(
+                    f"Executing TTL-preserving increment for key={op['key']}, "
+                    f"increment={op['increment_value']}, ttl={ttl_value}"
+                )
+                keys.append(op["key"])
+                args.extend([op["increment_value"], ttl_value])
+
+            await self.token_increment_script(
+                keys=keys,
+                args=args,
+            )
+
+            verbose_proxy_logger.debug(
+                f"Successfully executed TTL-preserving increment for {len(pipeline_operations)} keys"
+            )
+
+        except Exception as e:
+            verbose_proxy_logger.warning(
+                f"TTL preservation failed, falling back to regular pipeline: {str(e)}"
+            )
+            # Fallback to regular pipeline on error
+            await self.internal_usage_cache.dual_cache.async_increment_cache_pipeline(
+                increment_list=pipeline_operations,
+                litellm_parent_otel_span=parent_otel_span,
+            )
+
     def get_rate_limit_type(self) -> Literal["output", "input", "total"]:
         from litellm.proxy.proxy_server import general_settings
 
@@ -713,9 +801,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
 
             # Execute all increments in a single pipeline
             if pipeline_operations:
-                await self.internal_usage_cache.dual_cache.async_increment_cache_pipeline(
-                    increment_list=pipeline_operations,
-                    litellm_parent_otel_span=litellm_parent_otel_span,
+                await self.async_increment_tokens_with_ttl_preservation(
+                    pipeline_operations=pipeline_operations,
+                    parent_otel_span=litellm_parent_otel_span,
                 )
 
         except Exception as e:

litellm/proxy/litellm_pre_call_utils.py

@@ -291,6 +291,17 @@ class LiteLLMProxyRequestSetup:
         if num_retries_header is not None:
             return int(num_retries_header)
         return None
+    
+    @staticmethod
+    def _get_spend_logs_metadata_from_request_headers(headers: dict) -> Optional[dict]:
+        """
+        Get the `spend_logs_metadata` from the request headers.
+        """
+        from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
+        spend_logs_metadata_header = headers.get("x-litellm-spend-logs-metadata", None)
+        if spend_logs_metadata_header is not None:
+            return safe_json_loads(spend_logs_metadata_header)
+        return None
 
     @staticmethod
     def _get_forwardable_headers(
@@ -459,6 +470,30 @@ class LiteLLMProxyRequestSetup:
             data["num_retries"] = num_retries
 
         return data
+    
+    @staticmethod
+    def add_litellm_metadata_from_request_headers(
+        headers: dict,
+        data: dict,
+        _metadata_variable_name: str,
+    ) -> dict:
+        """
+        Add litellm metadata from request headers
+
+        Relevant issue: https://github.com/BerriAI/litellm/issues/14008
+        """
+        from litellm.proxy._types import LitellmMetadataFromRequestHeaders
+        metadata_from_headers = LitellmMetadataFromRequestHeaders()
+        spend_logs_metadata = LiteLLMProxyRequestSetup._get_spend_logs_metadata_from_request_headers(headers)
+        if spend_logs_metadata is not None:
+            metadata_from_headers["spend_logs_metadata"] = spend_logs_metadata
+        
+        #########################################################################################
+        # Finally update the requests metadata with the `metadata_from_headers`
+        #########################################################################################
+        if isinstance(data[_metadata_variable_name], dict):
+            data[_metadata_variable_name].update(metadata_from_headers)
+        return data
 
     @staticmethod
     def get_sanitized_user_information_from_key(
@@ -642,7 +677,6 @@ async def add_litellm_data_to_request(  # noqa: PLR0915
     from litellm.proxy.proxy_server import llm_router, premium_user
     from litellm.types.proxy.litellm_pre_call_utils import SecretFields
 
-    safe_add_api_version_from_query_params(data, request)
 
     _headers = clean_headers(
         request.headers,
@@ -653,6 +687,24 @@ async def add_litellm_data_to_request(  # noqa: PLR0915
         ),
     )
 
+    ##########################################################
+    # Init - Proxy Server Request
+    # we do this as soon as entering so we track the original request
+    ##########################################################
+    data["proxy_server_request"] = {
+        "url": str(request.url),
+        "method": request.method,
+        "headers": _headers,
+        "body": copy.copy(data),  # use copy instead of deepcopy
+    }
+
+    safe_add_api_version_from_query_params(data, request)
+    _metadata_variable_name = _get_metadata_variable_name(request)
+    if data.get(_metadata_variable_name, None) is None:
+        data[_metadata_variable_name] = {}
+
+
+
     data.update(
         LiteLLMProxyRequestSetup.add_litellm_data_for_backend_llm_call(
             headers=_headers,
@@ -661,6 +713,14 @@ async def add_litellm_data_to_request(  # noqa: PLR0915
         )
     )
 
+    data.update(
+        LiteLLMProxyRequestSetup.add_litellm_metadata_from_request_headers(
+            headers=_headers,
+            data=data,
+            _metadata_variable_name=_metadata_variable_name,
+        )
+    )
+
     # check for forwardable headers
     data = LiteLLMProxyRequestSetup.add_headers_to_llm_call_by_model_group(
         data=data, headers=_headers, user_api_key_dict=user_api_key_dict
@@ -674,13 +734,6 @@ async def add_litellm_data_to_request(  # noqa: PLR0915
         if "user" not in data:
             data["user"] = user
 
-    # Include original request and headers in the data
-    data["proxy_server_request"] = {
-        "url": str(request.url),
-        "method": request.method,
-        "headers": _headers,
-        "body": copy.copy(data),  # use copy instead of deepcopy
-    }
 
     data["secret_fields"] = SecretFields(raw_headers=dict(request.headers))
 
@@ -711,11 +764,6 @@ async def add_litellm_data_to_request(  # noqa: PLR0915
 
     verbose_proxy_logger.debug("receiving data: %s", data)
 
-    _metadata_variable_name = _get_metadata_variable_name(request)
-
-    if data.get(_metadata_variable_name, None) is None:
-        data[_metadata_variable_name] = {}
-
     # Parse metadata if it's a string (e.g., from multipart/form-data)
     if "metadata" in data and data["metadata"] is not None:
         if isinstance(data["metadata"], str):

litellm/proxy/management_endpoints/key_management_endpoints.py

@@ -346,6 +346,35 @@ def handle_key_type(data: GenerateKeyRequest, data_json: dict) -> dict:
         data_json["allowed_routes"] = ["info_routes"]
     return data_json
 
+async def validate_team_id_used_in_service_account_request(
+    team_id: Optional[str],
+    prisma_client: Optional[PrismaClient],
+):
+    """
+    Validate team_id is used in the request body for generating a service account key
+    """
+    if team_id is None:
+        raise HTTPException(
+            status_code=400,
+            detail="team_id is required for service account keys. Please specify `team_id` in the request body.",
+        )
+    
+    if prisma_client is None:
+        raise HTTPException(
+            status_code=400,
+            detail="prisma_client is required for service account keys. Please specify `prisma_client` in the request body.",
+        )
+    
+    # check if team_id exists in the database
+    team = await prisma_client.db.litellm_teamtable.find_unique(
+        where={"team_id": team_id},
+    )
+    if team is None:
+        raise HTTPException(
+            status_code=400,
+            detail="team_id does not exist in the database. Please specify a valid `team_id` in the request body.",
+        )
+    return True
 
 async def _common_key_generation_helper(  # noqa: PLR0915
     data: GenerateKeyRequest,
@@ -372,9 +401,9 @@ async def _common_key_generation_helper(  # noqa: PLR0915
         and data.metadata.get("service_account_id") is not None
         and data.team_id is None
     ):
-        raise HTTPException(
-            status_code=400,
-            detail="team_id is required for service account keys. Please specify `team_id` in the request body.",
+        await validate_team_id_used_in_service_account_request(
+            team_id=data.team_id,
+            prisma_client=prisma_client,
         )
 
     # check if user set default key/generate params on config.yaml
@@ -756,6 +785,11 @@ async def generate_service_account_key_fn(
         user_custom_key_generate,
     )
 
+    await validate_team_id_used_in_service_account_request(
+        team_id=data.team_id,
+        prisma_client=prisma_client,
+    )
+
     verbose_proxy_logger.debug("entered /key/generate")
 
     if user_custom_key_generate is not None:
@@ -1566,14 +1600,12 @@ async def generate_key_helper_fn(  # noqa: PLR0915
     if duration is None:  # allow tokens that never expire
         expires = None
     else:
-        duration_s = duration_in_seconds(duration=duration)
-        expires = datetime.now(timezone.utc) + timedelta(seconds=duration_s)
+        expires = get_budget_reset_time(budget_duration=duration)
 
     if key_budget_duration is None:  # one-time budget
         key_reset_at = None
     else:
-        duration_s = duration_in_seconds(duration=key_budget_duration)
-        key_reset_at = datetime.now(timezone.utc) + timedelta(seconds=duration_s)
+        key_reset_at = get_budget_reset_time(budget_duration=key_budget_duration)
 
     if budget_duration is None:  # one-time budget
         reset_at = None