Merge branch 'main' into litellm_dev_09_10_2025_p1

.circleci/config.yml

@@ -671,6 +671,7 @@ jobs:
             pip install mypy
             pip install "google-generativeai==0.3.2"
             pip install "google-cloud-aiplatform==1.43.0"
+            pip install "google-genai==1.22.0"
             pip install pyarrow
             pip install "boto3==1.36.0"
             pip install "aioboto3==13.4.0"

README.md

@@ -25,7 +25,7 @@
     <a href="https://discord.gg/wuPM9dRgDw">
         <img src="https://img.shields.io/static/v1?label=Chat%20on&message=Discord&color=blue&logo=Discord&style=flat-square" alt="Discord">
     </a>
-    <a href="https://join.slack.com/share/enQtOTE0ODczMzk2Nzk4NC01YjUxNjY2YjBlYTFmNDRiZTM3NDFiYTM3MzVkODFiMDVjOGRjMmNmZTZkZTMzOWQzZGQyZWIwYjQ0MWExYmE3">
+    <a href="https://www.litellm.ai/support">
         <img src="https://img.shields.io/static/v1?label=Chat%20on&message=Slack&color=black&logo=Slack&style=flat-square" alt="Slack">
     </a>
 </h4>
@@ -408,7 +408,7 @@ All these checks must pass before your PR can be merged.
 
 - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version)
 - [Community Discord 💭](https://discord.gg/wuPM9dRgDw)
-- [Community Slack 💭](https://join.slack.com/share/enQtOTE0ODczMzk2Nzk4NC01YjUxNjY2YjBlYTFmNDRiZTM3NDFiYTM3MzVkODFiMDVjOGRjMmNmZTZkZTMzOWQzZGQyZWIwYjQ0MWExYmE3)
+- [Community Slack 💭](https://www.litellm.ai/support)
 - Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬
 - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai
 

cookbook/litellm_proxy_server/batch_api/bedrock/bedrock.py

@@ -0,0 +1,25 @@
+from openai import OpenAI
+
+client = OpenAI(
+    base_url="http://0.0.0.0:4000",
+    api_key="sk-1234",
+)
+
+BEDROCK_BATCH_MODEL = "bedrock/batch-anthropic.claude-3-5-sonnet-20240620-v1:0"
+
+# Upload file
+batch_input_file = client.files.create(
+    file=open("./bedrock_batch_completions.jsonl", "rb"),
+    purpose="batch",
+    extra_body={"target_model_names": BEDROCK_BATCH_MODEL}
+)
+print(batch_input_file)
+
+# Create batch
+batch = client.batches.create( 
+    input_file_id=batch_input_file.id,
+    endpoint="/v1/chat/completions",
+    completion_window="24h",
+    metadata={"description": "Test batch job"},
+)
+print(batch)

cookbook/litellm_proxy_server/batch_api/bedrock/bedrock_batch_completions.jsonl

@@ -0,0 +1,128 @@
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}

cookbook/litellm_proxy_server/mcp/mcp_with_litellm_proxy.py

@@ -0,0 +1,36 @@
+"""
+Use LiteLLM Proxy MCP Gateway to call MCP tools.
+
+When using LiteLLM Proxy, you can use the same MCP tools across all your LLM providers.
+"""
+import openai
+
+client = openai.OpenAI(
+    api_key="sk-1234", # paste your litellm proxy api key here
+    base_url="http://localhost:4000" # paste your litellm proxy base url here
+)
+print("Making API request to Responses API with MCP tools")
+
+response = client.responses.create(
+    model="gpt-5",
+    input=[
+        {
+            "role": "user",
+            "content": "give me TLDR of what BerriAI/litellm repo is about",
+            "type": "message"
+        }
+    ],
+    tools=[
+        {
+            "type": "mcp",
+            "server_label": "litellm",
+            "server_url": "litellm_proxy",
+            "require_approval": "never"
+        }
+    ],
+    stream=True,
+    tool_choice="required"
+)
+
+for chunk in response:
+    print("response chunk: ", chunk)

docs/my-website/docs/batches.md

@@ -7,7 +7,7 @@ Covers Batches, Files
 
 | Feature | Supported | Notes | 
 |-------|-------|-------|
-| Supported Providers | OpenAI, Azure, Vertex | - |
+| Supported Providers | OpenAI, Azure, Vertex, Bedrock | - |
 | ✨ Cost Tracking | ✅ | LiteLLM Enterprise only |
 | Logging | ✅ | Works across all logging integrations |
 
@@ -178,6 +178,7 @@ print("list_batches_response=", list_batches_response)
 ### [Azure OpenAI](./providers/azure#azure-batches-api)
 ### [OpenAI](#quick-start)
 ### [Vertex AI](./providers/vertex#batch-apis)
+### [Bedrock](./providers/bedrock_batches)
 
 
 ## How Cost Tracking for Batches API Works

docs/my-website/docs/completion/http_handler_config.md

@@ -0,0 +1,145 @@
+# Custom HTTP Handler
+
+Configure custom aiohttp sessions for better performance and control in LiteLLM completions.
+
+## Overview
+
+You can now inject custom `aiohttp.ClientSession` instances into LiteLLM for:
+- Custom connection pooling and timeouts
+- Corporate proxy and SSL configurations  
+- Performance optimization
+- Request monitoring
+
+## Basic Usage
+
+### Default (No Changes Required)
+```python
+import litellm
+
+# Works exactly as before
+response = await litellm.acompletion(
+    model="gpt-3.5-turbo",
+    messages=[{"role": "user", "content": "Hello!"}]
+)
+```
+
+### Custom Session
+```python
+import aiohttp
+import litellm
+from litellm.llms.custom_httpx.aiohttp_handler import BaseLLMAIOHTTPHandler
+
+# Create optimized session
+session = aiohttp.ClientSession(
+    timeout=aiohttp.ClientTimeout(total=180),
+    connector=aiohttp.TCPConnector(limit=300, limit_per_host=75)
+)
+
+# Replace global handler
+litellm.base_llm_aiohttp_handler = BaseLLMAIOHTTPHandler(client_session=session)
+
+# All completions now use your session
+response = await litellm.acompletion(model="gpt-3.5-turbo", messages=[...])
+```
+
+## Common Patterns
+
+### FastAPI Integration
+```python
+from contextlib import asynccontextmanager
+from fastapi import FastAPI
+import aiohttp
+import litellm
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    # Startup
+    session = aiohttp.ClientSession(
+        timeout=aiohttp.ClientTimeout(total=180),
+        connector=aiohttp.TCPConnector(limit=300)
+    )
+    litellm.base_llm_aiohttp_handler = BaseLLMAIOHTTPHandler(
+        client_session=session
+    )
+    yield
+    # Shutdown
+    await session.close()
+
+app = FastAPI(lifespan=lifespan)
+
+@app.post("/chat")
+async def chat(messages: list[dict]):
+    return await litellm.acompletion(model="gpt-3.5-turbo", messages=messages)
+```
+
+### Corporate Proxy
+```python
+import ssl
+
+# Custom SSL context
+ssl_context = ssl.create_default_context()
+ssl_context.load_cert_chain('cert.pem', 'key.pem')
+
+# Proxy session
+session = aiohttp.ClientSession(
+    connector=aiohttp.TCPConnector(ssl=ssl_context),
+    trust_env=True  # Use environment proxy settings
+)
+
+litellm.base_llm_aiohttp_handler = BaseLLMAIOHTTPHandler(client_session=session)
+```
+
+### High Performance
+```python
+# Optimized for high throughput
+session = aiohttp.ClientSession(
+    timeout=aiohttp.ClientTimeout(total=300),
+    connector=aiohttp.TCPConnector(
+        limit=1000,             # High connection limit
+        limit_per_host=200,     # Per host limit
+        ttl_dns_cache=600,      # DNS cache
+        keepalive_timeout=60,   # Keep connections alive
+        enable_cleanup_closed=True
+    )
+)
+
+litellm.base_llm_aiohttp_handler = BaseLLMAIOHTTPHandler(client_session=session)
+```
+
+## Constructor Options
+
+```python
+BaseLLMAIOHTTPHandler(
+    client_session=None,    # Custom aiohttp.ClientSession
+    transport=None,         # Advanced transport control
+    connector=None,         # Custom aiohttp.BaseConnector
+)
+```
+
+## Resource Management
+
+- **User sessions**: You manage the lifecycle (call `await session.close()`)
+- **Auto-created sessions**: Automatically cleaned up by the handler
+- **100% backward compatible**: Existing code works unchanged
+
+## Configuration Tips
+
+### Development
+```python
+session = aiohttp.ClientSession(
+    timeout=aiohttp.ClientTimeout(total=60),
+    connector=aiohttp.TCPConnector(limit=50)
+)
+```
+
+### Production
+```python
+session = aiohttp.ClientSession(
+    timeout=aiohttp.ClientTimeout(total=300),
+    connector=aiohttp.TCPConnector(
+        limit=1000,
+        limit_per_host=200,
+        keepalive_timeout=60
+    )
+)
+```

docs/my-website/docs/langchain/langchain.md

@@ -162,3 +162,321 @@ Get more details [here](../observability/lunary_integration.md)
 
 ## Use LangChain ChatLiteLLM + Langfuse
 Checkout this section [here](../observability/langfuse_integration#use-langchain-chatlitellm--langfuse) for more details on how to integrate Langfuse with ChatLiteLLM.
+
+## Using Tags with LangChain and LiteLLM
+
+Tags are a powerful feature in LiteLLM that allow you to categorize, filter, and track your LLM requests. When using LangChain with LiteLLM, you can pass tags through the `extra_body` parameter in the metadata.
+
+### Basic Tag Usage
+
+<Tabs>
+<TabItem value="openai" label="OpenAI">
+
+```python
+import os
+from langchain_openai import ChatOpenAI
+from langchain_core.messages import HumanMessage, SystemMessage
+
+os.environ['OPENAI_API_KEY'] = "sk-your-key-here"
+
+chat = ChatOpenAI(
+    model="gpt-4o",
+    temperature=0.7,
+    extra_body={
+        "metadata": {
+            "tags": ["production", "customer-support", "high-priority"]
+        }
+    }
+)
+
+messages = [
+    SystemMessage(content="You are a helpful customer support assistant."),
+    HumanMessage(content="How do I reset my password?")
+]
+
+response = chat.invoke(messages)
+print(response)
+```
+
+</TabItem>
+
+<TabItem value="anthropic" label="Anthropic">
+
+```python
+import os
+from langchain_openai import ChatOpenAI
+from langchain_core.messages import HumanMessage, SystemMessage
+
+os.environ['ANTHROPIC_API_KEY'] = "sk-ant-your-key-here"
+
+chat = ChatOpenAI(
+    model="claude-3-sonnet-20240229",
+    temperature=0.7,
+    extra_body={
+        "metadata": {
+            "tags": ["research", "analysis", "claude-model"]
+        }
+    }
+)
+
+messages = [
+    SystemMessage(content="You are a research analyst."),
+    HumanMessage(content="Analyze this market trend...")
+]
+
+response = chat.invoke(messages)
+print(response)
+```
+
+</TabItem>
+
+<TabItem value="litellm-proxy" label="LiteLLM Proxy">
+
+```python
+import os
+from langchain_openai import ChatOpenAI
+from langchain_core.messages import HumanMessage, SystemMessage
+
+# No API key needed when using proxy
+chat = ChatOpenAI(
+    openai_api_base="http://localhost:4000",  # Your proxy URL
+    model="gpt-4o",
+    temperature=0.7,
+    extra_body={
+        "metadata": {
+            "tags": ["proxy", "team-alpha", "feature-flagged"],
+            "generation_name": "customer-onboarding",
+            "trace_user_id": "user-12345"
+        }
+    }
+)
+
+messages = [
+    SystemMessage(content="You are an onboarding assistant."),
+    HumanMessage(content="Welcome our new customer!")
+]
+
+response = chat.invoke(messages)
+print(response)
+```
+
+</TabItem>
+</Tabs>
+
+### Advanced Tag Patterns
+
+#### Dynamic Tags Based on Context
+
+```python
+import os
+from langchain_openai import ChatOpenAI
+from langchain_core.messages import HumanMessage, SystemMessage
+
+def create_chat_with_tags(user_type: str, feature: str):
+    """Create a chat instance with dynamic tags based on context"""
+    
+    # Build tags dynamically
+    tags = ["langchain-integration"]
+    
+    if user_type == "premium":
+        tags.extend(["premium-user", "high-priority"])
+    elif user_type == "enterprise":
+        tags.extend(["enterprise", "custom-sla"])
+    else:
+        tags.append("standard-user")
+    
+    # Add feature-specific tags
+    if feature == "code-review":
+        tags.extend(["development", "code-analysis"])
+    elif feature == "content-gen":
+        tags.extend(["marketing", "content-creation"])
+    
+    return ChatOpenAI(
+        openai_api_base="http://localhost:4000",
+        model="gpt-4o",
+        temperature=0.7,
+        extra_body={
+            "metadata": {
+                "tags": tags,
+                "user_type": user_type,
+                "feature": feature,
+                "trace_user_id": f"user-{user_type}-{feature}"
+            }
+        }
+    )
+
+# Usage examples
+premium_chat = create_chat_with_tags("premium", "code-review")
+enterprise_chat = create_chat_with_tags("enterprise", "content-gen")
+
+messages = [HumanMessage(content="Help me with this task")]
+response = premium_chat.invoke(messages)
+```
+
+#### Tags for Cost Tracking and Analytics
+
+```python
+import os
+from langchain_openai import ChatOpenAI
+from langchain_core.messages import HumanMessage, SystemMessage
+
+# Tags for cost tracking
+cost_tracking_chat = ChatOpenAI(
+    openai_api_base="http://localhost:4000",
+    model="gpt-4o",
+    temperature=0.7,
+    extra_body={
+        "metadata": {
+            "tags": [
+                "cost-center-marketing",
+                "budget-q4-2024",
+                "project-launch-campaign",
+                "high-cost-model"  # Flag for expensive models
+            ],
+            "department": "marketing",
+            "project_id": "campaign-2024-q4",
+            "cost_threshold": "high"
+        }
+    }
+)
+
+messages = [
+    SystemMessage(content="You are a marketing copywriter."),
+    HumanMessage(content="Create compelling ad copy for our new product launch.")
+]
+
+response = cost_tracking_chat.invoke(messages)
+```
+
+#### Tags for A/B Testing
+
+```python
+import os
+from langchain_openai import ChatOpenAI
+from langchain_core.messages import HumanMessage, SystemMessage
+import random
+
+def create_ab_test_chat(test_variant: str = None):
+    """Create chat instance for A/B testing with appropriate tags"""
+    
+    if test_variant is None:
+        test_variant = random.choice(["variant-a", "variant-b"])
+    
+    return ChatOpenAI(
+        openai_api_base="http://localhost:4000",
+        model="gpt-4o",
+        temperature=0.7 if test_variant == "variant-a" else 0.9,  # Different temp for variants
+        extra_body={
+            "metadata": {
+                "tags": [
+                    "ab-test-experiment-1",
+                    f"variant-{test_variant}",
+                    "temperature-test",
+                    "user-experience"
+                ],
+                "experiment_id": "ab-test-001",
+                "variant": test_variant,
+                "test_group": "temperature-optimization"
+            }
+        }
+    )
+
+# Run A/B test
+variant_a_chat = create_ab_test_chat("variant-a")
+variant_b_chat = create_ab_test_chat("variant-b")
+
+test_message = [HumanMessage(content="Explain quantum computing in simple terms")]
+
+response_a = variant_a_chat.invoke(test_message)
+response_b = variant_b_chat.invoke(test_message)
+```
+
+### Tag Best Practices
+
+#### 1. **Consistent Naming Convention**
+```python
+# ✅ Good: Consistent, descriptive tags
+tags = ["production", "api-v2", "customer-support", "urgent"]
+
+# ❌ Avoid: Inconsistent or unclear tags
+tags = ["prod", "v2", "support", "urgent123"]
+```
+
+#### 2. **Hierarchical Tags**
+```python
+# ✅ Good: Hierarchical structure
+tags = ["env:production", "team:backend", "service:api", "priority:high"]
+
+# This allows for easy filtering and grouping
+```
+
+#### 3. **Include Context Information**
+```python
+extra_body={
+    "metadata": {
+        "tags": ["production", "user-onboarding"],
+        "user_id": "user-12345",
+        "session_id": "session-abc123",
+        "feature_flag": "new-onboarding-flow",
+        "environment": "production"
+    }
+}
+```
+
+#### 4. **Tag Categories**
+Consider organizing tags into categories:
+- **Environment**: `production`, `staging`, `development`
+- **Team/Service**: `backend`, `frontend`, `api`, `worker`
+- **Feature**: `authentication`, `payment`, `notification`
+- **Priority**: `critical`, `high`, `medium`, `low`
+- **User Type**: `premium`, `enterprise`, `free`
+
+### Using Tags with LiteLLM Proxy
+
+When using tags with LiteLLM Proxy, you can:
+
+1. **Filter requests** based on tags
+2. **Track costs** by tags in spend reports
+3. **Apply routing rules** based on tags
+4. **Monitor usage** with tag-based analytics
+
+#### Example Proxy Configuration with Tags
+
+```yaml
+# config.yaml
+model_list:
+  - model_name: gpt-4o
+    litellm_params:
+      model: gpt-4o
+      api_key: your-key
+
+# Tag-based routing rules
+tag_routing:
+  - tags: ["premium", "high-priority"]
+    models: ["gpt-4o", "claude-3-opus"]
+  - tags: ["standard"]
+    models: ["gpt-3.5-turbo", "claude-3-haiku"]
+```
+
+### Monitoring and Analytics
+
+Tags enable powerful analytics capabilities:
+
+```python
+# Example: Get spend reports by tags
+import requests
+
+response = requests.get(
+    "http://localhost:4000/global/spend/report",
+    headers={"Authorization": "Bearer sk-your-key"},
+    params={
+        "start_date": "2024-01-01",
+        "end_date": "2024-12-31",
+        "group_by": "tags"
+    }
+)
+
+spend_by_tags = response.json()
+```
+
+This documentation covers the essential patterns for using tags effectively with LangChain and LiteLLM, enabling better organization, tracking, and analytics of your LLM requests.

docs/my-website/docs/mcp.md

@@ -195,70 +195,169 @@ litellm_settings:
 
 ## Using your MCP
 
+### Use on LiteLLM UI 
+
+Follow this walkthrough to use your MCP on LiteLLM UI
+
+<iframe width="840" height="500" src="https://www.loom.com/embed/57e0763267254bc79dbe6658d0b8758c" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>
+
+### Use with Responses API
+
+Replace `http://localhost:4000` with your LiteLLM Proxy base URL.
+
+Demo Video Using Responses API with LiteLLM Proxy: [Demo video here](https://www.loom.com/share/34587e618c5c47c0b0d67b4e4d02718f?sid=2caf3d45-ead4-4490-bcc1-8d6dd6041c02)
+
+
 <Tabs>
-<TabItem value="openai" label="OpenAI API">
-
-#### Connect via OpenAI Responses API
-
-Use the OpenAI Responses API to connect to your LiteLLM MCP server:
+<TabItem value="curl" label="cURL">
 
 ```bash title="cURL Example" showLineNumbers
-curl --location 'https://api.openai.com/v1/responses' \
+curl --location 'http://localhost:4000/v1/responses' \
 --header 'Content-Type: application/json' \
---header "Authorization: Bearer $OPENAI_API_KEY" \
+--header "Authorization: Bearer sk-1234" \
 --data '{
-    "model": "gpt-4o",
+    "model": "gpt-5",
+    "input": [
+    {
+      "role": "user",
+      "content": "give me TLDR of what BerriAI/litellm repo is about",
+      "type": "message"
+    }
+  ],
     "tools": [
         {
             "type": "mcp",
             "server_label": "litellm",
             "server_url": "litellm_proxy",
-            "require_approval": "never",
-            "headers": {
-                "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY"
-            }
+            "require_approval": "never"
         }
     ],
-    "input": "Run available tools",
+    "stream": true,
     "tool_choice": "required"
 }'
 ```
 
 </TabItem>
+<TabItem value="python" label="Python SDK">
 
-<TabItem value="litellm" label="LiteLLM Proxy">
+```python title="Python SDK Example" showLineNumbers
+"""
+Use LiteLLM Proxy MCP Gateway to call MCP tools.
 
-#### Connect via LiteLLM Proxy Responses API
+When using LiteLLM Proxy, you can use the same MCP tools across all your LLM providers.
+"""
+import openai
 
-Use this when calling LiteLLM Proxy for LLM API requests to `/v1/responses` endpoint.
+client = openai.OpenAI(
+    api_key="sk-1234", # paste your litellm proxy api key here
+    base_url="http://localhost:4000" # paste your litellm proxy base url here
+)
+print("Making API request to Responses API with MCP tools")
 
-```bash title="cURL Example" showLineNumbers
-curl --location '<your-litellm-proxy-base-url>/v1/responses' \
---header 'Content-Type: application/json' \
---header "Authorization: Bearer $LITELLM_API_KEY" \
---data '{
-    "model": "gpt-4o",
-    "tools": [
+response = client.responses.create(
+    model="gpt-5",
+    input=[
+        {
+            "role": "user",
+            "content": "give me TLDR of what BerriAI/litellm repo is about",
+            "type": "message"
+        }
+    ],
+    tools=[
         {
             "type": "mcp",
             "server_label": "litellm",
             "server_url": "litellm_proxy",
-            "require_approval": "never",
-            "headers": {
-                "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY"
-            }
+            "require_approval": "never"
         }
     ],
-    "input": "Run available tools",
+    stream=True,
+    tool_choice="required"
+)
+
+for chunk in response:
+    print("response chunk: ", chunk)
+```
+
+</TabItem>
+</Tabs>
+
+#### Specifying MCP Tools
+
+You can specify which MCP tools are available by using the `allowed_tools` parameter. This allows you to restrict access to specific tools within an MCP server.
+
+To get the list of allowed tools when using LiteLLM MCP Gateway, you can naigate to the LiteLLM UI on MCP Servers > MCP Tools > Click the Tool > Copy Tool Name.
+
+<Tabs>
+<TabItem value="curl" label="cURL">
+
+```bash title="cURL Example with allowed_tools" showLineNumbers
+curl --location 'http://localhost:4000/v1/responses' \
+--header 'Content-Type: application/json' \
+--header "Authorization: Bearer sk-1234" \
+--data '{
+    "model": "gpt-5",
+    "input": [
+    {
+      "role": "user",
+      "content": "give me TLDR of what BerriAI/litellm repo is about",
+      "type": "message"
+    }
+  ],
+    "tools": [
+        {
+            "type": "mcp",
+            "server_label": "litellm",
+            "server_url": "litellm_proxy/mcp",
+            "require_approval": "never",
+            "allowed_tools": ["GitMCP-fetch_litellm_documentation"]
+        }
+    ],
+    "stream": true,
     "tool_choice": "required"
 }'
 ```
 
 </TabItem>
+<TabItem value="python" label="Python SDK">
 
-<TabItem value="cursor" label="Cursor IDE">
+```python title="Python SDK Example with allowed_tools" showLineNumbers
+import openai
 
-#### Connect via Cursor IDE
+client = openai.OpenAI(
+    api_key="sk-1234",
+    base_url="http://localhost:4000"
+)
+
+response = client.responses.create(
+    model="gpt-5",
+    input=[
+        {
+            "role": "user",
+            "content": "give me TLDR of what BerriAI/litellm repo is about",
+            "type": "message"
+        }
+    ],
+    tools=[
+        {
+            "type": "mcp",
+            "server_label": "litellm",
+            "server_url": "litellm_proxy/mcp",
+            "require_approval": "never",
+            "allowed_tools": ["GitMCP-fetch_litellm_documentation"]
+        }
+    ],
+    stream=True,
+    tool_choice="required"
+)
+
+print(response)
+```
+
+</TabItem>
+</Tabs>
+
+### Use with Cursor IDE
 
 Use tools directly from Cursor IDE with LiteLLM MCP:
 
@@ -281,9 +380,6 @@ Use tools directly from Cursor IDE with LiteLLM MCP:
 }
 ```
 
-</TabItem>
-</Tabs>
-
 #### How it works when server_url="litellm_proxy"
 
 When server_url="litellm_proxy", LiteLLM bridges non-MCP providers to your MCP tools.

docs/my-website/docs/providers/bedrock_batches.md

@@ -0,0 +1,180 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Bedrock Batches
+
+Use Amazon Bedrock Batch Inference API through LiteLLM.
+
+| Property | Details |
+|----------|---------|
+| Description | Amazon Bedrock Batch Inference allows you to run inference on large datasets asynchronously |
+| Provider Doc | [AWS Bedrock Batch Inference ↗](https://docs.aws.amazon.com/bedrock/latest/userguide/batch-inference.html) |
+
+## Overview
+
+Use this to:
+
+- Run batch inference on large datasets with Bedrock models
+- Control batch model access by key/user/team (same as chat completion models)
+- Manage S3 storage for batch input/output files
+
+## (Proxy Admin) Usage
+
+Here's how to give developers access to your Bedrock Batch models.
+
+### 1. Setup config.yaml
+
+- Specify `mode: batch` for each model: Allows developers to know this is a batch model
+- Configure S3 bucket and AWS credentials for batch operations
+
+```yaml showLineNumbers title="litellm_config.yaml"
+model_list:
+  - model_name: "bedrock-batch-claude"
+    litellm_params:
+      model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0
+      #########################################################
+      ########## batch specific params ########################
+      s3_bucket_name: litellm-proxy
+      s3_region_name: us-west-2
+      s3_access_key_id: os.environ/AWS_ACCESS_KEY_ID
+      s3_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
+      aws_batch_role_arn: arn:aws:iam::888602223428:role/service-role/AmazonBedrockExecutionRoleForAgents_BB9HNW6V4CV
+    model_info: 
+      mode: batch # 👈 SPECIFY MODE AS BATCH, to tell user this is a batch model
+```
+
+**Required Parameters:**
+
+| Parameter | Description |
+|-----------|-------------|
+| `s3_bucket_name` | S3 bucket for batch input/output files |
+| `s3_region_name` | AWS region for S3 bucket |
+| `s3_access_key_id` | AWS access key for S3 bucket |
+| `s3_secret_access_key` | AWS secret key for S3 bucket |
+| `aws_batch_role_arn` | IAM role ARN for Bedrock batch operations. Bedrock Batch APIs require an IAM role ARN to be set. |
+| `mode: batch` | Indicates to LiteLLM this is a batch model |
+
+### 2. Create Virtual Key
+
+```bash showLineNumbers title="create_virtual_key.sh"
+curl -L -X POST 'https://{PROXY_BASE_URL}/key/generate' \
+-H 'Authorization: Bearer ${PROXY_API_KEY}' \
+-H 'Content-Type: application/json' \
+-d '{"models": ["bedrock-batch-claude"]}'
+```
+
+You can now use the virtual key to access the batch models (See Developer flow).
+
+## (Developer) Usage
+
+Here's how to create a LiteLLM managed file and execute Bedrock Batch CRUD operations with the file.
+
+### 1. Create request.jsonl
+
+- Check models available via `/model_group/info`
+- See all models with `mode: batch`
+- Set `model` in .jsonl to the model from `/model_group/info`
+
+```json showLineNumbers title="bedrock_batch_completions.jsonl"
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock-batch-claude", "messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "Hello world!"}], "max_tokens": 1000}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock-batch-claude", "messages": [{"role": "system", "content": "You are an unhelpful assistant."}, {"role": "user", "content": "Hello world!"}], "max_tokens": 1000}}
+```
+
+Expectation:
+
+- LiteLLM translates this to the bedrock deployment specific value (e.g. `bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0`)
+
+### 2. Upload File
+
+Specify `target_model_names: "<model-name>"` to enable LiteLLM managed files and request validation.
+
+model-name should be the same as the model-name in the request.jsonl
+
+<Tabs>
+<TabItem value="python" label="Python">
+
+```python showLineNumbers title="bedrock_batch.py"
+from openai import OpenAI
+
+client = OpenAI(
+    base_url="http://0.0.0.0:4000",
+    api_key="sk-1234",
+)
+
+# Upload file
+batch_input_file = client.files.create(
+    file=open("./bedrock_batch_completions.jsonl", "rb"), # {"model": "bedrock-batch-claude"} <-> {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0"}
+    purpose="batch",
+    extra_body={"target_model_names": "bedrock-batch-claude"}
+)
+print(batch_input_file)
+```
+
+</TabItem>
+<TabItem value="curl" label="Curl">
+
+```bash showLineNumbers title="Upload File"
+curl http://localhost:4000/v1/files \
+    -H "Authorization: Bearer sk-1234" \
+    -F purpose="batch" \
+    -F file="@bedrock_batch_completions.jsonl" \
+    -F extra_body='{"target_model_names": "bedrock-batch-claude"}'
+```
+
+</TabItem>
+</Tabs>
+
+**Where is the file written?**:
+
+The file is written to S3 bucket specified in your config and prepared for Bedrock batch inference.
+
+### 3. Create the batch
+
+<Tabs>
+<TabItem value="python" label="Python">
+
+```python showLineNumbers title="bedrock_batch.py"
+...
+# Create batch
+batch = client.batches.create( 
+    input_file_id=batch_input_file.id,
+    endpoint="/v1/chat/completions",
+    completion_window="24h",
+    metadata={"description": "Test batch job"},
+)
+print(batch)
+```
+
+</TabItem>
+<TabItem value="curl" label="Curl">
+
+```bash showLineNumbers title="Create Batch Request"
+curl http://localhost:4000/v1/batches \
+    -H "Authorization: Bearer sk-1234" \
+    -H "Content-Type: application/json" \
+    -d '{
+        "input_file_id": "file-abc123",
+        "endpoint": "/v1/chat/completions",
+        "completion_window": "24h",
+        "metadata": {"description": "Test batch job"}
+    }'
+```
+
+</TabItem>
+</Tabs>
+
+## FAQ
+
+### Where are my files written?
+
+When a `target_model_names` is specified, the file is written to the S3 bucket configured in your Bedrock batch model configuration.
+
+### What models are supported?
+
+LiteLLM only supports Bedrock Anthropic Models for Batch API. If you want other bedrock models file an issue [here](https://github.com/BerriAI/litellm/issues/new/choose).
+
+## Further Reading
+
+- [AWS Bedrock Batch Inference Documentation](https://docs.aws.amazon.com/bedrock/latest/userguide/batch-inference.html)
+- [LiteLLM Managed Batches](../proxy/managed_batches)
+- [LiteLLM Authentication to Bedrock](https://docs.litellm.ai/docs/providers/bedrock#boto3---authentication)

docs/my-website/docs/providers/dashscope.md

@@ -1,4 +1,4 @@
-# Dashscope
+# Dashscope (Qwen API)
 https://dashscope.console.aliyun.com/
 
 **We support ALL Qwen models, just set `dashscope/` as a prefix when sending completion requests**

docs/my-website/docs/proxy/admin_ui_sso.md

@@ -4,6 +4,10 @@ import TabItem from '@theme/TabItem';
 
 # ✨ SSO for Admin UI
 
+:::info
+From v1.76.0, SSO is now Free for up to 5 users.
+:::
+
 :::info
 
 ✨ SSO is on LiteLLM Enterprise

docs/my-website/docs/proxy/config_settings.md

@@ -21,7 +21,7 @@ litellm_settings:
   failure_callback: ["sentry"]  # list of failure callbacks
   callbacks: ["otel"]  # list of callbacks - runs on success and failure
   service_callbacks: ["datadog", "prometheus"]  # logs redis, postgres failures on datadog, prometheus
-  turn_off_message_logging: boolean  # prevent the messages and responses from being logged to on your callbacks, but request metadata will still be logged.
+  turn_off_message_logging: boolean  # prevent the messages and responses from being logged to on your callbacks, but request metadata will still be logged. Useful for privacy/compliance when handling sensitive data.
   redact_user_api_key_info: boolean  # Redact information about the user api key (hashed token, user_id, team id, etc.), from logs. Currently supported for Langfuse, OpenTelemetry, Logfire, ArizeAI logging.
   langfuse_default_tags: ["cache_hit", "cache_key", "proxy_base_url", "user_api_key_alias", "user_api_key_user_id", "user_api_key_user_email", "user_api_key_team_alias", "semantic-similarity", "proxy_base_url"] # default tags for Langfuse Logging
   
@@ -131,7 +131,7 @@ general_settings:
 | failure_callback | array of strings | List of failure callbacks [Doc Proxy logging callbacks](logging), [Doc Metrics](prometheus) |
 | callbacks | array of strings | List of callbacks - runs on success and failure [Doc Proxy logging callbacks](logging), [Doc Metrics](prometheus) |
 | service_callbacks | array of strings | System health monitoring - Logs redis, postgres failures on specified services (e.g. datadog, prometheus) [Doc Metrics](prometheus) |
-| turn_off_message_logging | boolean | If true, prevents messages and responses from being logged to callbacks, but request metadata will still be logged [Proxy Logging](logging) |
+| turn_off_message_logging | boolean | If true, prevents messages and responses from being logged to callbacks, but request metadata will still be logged. Useful for privacy/compliance when handling sensitive data [Proxy Logging](logging) |
 | modify_params | boolean | If true, allows modifying the parameters of the request before it is sent to the LLM provider |
 | enable_preview_features | boolean | If true, enables preview features - e.g. Azure O1 Models with streaming support.|
 | redact_user_api_key_info | boolean | If true, redacts information about the user api key from logs [Proxy Logging](logging#redacting-userapikeyinfo) |
@@ -473,6 +473,7 @@ router_settings:
 | EMAIL_SIGNATURE | Custom HTML footer/signature for all emails. Can include HTML tags for formatting and links.
 | EMAIL_SUBJECT_INVITATION | Custom subject template for invitation emails. 
 | EMAIL_SUBJECT_KEY_CREATED | Custom subject template for key creation emails. 
+| EXPERIMENTAL_MULTI_INSTANCE_RATE_LIMITING | Flag to enable new multi-instance rate limiting. **Default is False**
 | FIREWORKS_AI_4_B | Size parameter for Fireworks AI 4B model. Default is 4
 | FIREWORKS_AI_16_B | Size parameter for Fireworks AI 16B model. Default is 16
 | FIREWORKS_AI_56_B_MOE | Size parameter for Fireworks AI 56B MOE model. Default is 56

docs/my-website/docs/proxy/debugging.md

@@ -11,13 +11,13 @@ The proxy also supports json logs. [See here](#json-logs)
 
 **via cli**
 
-```bash
+```bash showLineNumbers
 $ litellm --debug
 ```
 
 **via env**
 
-```python
+```python showLineNumbers
 os.environ["LITELLM_LOG"] = "INFO"
 ```
 
@@ -25,25 +25,25 @@ os.environ["LITELLM_LOG"] = "INFO"
 
 **via cli**
 
-```bash
+```bash showLineNumbers
 $ litellm --detailed_debug
 ```
 
 **via env**
 
-```python
+```python showLineNumbers
 os.environ["LITELLM_LOG"] = "DEBUG"
 ```
 
 ### Debug Logs 
 
 Run the proxy with `--detailed_debug` to view detailed debug logs
-```shell
+```shell showLineNumbers
 litellm --config /path/to/config.yaml --detailed_debug
 ```
 
 When making requests you should see the POST request sent by LiteLLM to the LLM on the Terminal output
-```shell
+```shell showLineNumbers
 POST Request Sent from LiteLLM:
 curl -X POST \
 https://api.openai.com/v1/chat/completions \
@@ -51,25 +51,63 @@ https://api.openai.com/v1/chat/completions \
 -d '{"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "this is a test request, write a short poem"}]}'
 ```
 
+## Debug single request
+
+Pass in `litellm_request_debug=True` in the request body
+
+```bash showLineNumbers
+curl -L -X POST 'http://0.0.0.0:4000/chat/completions' \
+-H 'Content-Type: application/json' \
+-H 'Authorization: Bearer sk-1234' \
+-d '{ 
+    "model":"fake-openai-endpoint",
+    "messages": [{"role": "user","content": "How many r in the word strawberry?"}],
+    "litellm_request_debug": true
+}'
+```
+
+This will emit the raw request sent by LiteLLM to the API Provider and raw response received from the API Provider for **just** this request in the logs. 
+
+
+```bash showLineNumbers
+INFO:     Uvicorn running on http://0.0.0.0:4000 (Press CTRL+C to quit)
+20:14:06 - LiteLLM:WARNING: litellm_logging.py:938 - 
+
+POST Request Sent from LiteLLM:
+curl -X POST \
+https://exampleopenaiendpoint-production.up.railway.app/chat/completions \
+-H 'Authorization: Be****ey' -H 'Content-Type: application/json' \
+-d '{'model': 'fake', 'messages': [{'role': 'user', 'content': 'How many r in the word strawberry?'}], 'stream': False}'
+
+
+20:14:06 - LiteLLM:WARNING: litellm_logging.py:1015 - RAW RESPONSE:
+{"id":"chatcmpl-817fc08f0d6c451485d571dab39b26a1","object":"chat.completion","created":1677652288,"model":"gpt-3.5-turbo-0301","system_fingerprint":"fp_44709d6fcb","choices":[{"index":0,"message":{"role":"assistant","content":"\n\nHello there, how may I assist you today?"},"logprobs":null,"finish_reason":"stop"}],"usage":{"prompt_tokens":9,"completion_tokens":12,"total_tokens":21}}
+
+
+INFO:     127.0.0.1:56155 - "POST /chat/completions HTTP/1.1" 200 OK
+
+```
+
+
 ## JSON LOGS
 
 Set `JSON_LOGS="True"` in your env:
 
-```bash
+```bash showLineNumbers
 export JSON_LOGS="True"
 ```
 **OR**
 
 Set `json_logs: true` in your yaml: 
 
-```yaml
+```yaml showLineNumbers
 litellm_settings:
     json_logs: true
 ```
 
 Start proxy 
 
-```bash
+```bash showLineNumbers
 $ litellm
 ```
 
@@ -80,7 +118,7 @@ The proxy will now all logs in json format.
 Turn off fastapi's default 'INFO' logs 
 
 1. Turn on 'json logs' 
-```yaml
+```yaml showLineNumbers
 litellm_settings:
     json_logs: true
 ```
@@ -89,20 +127,20 @@ litellm_settings:
 
 Only get logs if an error occurs. 
 
-```bash
+```bash showLineNumbers
 LITELLM_LOG="ERROR"
 ```
 
 3. Start proxy 
 
 
-```bash
+```bash showLineNumbers
 $ litellm
 ```
 
 Expected Output: 
 
-```bash
+```bash showLineNumbers
 # no info statements
 ```
 
@@ -119,14 +157,14 @@ This can be caused due to all your models hitting rate limit errors, causing the
 How to control this? 
 - Adjust the cooldown time
 
-```yaml
+```yaml showLineNumbers
 router_settings:
     cooldown_time: 0 # 👈 KEY CHANGE
 ```
 
 - Disable Cooldowns [NOT RECOMMENDED]
 
-```yaml
+```yaml showLineNumbers
 router_settings:
     disable_cooldowns: True
 ```

docs/my-website/docs/proxy/forward_client_headers.md

@@ -0,0 +1,212 @@
+# Forward Client Headers to LLM API
+
+Control which model groups can forward client headers to the underlying LLM provider APIs.
+
+## Overview
+
+By default, LiteLLM does not forward client headers to LLM provider APIs for security reasons. However, you can selectively enable header forwarding for specific model groups using the `forward_client_headers_to_llm_api` setting.
+
+## Configuration
+
+## Enable Globally
+
+```yaml
+general_settings:
+  forward_client_headers_to_llm_api: true
+```
+
+## Enable for a Model Group
+
+Add the `forward_client_headers_to_llm_api` setting under `model_group_settings` in your configuration:
+
+```yaml
+model_list:
+  - model_name: gpt-4o-mini
+    litellm_params:
+      model: openai/gpt-4o-mini
+      api_key: "your-api-key"
+  - model_name: "wildcard-models/*"
+    litellm_params:
+      model: "openai/*"
+      api_key: "your-api-key"
+
+litellm_settings:
+  model_group_settings:
+    forward_client_headers_to_llm_api:
+      - gpt-4o-mini
+      - wildcard-models/*
+```
+
+## Supported Model Patterns
+
+The configuration supports various model matching patterns:
+
+### 1. Exact Model Names
+```yaml
+forward_client_headers_to_llm_api:
+  - gpt-4o-mini
+  - claude-3-sonnet
+```
+
+### 2. Wildcard Patterns
+```yaml
+forward_client_headers_to_llm_api:
+  - "openai/*"          # All OpenAI models
+  - "anthropic/*"       # All Anthropic models
+  - "wildcard-group/*"  # All models in wildcard-group
+```
+
+### 3. Team Model Aliases
+If your team has model aliases configured, the forwarding will work with both the original model name and the alias.
+
+## Forwarded Headers
+
+When enabled for a model group, LiteLLM forwards the following types of headers:
+
+### Custom Headers (x- prefix)
+- Any header starting with `x-` (except `x-stainless-*` which can cause OpenAI SDK issues)
+- Examples: `x-custom-header`, `x-request-id`, `x-trace-id`
+
+### Provider-Specific Headers
+- **Anthropic**: `anthropic-beta` headers
+- **OpenAI**: `openai-organization` (when enabled via `forward_openai_org_id: true`)
+
+### User Information Headers (Optional)
+When `add_user_information_to_llm_headers` is enabled, LiteLLM adds:
+- `x-litellm-user-id`
+- `x-litellm-org-id`
+- Other user metadata as `x-litellm-*` headers
+
+## Security Considerations
+
+⚠️ **Important Security Notes:**
+
+1. **Sensitive Data**: Only enable header forwarding for trusted model groups, as headers may contain sensitive information
+2. **API Keys**: Never include API keys or secrets in forwarded headers
+3. **PII**: Be cautious about forwarding headers that might contain personally identifiable information
+4. **Provider Limits**: Some providers have restrictions on custom headers
+
+## Example Use Cases
+
+### 1. Request Tracing
+Forward tracing headers to track requests across your system:
+
+```bash
+curl -X POST "https://your-proxy.com/v1/chat/completions" \
+  -H "Authorization: Bearer your-key" \
+  -H "x-trace-id: abc123" \
+  -H "x-request-source: mobile-app" \
+  -d '{
+    "model": "gpt-4o-mini",
+    "messages": [{"role": "user", "content": "Hello"}]
+  }'
+```
+
+### 2. Custom Metadata
+Pass custom metadata to your LLM provider:
+
+```bash
+curl -X POST "https://your-proxy.com/v1/chat/completions" \
+  -H "Authorization: Bearer your-key" \
+  -H "x-customer-id: customer-123" \
+  -H "x-environment: production" \
+  -d '{
+    "model": "gpt-4o-mini", 
+    "messages": [{"role": "user", "content": "Hello"}]
+  }'
+```
+
+### 3. Anthropic Beta Features
+Enable beta features for Anthropic models:
+
+```bash
+curl -X POST "https://your-proxy.com/v1/chat/completions" \
+  -H "Authorization: Bearer your-key" \
+  -H "anthropic-beta: tools-2024-04-04" \
+  -d '{
+    "model": "claude-3-sonnet",
+    "messages": [{"role": "user", "content": "Hello"}]
+  }'
+```
+
+## Complete Configuration Example
+
+```yaml
+model_list:
+  # Fixed model with header forwarding
+  - model_name: byok-fixed-gpt-4o-mini
+    litellm_params:
+      model: openai/gpt-4o-mini
+      api_base: "https://your-openai-endpoint.com"
+      api_key: "your-api-key"
+      
+  # Wildcard model group with header forwarding
+  - model_name: "byok-wildcard/*"
+    litellm_params:
+      model: "openai/*"
+      api_base: "https://your-openai-endpoint.com"
+      api_key: "your-api-key"
+      
+  # Standard model without header forwarding
+  - model_name: standard-gpt-4
+    litellm_params:
+      model: openai/gpt-4
+      api_key: "your-api-key"
+
+litellm_settings:
+  # Enable user info headers globally (optional)
+  add_user_information_to_llm_headers: true
+  
+  model_group_settings:
+    forward_client_headers_to_llm_api:
+      - byok-fixed-gpt-4o-mini
+      - byok-wildcard/*
+      # Note: standard-gpt-4 is NOT included, so no headers forwarded
+
+general_settings:
+  # Enable OpenAI organization header forwarding (optional)
+  forward_openai_org_id: true
+```
+
+## Testing Header Forwarding
+
+To test if headers are being forwarded:
+
+1. **Enable Debug Logging**: Set `set_verbose: true` in your config
+2. **Check Provider Logs**: Monitor your LLM provider's request logs
+3. **Use Webhook Sites**: For testing, you can use webhook.site URLs as api_base to see forwarded headers
+
+## Troubleshooting
+
+### Headers Not Being Forwarded
+
+1. **Check Model Name**: Ensure the model name in your request matches the configuration
+2. **Verify Pattern Matching**: Wildcard patterns must match exactly
+3. **Review Logs**: Enable verbose logging to see header processing
+
+### Provider Errors
+
+1. **Invalid Headers**: Some providers reject unknown headers
+2. **Header Limits**: Providers may have limits on header count/size
+3. **Authentication**: Ensure forwarded headers don't conflict with authentication
+
+## Related Features
+
+- [Request Headers](./request_headers.md) - Complete list of supported request headers
+- [Response Headers](./response_headers.md) - Headers returned by LiteLLM
+- [Team Model Aliases](./team_model_add.md) - Configure model aliases for teams
+- [Model Access Control](./model_access.md) - Control which users can access which models
+
+## API Reference
+
+The header forwarding is controlled by the `ModelGroupSettings` configuration:
+
+```python
+class ModelGroupSettings(BaseModel):
+    forward_client_headers_to_llm_api: Optional[List[str]] = None
+```
+
+Where each string in the list can be:
+- An exact model name (e.g., `"gpt-4o-mini"`)
+- A wildcard pattern (e.g., `"openai/*"`)
+- A model group name (e.g., `"my-model-group/*"`)

docs/my-website/docs/proxy/guardrails/noma_security.md

@@ -135,6 +135,7 @@ guardrails:
       # application_id: "my-app"
       # monitor_mode: false
       # block_failures: true
+      # anonymize_input: false
 ```
 
 ### Required Parameters
@@ -147,6 +148,7 @@ guardrails:
 - **`application_id`**: Your application identifier (defaults to `"litellm"`)
 - **`monitor_mode`**: If `true`, logs violations without blocking (defaults to `false`)
 - **`block_failures`**: If `true`, blocks requests when guardrail API failures occur (defaults to `true`)
+- **`anonymize_input`**: If `true`, replaces sensitive content with anonymized version (defaults to `false`)
 
 ## Environment Variables
 
@@ -158,6 +160,7 @@ export NOMA_API_BASE="https://api.noma.security/"   # Optional
 export NOMA_APPLICATION_ID="my-app"                 # Optional
 export NOMA_MONITOR_MODE="false"                    # Optional
 export NOMA_BLOCK_FAILURES="true"                   # Optional
+export NOMA_ANONYMIZE_INPUT="false"                 # Optional
 ```
 
 ## Advanced Configuration
@@ -190,6 +193,20 @@ guardrails:
       block_failures: false  # Allow requests to proceed if guardrail API fails
 ```
 
+### Content Anonymization
+
+Enable anonymization to replace sensitive content instead of blocking:
+
+```yaml
+guardrails:
+  - guardrail_name: "noma-anonymize"
+    litellm_params:
+      guardrail: noma
+      mode: "pre_call"
+      api_key: os.environ/NOMA_API_KEY
+      anonymize_input: true  # Replace sensitive data with anonymized version
+```
+
 ### Multiple Guardrails
 
 Apply different configurations for input and output:

docs/my-website/docs/proxy/logging.md

@@ -60,7 +60,7 @@ components in your system, including in logging tools.
 
 ### Redact Messages, Response Content
 
-Set `litellm.turn_off_message_logging=True` This will prevent the messages and responses from being logged to your logging provider, but request metadata - e.g. spend, will still be tracked.
+Set `litellm.turn_off_message_logging=True` This will prevent the messages and responses from being logged to your logging provider, but request metadata - e.g. spend, will still be tracked. Useful for privacy/compliance when handling sensitive data.
 
 <Tabs>
 

docs/my-website/docs/proxy/request_headers.md

@@ -2,6 +2,10 @@
 
 Special headers that are supported by LiteLLM.
 
+## Header Forwarding
+
+By default, LiteLLM does not forward client headers to LLM provider APIs. However, you can selectively enable header forwarding for specific model groups. [Learn more about configuring header forwarding](./forward_client_headers.md).
+
 ## LiteLLM Headers
 
 `x-litellm-timeout` Optional[float]: The timeout for the request in seconds.
@@ -21,11 +25,15 @@ Special headers that are supported by LiteLLM.
 `anthropic-version` Optional[str]: The version of the Anthropic API to use.  
 `anthropic-beta` Optional[str]: The beta version of the Anthropic API to use.
     - For `/v1/messages` endpoint, this will always be forward the header to the underlying model.
-    - For `/chat/completions` endpoint, this will only be forwarded if `forward_client_headers_to_llm_api` is true.
+    - For `/chat/completions` endpoint, this will only be forwarded if the model is configured in `forward_client_headers_to_llm_api`. [Learn more](./forward_client_headers.md)
 
 ## OpenAI Headers
 
 `openai-organization` Optional[str]: The organization to use for the OpenAI API. (currently needs to be enabled via `general_settings::forward_openai_org_id: true`)
 
+## Custom Headers
+
+Custom headers starting with `x-` can be forwarded to LLM provider APIs when the model is configured in `forward_client_headers_to_llm_api`. [Learn more about header forwarding configuration](./forward_client_headers.md).
+
 
 

docs/my-website/docs/proxy/user_keys.md

@@ -357,6 +357,106 @@ assert user.age == 25
 </TabItem>
 </Tabs>
 
+## Using Tags for Categorization and Tracking
+
+Tags allow you to categorize, filter, and track your LLM requests. Add tags to your metadata for better organization and analytics.
+
+<Tabs>
+<TabItem value="openai-python" label="OpenAI Python">
+
+```python
+import openai
+client = openai.OpenAI(
+    api_key="anything",
+    base_url="http://0.0.0.0:4000"
+)
+
+response = client.chat.completions.create(
+    model="gpt-3.5-turbo",
+    messages=[{"role": "user", "content": "Hello!"}],
+    extra_body={
+        "metadata": {
+            "tags": ["production", "customer-support", "urgent"],
+            "generation_name": "support-bot",
+            "trace_user_id": "user-123"
+        }
+    }
+)
+```
+
+</TabItem>
+
+<TabItem value="langchain-python" label="LangChain Python">
+
+```python
+from langchain_openai import ChatOpenAI
+from langchain_core.messages import HumanMessage
+
+chat = ChatOpenAI(
+    openai_api_base="http://0.0.0.0:4000",
+    model="gpt-4o",
+    extra_body={
+        "metadata": {
+            "tags": ["langchain-integration", "content-gen"],
+            "trace_user_id": "user-456"
+        }
+    }
+)
+
+response = chat.invoke([HumanMessage(content="Generate a blog post")])
+```
+
+</TabItem>
+
+<TabItem value="curl" label="Curl">
+
+```bash
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+    --header 'Content-Type: application/json' \
+    --data '{
+    "model": "gpt-3.5-turbo",
+    "messages": [{"role": "user", "content": "Hello!"}],
+    "metadata": {
+        "tags": ["api-test", "development"],
+        "trace_user_id": "test-user"
+    }
+}'
+```
+
+</TabItem>
+
+<TabItem value="openai-js" label="OpenAI JS">
+
+```js
+const { OpenAI } = require('openai');
+
+const openai = new OpenAI({
+  apiKey: "sk-1234",
+  baseURL: "http://0.0.0.0:4000"
+});
+
+async function main() {
+  const response = await openai.chat.completions.create({
+    messages: [{ role: 'user', content: 'Hello!' }],
+    model: 'gpt-3.5-turbo',
+    metadata: {
+      tags: ["javascript-client", "api-test"],
+      trace_user_id: "js-user-789"
+    }
+  });
+}
+```
+
+</TabItem>
+</Tabs>
+
+### Tag Benefits
+
+- **Cost Tracking**: Monitor spending by project/team/feature
+- **Analytics**: Filter requests by tags in logs and dashboards  
+- **Routing**: Use tags for conditional model routing
+- **Debugging**: Easier troubleshooting with categorized requests
+
 ### Response Format
 
 ```json

docs/my-website/sidebars.js

@@ -141,6 +141,7 @@ const sidebars = {
             "proxy/clientside_auth",
             "proxy/request_headers",
             "proxy/response_headers",
+            "proxy/forward_client_headers",
             "proxy/model_discovery",
           ],
         },
@@ -261,6 +262,7 @@ const sidebars = {
             "completion/input",
             "completion/output",
             "completion/usage",
+            "completion/http_handler_config",
           ],
         },
         "response_api",
@@ -409,6 +411,7 @@ const sidebars = {
           items: [
             "providers/bedrock",
             "providers/bedrock_agents",
+            "providers/bedrock_batches",
             "providers/bedrock_vector_store",
           ]
         },

litellm-js/spend-logs/package-lock.json

@@ -6,7 +6,7 @@
     "": {
       "dependencies": {
         "@hono/node-server": "^1.10.1",
-        "hono": "^4.6.5"
+        "hono": "^4.9.7"
       },
       "devDependencies": {
         "@types/node": "^20.11.17",
@@ -463,9 +463,10 @@
       }
     },
     "node_modules/hono": {
-      "version": "4.6.5",
-      "resolved": "https://registry.npmjs.org/hono/-/hono-4.6.5.tgz",
-      "integrity": "sha512-qsmN3V5fgtwdKARGLgwwHvcdLKursMd+YOt69eGpl1dUCJb8mCd7hZfyZnBYjxCegBG7qkJRQRUy2oO25yHcyQ==",
+      "version": "4.9.7",
+      "resolved": "https://registry.npmjs.org/hono/-/hono-4.9.7.tgz",
+      "integrity": "sha512-t4Te6ERzIaC48W3x4hJmBwgNlLhmiEdEE5ViYb02ffw4ignHNHa5IBtPjmbKstmtKa8X6C35iWwK4HaqvrzG9w==",
+      "license": "MIT",
       "engines": {
         "node": ">=16.9.0"
       }

litellm-js/spend-logs/package.json

@@ -4,7 +4,7 @@
   },
   "dependencies": {
     "@hono/node-server": "^1.10.1",
-    "hono": "^4.6.5"
+    "hono": "^4.9.7"
   },
   "devDependencies": {
     "@types/node": "^20.11.17",

litellm/batches/main.py

@@ -19,6 +19,7 @@ from typing import Any, Coroutine, Dict, Literal, Optional, Union, cast
 import httpx
 
 import litellm
+from litellm._logging import verbose_logger
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.llms.azure.batches.handler import AzureBatchesAPI
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
@@ -38,6 +39,7 @@ from litellm.utils import (
     ProviderConfigManager,
     client,
     get_litellm_params,
+    get_llm_provider,
     supports_httpx_timeout,
 )
 
@@ -49,6 +51,45 @@ base_llm_http_handler = BaseLLMHTTPHandler()
 #################################################
 
 
+def _resolve_timeout(
+    optional_params: GenericLiteLLMParams,
+    kwargs: Dict[str, Any],
+    custom_llm_provider: str,
+    default_timeout: float = 600.0,
+) -> float:
+    """
+    Resolve timeout value from various sources and handle httpx.Timeout objects.
+    
+    Args:
+        optional_params: GenericLiteLLMParams object containing timeout
+        kwargs: Additional kwargs that may contain request_timeout
+        custom_llm_provider: Provider name for httpx timeout support check
+        default_timeout: Default timeout value to use
+        
+    Returns:
+        Resolved timeout as float
+    """
+    timeout = optional_params.timeout or kwargs.get("request_timeout", default_timeout) or default_timeout
+    
+    # Handle httpx.Timeout objects
+    if isinstance(timeout, httpx.Timeout):
+        if supports_httpx_timeout(custom_llm_provider) is False:
+            # Extract read timeout for providers that don't support httpx.Timeout
+            read_timeout = timeout.read or default_timeout
+            return float(read_timeout)
+        else:
+            # For providers that support httpx.Timeout, we still need to return a float
+            # This case might need to be handled differently based on the actual use case
+            return float(timeout.read or default_timeout)
+    
+    # Handle None case
+    if timeout is None:
+        return float(default_timeout)
+    
+    # Handle numeric values (int, float, string representations)
+    return float(timeout)
+
+
 @client
 async def acreate_batch(
     completion_window: Literal["24h"],
@@ -118,13 +159,23 @@ def create_batch(
         litellm_call_id = kwargs.get("litellm_call_id", None)
         proxy_server_request = kwargs.get("proxy_server_request", None)
         model_info = kwargs.get("model_info", None)
+        model: Optional[str] = kwargs.get("model", None)
+        try:
+            if model is not None:
+                model, _, _, _ = get_llm_provider(
+                                model=model, 
+                                custom_llm_provider=None,
+                        )
+        except Exception as e:
+            verbose_logger.exception(f"litellm.batches.main.py::create_batch() - Error inferring custom_llm_provider - {str(e)}")
+            
         _is_async = kwargs.pop("acreate_batch", False) is True
         litellm_params = dict(GenericLiteLLMParams(**kwargs))
         litellm_logging_obj: LiteLLMLoggingObj = cast(LiteLLMLoggingObj, kwargs.get("litellm_logging_obj", None))
         ### TIMEOUT LOGIC ###
-        timeout = optional_params.timeout or kwargs.get("request_timeout", 600) or 600
+        timeout = _resolve_timeout(optional_params, kwargs, custom_llm_provider)
         litellm_logging_obj.update_environment_variables(
-            model=None,
+            model=model,
             user=None,
             optional_params=optional_params.model_dump(),
             litellm_params={
@@ -138,18 +189,6 @@ def create_batch(
             },
             custom_llm_provider=custom_llm_provider,
         )
-
-        if (
-            timeout is not None
-            and isinstance(timeout, httpx.Timeout)
-            and supports_httpx_timeout(custom_llm_provider) is False
-        ):
-            read_timeout = timeout.read or 600
-            timeout = read_timeout  # default 10 min timeout
-        elif timeout is not None and not isinstance(timeout, httpx.Timeout):
-            timeout = float(timeout)  # type: ignore
-        elif timeout is None:
-            timeout = 600.0
         
 
         _create_batch_request = CreateBatchRequest(
@@ -160,10 +199,13 @@ def create_batch(
             extra_headers=extra_headers,
             extra_body=extra_body,
         )
-        provider_config = ProviderConfigManager.get_provider_batches_config(
-            model="",
-            provider=LlmProviders(custom_llm_provider),
-        )
+        if model is not None:
+            provider_config = ProviderConfigManager.get_provider_batches_config(
+                model=model,
+                provider=LlmProviders(custom_llm_provider),
+            )
+        else:
+            provider_config = None
         if provider_config is not None:
             response = base_llm_http_handler.create_batch(
                 provider_config=provider_config,
@@ -179,6 +221,7 @@ def create_batch(
                 and isinstance(client, (HTTPHandler, AsyncHTTPHandler))
                 else None,
                 timeout=timeout,
+                model=model,
             )
             return response
         api_base: Optional[str] = None

litellm/constants.py

@@ -15,7 +15,7 @@ DEFAULT_SQS_FLUSH_INTERVAL_SECONDS = int(
     os.getenv("DEFAULT_SQS_FLUSH_INTERVAL_SECONDS", 10)
 )
 DEFAULT_NUM_WORKERS_LITELLM_PROXY = int(
-    os.getenv("DEFAULT_NUM_WORKERS_LITELLM_PROXY", os.cpu_count() or 4)
+    os.getenv("DEFAULT_NUM_WORKERS_LITELLM_PROXY", 1)
 )
 DEFAULT_SQS_BATCH_SIZE = int(os.getenv("DEFAULT_SQS_BATCH_SIZE", 512))
 SQS_SEND_MESSAGE_ACTION = "SendMessage"
@@ -60,7 +60,9 @@ DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_PRO = int(
     os.getenv("DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_PRO", 128)
 )
 DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE = int(
-    os.getenv("DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE", 512)
+    os.getenv(
+        "DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE", 512
+    )
 )
 
 # Generic fallback for unknown models
@@ -949,7 +951,9 @@ LITELLM_CLI_SESSION_TOKEN_PREFIX = "litellm-session-token"
 DB_SPEND_UPDATE_JOB_NAME = "db_spend_update_job"
 PROMETHEUS_EMIT_BUDGET_METRICS_JOB_NAME = "prometheus_emit_budget_metrics"
 CLOUDZERO_EXPORT_USAGE_DATA_JOB_NAME = "cloudzero_export_usage_data"
-CLOUDZERO_MAX_FETCHED_DATA_RECORDS = int(os.getenv("CLOUDZERO_MAX_FETCHED_DATA_RECORDS", 50000))
+CLOUDZERO_MAX_FETCHED_DATA_RECORDS = int(
+    os.getenv("CLOUDZERO_MAX_FETCHED_DATA_RECORDS", 50000)
+)
 SPEND_LOG_CLEANUP_JOB_NAME = "spend_log_cleanup"
 SPEND_LOG_RUN_LOOPS = int(os.getenv("SPEND_LOG_RUN_LOOPS", 500))
 SPEND_LOG_CLEANUP_BATCH_SIZE = int(os.getenv("SPEND_LOG_CLEANUP_BATCH_SIZE", 1000))

litellm/cost_calculator.py

@@ -344,6 +344,11 @@ def cost_per_token(  # noqa: PLR0915
         return perplexity_cost_per_token(model=model, usage=usage_block)
     elif custom_llm_provider == "xai":
         return xai_cost_per_token(model=model, usage=usage_block)
+    elif custom_llm_provider == "dashscope":
+        from litellm.llms.dashscope.cost_calculator import (
+            cost_per_token as dashscope_cost_per_token,
+        )
+        return dashscope_cost_per_token(model=model, usage=usage_block)
     else:
         model_info = _cached_get_model_info_helper(
             model=model, custom_llm_provider=custom_llm_provider

litellm/experimental_mcp_client/tools.py

@@ -17,22 +17,60 @@ from litellm.types.utils import ChatCompletionMessageToolCall
 ########################################################
 def transform_mcp_tool_to_openai_tool(mcp_tool: MCPTool) -> ChatCompletionToolParam:
     """Convert an MCP tool to an OpenAI tool."""
+    normalized_parameters = _normalize_mcp_input_schema(mcp_tool.inputSchema)
+    
     return ChatCompletionToolParam(
         type="function",
         function=FunctionDefinition(
             name=mcp_tool.name,
             description=mcp_tool.description or "",
-            parameters=mcp_tool.inputSchema,
+            parameters=normalized_parameters,
             strict=False,
         ),
     )
 
 
+def _normalize_mcp_input_schema(input_schema: dict) -> dict:
+    """
+    Normalize MCP input schema to ensure it's valid for OpenAI function calling.
+    
+    OpenAI requires that function parameters have:
+    - type: 'object'
+    - properties: dict (can be empty)
+    - additionalProperties: false (recommended)
+    """
+    if not input_schema:
+        return {
+            "type": "object",
+            "properties": {},
+            "additionalProperties": False
+        }
+    
+    # Make a copy to avoid modifying the original
+    normalized_schema = dict(input_schema)
+    
+    # Ensure type is 'object'
+    if "type" not in normalized_schema:
+        normalized_schema["type"] = "object"
+    
+    # Ensure properties exists (can be empty)
+    if "properties" not in normalized_schema:
+        normalized_schema["properties"] = {}
+    
+    # Add additionalProperties if not present (recommended by OpenAI)
+    if "additionalProperties" not in normalized_schema:
+        normalized_schema["additionalProperties"] = False
+    
+    return normalized_schema
+
+
 def transform_mcp_tool_to_openai_responses_api_tool(mcp_tool: MCPTool) -> FunctionToolParam:
     """Convert an MCP tool to an OpenAI Responses API tool."""
+    normalized_parameters = _normalize_mcp_input_schema(mcp_tool.inputSchema)
+    
     return FunctionToolParam(
         name=mcp_tool.name,
-        parameters=mcp_tool.inputSchema,
+        parameters=normalized_parameters,
         strict=False,
         type="function",
         description=mcp_tool.description or "",

litellm/files/utils.py

@@ -0,0 +1,27 @@
+from typing import Optional
+
+from litellm.types.llms.openai import CreateFileRequest
+from litellm.types.utils import ExtractedFileData
+
+
+class FilesAPIUtils:
+    """
+    Utils for files API interface on litellm
+    """
+    @staticmethod
+    def is_batch_jsonl_file(create_file_data: CreateFileRequest, extracted_file_data: ExtractedFileData) -> bool:
+        """
+        Check if the file is a batch jsonl file
+        """
+        return (
+            create_file_data.get("purpose") == "batch"
+            and FilesAPIUtils.valid_content_type(extracted_file_data.get("content_type"))
+            and extracted_file_data.get("content") is not None
+        )
+    
+    @staticmethod
+    def valid_content_type(content_type: Optional[str]) -> bool:
+        """
+        Check if the content type is valid
+        """
+        return content_type in set(["application/jsonl", "application/octet-stream"])

litellm/google_genai/main.py

@@ -224,6 +224,9 @@ async def agenerate_content(
         loop = asyncio.get_event_loop()
         kwargs["agenerate_content"] = True
 
+        # Handle generationConfig parameter from kwargs for backward compatibility
+        if "generationConfig" in kwargs and config is None:
+            config = kwargs.pop("generationConfig")
         # get custom llm provider so we can use this for mapping exceptions
         if custom_llm_provider is None:
             _, custom_llm_provider, _, _ = litellm.get_llm_provider(
@@ -288,6 +291,9 @@ def generate_content(
     try:
         _is_async = kwargs.pop("agenerate_content", False) is True
 
+        # Handle generationConfig parameter from kwargs for backward compatibility
+        if "generationConfig" in kwargs and config is None:
+            config = kwargs.pop("generationConfig")
         # Check for mock response first
         litellm_params = GenericLiteLLMParams(**kwargs)
         if litellm_params.mock_response and isinstance(
@@ -374,6 +380,9 @@ async def agenerate_content_stream(
     try:
         kwargs["agenerate_content_stream"] = True
 
+        # Handle generationConfig parameter from kwargs for backward compatibility
+        if "generationConfig" in kwargs and config is None:
+            config = kwargs.pop("generationConfig")
         # get custom llm provider so we can use this for mapping exceptions
         if custom_llm_provider is None:
             _, custom_llm_provider, _, _ = litellm.get_llm_provider(
@@ -461,6 +470,9 @@ def generate_content_stream(
         # Remove any async-related flags since this is the sync function
         _is_async = kwargs.pop("agenerate_content_stream", False)
 
+        # Handle generationConfig parameter from kwargs for backward compatibility
+        if "generationConfig" in kwargs and config is None:
+            config = kwargs.pop("generationConfig")
         # Setup the call
         setup_result = GenerateContentHelper.setup_generate_content_call(
             model=model,

litellm/litellm_core_utils/get_litellm_params.py

@@ -62,6 +62,7 @@ def get_litellm_params(
     use_litellm_proxy: Optional[bool] = None,
     api_version: Optional[str] = None,
     max_retries: Optional[int] = None,
+    litellm_request_debug: Optional[bool] = None,
     **kwargs,
 ) -> dict:
     litellm_params = {
@@ -118,5 +119,6 @@ def get_litellm_params(
         "vertex_credentials": kwargs.get("vertex_credentials"),
         "vertex_project": kwargs.get("vertex_project"),
         "use_litellm_proxy": use_litellm_proxy,
+        "litellm_request_debug": litellm_request_debug,
     }
     return litellm_params

litellm/litellm_core_utils/litellm_logging.py

@@ -245,6 +245,7 @@ class Logging(LiteLLMLoggingBaseClass):
     global supabaseClient, promptLayerLogger, weightsBiasesLogger, logfireLogger, capture_exception, add_breadcrumb, lunaryLogger, logfireLogger, prometheusLogger, slack_app
     custom_pricing: bool = False
     stream_options = None
+    litellm_request_debug: bool = False
 
     def __init__(
         self,
@@ -470,6 +471,7 @@ class Logging(LiteLLMLoggingBaseClass):
             **self.litellm_params,
             **scrub_sensitive_keys_in_metadata(litellm_params),
         }
+        self.litellm_request_debug = litellm_params.get("litellm_request_debug", False)
         self.logger_fn = litellm_params.get("logger_fn", None)
         verbose_logger.debug(f"self.optional_params: {self.optional_params}")
 
@@ -907,13 +909,19 @@ class Logging(LiteLLMLoggingBaseClass):
 
         Prints the RAW curl command sent from LiteLLM
         """
-        if _is_debugging_on():
+        if _is_debugging_on() or self.litellm_request_debug:
             if json_logs:
                 masked_headers = self._get_masked_headers(headers)
-                verbose_logger.debug(
-                    "POST Request Sent from LiteLLM",
-                    extra={"api_base": {api_base}, **masked_headers},
-                )
+                if self.litellm_request_debug:
+                    verbose_logger.warning(  # .warning ensures this shows up in all environments
+                        "POST Request Sent from LiteLLM",
+                        extra={"api_base": {api_base}, **masked_headers},
+                    )
+                else:
+                    verbose_logger.debug(
+                        "POST Request Sent from LiteLLM",
+                        extra={"api_base": {api_base}, **masked_headers},
+                    )
             else:
                 headers = additional_args.get("headers", {})
                 if headers is None:
@@ -926,7 +934,12 @@ class Logging(LiteLLMLoggingBaseClass):
                     additional_args=additional_args,
                     data=data,
                 )
-                verbose_logger.debug(f"\033[92m{curl_command}\033[0m\n")
+                if self.litellm_request_debug:
+                    verbose_logger.warning(
+                        f"\033[92m{curl_command}\033[0m\n"
+                    )  # .warning ensures this shows up in all environments
+                else:
+                    verbose_logger.debug(f"\033[92m{curl_command}\033[0m\n")
 
     def _get_request_body(self, data: dict) -> str:
         return str(data)
@@ -983,8 +996,14 @@ class Logging(LiteLLMLoggingBaseClass):
             self.model_call_details["additional_args"] = additional_args
             self.model_call_details["log_event_type"] = "post_api_call"
 
+            if self.litellm_request_debug:
+                attr = "warning"
+            else:
+                attr = "debug"
+
             if json_logs:
-                verbose_logger.debug(
+                callattr = getattr(verbose_logger, attr)
+                callattr(
                     "RAW RESPONSE:\n{}\n\n".format(
                         self.model_call_details.get(
                             "original_response", self.model_call_details
@@ -992,7 +1011,8 @@ class Logging(LiteLLMLoggingBaseClass):
                     ),
                 )
             else:
-                print_verbose(
+                callattr = getattr(verbose_logger, attr)
+                callattr(
                     "RAW RESPONSE:\n{}\n\n".format(
                         self.model_call_details.get(
                             "original_response", self.model_call_details
@@ -1714,12 +1734,16 @@ class Logging(LiteLLMLoggingBaseClass):
                             response_obj=result,
                             start_time=start_time,
                             end_time=end_time,
-                            litellm_call_id=current_call_id
-                            if (
-                                current_call_id := litellm_params.get("litellm_call_id")
-                            )
-                            is not None
-                            else str(uuid.uuid4()),
+                            litellm_call_id=(
+                                current_call_id
+                                if (
+                                    current_call_id := litellm_params.get(
+                                        "litellm_call_id"
+                                    )
+                                )
+                                is not None
+                                else str(uuid.uuid4())
+                            ),
                             print_verbose=print_verbose,
                         )
                     if callback == "wandb" and weightsBiasesLogger is not None:
@@ -3367,6 +3391,7 @@ def _init_custom_logger_compatible_class(  # noqa: PLR0915
             return galileo_logger  # type: ignore
         elif logging_integration == "cloudzero":
             from litellm.integrations.cloudzero.cloudzero import CloudZeroLogger
+
             for callback in _in_memory_loggers:
                 if isinstance(callback, CloudZeroLogger):
                     return callback  # type: ignore
@@ -3594,6 +3619,7 @@ def get_custom_logger_compatible_class(  # noqa: PLR0915
                     return callback
         elif logging_integration == "cloudzero":
             from litellm.integrations.cloudzero.cloudzero import CloudZeroLogger
+
             for callback in _in_memory_loggers:
                 if isinstance(callback, CloudZeroLogger):
                     return callback
@@ -4504,7 +4530,7 @@ def get_standard_logging_object_payload(
 
 def emit_standard_logging_payload(payload: StandardLoggingPayload):
     if os.getenv("LITELLM_PRINT_STANDARD_LOGGING_PAYLOAD"):
-        print(json.dumps(payload, indent=4)) # noqa
+        print(json.dumps(payload, indent=4))  # noqa
 
 
 def get_standard_logging_metadata(

litellm/litellm_core_utils/logging_worker.py

@@ -1,10 +1,21 @@
 import asyncio
 import contextlib
-from typing import Coroutine, Optional
+import contextvars
+from typing import Coroutine, Optional, TypedDict
 
 from litellm._logging import verbose_logger
 
 
+class LoggingTask(TypedDict):
+    """
+    A logging task with its associated context to ensure logging is executed in
+    the original task's context.
+    """
+
+    coroutine: Coroutine
+    context: contextvars.Context
+
+
 class LoggingWorker:
     """
     A simple, async logging worker that processes log coroutines in the background.
@@ -13,77 +24,84 @@ class LoggingWorker:
     This leads to a +200 RPS performance improvement when using LiteLLM Python SDK or Proxy Server.
     - Use this to queue coroutine tasks that are not critical to the main flow of the application. e.g Success/Error callbacks, logging, etc.
     """
+
     LOGGING_WORKER_MAX_QUEUE_SIZE = 50_000
     LOGGING_WORKER_MAX_TIME_PER_COROUTINE = 20.0
 
     MAX_ITERATIONS_TO_CLEAR_QUEUE = 200
     MAX_TIME_TO_CLEAR_QUEUE = 5.0
-    
+
     def __init__(
-        self, 
-        timeout: float = LOGGING_WORKER_MAX_TIME_PER_COROUTINE, 
+        self,
+        timeout: float = LOGGING_WORKER_MAX_TIME_PER_COROUTINE,
         max_queue_size: int = LOGGING_WORKER_MAX_QUEUE_SIZE,
     ):
         self.timeout = timeout
         self.max_queue_size = max_queue_size
-        self._queue: Optional[asyncio.Queue] = None
+        self._queue: Optional[asyncio.Queue[LoggingTask]] = None
         self._worker_task: Optional[asyncio.Task] = None
-    
+
     def _ensure_queue(self) -> None:
         """Initialize the queue if it doesn't exist."""
         if self._queue is None:
             self._queue = asyncio.Queue(maxsize=self.max_queue_size)
-    
+
     def start(self) -> None:
         """Start the logging worker. Idempotent - safe to call multiple times."""
         self._ensure_queue()
         if self._worker_task is None or self._worker_task.done():
             self._worker_task = asyncio.create_task(self._worker_loop())
-    
+
     async def _worker_loop(self) -> None:
         """Main worker loop that processes log coroutines sequentially."""
         try:
             if self._queue is None:
                 return
-            
+
             while True:
                 # Process one coroutine at a time to keep event loop load predictable
-                coroutine = await self._queue.get()
+                task = await self._queue.get()
                 try:
-                    await asyncio.wait_for(coroutine, timeout=self.timeout)
+                    # Run the coroutine in its original context
+                    await asyncio.wait_for(
+                        task["context"].run(asyncio.create_task, task["coroutine"]),
+                        timeout=self.timeout,
+                    )
                 except Exception as e:
                     verbose_logger.exception(f"LoggingWorker error: {e}")
                     pass
                 finally:
                     self._queue.task_done()
-                    
+
         except asyncio.CancelledError:
             verbose_logger.debug("LoggingWorker cancelled during shutdown")
             # Attempt to clear remaining items to prevent "never awaited" warnings
             await self.clear_queue()
-    
+
     def enqueue(self, coroutine: Coroutine) -> None:
         """
-        Add a coroutine to the logging queue. 
+        Add a coroutine to the logging queue.
         Hot path: never blocks, drops logs if queue is full.
         """
         if self._queue is None:
             return
-        
+
         try:
-            self._queue.put_nowait(coroutine)
+            # Capture the current context when enqueueing
+            task = LoggingTask(coroutine=coroutine, context=contextvars.copy_context())
+            self._queue.put_nowait(task)
         except asyncio.QueueFull as e:
             verbose_logger.exception(f"LoggingWorker queue is full: {e}")
             # Drop logs on overload to protect request throughput
             pass
-    
+
     def ensure_initialized_and_enqueue(self, async_coroutine: Coroutine):
         """
         Ensure the logging worker is initialized and enqueue the coroutine.
         """
         self.start()
         self.enqueue(async_coroutine)
-    
+
     async def stop(self) -> None:
         """Stop the logging worker and clean up resources."""
         if self._worker_task:
@@ -91,34 +109,42 @@ class LoggingWorker:
             with contextlib.suppress(Exception):
                 await self._worker_task
             self._worker_task = None
-    
+
     async def flush(self) -> None:
         """Flush the logging queue."""
         if self._queue is None:
             return
         while not self._queue.empty():
             await self._queue.join()
-    
+
     async def clear_queue(self):
         """
         Clear the queue with a maximum time limit.
         """
         if self._queue is None:
             return
-        
+
         start_time = asyncio.get_event_loop().time()
-        
+
         for _ in range(self.MAX_ITERATIONS_TO_CLEAR_QUEUE):
             # Check if we've exceeded the maximum time
-            if asyncio.get_event_loop().time() - start_time >= self.MAX_TIME_TO_CLEAR_QUEUE:
-                verbose_logger.warning(f"clear_queue exceeded max_time of {self.MAX_TIME_TO_CLEAR_QUEUE}s, stopping early")
+            if (
+                asyncio.get_event_loop().time() - start_time
+                >= self.MAX_TIME_TO_CLEAR_QUEUE
+            ):
+                verbose_logger.warning(
+                    f"clear_queue exceeded max_time of {self.MAX_TIME_TO_CLEAR_QUEUE}s, stopping early"
+                )
                 break
-                
+
             try:
-                coroutine = self._queue.get_nowait()
+                task = self._queue.get_nowait()
                 # Await the coroutine to properly execute and avoid "never awaited" warnings
                 try:
-                    await asyncio.wait_for(coroutine, timeout=self.timeout)
+                    await asyncio.wait_for(
+                        task["context"].run(asyncio.create_task, task["coroutine"]),
+                        timeout=self.timeout,
+                    )
                 except Exception:
                     # Suppress errors during cleanup
                     pass
@@ -129,4 +155,3 @@ class LoggingWorker:
 
 # Global instance for backward compatibility
 GLOBAL_LOGGING_WORKER = LoggingWorker()
-

litellm/litellm_core_utils/streaming_handler.py

@@ -1937,7 +1937,7 @@ class CustomStreamWrapper:
                 )
             ## Map to OpenAI Exception
             try:
-                exception_type(
+                raise exception_type(
                     model=self.model,
                     custom_llm_provider=self.custom_llm_provider,
                     original_exception=e,

litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py

@@ -28,10 +28,6 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
         TextBlock,
     )
 
-    def __init__(self, completion_stream: Any, model: str):
-        super().__init__(completion_stream)
-        self.model = model
-
     sent_first_chunk: bool = False
     sent_content_block_start: bool = False
     sent_content_block_finish: bool = False
@@ -39,6 +35,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
     sent_last_message: bool = False
     holding_chunk: Optional[Any] = None
     holding_stop_reason_chunk: Optional[Any] = None
+    queued_usage_chunk: bool = False
     current_content_block_index: int = 0
     current_content_block_start: ContentBlockContentBlockDict = TextBlock(
         type="text",
@@ -47,6 +44,10 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
     pending_new_content_block: bool = False
     chunk_queue: deque = deque()  # Queue for buffering multiple chunks
 
+    def __init__(self, completion_stream: Any, model: str):
+        super().__init__(completion_stream)
+        self.model = model
+
     def __next__(self):
         from .transformation import LiteLLMAnthropicMessagesAdapter
 
@@ -217,77 +218,83 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
 
                     # Queue the merged chunk and reset
                     self.chunk_queue.append(merged_chunk)
+                    self.queued_usage_chunk = True
                     self.holding_stop_reason_chunk = None
                     return self.chunk_queue.popleft()
 
                 # Check if this processed chunk has a stop_reason - hold it for next chunk
 
-                if should_start_new_block and not self.sent_content_block_finish:
-                    # Queue the sequence: content_block_stop -> content_block_start -> current_chunk
+                if not self.queued_usage_chunk:
+                    if should_start_new_block and not self.sent_content_block_finish:
+                        # Queue the sequence: content_block_stop -> content_block_start -> current_chunk
 
-                    # 1. Stop current content block
-                    self.chunk_queue.append(
-                        {
-                            "type": "content_block_stop",
-                            "index": max(self.current_content_block_index - 1, 0),
-                        }
-                    )
+                        # 1. Stop current content block
+                        self.chunk_queue.append(
+                            {
+                                "type": "content_block_stop",
+                                "index": max(self.current_content_block_index - 1, 0),
+                            }
+                        )
 
-                    # 2. Start new content block
-                    self.chunk_queue.append(
-                        {
-                            "type": "content_block_start",
-                            "index": self.current_content_block_index,
-                            "content_block": self.current_content_block_start,
-                        }
-                    )
+                        # 2. Start new content block
+                        self.chunk_queue.append(
+                            {
+                                "type": "content_block_start",
+                                "index": self.current_content_block_index,
+                                "content_block": self.current_content_block_start,
+                            }
+                        )
 
-                    # 3. Queue the current chunk (don't lose it!)
-                    self.chunk_queue.append(processed_chunk)
-
-                    # Reset state for new block
-                    self.sent_content_block_finish = False
-
-                    # Return the first queued item
-                    return self.chunk_queue.popleft()
-
-                if (
-                    processed_chunk["type"] == "message_delta"
-                    and self.sent_content_block_finish is False
-                ):
-                    # Queue both the content_block_stop and the holding chunk
-                    self.chunk_queue.append(
-                        {
-                            "type": "content_block_stop",
-                            "index": self.current_content_block_index,
-                        }
-                    )
-                    self.sent_content_block_finish = True
-                    if processed_chunk.get("delta", {}).get("stop_reason") is not None:
-
-                        self.holding_stop_reason_chunk = processed_chunk
-                    else:
+                        # 3. Queue the current chunk (don't lose it!)
                         self.chunk_queue.append(processed_chunk)
-                    return self.chunk_queue.popleft()
-                elif self.holding_chunk is not None:
-                    # Queue both chunks
-                    self.chunk_queue.append(self.holding_chunk)
-                    self.chunk_queue.append(processed_chunk)
-                    self.holding_chunk = None
-                    return self.chunk_queue.popleft()
-                else:
-                    # Queue the current chunk
-                    self.chunk_queue.append(processed_chunk)
-                    return self.chunk_queue.popleft()
+
+                        # Reset state for new block
+                        self.sent_content_block_finish = False
+
+                        # Return the first queued item
+                        return self.chunk_queue.popleft()
+
+                    if (
+                        processed_chunk["type"] == "message_delta"
+                        and self.sent_content_block_finish is False
+                    ):
+                        # Queue both the content_block_stop and the holding chunk
+                        self.chunk_queue.append(
+                            {
+                                "type": "content_block_stop",
+                                "index": self.current_content_block_index,
+                            }
+                        )
+                        self.sent_content_block_finish = True
+                        if (
+                            processed_chunk.get("delta", {}).get("stop_reason")
+                            is not None
+                        ):
+
+                            self.holding_stop_reason_chunk = processed_chunk
+                        else:
+                            self.chunk_queue.append(processed_chunk)
+                        return self.chunk_queue.popleft()
+                    elif self.holding_chunk is not None:
+                        # Queue both chunks
+                        self.chunk_queue.append(self.holding_chunk)
+                        self.chunk_queue.append(processed_chunk)
+                        self.holding_chunk = None
+                        return self.chunk_queue.popleft()
+                    else:
+                        # Queue the current chunk
+                        self.chunk_queue.append(processed_chunk)
+                        return self.chunk_queue.popleft()
 
             # Handle any remaining held chunks after stream ends
-            if self.holding_stop_reason_chunk is not None:
-                self.chunk_queue.append(self.holding_stop_reason_chunk)
-                self.holding_stop_reason_chunk = None
+            if not self.queued_usage_chunk:
+                if self.holding_stop_reason_chunk is not None:
+                    self.chunk_queue.append(self.holding_stop_reason_chunk)
+                    self.holding_stop_reason_chunk = None
 
-            if self.holding_chunk is not None:
-                self.chunk_queue.append(self.holding_chunk)
-                self.holding_chunk = None
+                if self.holding_chunk is not None:
+                    self.chunk_queue.append(self.holding_chunk)
+                    self.holding_chunk = None
 
             if not self.sent_last_message:
                 self.sent_last_message = True

litellm/llms/bedrock/batches/transformation.py

@@ -124,15 +124,13 @@ class BedrockBatchesConfig(BaseAWSLLM, BaseBatchesConfig):
                 "AWS IAM role ARN is required for Bedrock batch jobs. "
                 "Set 'aws_batch_role_arn' in litellm_params or AWS_BATCH_ROLE_ARN env var"
             )
+
         
-        # Get the actual Bedrock model ID using common utility
-        bedrock_model_id = self.common_utils.extract_model_from_s3_file_path(input_file_id, optional_params)
-        
-        if not bedrock_model_id:
-            raise ValueError("Could not determine Bedrock model ID. Ensure the model is specified in the input file or passed as a parameter.")
+        if not model:
+            raise ValueError("Could not determine Bedrock model ID. Please pass `model` in your request body.")
         
         # Generate job name with the correct model ID using common utility
-        job_name = self.common_utils.generate_unique_job_name(bedrock_model_id, prefix="litellm")
+        job_name = self.common_utils.generate_unique_job_name(model, prefix="litellm")
         output_key = f"litellm-batch-outputs/{job_name}/"
         
         # Build input data config
@@ -151,7 +149,7 @@ class BedrockBatchesConfig(BaseAWSLLM, BaseBatchesConfig):
         
         # Create Bedrock batch request with proper typing
         bedrock_request: BedrockCreateBatchRequest = {
-            "modelId": bedrock_model_id,
+            "modelId": model,
             "jobName": job_name,
             "inputDataConfig": input_data_config,
             "outputDataConfig": output_data_config,

litellm/llms/bedrock/files/transformation.py

@@ -6,6 +6,8 @@ from typing import Any, Dict, List, Optional, Tuple, Union
 
 from httpx import Headers, Response
 
+from litellm._logging import verbose_logger
+from litellm.files.utils import FilesAPIUtils
 from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.llms.base_llm.files.transformation import (
@@ -21,6 +23,7 @@ from litellm.types.llms.openai import (
     PathLike,
 )
 from litellm.types.utils import ExtractedFileData, LlmProviders
+from litellm.utils import get_llm_provider
 
 from ..base_aws_llm import BaseAWSLLM
 from ..common_utils import BedrockError
@@ -111,6 +114,10 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
         # Remove bedrock/ prefix if present
         if _model.startswith("bedrock/"):
             _model = _model[8:]
+        
+        # Replace colons with hyphens for Bedrock S3 URI compliance
+        _model = _model.replace(":", "-")
+        
         object_name = f"litellm-bedrock-files-{_model}-{uuid.uuid4()}.jsonl"
         return object_name
 
@@ -191,24 +198,6 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
     ) -> dict:
         return optional_params
 
-    def _get_bedrock_provider_from_model(self, model: str) -> Optional[str]:
-        """
-        Extract provider from Bedrock model name
-        """
-        if model.startswith("anthropic."):
-            return "anthropic"
-        elif model.startswith("cohere."):
-            return "cohere"
-        elif model.startswith("meta.") or model.startswith("llama"):
-            return "meta"
-        elif model.startswith("mistral."):
-            return "mistral"
-        elif model.startswith("ai21."):
-            return "ai21"
-        elif model.startswith("amazon."):
-            return "amazon"
-        else:
-            return None
 
     def _map_openai_to_bedrock_params(
         self,
@@ -218,11 +207,12 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
         """
         Transform OpenAI request body to Bedrock-compatible modelInput parameters using existing transformation logic
         """
+        from litellm.types.utils import LlmProviders
         _model = openai_request_body.get("model", "")
         messages = openai_request_body.get("messages", [])
         
         # Use existing Anthropic transformation logic for Anthropic models
-        if provider == "anthropic":
+        if provider == LlmProviders.ANTHROPIC:
             from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import (
                 AmazonAnthropicClaudeConfig,
             )
@@ -231,16 +221,22 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
             
             # Extract optional params (everything except model and messages)
             optional_params = {k: v for k, v in openai_request_body.items() if k not in ["model", "messages"]}
+            mapped_params = anthropic_config.map_openai_params(
+                non_default_params={},
+                optional_params=optional_params,
+                model=_model,
+                drop_params=False
+            )
             
             # Transform using existing Anthropic logic
             bedrock_params = anthropic_config.transform_request(
                 model=_model,
                 messages=messages,
-                optional_params=optional_params,
+                optional_params=mapped_params,
                 litellm_params={},
                 headers={}
             )
-            
+
             return bedrock_params
         else:
             # For other providers, use basic mapping
@@ -278,9 +274,17 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
             # Extract the request body from OpenAI format
             openai_body = _openai_jsonl_content.get("body", {})
             model = openai_body.get("model", "")
+
+            try:
+                model, _, _, _ = get_llm_provider(
+                            model=model, 
+                            custom_llm_provider=None, 
+                    )
+            except Exception as e:
+                verbose_logger.exception(f"litellm.llms.bedrock.files.transformation.py::_transform_openai_jsonl_content_to_bedrock_jsonl_content() - Error inferring custom_llm_provider - {str(e)}")
             
             # Determine provider from model name
-            provider = self._get_bedrock_provider_from_model(model)
+            provider = self.get_bedrock_invoke_provider(model)
             
             # Transform to Bedrock modelInput format
             model_input = self._map_openai_to_bedrock_params(
@@ -315,11 +319,13 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
         extracted_file_data = extract_file_data(file_data)
         extracted_file_data_content = extracted_file_data.get("content")
         
+        if extracted_file_data_content is None:
+            raise ValueError("file content is required")
+            
         # Get and transform the file content
-        if (
-            create_file_data.get("purpose") == "batch"
-            and extracted_file_data.get("content_type") == "application/jsonl"
-            and extracted_file_data_content is not None
+        if FilesAPIUtils.is_batch_jsonl_file(
+            create_file_data=create_file_data,
+            extracted_file_data=extracted_file_data,
         ):
             ## Transform JSONL content to Bedrock format
             original_file_content = self._get_content_from_openai_file(
@@ -357,6 +363,8 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
             api_base=api_base,
             optional_params=optional_params,
         )
+
+        litellm_params["upload_url"] = api_base
         
         # Return a dict that tells the HTTP handler exactly what to do
         return {
@@ -440,6 +448,56 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
         
         return dict(aws_request.headers), signed_body
 
+    def _convert_https_url_to_s3_uri(self, https_url: str) -> tuple[str, str]:
+        """
+        Convert HTTPS S3 URL to s3:// URI format.
+        
+        Args:
+            https_url: HTTPS S3 URL (e.g., "https://s3.us-west-2.amazonaws.com/bucket/key")
+        
+        Returns:
+            Tuple of (s3_uri, filename)
+        
+        Example:
+            Input: "https://s3.us-west-2.amazonaws.com/litellm-proxy/file.jsonl"
+            Output: ("s3://litellm-proxy/file.jsonl", "file.jsonl")
+        """
+        import re
+
+        # Match HTTPS S3 URL patterns
+        # Pattern 1: https://s3.region.amazonaws.com/bucket/key
+        # Pattern 2: https://bucket.s3.region.amazonaws.com/key
+        
+        pattern1 = r"https://s3\.([^.]+)\.amazonaws\.com/([^/]+)/(.+)"
+        pattern2 = r"https://([^.]+)\.s3\.([^.]+)\.amazonaws\.com/(.+)"
+        
+        match1 = re.match(pattern1, https_url)
+        match2 = re.match(pattern2, https_url)
+        
+        if match1:
+            # Pattern: https://s3.region.amazonaws.com/bucket/key
+            region, bucket, key = match1.groups()
+            s3_uri = f"s3://{bucket}/{key}"
+        elif match2:
+            # Pattern: https://bucket.s3.region.amazonaws.com/key
+            bucket, region, key = match2.groups()
+            s3_uri = f"s3://{bucket}/{key}"
+        else:
+            # Fallback: try to extract bucket and key from URL path
+            from urllib.parse import urlparse
+            parsed = urlparse(https_url)
+            path_parts = parsed.path.lstrip('/').split('/', 1)
+            if len(path_parts) >= 2:
+                bucket, key = path_parts[0], path_parts[1]
+                s3_uri = f"s3://{bucket}/{key}"
+            else:
+                raise ValueError(f"Unable to parse S3 URL: {https_url}")
+        
+        # Extract filename from key
+        filename = key.split("/")[-1] if "/" in key else key
+        
+        return s3_uri, filename
+
     def transform_create_file_response(
         self,
         model: Optional[str],
@@ -452,21 +510,18 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
         """
         # For S3 uploads, we typically get an ETag and other metadata
         response_headers = raw_response.headers
-        
         # Extract S3 object information from the response
         # S3 PUT object returns ETag and other metadata in headers
         content_length = response_headers.get("Content-Length", "0")
         
-        # Extract bucket and key from the request URL or litellm_params
-        bucket_name = litellm_params.get("s3_bucket_name") or os.getenv("AWS_S3_BUCKET_NAME")
-        
-        # Generate file ID in S3 format
-        object_key = getattr(logging_obj, 'object_key', None) or f"file-{int(time.time())}"
-        file_id = f"s3://{bucket_name}/{object_key}"
-        
-        # Extract filename from object key
-        filename = object_key.split("/")[-1] if "/" in object_key else object_key
-        
+        # Use the actual upload URL that was used for the S3 upload
+        upload_url = litellm_params.get("upload_url")
+        file_id: str = ""
+        filename: str = ""
+        if upload_url:
+            # Convert HTTPS S3 URL to s3:// URI format
+            file_id, filename = self._convert_https_url_to_s3_uri(upload_url)
+
         return OpenAIFileObject(
             purpose="batch",  # Default purpose for Bedrock files
             id=file_id,

litellm/llms/custom_httpx/aiohttp_handler.py

@@ -17,6 +17,7 @@ from litellm.llms.custom_httpx.http_handler import (
     HTTPHandler,
     _get_httpx_client,
 )
+from litellm.llms.custom_httpx.aiohttp_transport import LiteLLMAiohttpTransport
 from litellm.types.llms.openai import FileTypes
 from litellm.types.utils import HttpHandlerRequestFields, ImageResponse, LlmProviders
 from litellm.utils import CustomStreamWrapper, ModelResponse, ProviderConfigManager
@@ -32,8 +33,71 @@ DEFAULT_TIMEOUT = 600
 
 
 class BaseLLMAIOHTTPHandler:
-    def __init__(self):
-        self.client_session: Optional[aiohttp.ClientSession] = None
+    def __init__(
+        self,
+        client_session: Optional[aiohttp.ClientSession] = None,
+        transport: Optional[LiteLLMAiohttpTransport] = None,
+        connector: Optional[aiohttp.BaseConnector] = None,
+    ):
+        self.client_session = client_session
+        self._owns_session = (
+            client_session is None
+        )  # Track if we own the session for cleanup
+
+        self.transport = transport
+        self._owns_transport = (
+            transport is None
+        )  # Track if we own the transport for cleanup
+
+        self.connector = connector
+        self._owns_connector = (
+            connector is None
+        )  # Track if we own the connector for cleanup
+
+    def _get_or_create_transport(self) -> Optional[LiteLLMAiohttpTransport]:
+        """Get existing transport or create a new one if needed."""
+        if self.transport:
+            return self.transport
+
+        # Create a transport using AsyncHTTPHandler's logic
+        try:
+            self.transport = AsyncHTTPHandler._create_aiohttp_transport()
+            self._owns_transport = True
+            return self.transport
+        except Exception:
+            # If transport creation fails, return None (will use direct session)
+            return None
+
+    def _get_connector(self) -> Optional[aiohttp.BaseConnector]:
+        """Get or create a connector for the client session."""
+        if self.connector:
+            return self.connector
+        elif self.transport and hasattr(self.transport, "client"):
+            # Extract connector from transport if available
+            client = self.transport.client
+            if callable(client):
+                # If client is a factory, we can't extract connector directly
+                return None
+            elif hasattr(client, "connector"):
+                return client.connector
+        return None
+
+    def _create_client_session_with_transport(self) -> ClientSession:
+        """Create a new client session using transport or connector configuration."""
+        connector = self._get_connector()
+
+        if self.transport and hasattr(self.transport, "_get_valid_client_session"):
+            # Use transport's session creation if available
+            session = self.transport._get_valid_client_session()
+            return session
+        elif connector:
+            # Use provided connector
+            session = aiohttp.ClientSession(connector=connector)
+            return session
+        else:
+            # Default session creation
+            session = aiohttp.ClientSession()
+            return session
 
     def _get_async_client_session(
         self, dynamic_client_session: Optional[ClientSession] = None
@@ -43,15 +107,33 @@ class BaseLLMAIOHTTPHandler:
         elif self.client_session:
             return self.client_session
         else:
-            # init client session, and then return new session
-            self.client_session = aiohttp.ClientSession()
+            # Create client session using transport/connector if available
+            self.client_session = self._create_client_session_with_transport()
+            self._owns_session = True  # We created this session, so we own it
             return self.client_session
 
     async def close(self):
-        """Close the aiohttp client session if it exists."""
-        if self.client_session and not self.client_session.closed:
+        """Close the aiohttp client session and transport if we own them."""
+        # Close client session if we own it
+        if (
+            self.client_session
+            and not self.client_session.closed
+            and self._owns_session
+        ):
             await self.client_session.close()
 
+        # Close transport if we own it
+        if (
+            self.transport
+            and self._owns_transport
+            and hasattr(self.transport, "aclose")
+        ):
+            try:
+                await self.transport.aclose()
+            except Exception:
+                # Ignore errors during transport cleanup
+                pass
+
     async def _make_common_async_call(
         self,
         async_client_session: Optional[ClientSession],

litellm/llms/custom_httpx/llm_http_handler.py

@@ -2201,7 +2201,6 @@ class BaseLLMHTTPHandler:
             litellm_params=litellm_params,
             optional_params={},
         )
-
         if _is_async:
             return self.async_create_file(
                 transformed_request=transformed_request,
@@ -2218,6 +2217,7 @@ class BaseLLMHTTPHandler:
             sync_httpx_client = _get_httpx_client()
         else:
             sync_httpx_client = client
+        
 
         if isinstance(transformed_request, dict) and "method" in transformed_request:
             # Handle pre-signed requests (e.g., from Bedrock S3 uploads)
@@ -2283,11 +2283,15 @@ class BaseLLMHTTPHandler:
                     provider_config=provider_config,
                 )
 
+        # Store the upload URL in litellm_params for the transformation method
+        litellm_params_with_url = dict(litellm_params)
+        litellm_params_with_url["upload_url"] = api_base
+        
         return provider_config.transform_create_file_response(
             model=None,
             raw_response=upload_response,
             logging_obj=logging_obj,
-            litellm_params=litellm_params,
+            litellm_params=litellm_params_with_url,
         )
 
     async def async_create_file(
@@ -2408,15 +2412,19 @@ class BaseLLMHTTPHandler:
         _is_async: bool = False,
         client: Optional[Union["HTTPHandler", "AsyncHTTPHandler"]] = None,
         timeout: Optional[Union[float, httpx.Timeout]] = None,
+        model: Optional[str] = None,
     ) -> Union["LiteLLMBatch", Coroutine[Any, Any, "LiteLLMBatch"]]:
         """
         Creates a batch using provider-specific batch creation process
         """
         # get config from model, custom llm provider
+        if model is None:
+            raise ValueError("model is required for create_batch")
+            
         headers = provider_config.validate_environment(
             api_key=api_key,
             headers=headers,
-            model="",
+            model=model,
             messages=[],
             optional_params={},
             litellm_params=litellm_params,
@@ -2425,7 +2433,7 @@ class BaseLLMHTTPHandler:
         api_base = provider_config.get_complete_batch_url(
             api_base=api_base,
             api_key=api_key,
-            model="",
+            model=model,
             optional_params={},
             litellm_params=litellm_params,
             data=create_batch_data,
@@ -2435,7 +2443,7 @@ class BaseLLMHTTPHandler:
 
         # Get the transformed request data
         transformed_request = provider_config.transform_create_batch_request(
-            model="",
+            model=model,
             create_batch_data=create_batch_data,
             litellm_params=litellm_params,
             optional_params={},
@@ -2495,7 +2503,7 @@ class BaseLLMHTTPHandler:
         litellm_params_with_request = {**litellm_params, "original_batch_request": create_batch_data}
         
         return provider_config.transform_create_batch_response(
-            model=None,
+            model=model,
             raw_response=batch_response,
             logging_obj=logging_obj,
             litellm_params=litellm_params_with_request,
@@ -2512,6 +2520,7 @@ class BaseLLMHTTPHandler:
         client: Optional[Union["HTTPHandler", "AsyncHTTPHandler"]] = None,
         timeout: Optional[Union[float, httpx.Timeout]] = None,
         create_batch_data: Optional["CreateBatchRequest"] = None,
+        model: Optional[str] = None,
     ):
         """
         Async version of create_batch
@@ -2572,7 +2581,7 @@ class BaseLLMHTTPHandler:
         litellm_params_with_request = {**litellm_params, "original_batch_request": create_batch_data or {}}
         
         return provider_config.transform_create_batch_response(
-            model=None,
+            model=model,
             raw_response=batch_response,
             logging_obj=logging_obj,
             litellm_params=litellm_params_with_request,

litellm/llms/dashscope/cost_calculator.py

@@ -1,21 +1,155 @@
 """
-Cost calculator for DeepSeek Chat models. 
+Cost calculator for Dashscope Chat models. 
 
-Handles prompt caching scenario.
+Handles tiered pricing and prompt caching scenarios.
 """
 
-from typing import Tuple
+from dataclasses import dataclass
+from typing import List, Optional, Tuple
 
-from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
-from litellm.types.utils import Usage
+from litellm.types.utils import ModelInfo, Usage
+from litellm.utils import get_model_info
+
+
+@dataclass
+class TokenBreakdown:
+    """Token breakdown for cost calculation."""
+    text_tokens: int
+    cached_tokens: int
+    completion_tokens: int
+    reasoning_tokens: int
+
+
+def _extract_token_breakdown(usage: Usage) -> TokenBreakdown:
+    """Extract token counts from usage, handling cached and reasoning tokens."""
+    cached_tokens = 0
+    if usage.prompt_tokens_details and hasattr(usage.prompt_tokens_details, "cached_tokens"):
+        cached_tokens = usage.prompt_tokens_details.cached_tokens or 0
+    
+    text_tokens = usage.prompt_tokens - cached_tokens
+    
+    reasoning_tokens = 0
+    if (hasattr(usage, "completion_tokens_details") and 
+        usage.completion_tokens_details and
+        hasattr(usage.completion_tokens_details, "reasoning_tokens")):
+        reasoning_tokens = usage.completion_tokens_details.reasoning_tokens or 0
+    
+    completion_tokens = (usage.completion_tokens or 0) - reasoning_tokens
+    
+    return TokenBreakdown(text_tokens, cached_tokens, completion_tokens, reasoning_tokens)
+
+
+def _calculate_tiered_cost(
+    tokens: int, 
+    tiered_pricing: List[dict], 
+    cost_key: str,
+    fallback_cost_key: Optional[str] = None
+) -> float:
+    """Calculate cost using tiered pricing structure.
+    
+    Finds the appropriate tier based on token count and applies that tier's rate to all tokens.
+    """
+    if not tiered_pricing or tokens <= 0:
+        return 0.0
+    
+    # Find the appropriate tier for the token count
+    for tier in tiered_pricing:
+        tier_range = tier.get("range", [])
+        if len(tier_range) != 2:
+            continue
+            
+        range_start, range_end = tier_range
+        
+        # Check if tokens fall within this tier's range
+        if range_start <= tokens <= range_end:
+            cost_per_token = tier.get(cost_key) or tier.get(fallback_cost_key, 0)
+            return tokens * cost_per_token
+    
+    # If no tier matches, use the last tier (highest tier)
+    if tiered_pricing:
+        last_tier = tiered_pricing[-1]
+        cost_per_token = last_tier.get(cost_key) or last_tier.get(fallback_cost_key, 0)
+        return tokens * cost_per_token
+    
+    return 0.0
+
+
+def _calculate_flat_cost(tokens: int, cost_per_token: float) -> float:
+    """Calculate cost using flat pricing."""
+    return tokens * cost_per_token
+
+
+def _calculate_prompt_cost(breakdown: TokenBreakdown, model_info: ModelInfo, tiered_pricing: Optional[List[dict]]) -> float:
+    """Calculate total prompt cost including cached tokens."""
+    if tiered_pricing:
+        text_cost = _calculate_tiered_cost(
+            tokens=breakdown.text_tokens, 
+            tiered_pricing=tiered_pricing, 
+            cost_key="input_cost_per_token"
+        )
+        cache_cost = _calculate_tiered_cost(
+            tokens=breakdown.cached_tokens, 
+            tiered_pricing=tiered_pricing, 
+            cost_key="cache_read_input_token_cost"
+        )
+        return text_cost + cache_cost
+    
+    input_cost = model_info.get("input_cost_per_token", 0.0)
+    cache_cost = model_info.get("cache_read_input_token_cost", input_cost) or input_cost
+    
+    return (_calculate_flat_cost(tokens=breakdown.text_tokens, cost_per_token=input_cost) + 
+            _calculate_flat_cost(tokens=breakdown.cached_tokens, cost_per_token=cache_cost))
+
+
+def _calculate_completion_cost(breakdown: TokenBreakdown, model_info: ModelInfo, tiered_pricing: Optional[List[dict]]) -> float:
+    """Calculate total completion cost including reasoning tokens."""
+    if tiered_pricing:
+        completion_cost = _calculate_tiered_cost(
+            tokens=breakdown.completion_tokens, 
+            tiered_pricing=tiered_pricing, 
+            cost_key="output_cost_per_token"
+        )
+        reasoning_cost = _calculate_tiered_cost(
+            tokens=breakdown.reasoning_tokens, 
+            tiered_pricing=tiered_pricing, 
+            cost_key="output_cost_per_reasoning_token",
+            fallback_cost_key="output_cost_per_token"
+        )
+        return completion_cost + reasoning_cost
+    
+    output_cost = model_info.get("output_cost_per_token", 0.0)
+    reasoning_cost = model_info.get("output_cost_per_reasoning_token", output_cost) or output_cost
+    
+    return (_calculate_flat_cost(tokens=breakdown.completion_tokens, cost_per_token=output_cost) + 
+            _calculate_flat_cost(tokens=breakdown.reasoning_tokens, cost_per_token=reasoning_cost))
 
 
 def cost_per_token(model: str, usage: Usage) -> Tuple[float, float]:
     """
-    Calculates the cost per token for a given model, prompt tokens, and completion tokens.
-
-    Follows the same logic as Anthropic's cost per token calculation.
+    Calculate cost per token for Dashscope models.
+    
+    Supports both tiered and flat pricing with cached and reasoning tokens.
+    
+    Args:
+        model: Model name without provider prefix
+        usage: LiteLLM Usage block
+        
+    Returns:
+        Tuple[float, float] - (prompt_cost_in_usd, completion_cost_in_usd)
     """
-    return generic_cost_per_token(
-        model=model, usage=usage, custom_llm_provider="deepseek"
+    model_info = get_model_info(model=model, custom_llm_provider="dashscope")
+    breakdown = _extract_token_breakdown(usage)
+    tiered_pricing = model_info.get("tiered_pricing") if isinstance(model_info.get("tiered_pricing"), list) else None
+    
+    prompt_cost = _calculate_prompt_cost(
+        breakdown=breakdown, 
+        model_info=model_info, 
+        tiered_pricing=tiered_pricing
     )
+    completion_cost = _calculate_completion_cost(
+        breakdown=breakdown, 
+        model_info=model_info, 
+        tiered_pricing=tiered_pricing
+    )
+    
+    return prompt_cost, completion_cost

litellm/llms/databricks/chat/transformation.py

@@ -169,12 +169,20 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
         if tool is None:
             return None
 
+        # Build DatabricksFunction explicitly to avoid parameter conflicts
+        function_params: DatabricksFunction = {
+            "name": tool["name"],
+            "parameters": cast(dict, tool.get("input_schema") or {})
+        }
+        
+        # Only add description if it exists
+        description = tool.get("description")
+        if description is not None:
+            function_params["description"] = cast(Union[dict, str], description)
+
         return DatabricksTool(
             type="function",
-            function=DatabricksFunction(
-                name=tool["name"],
-                parameters=cast(dict, tool.get("input_schema") or {}),
-            ),
+            function=function_params,
         )
 
     def _map_openai_to_dbrx_tool(self, model: str, tools: List) -> List[DatabricksTool]:
@@ -331,8 +339,9 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
         elif isinstance(content, list):
             content_str = ""
             for item in content:
-                if item["type"] == "text":
-                    content_str += item["text"]
+                if item.get("type") == "text":
+                    text_value = item.get("text", "")
+                    content_str += str(text_value) if text_value is not None else ""
             return content_str
         else:
             raise Exception(f"Unsupported content type: {type(content)}")
@@ -361,19 +370,21 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
         reasoning_content: Optional[str] = None
         if isinstance(content, list):
             for item in content:
-                if item["type"] == "reasoning":
-                    for sum in item["summary"]:
-                        if reasoning_content is None:
-                            reasoning_content = ""
-                        reasoning_content += sum["text"]
-                        thinking_block = ChatCompletionThinkingBlock(
-                            type="thinking",
-                            thinking=sum.get("text", ""),
-                            signature=sum.get("signature", ""),
-                        )
-                        if thinking_blocks is None:
-                            thinking_blocks = []
-                        thinking_blocks.append(thinking_block)
+                if item.get("type") == "reasoning":
+                    summary_list = item.get("summary", [])
+                    if isinstance(summary_list, list):
+                        for sum in summary_list:
+                            if reasoning_content is None:
+                                reasoning_content = ""
+                            reasoning_content += sum["text"]
+                            thinking_block = ChatCompletionThinkingBlock(
+                                type="thinking",
+                                thinking=sum.get("text", ""),
+                                signature=sum.get("signature", ""),
+                            )
+                            if thinking_blocks is None:
+                                thinking_blocks = []
+                            thinking_blocks.append(thinking_block)
         return reasoning_content, thinking_blocks
 
     @staticmethod

litellm/llms/gemini/chat/transformation.py

@@ -1,10 +1,13 @@
-from typing import List, Optional
+from typing import List, Optional, cast
 
 from litellm.litellm_core_utils.prompt_templates.factory import (
     convert_generic_image_chunk_to_openai_image_obj,
     convert_to_anthropic_image_obj,
 )
-from litellm.types.llms.openai import AllMessageValues
+from litellm.litellm_core_utils.prompt_templates.image_handling import (
+    convert_url_to_base64,
+)
+from litellm.types.llms.openai import AllMessageValues, ChatCompletionFileObject
 from litellm.types.llms.vertex_ai import ContentType, PartType
 from litellm.utils import supports_reasoning
 
@@ -99,7 +102,8 @@ class GoogleAIStudioGeminiConfig(VertexGeminiConfig):
         self, messages: List[AllMessageValues]
     ) -> List[ContentType]:
         """
-        Google AI Studio Gemini does not support image urls in messages.
+        Google AI Studio Gemini does not support HTTP/HTTPS URLs for files.
+        Convert them to base64 data instead.
         """
         for message in messages:
             _message_content = message.get("content")
@@ -124,4 +128,16 @@ class GoogleAIStudioGeminiConfig(VertexGeminiConfig):
                                     image_obj
                                 )
                             )
+                    elif element.get("type") == "file":
+                        file_element = cast(ChatCompletionFileObject, element)
+                        file_id = file_element["file"].get("file_id")
+                        if file_id and ("http://" in file_id or "https://" in file_id):
+                            # Convert HTTP/HTTPS file URL to base64 data
+                            try:
+                                base64_data = convert_url_to_base64(file_id)
+                                file_element["file"]["file_data"] = base64_data  # type: ignore
+                                file_element["file"].pop("file_id", None)  # type: ignore
+                            except Exception:
+                                # If conversion fails, leave as is and let the API handle it
+                                pass
         return _gemini_convert_messages_with_history(messages=messages)

litellm/llms/gemini/count_tokens/handler.py

@@ -11,7 +11,39 @@ if TYPE_CHECKING:
 else:
     GenerateContentContentListUnionDict = Any
 
+
 class GoogleAIStudioTokenCounter:
+    def _clean_contents_for_gemini_api(self, contents: Any) -> Any:
+        """
+        Clean up contents to remove unsupported fields for the Gemini API.
+
+        The Google Gemini API doesn't recognize the 'id' field in function responses,
+        so we need to remove it to prevent 400 Bad Request errors.
+
+        Args:
+            contents: The contents to clean up
+
+        Returns:
+            Cleaned contents with unsupported fields removed
+        """
+        import copy
+
+        from google.genai.types import FunctionResponse
+
+        cleaned_contents = copy.deepcopy(contents)
+
+        for content in cleaned_contents:
+            parts = content["parts"]
+            for part in parts:
+                if "functionResponse" in part:
+                    function_response_data = part["functionResponse"]
+                    function_response_part = FunctionResponse(**function_response_data)
+                    function_response_part.id = None
+                    part["functionResponse"] = function_response_part.model_dump(
+                        exclude_none=True
+                    )
+
+        return cleaned_contents
 
     def _construct_url(self, model: str, api_base: Optional[str] = None) -> str:
         """
@@ -20,7 +52,6 @@ class GoogleAIStudioTokenCounter:
         base_url = api_base or "https://generativelanguage.googleapis.com"
         return f"{base_url}/v1beta/models/{model}:countTokens"
 
-        
     async def validate_environment(
         self,
         api_base: Optional[str] = None,
@@ -33,7 +64,8 @@ class GoogleAIStudioTokenCounter:
         Returns a Tuple of headers and url for the Google Gen AI Studio countTokens endpoint.
         """
         from litellm.llms.gemini.google_genai.transformation import GoogleGenAIConfig
-        headers =  GoogleGenAIConfig().validate_environment(
+
+        headers = GoogleGenAIConfig().validate_environment(
             api_key=api_key,
             headers=headers,
             model=model,
@@ -54,7 +86,7 @@ class GoogleAIStudioTokenCounter:
     ) -> Dict[str, Any]:
         """
         Count tokens using Google Gen AI Studio countTokens endpoint.
-        
+
         Args:
             contents: The content to count tokens for (Google Gen AI format)
                     Example: [{"parts": [{"text": "Hello world"}]}]
@@ -63,7 +95,7 @@ class GoogleAIStudioTokenCounter:
             api_base: Optional API base URL (defaults to Google Gen AI Studio)
             timeout: Optional timeout for the request
             **kwargs: Additional parameters
-            
+
         Returns:
             Dict containing token count information from Google Gen AI Studio API.
             Example response:
@@ -77,14 +109,13 @@ class GoogleAIStudioTokenCounter:
                     }
                 ]
             }
-            
+
         Raises:
             ValueError: If API key is missing
             litellm.APIError: If the API call fails
             litellm.APIConnectionError: If the connection fails
             Exception: For any other unexpected errors
         """
-        # Set up API base URL
 
         # Prepare headers
         headers, url = await self.validate_environment(
@@ -94,46 +125,40 @@ class GoogleAIStudioTokenCounter:
             model=model,
             litellm_params=kwargs,
         )
-        
-        # Prepare request body
-        request_body = {
-            "contents": contents
-        }
-        
+
+        # Prepare request body - clean up contents to remove unsupported fields
+        cleaned_contents = self._clean_contents_for_gemini_api(contents)
+        request_body = {"contents": cleaned_contents}
+
         async_httpx_client = get_async_httpx_client(
             llm_provider=LlmProviders.GEMINI,
         )
 
         try:
             response = await async_httpx_client.post(
-                url=url,
-                headers=headers,
-                json=request_body
+                url=url, headers=headers, json=request_body
             )
-            
+
             # Check for HTTP errors
             response.raise_for_status()
-            
+
             # Parse response
             result = response.json()
             return result
-                
+
         except httpx.HTTPStatusError as e:
             error_msg = f"Google Gen AI Studio API error: {e.response.status_code} - {e.response.text}"
             raise litellm.APIError(
                 message=error_msg,
                 llm_provider="gemini",
                 model=model,
-                status_code=e.response.status_code
+                status_code=e.response.status_code,
             ) from e
         except httpx.RequestError as e:
             error_msg = f"Request to Google Gen AI Studio failed: {str(e)}"
             raise litellm.APIConnectionError(
-                message=error_msg,
-                llm_provider="gemini",
-                model=model
+                message=error_msg, llm_provider="gemini", model=model
             ) from e
         except Exception as e:
             error_msg = f"Unexpected error during token counting: {str(e)}"
             raise Exception(error_msg) from e
-

litellm/llms/lm_studio/chat/transformation.py

@@ -15,8 +15,8 @@ class LMStudioChatConfig(OpenAIGPTConfig):
     ) -> Tuple[Optional[str], Optional[str]]:
         api_base = api_base or get_secret_str("LM_STUDIO_API_BASE")  # type: ignore
         dynamic_api_key = (
-            api_key or get_secret_str("LM_STUDIO_API_KEY") or " "
-        )  # vllm does not require an api key
+            api_key or get_secret_str("LM_STUDIO_API_KEY") or "fake-api-key"
+        )  # LM Studio does not require an api key, but OpenAI client requires non-None value
         return api_base, dynamic_api_key
     
     def map_openai_params(

litellm/llms/openai/responses/transformation.py

@@ -272,6 +272,14 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
             ResponsesAPIStreamEvents.WEB_SEARCH_CALL_IN_PROGRESS: WebSearchCallInProgressEvent,
             ResponsesAPIStreamEvents.WEB_SEARCH_CALL_SEARCHING: WebSearchCallSearchingEvent,
             ResponsesAPIStreamEvents.WEB_SEARCH_CALL_COMPLETED: WebSearchCallCompletedEvent,
+            ResponsesAPIStreamEvents.MCP_LIST_TOOLS_IN_PROGRESS: MCPListToolsInProgressEvent,
+            ResponsesAPIStreamEvents.MCP_LIST_TOOLS_COMPLETED: MCPListToolsCompletedEvent,
+            ResponsesAPIStreamEvents.MCP_LIST_TOOLS_FAILED: MCPListToolsFailedEvent,
+            ResponsesAPIStreamEvents.MCP_CALL_IN_PROGRESS: MCPCallInProgressEvent,
+            ResponsesAPIStreamEvents.MCP_CALL_ARGUMENTS_DELTA: MCPCallArgumentsDeltaEvent,
+            ResponsesAPIStreamEvents.MCP_CALL_ARGUMENTS_DONE: MCPCallArgumentsDoneEvent,
+            ResponsesAPIStreamEvents.MCP_CALL_COMPLETED: MCPCallCompletedEvent,
+            ResponsesAPIStreamEvents.MCP_CALL_FAILED: MCPCallFailedEvent,
             ResponsesAPIStreamEvents.ERROR: ErrorEvent,
         }
 

litellm/llms/vertex_ai/files/transformation.py

@@ -6,6 +6,7 @@ from typing import Any, Dict, List, Optional, Tuple, Union
 
 from httpx import Headers, Response
 
+from litellm.files.utils import FilesAPIUtils
 from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.llms.base_llm.files.transformation import (
@@ -260,10 +261,13 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
             raise ValueError("file is required")
         extracted_file_data = extract_file_data(file_data)
         extracted_file_data_content = extracted_file_data.get("content")
-        if (
-            create_file_data.get("purpose") == "batch"
-            and extracted_file_data.get("content_type") == "application/jsonl"
-            and extracted_file_data_content is not None
+        
+        if extracted_file_data_content is None:
+            raise ValueError("file content is required")
+            
+        if FilesAPIUtils.is_batch_jsonl_file(
+            create_file_data=create_file_data,
+            extracted_file_data=extracted_file_data,
         ):
             ## 1. If jsonl, check if there's a model name
             file_content = self._get_content_from_openai_file(

litellm/llms/vertex_ai/google_genai/transformation.py

@@ -1,7 +1,7 @@
 """
 Transformation for Calling Google models in their native format.
 """
-from typing import Literal, Optional, Union
+from typing import Dict, Literal, Optional, Union
 
 from litellm.llms.gemini.google_genai.transformation import GoogleGenAIConfig
 from litellm.types.router import GenericLiteLLMParams
@@ -11,20 +11,20 @@ class VertexAIGoogleGenAIConfig(GoogleGenAIConfig):
     """
     Configuration for calling Google models in their native format.
     """
+
     HEADER_NAME = "Authorization"
     BEARER_PREFIX = "Bearer"
-    
+
     @property
     def custom_llm_provider(self) -> Literal["gemini", "vertex_ai"]:
         return "vertex_ai"
-    
 
     def validate_environment(
-        self, 
+        self,
         api_key: Optional[str],
         headers: Optional[dict],
         model: str,
-        litellm_params: Optional[Union[GenericLiteLLMParams, dict]]
+        litellm_params: Optional[Union[GenericLiteLLMParams, dict]],
     ) -> dict:
         default_headers = {
             "Content-Type": "application/json",
@@ -36,4 +36,65 @@ class VertexAIGoogleGenAIConfig(GoogleGenAIConfig):
             default_headers.update(headers)
 
         return default_headers
-    
+
+    def _camel_to_snake(self, camel_str: str) -> str:
+        """Convert camelCase to snake_case"""
+        import re
+
+        return re.sub(r"(?<!^)(?=[A-Z])", "_", camel_str).lower()
+
+    def map_generate_content_optional_params(
+        self,
+        generate_content_config_dict,
+        model: str,
+    ):
+        """
+        Map Google GenAI parameters to provider-specific format.
+
+        Args:
+            generate_content_optional_params: Optional parameters for generate content
+            model: The model name
+
+        Returns:
+            Mapped parameters for the provider
+        """
+        from litellm.types.google_genai.main import GenerateContentConfigDict
+
+        _generate_content_config_dict = GenerateContentConfigDict()
+
+        for param, value in generate_content_config_dict.items():
+            camel_case_key = self._camel_to_snake(param)
+            _generate_content_config_dict[camel_case_key] = value
+        return dict(_generate_content_config_dict)
+
+    def transform_generate_content_request(
+        self,
+        model: str,
+        contents: any,
+        tools: Optional[any],
+        generate_content_config_dict: Dict,
+        system_instruction: Optional[any] = None,
+    ) -> dict:
+        """
+        Transform the generate content request for Vertex AI.
+        Since Vertex AI natively supports Google GenAI format, we can pass most fields directly.
+        """
+        # Build the request in Google GenAI format that Vertex AI expects
+        result = {
+            "model": model,
+            "contents": contents,
+        }
+
+        # Add tools if provided
+        if tools:
+            result["tools"] = tools
+
+        # Add systemInstruction if provided
+        if system_instruction:
+            result["systemInstruction"] = system_instruction
+
+        # Handle generationConfig - Vertex AI expects it in the same format
+        if generate_content_config_dict:
+            result["generationConfig"] = generate_content_config_dict
+
+        return result

litellm/main.py

@@ -150,9 +150,9 @@ from .llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
 from .llms.custom_llm import CustomLLM, custom_chat_llm_router
 from .llms.databricks.embed.handler import DatabricksEmbeddingHandler
 from .llms.deprecated_providers import aleph_alpha, palm
+from .llms.gemini.common_utils import get_api_key_from_env
 from .llms.groq.chat.handler import GroqChatCompletion
 from .llms.heroku.chat.transformation import HerokuChatConfig
-from .llms.gemini.common_utils import get_api_key_from_env
 from .llms.huggingface.embedding.handler import HuggingFaceEmbedding
 from .llms.nlp_cloud.chat.handler import completion as nlp_cloud_chat_completion
 from .llms.oci.chat.transformation import OCIChatConfig
@@ -358,7 +358,9 @@ async def acompletion(
     logprobs: Optional[bool] = None,
     top_logprobs: Optional[int] = None,
     deployment_id=None,
-    reasoning_effort: Optional[Literal["none", "minimal", "low", "medium", "high", "default"]] = None,
+    reasoning_effort: Optional[
+        Literal["none", "minimal", "low", "medium", "high", "default"]
+    ] = None,
     safety_identifier: Optional[str] = None,
     # set api_base, api_version, api_key
     base_url: Optional[str] = None,
@@ -504,7 +506,9 @@ async def acompletion(
     }
     if custom_llm_provider is None:
         _, custom_llm_provider, _, _ = get_llm_provider(
-            model=model, custom_llm_provider=custom_llm_provider, api_base=completion_kwargs.get("base_url", None)
+            model=model,
+            custom_llm_provider=custom_llm_provider,
+            api_base=completion_kwargs.get("base_url", None),
         )
 
     fallbacks = fallbacks or litellm.model_fallbacks
@@ -899,7 +903,9 @@ def completion(  # type: ignore # noqa: PLR0915
     logit_bias: Optional[dict] = None,
     user: Optional[str] = None,
     # openai v1.0+ new params
-    reasoning_effort: Optional[Literal["none", "minimal", "low", "medium", "high", "default"]] = None,
+    reasoning_effort: Optional[
+        Literal["none", "minimal", "low", "medium", "high", "default"]
+    ] = None,
     response_format: Optional[Union[dict, Type[BaseModel]]] = None,
     seed: Optional[int] = None,
     tools: Optional[List] = None,
@@ -1116,10 +1122,12 @@ def completion(  # type: ignore # noqa: PLR0915
         )
 
         if provider_specific_header is not None:
-            headers.update(ProviderSpecificHeaderUtils.get_provider_specific_headers(
-                provider_specific_header=provider_specific_header,
-                custom_llm_provider=custom_llm_provider,
-            ))
+            headers.update(
+                ProviderSpecificHeaderUtils.get_provider_specific_headers(
+                    provider_specific_header=provider_specific_header,
+                    custom_llm_provider=custom_llm_provider,
+                )
+            )
 
         if model_response is not None and hasattr(model_response, "_hidden_params"):
             model_response._hidden_params["custom_llm_provider"] = custom_llm_provider
@@ -1325,6 +1333,7 @@ def completion(  # type: ignore # noqa: PLR0915
             azure_scope=kwargs.get("azure_scope"),
             max_retries=max_retries,
             timeout=timeout,
+            litellm_request_debug=kwargs.get("litellm_request_debug", False),
         )
         cast(LiteLLMLoggingObj, logging).update_environment_variables(
             model=model,
@@ -2712,9 +2721,7 @@ def completion(  # type: ignore # noqa: PLR0915
             )
 
             api_key = (
-                api_key
-                or litellm.api_key
-                or get_secret("VERCEL_AI_GATEWAY_API_KEY")
+                api_key or litellm.api_key or get_secret("VERCEL_AI_GATEWAY_API_KEY")
             )
 
             vercel_site_url = get_secret("VERCEL_SITE_URL") or "https://litellm.ai"
@@ -2730,7 +2737,7 @@ def completion(  # type: ignore # noqa: PLR0915
                 vercel_headers.update(_headers)
 
             headers = vercel_headers
-            
+
             ## Load Config
             config = litellm.VercelAIGatewayConfig.get_config()
             for k, v in config.items():
@@ -3712,7 +3719,9 @@ async def aembedding(*args, **kwargs) -> EmbeddingResponse:
         func_with_context = partial(ctx.run, func)
 
         _, custom_llm_provider, _, _ = get_llm_provider(
-            model=model, custom_llm_provider=custom_llm_provider, api_base=kwargs.get("api_base", None)
+            model=model,
+            custom_llm_provider=custom_llm_provider,
+            api_base=kwargs.get("api_base", None),
         )
 
         # Await normally
@@ -3854,7 +3863,7 @@ def embedding(  # noqa: PLR0915
     max_retries = kwargs.get("max_retries", None)
     litellm_logging_obj: LiteLLMLoggingObj = kwargs.get("litellm_logging_obj")  # type: ignore
     mock_response: Optional[List[float]] = kwargs.get("mock_response", None)  # type: ignore
-    azure_ad_token_provider = kwargs.pop("azure_ad_token_provider", None)
+    azure_ad_token_provider = kwargs.get("azure_ad_token_provider", None)
     aembedding = kwargs.get("aembedding", None)
     extra_headers = kwargs.get("extra_headers", None)
     headers = kwargs.get("headers", None)
@@ -5780,7 +5789,14 @@ async def ahealth_check(
                 input=input or ["test"],
             ),
             "audio_speech": lambda: litellm.aspeech(
-                **{**_filter_model_params(model_params), **({"voice": "alloy"} if "voice" not in _filter_model_params(model_params) else {})},
+                **{
+                    **_filter_model_params(model_params),
+                    **(
+                        {"voice": "alloy"}
+                        if "voice" not in _filter_model_params(model_params)
+                        else {}
+                    ),
+                },
                 input=prompt or "test",
             ),
             "audio_transcription": lambda: litellm.atranscription(

litellm/proxy/_experimental/mcp_server/mcp_server_manager.py

@@ -241,6 +241,9 @@ class MCPServerManager:
                 transport=server_config.get("transport", MCPTransport.http),
                 spec_version=server_config.get("spec_version", MCPSpecVersion.jun_2025),
                 auth_type=server_config.get("auth_type", None),
+                authentication_token=server_config.get(
+                    "authentication_token", server_config.get("auth_value", None)
+                ),
                 mcp_info=mcp_info,
                 access_groups=server_config.get("access_groups", None),
             )
@@ -716,8 +719,8 @@ class MCPServerManager:
             tasks = []
             if proxy_logging_obj:
                 # Create synthetic LLM data for during hook processing
-                from litellm.types.mcp import MCPDuringCallRequestObject
                 from litellm.types.llms.base import HiddenParams
+                from litellm.types.mcp import MCPDuringCallRequestObject
                 
                 request_obj = MCPDuringCallRequestObject(
                     tool_name=name,

litellm/proxy/_experimental/mcp_server/server.py

@@ -215,9 +215,9 @@ if MCP_AVAILABLE:
         """
         from fastapi import Request
 
+        from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException
         from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request
         from litellm.proxy.proxy_server import proxy_config
-        from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException
 
         # Validate arguments
         user_api_key_auth, mcp_auth_header, _, mcp_server_auth_headers, mcp_protocol_version = get_auth_context()
@@ -279,33 +279,15 @@ if MCP_AVAILABLE:
     ############ Helper Functions ##########################
     ########################################################
 
-    async def _get_tools_from_mcp_servers(
-        user_api_key_auth: Optional[UserAPIKeyAuth],
-        mcp_auth_header: Optional[str],
+    async def _get_allowed_mcp_servers_from_mcp_server_names(
         mcp_servers: Optional[List[str]],
-        mcp_server_auth_headers: Optional[Dict[str, str]] = None,
-        mcp_protocol_version: Optional[str] = None,
-    ) -> List[MCPTool]:
+        allowed_mcp_servers: List[str],
+    ) -> List[str]:
         """
-        Helper method to fetch tools from MCP servers based on server filtering criteria.
-
-        Args:
-            user_api_key_auth: User authentication info for access control
-            mcp_auth_header: Optional auth header for MCP server (deprecated)
-            mcp_servers: Optional list of server names/aliases to filter by
-            mcp_server_auth_headers: Optional dict of server-specific auth headers {server_alias: auth_value}
-
-        Returns:
-            List[MCPTool]: Combined list of tools from filtered servers
+        Get the filtered MCP servers from the MCP server names
         """
-        if not MCP_AVAILABLE:
-            return []
-
-        # Get allowed MCP servers based on user permissions
-        allowed_mcp_servers = await global_mcp_server_manager.get_allowed_mcp_servers(user_api_key_auth)
-
-        filtered_server_ids = set()
-
+        from typing import Set
+        filtered_server_ids: Set[str] = set()
         # Filter servers based on mcp_servers parameter if provided
         if mcp_servers is not None:
             for server_or_group in mcp_servers:
@@ -336,6 +318,40 @@ if MCP_AVAILABLE:
 
         if filtered_server_ids:
             allowed_mcp_servers = list(filtered_server_ids)
+        
+        return allowed_mcp_servers
+
+    async def _get_tools_from_mcp_servers(
+        user_api_key_auth: Optional[UserAPIKeyAuth],
+        mcp_auth_header: Optional[str],
+        mcp_servers: Optional[List[str]],
+        mcp_server_auth_headers: Optional[Dict[str, str]] = None,
+        mcp_protocol_version: Optional[str] = None,
+    ) -> List[MCPTool]:
+        """
+        Helper method to fetch tools from MCP servers based on server filtering criteria.
+
+        Args:
+            user_api_key_auth: User authentication info for access control
+            mcp_auth_header: Optional auth header for MCP server (deprecated)
+            mcp_servers: Optional list of server names/aliases to filter by
+            mcp_server_auth_headers: Optional dict of server-specific auth headers {server_alias: auth_value}
+
+        Returns:
+            List[MCPTool]: Combined list of tools from filtered servers
+        """
+        if not MCP_AVAILABLE:
+            return []
+
+        # Get allowed MCP servers based on user permissions
+        allowed_mcp_servers = await global_mcp_server_manager.get_allowed_mcp_servers(user_api_key_auth)
+
+        if mcp_servers is not None:
+            allowed_mcp_servers = await _get_allowed_mcp_servers_from_mcp_server_names(
+                mcp_servers=mcp_servers,
+                allowed_mcp_servers=allowed_mcp_servers,
+            )
+
 
         # Get tools from each allowed server
         all_tools = []
@@ -556,20 +572,25 @@ if MCP_AVAILABLE:
         except Exception as e:
             return [TextContent(text=f"Error: {str(e)}", type="text")]
 
-    async def extract_mcp_auth_context(scope, path):
+    def _get_mcp_servers_in_path(path: str) -> Optional[List[str]]:
         """
-        Extracts mcp_servers from the path and processes the MCP request for auth context.
-        Returns: (user_api_key_auth, mcp_auth_header, mcp_servers, mcp_server_auth_headers)
+        Get the MCP servers from the path
         """
         import re
-
-        mcp_servers_from_path = None
+        mcp_servers_from_path: Optional[List[str]] = None
         mcp_path_match = re.match(r"^/mcp/([^/]+)(/.*)?$", path)
         if mcp_path_match:
             mcp_servers_str = mcp_path_match.group(1)
             if mcp_servers_str:
                 mcp_servers_from_path = [s.strip() for s in mcp_servers_str.split(",") if s.strip()]
+        return mcp_servers_from_path
 
+    async def extract_mcp_auth_context(scope, path):
+        """
+        Extracts mcp_servers from the path and processes the MCP request for auth context.
+        Returns: (user_api_key_auth, mcp_auth_header, mcp_servers, mcp_server_auth_headers)
+        """
+        mcp_servers_from_path = _get_mcp_servers_in_path(path)
         if mcp_servers_from_path is not None:
             (
                 user_api_key_auth,

litellm/proxy/_experimental/out/index.txt

@@ -1,7 +1,7 @@
 2:I[19107,[],"ClientPageRoot"]
-3:I[75832,["665","static/chunks/3014691f-b7b79b78e27792f3.js","990","static/chunks/13b76428-ebdf3012af0e4489.js","50","static/chunks/50-fe160ecfa8bc4059.js","521","static/chunks/521-d97d355792d44830.js","866","static/chunks/866-3523e0e07cf314f6.js","220","static/chunks/220-1c8d82f7ce7658c4.js","154","static/chunks/154-fff436ed72b19a24.js","162","static/chunks/162-714ca0ed10a07f66.js","172","static/chunks/172-0f7049c565983c4d.js","931","static/chunks/app/page-8dc8d9524a1f3965.js"],"default",1]
+3:I[30628,["665","static/chunks/3014691f-b7b79b78e27792f3.js","990","static/chunks/13b76428-ebdf3012af0e4489.js","50","static/chunks/50-fe160ecfa8bc4059.js","521","static/chunks/521-d97d355792d44830.js","866","static/chunks/866-9e1803a09e9ae8da.js","220","static/chunks/220-5061c4cea850d728.js","154","static/chunks/154-fff436ed72b19a24.js","162","static/chunks/162-4e7640b4d68e1ae4.js","172","static/chunks/172-0f7049c565983c4d.js","931","static/chunks/app/page-127adcf8da2b5294.js"],"default",1]
 4:I[4707,[],""]
 5:I[36423,[],""]
-0:["0GF-OyXnYlAPMWfyPAZSs",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/31b7f215e119031e.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/060d5ddee53e45ce.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_b0dd8a","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
+0:["FMlZjJYLUentCU02Wj6R_",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/31b7f215e119031e.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/c528590c6415a94c.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_b0dd8a","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
 6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"LiteLLM Dashboard"}],["$","meta","3",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","4",{"rel":"icon","href":"/favicon.ico","type":"image/x-icon","sizes":"16x16"}],["$","link","5",{"rel":"icon","href":"./favicon.ico"}],["$","meta","6",{"name":"next-size-adjust"}]]
 1:null

litellm/proxy/_experimental/out/model_hub.txt

@@ -1,7 +1,7 @@
 2:I[19107,[],"ClientPageRoot"]
-3:I[52829,["50","static/chunks/50-fe160ecfa8bc4059.js","521","static/chunks/521-d97d355792d44830.js","154","static/chunks/154-fff436ed72b19a24.js","162","static/chunks/162-714ca0ed10a07f66.js","418","static/chunks/app/model_hub/page-d6e5fb7de2cedde9.js"],"default",1]
+3:I[52829,["50","static/chunks/50-fe160ecfa8bc4059.js","521","static/chunks/521-d97d355792d44830.js","154","static/chunks/154-fff436ed72b19a24.js","162","static/chunks/162-4e7640b4d68e1ae4.js","418","static/chunks/app/model_hub/page-d6e5fb7de2cedde9.js"],"default",1]
 4:I[4707,[],""]
 5:I[36423,[],""]
-0:["0GF-OyXnYlAPMWfyPAZSs",[[["",{"children":["model_hub",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],["",{"children":["model_hub",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","model_hub","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/31b7f215e119031e.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/060d5ddee53e45ce.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_b0dd8a","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
+0:["FMlZjJYLUentCU02Wj6R_",[[["",{"children":["model_hub",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],["",{"children":["model_hub",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","model_hub","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/31b7f215e119031e.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/c528590c6415a94c.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_b0dd8a","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
 6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"LiteLLM Dashboard"}],["$","meta","3",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","4",{"rel":"icon","href":"/favicon.ico","type":"image/x-icon","sizes":"16x16"}],["$","link","5",{"rel":"icon","href":"./favicon.ico"}],["$","meta","6",{"name":"next-size-adjust"}]]
 1:null

litellm/proxy/_experimental/out/model_hub_table.txt

@@ -1,7 +1,7 @@
 2:I[19107,[],"ClientPageRoot"]
-3:I[22775,["50","static/chunks/50-fe160ecfa8bc4059.js","521","static/chunks/521-d97d355792d44830.js","866","static/chunks/866-3523e0e07cf314f6.js","154","static/chunks/154-fff436ed72b19a24.js","162","static/chunks/162-714ca0ed10a07f66.js","172","static/chunks/172-0f7049c565983c4d.js","25","static/chunks/app/model_hub_table/page-e06e934de1021ee4.js"],"default",1]
+3:I[22775,["50","static/chunks/50-fe160ecfa8bc4059.js","521","static/chunks/521-d97d355792d44830.js","866","static/chunks/866-9e1803a09e9ae8da.js","154","static/chunks/154-fff436ed72b19a24.js","162","static/chunks/162-4e7640b4d68e1ae4.js","172","static/chunks/172-0f7049c565983c4d.js","25","static/chunks/app/model_hub_table/page-e06e934de1021ee4.js"],"default",1]
 4:I[4707,[],""]
 5:I[36423,[],""]
-0:["0GF-OyXnYlAPMWfyPAZSs",[[["",{"children":["model_hub_table",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],["",{"children":["model_hub_table",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","model_hub_table","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/31b7f215e119031e.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/060d5ddee53e45ce.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_b0dd8a","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
+0:["FMlZjJYLUentCU02Wj6R_",[[["",{"children":["model_hub_table",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],["",{"children":["model_hub_table",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","model_hub_table","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/31b7f215e119031e.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/c528590c6415a94c.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_b0dd8a","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
 6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"LiteLLM Dashboard"}],["$","meta","3",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","4",{"rel":"icon","href":"/favicon.ico","type":"image/x-icon","sizes":"16x16"}],["$","link","5",{"rel":"icon","href":"./favicon.ico"}],["$","meta","6",{"name":"next-size-adjust"}]]
 1:null

litellm/proxy/_experimental/out/onboarding.txt

@@ -2,6 +2,6 @@
 3:I[12011,["665","static/chunks/3014691f-b7b79b78e27792f3.js","50","static/chunks/50-fe160ecfa8bc4059.js","154","static/chunks/154-fff436ed72b19a24.js","461","static/chunks/app/onboarding/page-3c5840c907b0a5c8.js"],"default",1]
 4:I[4707,[],""]
 5:I[36423,[],""]
-0:["0GF-OyXnYlAPMWfyPAZSs",[[["",{"children":["onboarding",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],["",{"children":["onboarding",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","onboarding","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/31b7f215e119031e.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/060d5ddee53e45ce.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_b0dd8a","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
+0:["FMlZjJYLUentCU02Wj6R_",[[["",{"children":["onboarding",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],["",{"children":["onboarding",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","onboarding","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/31b7f215e119031e.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/c528590c6415a94c.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_b0dd8a","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
 6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"LiteLLM Dashboard"}],["$","meta","3",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","4",{"rel":"icon","href":"/favicon.ico","type":"image/x-icon","sizes":"16x16"}],["$","link","5",{"rel":"icon","href":"./favicon.ico"}],["$","meta","6",{"name":"next-size-adjust"}]]
 1:null

litellm/proxy/_new_secret_config.yaml

@@ -18,4 +18,4 @@ router_settings:
   model_group_alias: {"my-fake-gpt-4": "fake-openai-endpoint"}
 
 litellm_settings:
-  callbacks: ["prometheus"]
+  callbacks: ["prometheus"]

litellm/proxy/_types.py

@@ -1617,6 +1617,20 @@ class ConfigList(LiteLLMPydanticObjectBase):
     )
 
 
+class UserHeaderMapping(LiteLLMPydanticObjectBase):
+    """
+    Map an incoming HTTP header to a LiteLLM user role.
+    """
+    header_name: str
+    litellm_user_role: Literal[
+        LitellmUserRoles.INTERNAL_USER,
+        LitellmUserRoles.CUSTOMER,
+    ]
+
+    model_config = {
+        "extra": "forbid",
+    }
+
 class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
     """
     Documents all the fields supported by `general_settings` in config.yaml
@@ -1721,6 +1735,11 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
         default=None,
         description="Set-up pass-through endpoints for provider-specific endpoints. Docs - https://docs.litellm.ai/docs/proxy/pass_through",
     )
+    user_header_name: Optional[str] = Field(
+        None,
+        description="[DEPRECATED] Use 'user_header_mappings' instead. When set, the header value is treated as the end user id unless overridden by user_header_mappings.",
+    )
+    user_header_mappings: Optional[List[UserHeaderMapping]] = None
 
 
 class ConfigYAML(LiteLLMPydanticObjectBase):

litellm/proxy/auth/auth_checks.py

@@ -1211,7 +1211,6 @@ def _check_model_access_helper(
     models: List[str],
     team_model_aliases: Optional[Dict[str, str]] = None,
     team_id: Optional[str] = None,
-    object_type: Literal["user", "team", "key", "org"] = "user",
 ) -> bool:
     ## check if model in allowed model names
     from collections import defaultdict
@@ -1316,7 +1315,6 @@ def _can_object_call_model(
             models=models,
             team_model_aliases=team_model_aliases,
             team_id=team_id,
-            object_type=object_type,
         ):
             return True
 

litellm/proxy/auth/auth_utils.py

@@ -473,6 +473,22 @@ def _has_user_setup_sso():
 
     return sso_setup
 
+def get_customer_user_header_from_mapping(user_id_mapping) -> Optional[str]:
+    """Return the header_name mapped to CUSTOMER role, if any (dict-based)."""
+    if not user_id_mapping:
+        return None
+    items = user_id_mapping if isinstance(user_id_mapping, list) else [user_id_mapping]
+    for item in items:
+        if not isinstance(item, dict):
+            continue
+        role = item.get("litellm_user_role")
+        header_name = item.get("header_name")
+        if role is None or not header_name:
+            continue
+        if str(role).lower() == str(LitellmUserRoles.CUSTOMER).lower():
+            return header_name
+    return None
+
 
 def get_end_user_id_from_request_body(
     request_body: dict, request_headers: Optional[dict] = None
@@ -481,20 +497,34 @@ def get_end_user_id_from_request_body(
     # and to ensure it's fetched at runtime.
     from litellm.proxy.proxy_server import general_settings
 
-    # Check 1: Custom Header from general_settings.user_header_name (only if request_headers is provided)
+    # Check 1 : Follow the user header mappings feature, if not found, then check for deprecated user_header_name (only if request_headers is provided)
     # User query: "system not respecting user_header_name property"
     # This implies the key in general_settings is 'user_header_name'.
     if request_headers is not None:
-        user_id_header_config_key = "user_header_name"
+        custom_header_name_to_check: Optional[str] = None
 
-        custom_header_name_to_check = general_settings.get(user_id_header_config_key)
+        # Prefer user mappings (new behavior)
+        user_id_mapping = general_settings.get("user_header_mappings", None)
+        if user_id_mapping:
+            custom_header_name_to_check = get_customer_user_header_from_mapping(
+                user_id_mapping
+            )
 
-        if custom_header_name_to_check and isinstance(custom_header_name_to_check, str):
+        # Fallback to deprecated user_header_name if mapping did not specify
+        if not custom_header_name_to_check:
+            user_id_header_config_key = "user_header_name"
+            value = general_settings.get(user_id_header_config_key)
+            if isinstance(value, str) and value.strip() != "":
+                custom_header_name_to_check = value
+
+        # If we have a header name to check, try to read it from request headers
+        if isinstance(custom_header_name_to_check, str):
             for header_name, header_value in request_headers.items():
                 if header_name.lower() == custom_header_name_to_check.lower():
                     user_id_from_header = header_value
-                    if user_id_from_header.strip():
-                        return str(user_id_from_header)
+                    user_id_str = str(user_id_from_header) if user_id_from_header is not None else ""
+                    if user_id_str.strip():
+                        return user_id_str
 
     # Check 2: 'user' field in request_body (commonly OpenAI)
     if "user" in request_body and request_body["user"] is not None:

litellm/proxy/example_config_yaml/oai_misc_config.yaml

@@ -18,6 +18,19 @@ model_list:
     litellm_params:
       model: "groq/*"
       api_key: os.environ/GROQ_API_KEY
+  - model_name: bedrock/batch-anthropic.claude-3-5-sonnet-20240620-v1:0
+    litellm_params:
+      model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0
+      #########################################################
+      ########## batch specific params ########################
+      s3_bucket_name: litellm-proxy
+      s3_region_name: us-west-2
+      s3_access_key_id: os.environ/AWS_ACCESS_KEY_ID
+      s3_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
+      aws_batch_role_arn: arn:aws:iam::888602223428:role/service-role/AmazonBedrockExecutionRoleForAgents_BB9HNW6V4CV
+    model_info: 
+      mode: batch
+
 litellm_settings:
   # set_verbose: True  # Uncomment this if you want to see verbose logs; not recommended in production
   drop_params: True

litellm/proxy/guardrails/guardrail_hooks/noma/__init__.py

@@ -18,6 +18,7 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail"
         application_id=litellm_params.application_id,
         monitor_mode=litellm_params.monitor_mode,
         block_failures=litellm_params.block_failures,
+        anonymize_input=litellm_params.anonymize_input,
         event_hook=litellm_params.mode,
         default_on=litellm_params.default_on,
     )

litellm/proxy/guardrails/guardrail_hooks/noma/noma.py

@@ -5,9 +5,10 @@
 #
 # +-------------------------------------------------------------+
 
+import asyncio
 import copy
 import os
-from typing import Any, Dict, Literal, Optional, Union
+from typing import Any, Dict, Final, Literal, Optional, Union
 from urllib.parse import urljoin
 
 from fastapi import HTTPException
@@ -24,6 +25,15 @@ from litellm.proxy._types import UserAPIKeyAuth
 from litellm.types.guardrails import GuardrailEventHooks
 from litellm.types.utils import EmbeddingResponse, ImageResponse
 
+# Constants
+USER_ROLE: Final[Literal["user"]] = "user"
+ASSISTANT_ROLE: Final[Literal["assistant"]] = "assistant"
+SENSITIVE_DATA_DETECTOR_KEYS: Final[list[str]] = ["sensitiveData", "dataDetector"]
+
+# Type aliases
+MessageRole = Literal["user", "assistant"]
+LLMResponse = Union[Any, ModelResponse, EmbeddingResponse, ImageResponse]
+
 
 class NomaBlockedMessage(HTTPException):
     """Exception raised when Noma guardrail blocks a message"""
@@ -77,6 +87,7 @@ class NomaBlockedMessage(HTTPException):
                 "allowedTopics",
                 "bannedTopics",
                 "topicGuardrails",
+                "topicDetector",  # Mock name for tests
             ] and isinstance(value, dict):
                 filtered_topics = {}
                 for topic, topic_result in value.items():
@@ -86,7 +97,7 @@ class NomaBlockedMessage(HTTPException):
                 if filtered_topics:
                     result[key] = filtered_topics
 
-            elif key == "sensitiveData" and isinstance(value, dict):
+            elif key in SENSITIVE_DATA_DETECTOR_KEYS and isinstance(value, dict):
                 filtered_sensitive = {}
                 for data_type, data_result in value.items():
                     if self._is_result_true(data_result):
@@ -135,6 +146,7 @@ class NomaGuardrail(CustomGuardrail):
         application_id: Optional[str] = None,
         monitor_mode: Optional[bool] = None,
         block_failures: Optional[bool] = None,
+        anonymize_input: Optional[bool] = None,
         **kwargs,
     ):
         self.async_handler = get_async_httpx_client(
@@ -162,8 +174,326 @@ class NomaGuardrail(CustomGuardrail):
         else:
             self.block_failures = block_failures
 
+        if anonymize_input is None:
+            self.anonymize_input = (
+                os.environ.get("NOMA_ANONYMIZE_INPUT", "false").lower() == "true"
+            )
+        else:
+            self.anonymize_input = anonymize_input
+
         super().__init__(**kwargs)
 
+    def _create_background_noma_check(
+        self,
+        coro,
+    ) -> None:
+        """Create a background task for Noma API calls without blocking the main flow"""
+        try:
+            asyncio.create_task(coro)
+        except Exception as e:
+            verbose_proxy_logger.error(
+                f"Failed to create background Noma task: {str(e)}"
+            )
+
+    async def _process_user_message_check(
+        self,
+        request_data: dict,
+        user_auth: UserAPIKeyAuth,
+    ) -> Optional[str]:
+        """Shared logic for processing user message checks"""
+        extra_data = self.get_guardrail_dynamic_request_body_params(request_data)
+
+        user_message = await self._extract_user_message(request_data)
+        if not user_message:
+            return None
+
+        payload = {"request": {"text": user_message}}
+        response_json = await self._call_noma_api(
+            payload=payload,
+            llm_request_id=None,
+            request_data=request_data,
+            user_auth=user_auth,
+            extra_data=extra_data,
+        )
+
+        if self.monitor_mode:
+            await self._handle_verdict_background(
+                USER_ROLE, user_message, response_json
+            )
+            return user_message
+
+        # Check if we should anonymize content
+        if self._should_anonymize(response_json, USER_ROLE):
+            anonymized_content = self._extract_anonymized_content(
+                response_json, USER_ROLE
+            )
+            if anonymized_content:
+                # Replace the user message content with anonymized version
+                self._replace_user_message_content(request_data, anonymized_content)
+                verbose_proxy_logger.debug(
+                    f"Noma guardrail anonymized user message: {anonymized_content}"
+                )
+                return anonymized_content
+
+        await self._check_verdict(USER_ROLE, user_message, response_json)
+        return user_message
+
+    async def _process_llm_response_check(
+        self,
+        request_data: dict,
+        response: LLMResponse,
+        user_auth: UserAPIKeyAuth,
+    ) -> Optional[str]:
+        """Shared logic for processing LLM response checks"""
+        extra_data = self.get_guardrail_dynamic_request_body_params(request_data)
+
+        if not isinstance(response, litellm.ModelResponse):
+            return None
+
+        content = None
+        for choice in response.choices:
+            if isinstance(choice, litellm.Choices) and choice.message.content:
+                content = choice.message.content
+                break
+
+        if not content or not isinstance(content, str):
+            return None
+
+        payload = {"response": {"text": content}}
+
+        response_json = await self._call_noma_api(
+            payload=payload,
+            llm_request_id=response.id,
+            request_data=request_data,
+            user_auth=user_auth,
+            extra_data=extra_data,
+        )
+
+        if self.monitor_mode:
+            await self._handle_verdict_background(
+                ASSISTANT_ROLE, content, response_json
+            )
+            return content
+
+        # Check if we should anonymize content
+        if self._should_anonymize(response_json, ASSISTANT_ROLE):
+            anonymized_content = self._extract_anonymized_content(
+                response_json, ASSISTANT_ROLE
+            )
+            if anonymized_content:
+                # Replace the LLM response content with anonymized version
+                self._replace_llm_response_content(response, anonymized_content)
+                verbose_proxy_logger.debug(
+                    f"Noma guardrail anonymized LLM response: {anonymized_content}"
+                )
+                return anonymized_content
+
+        await self._check_verdict(ASSISTANT_ROLE, content, response_json)
+        return content
+
+    def _should_only_sensitive_data_failed(self, classification_obj: dict) -> bool:
+        """
+        Check if only sensitive data detectors (PII, PCI, secrets) have result=true in the classification.
+
+        Args:
+            classification_obj: The prompt or response classification object from Noma API
+
+        Returns:
+            True if only sensitiveData detectors have result=true, False otherwise
+        """
+        if not classification_obj:
+            return False
+
+        # Track which detectors have result=true (detected violations)
+        failed_detectors = []
+        sensitive_data_detected = False
+
+        for key, value in classification_obj.items():
+            if key in SENSITIVE_DATA_DETECTOR_KEYS and isinstance(value, dict):
+                # Check if any sensitive data detector has result=true
+                for data_type, data_result in value.items():
+                    if self._is_result_true(data_result):
+                        sensitive_data_detected = True
+                        # Don't add to failed_detectors as we want to allow these
+
+            elif isinstance(value, dict) and "result" in value:
+                # Check other detectors - these should NOT have result=true
+                if self._is_result_true(value):
+                    failed_detectors.append(key)
+
+            elif isinstance(value, dict):
+                # Handle nested detectors
+                for nested_key, nested_value in value.items():
+                    if self._is_result_true(nested_value):
+                        failed_detectors.append(f"{key}.{nested_key}")
+
+        # Return True only if sensitive data was detected AND no other detectors have result=true
+        return sensitive_data_detected and len(failed_detectors) == 0
+
+    def _extract_anonymized_content(
+        self, response_json: dict, message_type: MessageRole
+    ) -> Optional[str]:
+        """
+        Extract anonymized content from Noma API response.
+
+        Args:
+            response_json: The full response from Noma API
+            message_type: Either 'user' or 'assistant' to determine which content to extract
+
+        Returns:
+            The anonymized content string if available, None otherwise
+        """
+        original_response = response_json.get("originalResponse", {})
+
+        if message_type == USER_ROLE:
+            prompt_data = original_response.get("prompt", {})
+            anonymized_data = prompt_data.get("anonymizedContent", {})
+            return anonymized_data.get("anonymized")
+        elif message_type == ASSISTANT_ROLE:
+            response_data = original_response.get("response", {})
+            anonymized_data = response_data.get("anonymizedContent", {})
+            return anonymized_data.get("anonymized")
+
+        return None
+
+    def _should_anonymize(self, response_json: dict, message_type: MessageRole) -> bool:
+        """
+        Determine if content should be anonymized based on Noma API response.
+
+        Logic:
+        - If verdict=True: Content is safe, anonymize if anonymized version exists
+        - If verdict=False: Check if only sensitiveData detectors have result=True
+          - If yes: Anonymize
+          - If no: Block (other violations detected)
+
+        Args:
+            response_json: The full response from Noma API
+            message_type: Either 'user' or 'assistant' to determine which classification to check
+
+        Returns:
+            True if content should be anonymized, False if it should be blocked
+        """
+        # Only anonymize in blocking mode when anonymize_input is enabled
+        if self.monitor_mode or not self.anonymize_input:
+            return False
+
+        verdict = response_json.get("verdict", True)
+        # If verdict is True, anonymize (content is considered safe)
+        if verdict:
+            return True
+
+        # If verdict is False, check if only sensitive data detectors have result=True
+        original_response = response_json.get("originalResponse", {})
+
+        if message_type == USER_ROLE:
+            classification_obj = original_response.get("prompt", {})
+        elif message_type == ASSISTANT_ROLE:
+            classification_obj = original_response.get("response", {})
+        else:
+            return False
+
+        # Anonymize only if solely sensitive data (PII/PCI/secrets) was detected
+        return self._should_only_sensitive_data_failed(classification_obj)
+
+    def _is_result_true(self, result_obj: Optional[Dict[str, Any]]) -> bool:
+        """
+        Check if a result object has a "result" field that is True.
+
+        Args:
+            result_obj: A dictionary that may contain a "result" field
+
+        Returns:
+            True if the "result" field exists and is True, False otherwise
+        """
+        if not result_obj or not isinstance(result_obj, dict):
+            return False
+
+        return result_obj.get("result") is True
+
+    def _replace_user_message_content(
+        self, request_data: dict, anonymized_content: str
+    ):
+        """
+        Replace the user message content in request data with anonymized version.
+
+        Args:
+            request_data: The original request data
+            anonymized_content: The anonymized content to replace with
+        """
+        messages = request_data.get("messages", [])
+        if not messages:
+            return
+
+        # Find and replace the last user message
+        for i in range(len(messages) - 1, -1, -1):
+            if messages[i].get("role") == USER_ROLE:
+                messages[i]["content"] = anonymized_content
+                break
+
+    def _replace_llm_response_content(
+        self, response: LLMResponse, anonymized_content: str
+    ):
+        """
+        Replace the LLM response content with anonymized version.
+
+        Args:
+            response: The original LLM response
+            anonymized_content: The anonymized content to replace with
+        """
+        if not isinstance(response, litellm.ModelResponse):
+            return
+
+        # Replace content in all choices
+        for choice in response.choices:
+            if isinstance(choice, litellm.Choices) and choice.message.content:
+                choice.message.content = anonymized_content
+
+    async def _check_user_message_background(
+        self,
+        request_data: dict,
+        user_auth: UserAPIKeyAuth,
+    ) -> None:
+        """Check user message in background for monitor mode - non-blocking"""
+        try:
+            await self._process_user_message_check(request_data, user_auth)
+        except Exception as e:
+            verbose_proxy_logger.error(
+                f"Noma background user message check failed: {str(e)}"
+            )
+
+    async def _check_llm_response_background(
+        self,
+        request_data: dict,
+        response: LLMResponse,
+        user_auth: UserAPIKeyAuth,
+    ) -> None:
+        """Check LLM response in background for monitor mode - non-blocking"""
+        try:
+            await self._process_llm_response_check(request_data, response, user_auth)
+        except Exception as e:
+            verbose_proxy_logger.error(
+                f"Noma background response check failed: {str(e)}"
+            )
+
+    async def _handle_verdict_background(
+        self,
+        type: MessageRole,
+        message: str,
+        response_json: dict,
+    ) -> None:
+        """Handle verdict from Noma API in background - logging only, never blocks"""
+        try:
+            if not response_json.get("verdict", True):
+                msg = f"Noma guardrail blocked {type} message: {message}"
+                verbose_proxy_logger.warning(msg)
+            else:
+                msg = f"Noma guardrail allowed {type} message: {message}"
+                verbose_proxy_logger.info(msg)
+        except Exception as e:
+            verbose_proxy_logger.error(
+                f"Noma background verdict handling failed: {str(e)}"
+            )
+
     async def async_pre_call_hook(
         self,
         user_api_key_dict: UserAPIKeyAuth,
@@ -191,6 +521,18 @@ class NomaGuardrail(CustomGuardrail):
         ):
             return data
 
+        # In monitor mode, run Noma check in background and return immediately
+        if self.monitor_mode:
+            try:
+                self._create_background_noma_check(
+                    self._check_user_message_background(data, user_api_key_dict)
+                )
+            except Exception as e:
+                verbose_proxy_logger.error(
+                    f"Failed to start background Noma pre-call check: {str(e)}"
+                )
+            return data
+
         try:
             return await self._check_user_message(data, user_api_key_dict)
         except NomaBlockedMessage:
@@ -198,7 +540,7 @@ class NomaGuardrail(CustomGuardrail):
         except Exception as e:
             verbose_proxy_logger.error(f"Noma pre-call hook failed: {str(e)}")
 
-            if self.block_failures and not self.monitor_mode:
+            if self.block_failures:
                 raise
             return data
 
@@ -220,6 +562,18 @@ class NomaGuardrail(CustomGuardrail):
         if self.should_run_guardrail(data=data, event_type=event_type) is not True:
             return data
 
+        # In monitor mode, run Noma check in background and return immediately
+        if self.monitor_mode:
+            try:
+                self._create_background_noma_check(
+                    self._check_user_message_background(data, user_api_key_dict)
+                )
+            except Exception as e:
+                verbose_proxy_logger.error(
+                    f"Failed to start background Noma moderation check: {str(e)}"
+                )
+            return data
+
         try:
             return await self._check_user_message(data, user_api_key_dict)
         except NomaBlockedMessage:
@@ -227,7 +581,7 @@ class NomaGuardrail(CustomGuardrail):
         except Exception as e:
             verbose_proxy_logger.error(f"Noma moderation hook failed: {str(e)}")
 
-            if self.block_failures and not self.monitor_mode:
+            if self.block_failures:
                 raise
             return data
 
@@ -235,19 +589,33 @@ class NomaGuardrail(CustomGuardrail):
         self,
         data: dict,
         user_api_key_dict: UserAPIKeyAuth,
-        response: Union[Any, ModelResponse, EmbeddingResponse, ImageResponse],
+        response: LLMResponse,
     ):
         event_type: GuardrailEventHooks = GuardrailEventHooks.post_call
         if self.should_run_guardrail(data=data, event_type=event_type) is not True:
             return response
 
+        # In monitor mode, run Noma check in background and return immediately
+        if self.monitor_mode:
+            try:
+                self._create_background_noma_check(
+                    self._check_llm_response_background(
+                        data, response, user_api_key_dict
+                    )
+                )
+            except Exception as e:
+                verbose_proxy_logger.error(
+                    f"Failed to start background Noma post-call check: {str(e)}"
+                )
+            return response
+
         try:
             return await self._check_llm_response(data, response, user_api_key_dict)
         except NomaBlockedMessage:
             raise
         except Exception as e:
             verbose_proxy_logger.error(f"Noma post-call hook failed: {str(e)}")
-            if self.block_failures and not self.monitor_mode:
+            if self.block_failures:
                 raise
             return response
 
@@ -257,55 +625,24 @@ class NomaGuardrail(CustomGuardrail):
         user_auth: UserAPIKeyAuth,
     ) -> Union[Exception, str, dict, None]:
         """Check user message for policy violations"""
-        extra_data = self.get_guardrail_dynamic_request_body_params(request_data)
-
-        user_message = await self._extract_user_message(request_data)
+        user_message = await self._process_user_message_check(request_data, user_auth)
         if not user_message:
             return request_data
 
-        payload = {"request": {"text": user_message}}
-        response_json = await self._call_noma_api(
-            payload=payload,
-            llm_request_id=None,
-            request_data=request_data,
-            user_auth=user_auth,
-            extra_data=extra_data,
-        )
-        await self._check_verdict("user", user_message, response_json)
-
         return request_data
 
     async def _check_llm_response(
         self,
         request_data: dict,
-        response: Union[Any, ModelResponse, EmbeddingResponse, ImageResponse],
+        response: LLMResponse,
         user_auth: UserAPIKeyAuth,
     ) -> Union[Exception, ModelResponse, Any]:
         """Check LLM response for policy violations"""
-        extra_data = self.get_guardrail_dynamic_request_body_params(request_data)
-
-        if not isinstance(response, litellm.ModelResponse):
-            return response
-
-        content = None
-        for choice in response.choices:
-            if isinstance(choice, litellm.Choices) and choice.message.content:
-                content = choice.message.content
-                break
-
-        if not content or not isinstance(content, str):
-            return response
-
-        payload = {"response": {"text": content}}
-
-        response_json = await self._call_noma_api(
-            payload=payload,
-            llm_request_id=response.id,
-            request_data=request_data,
-            user_auth=user_auth,
-            extra_data=extra_data,
+        content = await self._process_llm_response_check(
+            request_data, response, user_auth
         )
-        await self._check_verdict("assistant", content, response_json)
+        if not content:
+            return response
 
         return response
 
@@ -316,7 +653,7 @@ class NomaGuardrail(CustomGuardrail):
             return None
 
         # Get the last user message
-        user_messages = [msg for msg in messages if msg.get("role") == "user"]
+        user_messages = [msg for msg in messages if msg.get("role") == USER_ROLE]
         if not user_messages:
             return None
 
@@ -371,7 +708,7 @@ class NomaGuardrail(CustomGuardrail):
 
     async def _check_verdict(
         self,
-        type: Literal["user", "assistant"],
+        type: MessageRole,
         message: str,
         response_json: dict,
     ) -> None:
@@ -379,11 +716,7 @@ class NomaGuardrail(CustomGuardrail):
         Check the verdict from the Noma API and raise an exception if needed
         """
         if not response_json.get("verdict", True):
-            msg = str.format(
-                "Noma guardrail blocked {type} message: {message}",
-                type=type,
-                message=message,
-            )
+            msg = f"Noma guardrail blocked {type} message: {message}"
 
             if self.monitor_mode:
                 verbose_proxy_logger.warning(msg)
@@ -392,11 +725,7 @@ class NomaGuardrail(CustomGuardrail):
                 original_response = response_json.get("originalResponse", {})
                 raise NomaBlockedMessage(original_response)
         else:
-            msg = str.format(
-                "Noma guardrail allowed {type} message: {message}",
-                type=type,
-                message=message,
-            )
+            msg = f"Noma guardrail allowed {type} message: {message}"
             if self.monitor_mode:
                 verbose_proxy_logger.info(msg)
             else:

litellm/proxy/hooks/parallel_request_limiter_v3.py

@@ -6,6 +6,7 @@ This is currently in development and not yet ready for production.
 
 import os
 from datetime import datetime
+from math import floor
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -17,7 +18,7 @@ from typing import (
     Union,
     cast,
 )
-from math import floor
+
 from fastapi import HTTPException
 
 from litellm import DualCache
@@ -95,6 +96,7 @@ end
 return results
 """
 
+
 class RateLimitDescriptorRateLimitObject(TypedDict, total=False):
     requests_per_unit: Optional[int]
     tokens_per_unit: Optional[int]
@@ -266,7 +268,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
             if current_limit is None or rate_limit_type is None:
                 continue
 
-            if counter_value is not None and int(counter_value) + 1 > current_limit:
+            if counter_value is not None and int(counter_value) > current_limit:
                 overall_code = "OVER_LIMIT"
                 item_code = "OVER_LIMIT"
 
@@ -480,10 +482,15 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
                     },
                 )
             )
-        
+
         # Team Member rate limits
-        if user_api_key_dict.user_id and (user_api_key_dict.team_member_rpm_limit is not None or user_api_key_dict.team_member_tpm_limit is not None):
-            team_member_value = f"{user_api_key_dict.team_id}:{user_api_key_dict.user_id}"
+        if user_api_key_dict.user_id and (
+            user_api_key_dict.team_member_rpm_limit is not None
+            or user_api_key_dict.team_member_tpm_limit is not None
+        ):
+            team_member_value = (
+                f"{user_api_key_dict.team_id}:{user_api_key_dict.user_id}"
+            )
             descriptors.append(
                 RateLimitDescriptor(
                     key="team_member",
@@ -557,13 +564,13 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
                 # Find which descriptor hit the limit
                 for i, status in enumerate(response["statuses"]):
                     if status["code"] == "OVER_LIMIT":
-                        descriptor = descriptors[floor(i/2)]
+                        descriptor = descriptors[floor(i / 2)]
                         raise HTTPException(
                             status_code=429,
                             detail=f"Rate limit exceeded for {descriptor['key']}: {descriptor['value']}. Remaining: {status['limit_remaining']}",
                             headers={
                                 "retry-after": str(self.window_size),
-                                "rate_limit_type": str(status["rate_limit_type"])
+                                "rate_limit_type": str(status["rate_limit_type"]),
                             },  # Retry after 1 minute
                         )
 
@@ -613,7 +620,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
 
         # Check if script is available
         if self.token_increment_script is None:
-            verbose_proxy_logger.debug("TTL preservation script not available, using regular pipeline")
+            verbose_proxy_logger.debug(
+                "TTL preservation script not available, using regular pipeline"
+            )
             await self.internal_usage_cache.dual_cache.async_increment_cache_pipeline(
                 increment_list=pipeline_operations,
                 litellm_parent_otel_span=parent_otel_span,
@@ -628,7 +637,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
             for op in pipeline_operations:
                 # Convert None TTL to 0 for Lua script
                 ttl_value = op["ttl"] if op["ttl"] is not None else 0
-                
+
                 verbose_proxy_logger.debug(
                     f"Executing TTL-preserving increment for key={op['key']}, "
                     f"increment={op['increment_value']}, ttl={ttl_value}"
@@ -693,16 +702,15 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
             )
 
             # Get metadata from kwargs
-            user_api_key = kwargs["litellm_params"]["metadata"].get("user_api_key")
-            user_api_key_user_id = kwargs["litellm_params"]["metadata"].get(
-                "user_api_key_user_id"
+            litellm_metadata = kwargs["litellm_params"]["metadata"]
+            if litellm_metadata is None:
+                return
+            user_api_key = litellm_metadata.get("user_api_key")
+            user_api_key_user_id = litellm_metadata.get("user_api_key_user_id")
+            user_api_key_team_id = litellm_metadata.get("user_api_key_team_id")
+            user_api_key_end_user_id = kwargs.get("user") or litellm_metadata.get(
+                "user_api_key_end_user_id"
             )
-            user_api_key_team_id = kwargs["litellm_params"]["metadata"].get(
-                "user_api_key_team_id"
-            )
-            user_api_key_end_user_id = kwargs.get("user") or kwargs["litellm_params"][
-                "metadata"
-            ].get("user_api_key_end_user_id")
             model_group = get_model_group_from_litellm_kwargs(kwargs)
 
             # Get total tokens from response

litellm/proxy/litellm_pre_call_utils.py

@@ -14,6 +14,7 @@ from litellm.proxy._types import (
     AddTeamCallback,
     CommonProxyErrors,
     LitellmDataForBackendLLMCall,
+    LitellmUserRoles,
     SpecialHeaders,
     TeamCallbackMetadata,
     UserAPIKeyAuth,
@@ -271,7 +272,7 @@ class LiteLLMProxyRequestSetup:
         if timeout_header is not None:
             return float(timeout_header)
         return None
-    
+
     @staticmethod
     def _get_stream_timeout_from_request(headers: dict) -> Optional[float]:
         """
@@ -291,13 +292,14 @@ class LiteLLMProxyRequestSetup:
         if num_retries_header is not None:
             return int(num_retries_header)
         return None
-    
+
     @staticmethod
     def _get_spend_logs_metadata_from_request_headers(headers: dict) -> Optional[dict]:
         """
         Get the `spend_logs_metadata` from the request headers.
         """
         from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
+
         spend_logs_metadata_header = headers.get("x-litellm-spend-logs-metadata", None)
         if spend_logs_metadata_header is not None:
             return safe_json_loads(spend_logs_metadata_header)
@@ -335,6 +337,30 @@ class LiteLLMProxyRequestSetup:
                 return value
         return None
 
+    @staticmethod
+    def add_internal_user_from_user_mapping(
+        general_settings: Optional[Dict],
+        user_api_key_dict: UserAPIKeyAuth,
+        headers: dict,
+    ) -> UserAPIKeyAuth:
+        if general_settings is None:
+            return user_api_key_dict
+        user_header_mapping = general_settings.get("user_header_mappings")
+        if not user_header_mapping:
+            return user_api_key_dict
+        header_name = LiteLLMProxyRequestSetup.get_internal_user_header_from_mapping(
+            user_header_mapping
+        )
+        if not header_name:
+            return user_api_key_dict
+        header_value = LiteLLMProxyRequestSetup._get_case_insensitive_header(
+            headers, header_name
+        )
+        if header_value:
+            user_api_key_dict.user_id = header_value
+            return user_api_key_dict
+        return user_api_key_dict
+
     @staticmethod
     def get_user_from_headers(
         headers: dict, general_settings: Optional[Dict] = None
@@ -412,15 +438,25 @@ class LiteLLMProxyRequestSetup:
         """
         Add headers to the LLM call by model group
         """
+        from litellm.proxy.auth.auth_checks import _check_model_access_helper
+        from litellm.proxy.proxy_server import llm_router
+
         data_model = data.get("model")
+
         if (
             data_model is not None
             and litellm.model_group_settings is not None
             and litellm.model_group_settings.forward_client_headers_to_llm_api
             is not None
-            and data_model
-            in litellm.model_group_settings.forward_client_headers_to_llm_api
+            and _check_model_access_helper(
+                model=data_model,
+                llm_router=llm_router,
+                models=litellm.model_group_settings.forward_client_headers_to_llm_api,
+                team_model_aliases=user_api_key_dict.team_model_aliases,
+                team_id=user_api_key_dict.team_id,
+            )  # handles aliases, wildcards, etc.
         ):
+
             _headers = LiteLLMProxyRequestSetup.add_headers_to_llm_call(
                 headers, user_api_key_dict
             )
@@ -428,6 +464,26 @@ class LiteLLMProxyRequestSetup:
                 data["headers"] = _headers
         return data
 
+    @staticmethod
+    def get_internal_user_header_from_mapping(user_header_mapping) -> Optional[str]:
+        if not user_header_mapping:
+            return None
+        items = (
+            user_header_mapping
+            if isinstance(user_header_mapping, list)
+            else [user_header_mapping]
+        )
+        for item in items:
+            if not isinstance(item, dict):
+                continue
+            role = item.get("litellm_user_role")
+            header_name = item.get("header_name")
+            if role is None or not header_name:
+                continue
+            if str(role).lower() == str(LitellmUserRoles.INTERNAL_USER).lower():
+                return header_name
+        return None
+
     @staticmethod
     def add_litellm_data_for_backend_llm_call(
         *,
@@ -460,8 +516,10 @@ class LiteLLMProxyRequestSetup:
         timeout = LiteLLMProxyRequestSetup._get_timeout_from_request(headers)
         if timeout is not None:
             data["timeout"] = timeout
-        
-        stream_timeout = LiteLLMProxyRequestSetup._get_stream_timeout_from_request(headers)
+
+        stream_timeout = LiteLLMProxyRequestSetup._get_stream_timeout_from_request(
+            headers
+        )
         if stream_timeout is not None:
             data["stream_timeout"] = stream_timeout
 
@@ -470,7 +528,7 @@ class LiteLLMProxyRequestSetup:
             data["num_retries"] = num_retries
 
         return data
-    
+
     @staticmethod
     def add_litellm_metadata_from_request_headers(
         headers: dict,
@@ -483,11 +541,16 @@ class LiteLLMProxyRequestSetup:
         Relevant issue: https://github.com/BerriAI/litellm/issues/14008
         """
         from litellm.proxy._types import LitellmMetadataFromRequestHeaders
+
         metadata_from_headers = LitellmMetadataFromRequestHeaders()
-        spend_logs_metadata = LiteLLMProxyRequestSetup._get_spend_logs_metadata_from_request_headers(headers)
+        spend_logs_metadata = (
+            LiteLLMProxyRequestSetup._get_spend_logs_metadata_from_request_headers(
+                headers
+            )
+        )
         if spend_logs_metadata is not None:
             metadata_from_headers["spend_logs_metadata"] = spend_logs_metadata
-        
+
         #########################################################################################
         # Finally update the requests metadata with the `metadata_from_headers`
         #########################################################################################
@@ -677,7 +740,6 @@ async def add_litellm_data_to_request(  # noqa: PLR0915
     from litellm.proxy.proxy_server import llm_router, premium_user
     from litellm.types.proxy.litellm_pre_call_utils import SecretFields
 
-
     _headers = clean_headers(
         request.headers,
         litellm_key_header_name=(
@@ -703,8 +765,6 @@ async def add_litellm_data_to_request(  # noqa: PLR0915
     if data.get(_metadata_variable_name, None) is None:
         data[_metadata_variable_name] = {}
 
-
-
     data.update(
         LiteLLMProxyRequestSetup.add_litellm_data_for_backend_llm_call(
             headers=_headers,
@@ -726,6 +786,10 @@ async def add_litellm_data_to_request(  # noqa: PLR0915
         data=data, headers=_headers, user_api_key_dict=user_api_key_dict
     )
 
+    user_api_key_dict = LiteLLMProxyRequestSetup.add_internal_user_from_user_mapping(
+        general_settings, user_api_key_dict, _headers
+    )
+
     # Parse user info from headers
     user = LiteLLMProxyRequestSetup.get_user_from_headers(_headers, general_settings)
     if user is not None:
@@ -734,7 +798,6 @@ async def add_litellm_data_to_request(  # noqa: PLR0915
         if "user" not in data:
             data["user"] = user
 
-
     data["secret_fields"] = SecretFields(raw_headers=dict(request.headers))
 
     ## Dynamic api version (Azure OpenAI endpoints) ##

litellm/proxy/management_endpoints/key_management_endpoints.py

@@ -346,6 +346,7 @@ def handle_key_type(data: GenerateKeyRequest, data_json: dict) -> dict:
         data_json["allowed_routes"] = ["info_routes"]
     return data_json
 
+
 async def validate_team_id_used_in_service_account_request(
     team_id: Optional[str],
     prisma_client: Optional[PrismaClient],
@@ -358,13 +359,13 @@ async def validate_team_id_used_in_service_account_request(
             status_code=400,
             detail="team_id is required for service account keys. Please specify `team_id` in the request body.",
         )
-    
+
     if prisma_client is None:
         raise HTTPException(
             status_code=400,
             detail="prisma_client is required for service account keys. Please specify `prisma_client` in the request body.",
         )
-    
+
     # check if team_id exists in the database
     team = await prisma_client.db.litellm_teamtable.find_unique(
         where={"team_id": team_id},
@@ -376,6 +377,7 @@ async def validate_team_id_used_in_service_account_request(
         )
     return True
 
+
 async def _common_key_generation_helper(  # noqa: PLR0915
     data: GenerateKeyRequest,
     user_api_key_dict: UserAPIKeyAuth,
@@ -551,6 +553,15 @@ async def _common_key_generation_helper(  # noqa: PLR0915
         prisma_client=prisma_client,
     )
 
+    # Validate user-provided key format
+    if data.key is not None and not data.key.startswith("sk-"):
+        raise HTTPException(
+            status_code=400,
+            detail={
+                "error": f"Invalid key format. LiteLLM Virtual Key must start with 'sk-'. Received: {data.key}"
+            },
+        )
+
     response = await generate_key_helper_fn(
         request_type="key", **data_json, table_name="key"
     )
@@ -2876,7 +2887,10 @@ async def unblock_key(
             param="key",
             code=status.HTTP_400_BAD_REQUEST,
         )
-    hashed_token = hash_token(token=data.key)
+    if data.key.startswith("sk-"):
+        hashed_token = hash_token(token=data.key)
+    else:
+        hashed_token = data.key
 
     if litellm.store_audit_logs is True:
         # make an audit log for key update

litellm/proxy/management_endpoints/mcp_management_endpoints.py

@@ -17,8 +17,8 @@ Endpoints here:
 """
 
 import importlib
-from typing import Iterable, List, Optional
 from datetime import datetime
+from typing import Iterable, List, Optional
 
 from fastapi import APIRouter, Depends, Header, HTTPException, Response, status
 from fastapi.responses import JSONResponse
@@ -26,7 +26,9 @@ from fastapi.responses import JSONResponse
 import litellm
 from litellm._logging import verbose_logger, verbose_proxy_logger
 from litellm.constants import LITELLM_PROXY_ADMIN_NAME
-from litellm.proxy._experimental.mcp_server.utils import validate_and_normalize_mcp_server_payload
+from litellm.proxy._experimental.mcp_server.utils import (
+    validate_and_normalize_mcp_server_payload,
+)
 
 router = APIRouter(prefix="/v1/mcp", tags=["mcp"])
 MCP_AVAILABLE: bool = True
@@ -94,34 +96,17 @@ if MCP_AVAILABLE:
         """
         Get all MCP tools available for the current key, including those from access groups
         """
-        from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import (
-            MCPRequestHandler,
-        )
-        from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
-            global_mcp_server_manager,
+        from litellm.proxy._experimental.mcp_server.server import _list_mcp_tools
+        tools = await _list_mcp_tools(
+            user_api_key_auth=user_api_key_dict,
+            mcp_auth_header=None,
+            mcp_servers=None,
+            mcp_server_auth_headers=None,
+            mcp_protocol_version=None,
         )
+        dumped_tools = [dict(tool) for tool in tools]
 
-        # This now includes both direct and access group servers
-        server_ids = await MCPRequestHandler._get_allowed_mcp_servers_for_key(user_api_key_dict)
-
-        tools = []
-        errors = []
-        for server_id in server_ids:
-            try:
-                server_tools = await global_mcp_server_manager.get_tools_for_server(server_id)
-                tools.extend(server_tools)
-                verbose_proxy_logger.debug(f"Successfully fetched {len(server_tools)} tools from server {server_id}")
-            except Exception as e:
-                error_msg = f"Failed to get tools from server {server_id}: {str(e)}"
-                verbose_proxy_logger.warning(error_msg)
-                errors.append(error_msg)
-                # Continue with other servers instead of failing completely
-
-        verbose_proxy_logger.debug(f"Available tools: {tools}")
-        if errors:
-            verbose_proxy_logger.warning(f"Some servers failed to respond: {errors}")
-
-        return {"tools": tools}
+        return {"tools": dumped_tools}
 
     @router.get(
         "/access_groups",
@@ -134,8 +119,10 @@ if MCP_AVAILABLE:
         """
         Get all available MCP access groups from the database AND config
         """
+        from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
+            global_mcp_server_manager,
+        )
         from litellm.proxy.proxy_server import prisma_client
-        from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager
 
         access_groups = set()
 

litellm/proxy/proxy_config.yaml

@@ -1,7 +1,13 @@
 model_list:
-  - model_name: db-openai-endpoint
+  - model_name: bedrock/batch-anthropic.claude-3-5-sonnet-20240620-v1:0
     litellm_params:
-      model: openai/*
-      api_base: https://exampleopenaiendpoint-production-0ee2.up.railway.app/
-litellm_settings:
-  callbacks: ["cloudzero"]
+      model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0
+      #########################################################
+      ########## batch specific params ########################
+      s3_bucket_name: litellm-proxy
+      s3_region_name: us-west-2
+      s3_access_key_id: os.environ/AWS_ACCESS_KEY_ID
+      s3_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
+      aws_batch_role_arn: arn:aws:iam::888602223428:role/service-role/AmazonBedrockExecutionRoleForAgents_BB9HNW6V4CV
+    model_info: 
+      mode: batch

litellm/proxy/route_llm_request.py

@@ -85,6 +85,7 @@ async def route_request(
     """
     team_id = get_team_id_from_data(data)
     router_model_names = llm_router.model_names if llm_router is not None else []
+
     if "api_key" in data or "api_base" in data:
         if llm_router is not None:
             return getattr(llm_router, f"{route_type}")(**data)
@@ -123,24 +124,20 @@ async def route_request(
             data["model"] in router_model_names
             or data["model"] in llm_router.get_model_ids()
         ):
-
             return getattr(llm_router, f"{route_type}")(**data)
 
         elif (
             llm_router.model_group_alias is not None
             and data["model"] in llm_router.model_group_alias
         ):
-
             return getattr(llm_router, f"{route_type}")(**data)
 
         elif data["model"] in llm_router.deployment_names:
-
             return getattr(llm_router, f"{route_type}")(
                 **data, specific_deployment=True
             )
 
         elif data["model"] not in router_model_names:
-
             if llm_router.router_general_settings.pass_through_all_models:
                 return getattr(litellm, f"{route_type}")(**data)
             elif (
@@ -162,7 +159,6 @@ async def route_request(
     elif user_model is not None:
         return getattr(litellm, f"{route_type}")(**data)
     elif route_type == "allm_passthrough_route":
-
         return getattr(litellm, f"{route_type}")(**data)
 
     # if no route found then it's a bad request

litellm/proxy/spend_tracking/spend_management_endpoints.py

@@ -10,6 +10,7 @@ from fastapi import APIRouter, Depends, HTTPException, status
 
 import litellm
 from litellm._logging import verbose_proxy_logger
+from litellm.router_strategy.budget_limiter import RouterBudgetLimiting
 from litellm.proxy._types import *
 from litellm.proxy._types import ProviderBudgetResponse, ProviderBudgetResponseObject
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
@@ -1659,6 +1660,9 @@ async def ui_view_spend_logs(  # noqa: PLR0915
     model: Optional[str] = fastapi.Query(
         default=None, description="Filter logs by model"
     ),
+    key_alias: Optional[str] = fastapi.Query(
+        default=None, description="Filter logs by key alias"
+    ),
 ):
     """
     View spend logs for UI with pagination support
@@ -1726,6 +1730,12 @@ async def ui_view_spend_logs(  # noqa: PLR0915
 
         if model is not None:
             where_conditions["model"] = model
+        
+        if key_alias is not None:
+            where_conditions["metadata"] = {
+                "path": ["user_api_key_alias"],
+                "string_contains": key_alias
+            }
 
         if min_spend is not None or max_spend is not None:
             where_conditions["spend"] = {}
@@ -2765,16 +2775,23 @@ async def provider_budgets() -> ProviderBudgetResponse:
 
         provider_budget_response_dict: Dict[str, ProviderBudgetResponseObject] = {}
         for _provider, _budget_info in provider_budget_config.items():
-            if llm_router.router_budget_logger is None:
+            router_budget_logger = next(
+                (
+                    cb
+                    for cb in (llm_router.optional_callbacks or [])
+                    if isinstance(cb, RouterBudgetLimiting)
+                ),
+                None,
+            )
+            if router_budget_logger is None:
                 raise ValueError("No router budget logger found")
             _provider_spend = (
-                await llm_router.router_budget_logger._get_current_provider_spend(
+                await router_budget_logger._get_current_provider_spend(_provider) or 0.0
+            )
+            _provider_budget_ttl = (
+                await router_budget_logger._get_current_provider_budget_reset_at(
                     _provider
                 )
-                or 0.0
-            )
-            _provider_budget_ttl = await llm_router.router_budget_logger._get_current_provider_budget_reset_at(
-                _provider
             )
             provider_budget_response_object = ProviderBudgetResponseObject(
                 budget_limit=_budget_info.max_budget,

litellm/responses/main.py

@@ -1,12 +1,24 @@
 import asyncio
 import contextvars
 from functools import partial
-from typing import Any, Coroutine, Dict, Iterable, List, Literal, Optional, Type, Union
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Coroutine,
+    Dict,
+    Iterable,
+    List,
+    Literal,
+    Optional,
+    Type,
+    Union,
+)
 
 import httpx
 from pydantic import BaseModel
 
 import litellm
+from litellm._logging import verbose_logger
 from litellm.constants import request_timeout
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
@@ -22,15 +34,27 @@ from litellm.types.llms.openai import (
     ResponseInputParam,
     ResponsesAPIOptionalRequestParams,
     ResponsesAPIResponse,
-    ResponseText,
     ToolChoice,
     ToolParam,
 )
+
+# Handle ResponseText import with fallback
+if TYPE_CHECKING:
+    from litellm.types.llms.openai import ResponseText
+else:
+    ResponseText = str  # Fallback for ResponseText import
 from litellm.types.responses.main import *
 from litellm.types.router import GenericLiteLLMParams
 from litellm.utils import ProviderConfigManager, client
 
-from .streaming_iterator import BaseResponsesAPIStreamingIterator
+if TYPE_CHECKING:
+    from mcp.types import Tool as MCPTool
+else:
+    MCPTool = Any
+
+from .streaming_iterator import (
+    BaseResponsesAPIStreamingIterator,
+)
 
 ####### ENVIRONMENT VARIABLES ###################
 # Initialize any necessary instances or variables here
@@ -141,17 +165,15 @@ async def aresponses_api_with_mcp(
         other_tools,
     ) = LiteLLM_Proxy_MCP_Handler._parse_mcp_tools(tools)
 
-    # Get available tools from MCP manager if we have MCP tools
-    openai_tools = []
-    mcp_tools_fetched = []
-    if mcp_tools_with_litellm_proxy:
-        user_api_key_auth = kwargs.get("user_api_key_auth")
-        mcp_tools_fetched = await LiteLLM_Proxy_MCP_Handler._get_mcp_tools_from_manager(
-            user_api_key_auth
-        )
-        openai_tools = LiteLLM_Proxy_MCP_Handler._transform_mcp_tools_to_openai(
-            mcp_tools_fetched
-        )
+    # Process MCP tools through the complete pipeline (fetch + filter + deduplicate + transform)
+    user_api_key_auth = kwargs.get("user_api_key_auth")
+    
+    # Get original MCP tools (for events) and OpenAI tools (for LLM) by reusing existing methods
+    original_mcp_tools = await LiteLLM_Proxy_MCP_Handler._process_mcp_tools_without_openai_transform(
+        user_api_key_auth=user_api_key_auth,
+        mcp_tools_with_litellm_proxy=mcp_tools_with_litellm_proxy
+    )
+    openai_tools = LiteLLM_Proxy_MCP_Handler._transform_mcp_tools_to_openai(original_mcp_tools)
 
     # Combine with other tools
     all_tools = openai_tools + other_tools if (openai_tools or other_tools) else None
@@ -182,23 +204,68 @@ async def aresponses_api_with_mcp(
         **kwargs,
     }
 
+    # Handle MCP streaming if requested
+    if stream and mcp_tools_with_litellm_proxy:
+        # Generate MCP discovery events using the already processed tools
+        import uuid
+
+        from litellm.responses.mcp.mcp_streaming_iterator import (
+            create_mcp_list_tools_events,
+        )
+        
+        base_item_id = f"mcp_{uuid.uuid4().hex[:8]}"
+        mcp_discovery_events = await create_mcp_list_tools_events(
+            mcp_tools_with_litellm_proxy=mcp_tools_with_litellm_proxy,
+            user_api_key_auth=user_api_key_auth,
+            base_item_id=base_item_id,
+            pre_processed_mcp_tools=original_mcp_tools
+        )
+        
+        return LiteLLM_Proxy_MCP_Handler._create_mcp_streaming_response(
+            input=input,
+            model=model,
+            all_tools=all_tools,
+            mcp_tools_with_litellm_proxy=mcp_tools_with_litellm_proxy,
+            mcp_discovery_events=mcp_discovery_events,
+            call_params=call_params,
+            previous_response_id=previous_response_id,
+            **kwargs
+        )
+    
+    # Determine if we should auto-execute tools
+    should_auto_execute = (
+        bool(mcp_tools_with_litellm_proxy)
+        and LiteLLM_Proxy_MCP_Handler._should_auto_execute_tools(
+            mcp_tools_with_litellm_proxy=mcp_tools_with_litellm_proxy
+        )
+    )
+    
+    # Prepare parameters for the initial call
+    initial_call_params = LiteLLM_Proxy_MCP_Handler._prepare_initial_call_params(
+        call_params=call_params,
+        should_auto_execute=should_auto_execute
+    )
+    
+    #########################################################
     # Make initial response API call
-    # TODO: if should auto-execute  is True, then this first response should not be streamed
+    #########################################################
     response = await aresponses(
         input=input,
         model=model,
         tools=all_tools,
         previous_response_id=previous_response_id,
-        **call_params,
+        **initial_call_params,
     )
 
-    # Check if we need to auto-execute tool calls (only for non-streaming responses)
+    verbose_logger.debug("Initial response %s", response)
+
+    #########################################################
+    # Auto-Execute Tools Handling
+    # If auto-execute tools is True, then we need to execute the tool calls
+    #########################################################
     if (
-        mcp_tools_with_litellm_proxy
+        should_auto_execute
         and isinstance(response, ResponsesAPIResponse)
-        and LiteLLM_Proxy_MCP_Handler._should_auto_execute_tools(
-            mcp_tools_with_litellm_proxy=mcp_tools_with_litellm_proxy
-        )
     ):  # type: ignore
         tool_calls = LiteLLM_Proxy_MCP_Handler._extract_tool_calls_from_response(
             response=response
@@ -217,20 +284,49 @@ async def aresponses_api_with_mcp(
                     response=response, tool_results=tool_results, original_input=input
                 )
 
+                # Prepare parameters for follow-up call (restores original stream setting)
+                follow_up_call_params = LiteLLM_Proxy_MCP_Handler._prepare_follow_up_call_params(
+                    call_params=call_params,
+                    original_stream_setting=stream or False
+                )
+                
+                # Create tool execution events for streaming if needed
+                tool_execution_events = []
+                if stream:
+                    tool_execution_events = LiteLLM_Proxy_MCP_Handler._create_tool_execution_events(
+                        tool_calls=tool_calls, 
+                        tool_results=tool_results
+                    )
+                
                 final_response = await LiteLLM_Proxy_MCP_Handler._make_follow_up_call(
                     follow_up_input=follow_up_input,
                     model=model,
                     all_tools=all_tools,
                     response_id=response.id,
-                    **call_params,
+                    **follow_up_call_params,
                 )
 
-                # Add custom output elements to the final response
-                if isinstance(final_response, ResponsesAPIResponse):
+                # If streaming and we have tool execution events, wrap the response
+                if stream and tool_execution_events and (hasattr(final_response, '__aiter__') or hasattr(final_response, '__iter__')):
+                    from litellm.responses.mcp.mcp_streaming_iterator import (
+                        MCPEnhancedStreamingIterator,
+                    )
+                    final_response = MCPEnhancedStreamingIterator(
+                        base_iterator=final_response,
+                        mcp_events=tool_execution_events
+                    )
+
+                # Add custom output elements to the final response (for non-streaming)
+                elif isinstance(final_response, ResponsesAPIResponse):
+                    # Fetch MCP tools again for output elements (without OpenAI transformation)
+                    mcp_tools_for_output = await LiteLLM_Proxy_MCP_Handler._process_mcp_tools_without_openai_transform(
+                        user_api_key_auth=user_api_key_auth,
+                        mcp_tools_with_litellm_proxy=mcp_tools_with_litellm_proxy
+                    )
                     final_response = (
                         LiteLLM_Proxy_MCP_Handler._add_mcp_output_elements_to_response(
                             response=final_response,
-                            mcp_tools_fetched=mcp_tools_fetched,
+                            mcp_tools_fetched=mcp_tools_for_output,
                             tool_results=tool_results,
                         )
                     )
@@ -401,13 +497,13 @@ def responses(
     Synchronous version of the Responses API.
     Uses the synchronous HTTP handler to make requests.
     """
+    local_vars = locals()
     from litellm.responses.mcp.litellm_proxy_mcp_handler import (
         LiteLLM_Proxy_MCP_Handler,
     )
 
-    local_vars = locals()
     try:
-        litellm_logging_obj: LiteLLMLoggingObj = kwargs.get("litellm_logging_obj")  # type: ignore
+        litellm_logging_obj: LiteLLMLoggingObj = kwargs.pop("litellm_logging_obj")  # type: ignore
         litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None)
         _is_async = kwargs.pop("aresponses", False) is True
 
@@ -448,7 +544,32 @@ def responses(
         #########################################################
         if LiteLLM_Proxy_MCP_Handler._should_use_litellm_mcp_gateway(tools=tools):
             return aresponses_api_with_mcp(
-                **local_vars,
+                input=input,
+                model=model,
+                include=include,
+                instructions=instructions,
+                max_output_tokens=max_output_tokens,
+                prompt=prompt,
+                metadata=metadata,
+                parallel_tool_calls=parallel_tool_calls,
+                previous_response_id=previous_response_id,
+                reasoning=reasoning,
+                store=store,
+                background=background,
+                stream=stream,
+                temperature=temperature,
+                text=text,
+                tool_choice=tool_choice,
+                tools=tools,
+                top_p=top_p,
+                truncation=truncation,
+                user=user,
+                extra_headers=extra_headers,
+                extra_query=extra_query,
+                extra_body=extra_body,
+                timeout=timeout,
+                custom_llm_provider=custom_llm_provider,
+                **kwargs,
             )
 
         # get provider config

litellm/responses/mcp/litellm_proxy_mcp_handler.py

@@ -1,10 +1,17 @@
-from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
+from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Tuple, Union
 
 from litellm._logging import verbose_logger
 from litellm.responses.main import aresponses
 from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator
 from litellm.types.llms.openai import ResponsesAPIResponse, ToolParam
 
+if TYPE_CHECKING:
+    from mcp.types import Tool as MCPTool
+else:
+    MCPTool = Any
+
+LITELLM_PROXY_MCP_SERVER_URL = "litellm_proxy"
+LITELLM_PROXY_MCP_SERVER_URL_PREFIX = f"{LITELLM_PROXY_MCP_SERVER_URL}/mcp/"
 
 class LiteLLM_Proxy_MCP_Handler:
     """
@@ -20,10 +27,10 @@ class LiteLLM_Proxy_MCP_Handler:
         """
         if tools:
             for tool in tools:
-                if (isinstance(tool, dict) and 
-                    tool.get("type") == "mcp" and 
-                    tool.get("server_url") == "litellm_proxy"):
-                    return True
+                if isinstance(tool, dict) and tool.get("type") == "mcp":
+                    server_url = tool.get("server_url", "")
+                    if isinstance(server_url, str) and server_url.startswith(LITELLM_PROXY_MCP_SERVER_URL):
+                        return True
         return False
     
     @staticmethod
@@ -39,23 +46,180 @@ class LiteLLM_Proxy_MCP_Handler:
         
         if tools:
             for tool in tools:
-                if (isinstance(tool, dict) and 
-                    tool.get("type") == "mcp" and 
-                    tool.get("server_url") == "litellm_proxy"):
-                    mcp_tools_with_litellm_proxy.append(tool)
+                if isinstance(tool, dict) and tool.get("type") == "mcp":
+                    server_url = tool.get("server_url", "")
+                    if isinstance(server_url, str) and server_url.startswith(LITELLM_PROXY_MCP_SERVER_URL):
+                        mcp_tools_with_litellm_proxy.append(tool)
+                    else:
+                        other_tools.append(tool)
                 else:
                     other_tools.append(tool)
         
         return mcp_tools_with_litellm_proxy, other_tools
     
     @staticmethod
-    async def _get_mcp_tools_from_manager(user_api_key_auth: Any) -> List[Any]:
-        """Get available tools from the MCP server manager."""
-        from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
-            global_mcp_server_manager,
+    async def _get_mcp_tools_from_manager(
+        user_api_key_auth: Any,
+        mcp_tools_with_litellm_proxy: Optional[Iterable[ToolParam]],
+    ) -> List[MCPTool]:
+        """
+        Get available tools from the MCP server manager.
+        
+        Args:
+            user_api_key_auth: User authentication info for access control
+            mcp_tools_with_litellm_proxy: ToolParam objects with server_url starting with "litellm_proxy"
+        """
+        from litellm.proxy._experimental.mcp_server.server import (
+            _get_tools_from_mcp_servers,
+        )
+        mcp_servers: List[str] = []
+        if mcp_tools_with_litellm_proxy:
+            for _tool in mcp_tools_with_litellm_proxy:
+                # if user specifies servers as server_url: litellm_proxy/mcp/zapier,github then return zapier,github
+                server_url = _tool.get("server_url", "") if isinstance(_tool, dict) else ""
+                if isinstance(server_url, str) and server_url.startswith(LITELLM_PROXY_MCP_SERVER_URL_PREFIX):
+                    mcp_servers.append(server_url.split("/")[-1])
+        
+        return await _get_tools_from_mcp_servers(
+            user_api_key_auth=user_api_key_auth,
+            mcp_auth_header=None,
+            mcp_servers=mcp_servers,
+            mcp_server_auth_headers=None,
+            mcp_protocol_version=None,
+        )
+    
+    @staticmethod
+    def _deduplicate_mcp_tools(mcp_tools: List[Any]) -> List[Any]:
+        """
+        Deduplicate MCP tools by name, keeping the first occurrence of each tool.
+        
+        Args:
+            mcp_tools: List of MCP tools that may contain duplicates
+            
+        Returns:
+            List of deduplicated MCP tools
+        """
+        seen_names = set()
+        deduplicated_tools = []
+        
+        for tool in mcp_tools:
+            tool_name = getattr(tool, 'name', None) if hasattr(tool, 'name') else tool.get('name') if isinstance(tool, dict) else None
+            if tool_name and tool_name not in seen_names:
+                seen_names.add(tool_name)
+                deduplicated_tools.append(tool)
+        
+        return deduplicated_tools
+
+    @staticmethod
+    def _filter_mcp_tools_by_allowed_tools(
+        mcp_tools: List[Any], 
+        mcp_tools_with_litellm_proxy: List[ToolParam]
+    ) -> List[Any]:
+        """Filter MCP tools based on allowed_tools parameter from the original tool configs."""
+        # Collect all allowed tool names from all MCP tool configs
+        allowed_tool_names = set()
+        for tool_config in mcp_tools_with_litellm_proxy:
+            if isinstance(tool_config, dict) and "allowed_tools" in tool_config:
+                allowed_tools = tool_config.get("allowed_tools", [])
+                if isinstance(allowed_tools, list):
+                    allowed_tool_names.update(allowed_tools)
+        
+        # If no allowed_tools specified, return all tools
+        if not allowed_tool_names:
+            return mcp_tools
+        
+        # Filter tools based on allowed names
+        filtered_tools = []
+        for mcp_tool in mcp_tools:
+            tool_name = getattr(mcp_tool, 'name', None) if hasattr(mcp_tool, 'name') else mcp_tool.get('name') if isinstance(mcp_tool, dict) else None
+            if tool_name and tool_name in allowed_tool_names:
+                filtered_tools.append(mcp_tool)
+        
+        return filtered_tools
+    
+    @staticmethod
+    async def _process_mcp_tools_to_openai_format(
+        user_api_key_auth: Any,
+        mcp_tools_with_litellm_proxy: List[ToolParam]
+    ) -> List[Any]:
+        """
+        Centralized method to process MCP tools through the complete pipeline:
+        1. Fetch tools from MCP manager
+        2. Filter based on allowed_tools parameter  
+        3. Deduplicate tools by name
+        4. Transform to OpenAI format
+        
+        Args:
+            user_api_key_auth: User authentication info for access control
+            mcp_tools_with_litellm_proxy: ToolParam objects with server_url starting with "litellm_proxy"
+            
+        Returns:
+            List of tools in OpenAI format ready to be sent to the LLM
+        """
+        if not mcp_tools_with_litellm_proxy:
+            return []
+            
+        # Step 1: Fetch MCP tools from manager
+        mcp_tools_fetched = await LiteLLM_Proxy_MCP_Handler._get_mcp_tools_from_manager(
+            user_api_key_auth=user_api_key_auth,
+            mcp_tools_with_litellm_proxy=mcp_tools_with_litellm_proxy,
         )
         
-        return await global_mcp_server_manager.list_tools(user_api_key_auth=user_api_key_auth)
+        # Step 2: Filter tools based on allowed_tools parameter
+        filtered_mcp_tools = LiteLLM_Proxy_MCP_Handler._filter_mcp_tools_by_allowed_tools(
+            mcp_tools=mcp_tools_fetched,
+            mcp_tools_with_litellm_proxy=mcp_tools_with_litellm_proxy,
+        )
+        
+        # Step 3: Deduplicate tools after filtering
+        deduplicated_mcp_tools = LiteLLM_Proxy_MCP_Handler._deduplicate_mcp_tools(
+            filtered_mcp_tools
+        )
+        
+        # Step 4: Transform to OpenAI format
+        openai_tools = LiteLLM_Proxy_MCP_Handler._transform_mcp_tools_to_openai(
+            deduplicated_mcp_tools
+        )
+        
+        return openai_tools
+    
+    @staticmethod
+    async def _process_mcp_tools_without_openai_transform(
+        user_api_key_auth: Any,
+        mcp_tools_with_litellm_proxy: List[ToolParam]
+    ) -> List[Any]:
+        """
+        Process MCP tools through filtering and deduplication pipeline without OpenAI transformation.
+        This is useful for cases where we need the original MCP tool objects (e.g., for events).
+        
+        Args:
+            user_api_key_auth: User authentication info for access control
+            mcp_tools_with_litellm_proxy: ToolParam objects with server_url starting with "litellm_proxy"
+            
+        Returns:
+            List of filtered and deduplicated MCP tools in their original format
+        """
+        if not mcp_tools_with_litellm_proxy:
+            return []
+            
+        # Step 1: Fetch MCP tools from manager
+        mcp_tools_fetched = await LiteLLM_Proxy_MCP_Handler._get_mcp_tools_from_manager(
+            user_api_key_auth=user_api_key_auth,
+            mcp_tools_with_litellm_proxy=mcp_tools_with_litellm_proxy,
+        )
+        
+        # Step 2: Filter tools based on allowed_tools parameter
+        filtered_mcp_tools = LiteLLM_Proxy_MCP_Handler._filter_mcp_tools_by_allowed_tools(
+            mcp_tools=mcp_tools_fetched,
+            mcp_tools_with_litellm_proxy=mcp_tools_with_litellm_proxy,
+        )
+        
+        # Step 3: Deduplicate tools after filtering
+        deduplicated_mcp_tools = LiteLLM_Proxy_MCP_Handler._deduplicate_mcp_tools(
+            filtered_mcp_tools
+        )
+        
+        return deduplicated_mcp_tools
     
     @staticmethod
     def _transform_mcp_tools_to_openai(mcp_tools: List[Any]) -> List[Any]:
@@ -178,11 +342,12 @@ class LiteLLM_Proxy_MCP_Handler:
         user_api_key_auth: Any
     ) -> List[Dict[str, Any]]:
         """Execute tool calls and return results."""
+        from fastapi import HTTPException
+
+        from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException
         from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
             global_mcp_server_manager,
         )
-        from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException
-        from fastapi import HTTPException
         
         tool_results = []
         tool_call_id: Optional[str] = None
@@ -331,6 +496,170 @@ class LiteLLM_Proxy_MCP_Handler:
             **call_params
         )
 
+    @staticmethod
+    def _create_mcp_streaming_response(
+        input: Union[str, Any],
+        model: str,
+        all_tools: Optional[List[Any]],
+        mcp_tools_with_litellm_proxy: List[Any],
+        mcp_discovery_events: List[Any],
+        call_params: Dict[str, Any],
+        previous_response_id: Optional[str],
+        **kwargs
+    ) -> Any:
+        """
+        Create MCP enhanced streaming response that handles the full MCP workflow.
+        
+        This creates a streaming iterator that:
+        1. Immediately emits MCP discovery events
+        2. Makes the LLM call and streams the response
+        3. Handles tool execution and follow-up calls
+        """
+        from litellm.responses.mcp.mcp_streaming_iterator import (
+            MCPEnhancedStreamingIterator,
+        )
+
+        # Build the complete request parameters by merging all sources
+        request_params = LiteLLM_Proxy_MCP_Handler._build_request_params(
+            input=input,
+            model=model,
+            all_tools=all_tools,
+            call_params=call_params,
+            previous_response_id=previous_response_id,
+            **kwargs
+        )
+        
+        # Create the enhanced streaming iterator that will handle everything
+        return MCPEnhancedStreamingIterator(
+            base_iterator=None,  # Will be created internally
+            mcp_events=mcp_discovery_events,  # Pre-generated MCP discovery events
+            mcp_tools_with_litellm_proxy=mcp_tools_with_litellm_proxy,
+            user_api_key_auth=kwargs.get("user_api_key_auth"),
+            original_request_params=request_params
+        )
+
+    @staticmethod
+    def _build_request_params(
+        input: Union[str, Any],
+        model: str,
+        all_tools: Optional[List[Any]],
+        call_params: Dict[str, Any],
+        previous_response_id: Optional[str],
+        **kwargs
+    ) -> Dict[str, Any]:
+        """
+        Build a clean request parameters dictionary for MCP streaming.
+        
+        Combines input, model, tools with call_params and additional kwargs
+        in a clean, maintainable way.
+        """
+        # Start with the core required parameters
+        request_params = {
+            'input': input,
+            'model': model,
+            'tools': all_tools,
+        }
+        
+        # Add previous_response_id if provided
+        if previous_response_id is not None:
+            request_params['previous_response_id'] = previous_response_id
+        
+        # Merge in all call_params (which contains most of the API parameters)
+        request_params.update(call_params)
+        
+        # Merge in any additional kwargs
+        request_params.update(kwargs)
+        
+        return request_params
+
+    @staticmethod
+    def _create_tool_execution_events(
+        tool_calls: List[Any], 
+        tool_results: List[Dict[str, Any]]
+    ) -> List[Any]:
+        """
+        Create MCP tool execution events for streaming.
+        
+        Args:
+            tool_calls: List of tool calls from the LLM response
+            tool_results: List of tool execution results
+            
+        Returns:
+            List of MCP tool execution events for streaming
+        """
+        import uuid
+
+        from litellm.responses.mcp.mcp_streaming_iterator import create_mcp_call_events
+        
+        tool_execution_events: List[Any] = []
+        
+        # Create events for each tool execution
+        for tool_result in tool_results:
+            tool_call_id = tool_result.get("tool_call_id", "unknown")
+            result_text = tool_result.get("result", "")
+            
+            # Extract tool name and arguments from tool calls
+            tool_name = "unknown"
+            tool_arguments = "{}"
+            for tool_call in tool_calls:
+                name, args, call_id = LiteLLM_Proxy_MCP_Handler._extract_tool_call_details(tool_call)
+                if call_id == tool_call_id:
+                    tool_name = name or "unknown"
+                    tool_arguments = args or "{}"
+                    break
+            
+            execution_events = create_mcp_call_events(
+                tool_name=tool_name,
+                tool_call_id=tool_call_id,
+                arguments=tool_arguments,  # Use actual arguments
+                result=result_text,
+                base_item_id=f"mcp_{uuid.uuid4().hex[:8]}",  # Unique ID for each tool call
+                sequence_start=len(tool_execution_events) + 1
+            )
+            tool_execution_events.extend(execution_events)
+        
+        return tool_execution_events
+
+    @staticmethod
+    def _prepare_initial_call_params(
+        call_params: Dict[str, Any],
+        should_auto_execute: bool
+    ) -> Dict[str, Any]:
+        """
+        Prepare call parameters for the initial LLM call.
+        
+        For auto-execute scenarios, we need to disable streaming for the initial call
+        so we can process the tool calls before streaming the final response.
+        """
+        initial_params = call_params.copy()
+        
+        if should_auto_execute:
+            # Disable streaming for initial call when auto-executing tools
+            initial_params["stream"] = False
+            
+        return initial_params
+
+    @staticmethod
+    def _prepare_follow_up_call_params(
+        call_params: Dict[str, Any],
+        original_stream_setting: bool
+    ) -> Dict[str, Any]:
+        """
+        Prepare call parameters for the follow-up LLM call after tool execution.
+        
+        Restores the original streaming setting and removes tool_choice since
+        we're now providing tool results, not requesting tool calls.
+        """
+        follow_up_params = call_params.copy()
+        
+        # Restore original streaming setting for follow-up call
+        follow_up_params["stream"] = original_stream_setting
+        
+        # Remove tool_choice since we're providing results, not requesting tool calls
+        follow_up_params.pop("tool_choice", None)
+        
+        return follow_up_params
+
     @staticmethod
     def _add_mcp_output_elements_to_response(
         response: ResponsesAPIResponse,

litellm/responses/mcp/mcp_streaming_iterator.py

@@ -0,0 +1,601 @@
+import uuid
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Dict,
+    List,
+    Optional,
+    Union,
+    cast,
+)
+
+from litellm._logging import verbose_logger
+from litellm.responses.streaming_iterator import (
+    BaseResponsesAPIStreamingIterator,
+)
+from litellm.types.llms.openai import (
+    MCPCallArgumentsDeltaEvent,
+    MCPCallArgumentsDoneEvent,
+    MCPCallCompletedEvent,
+    MCPCallFailedEvent,
+    MCPCallInProgressEvent,
+    MCPListToolsCompletedEvent,
+    MCPListToolsFailedEvent,
+    MCPListToolsInProgressEvent,
+    ResponsesAPIResponse,
+    ResponsesAPIStreamEvents,
+    ResponsesAPIStreamingResponse,
+    ToolParam,
+)
+
+if TYPE_CHECKING:
+    from mcp.types import Tool as MCPTool
+else:
+    MCPTool = Any
+
+
+async def create_mcp_list_tools_events(
+    mcp_tools_with_litellm_proxy: List[ToolParam],
+    user_api_key_auth: Any,
+    base_item_id: str,
+    pre_processed_mcp_tools: List[Any]
+) -> List[ResponsesAPIStreamingResponse]:
+    """Create MCP discovery events using pre-processed tools from the parent"""
+    
+    events: List[ResponsesAPIStreamingResponse] = []
+    
+    try:
+        # Extract MCP server names
+        mcp_servers = []
+        for tool in mcp_tools_with_litellm_proxy:
+            if isinstance(tool, dict) and "server_url" in tool:
+                server_url = tool.get("server_url")
+                if isinstance(server_url, str) and server_url.startswith("litellm_proxy/mcp/"):
+                    server_name = server_url.split("/")[-1]
+                    mcp_servers.append(server_name)
+        
+        # Emit list tools in progress event
+        in_progress_event = MCPListToolsInProgressEvent(
+            type=ResponsesAPIStreamEvents.MCP_LIST_TOOLS_IN_PROGRESS,
+            sequence_number=1,
+            output_index=0,
+            item_id=base_item_id,
+        )
+        events.append(in_progress_event)
+        
+        # Use the pre-processed MCP tools that were already fetched, filtered, and deduplicated by the parent
+        filtered_mcp_tools = pre_processed_mcp_tools
+        
+        # Convert tools to dict format for the event
+        mcp_tools_dict = []
+        for tool in filtered_mcp_tools:
+            if hasattr(tool, 'model_dump') and callable(getattr(tool, 'model_dump')):
+                # Type cast to help mypy understand this is safe after hasattr check
+                mcp_tools_dict.append(cast(Any, tool).model_dump())
+            elif hasattr(tool, '__dict__'):
+                mcp_tools_dict.append(tool.__dict__)
+            else:
+                mcp_tools_dict.append({"name": getattr(tool, 'name', str(tool))})
+        
+        # Emit list tools completed event
+        completed_event = MCPListToolsCompletedEvent(
+            type=ResponsesAPIStreamEvents.MCP_LIST_TOOLS_COMPLETED,
+            sequence_number=2,
+            output_index=0,
+            item_id=base_item_id,
+        )
+        events.append(completed_event)
+        
+        # Add output_item.done event with the actual tools list (matching OpenAI format)
+        from litellm.types.llms.openai import OutputItemDoneEvent
+
+        # Extract server label from the first MCP tool config
+        server_label = ""
+        if mcp_tools_with_litellm_proxy:
+            first_tool = mcp_tools_with_litellm_proxy[0]
+            if isinstance(first_tool, dict):
+                server_label_value = first_tool.get("server_label", "")
+                server_label = str(server_label_value) if server_label_value is not None else ""
+        
+        # Format tools for OpenAI output_item.done format
+        formatted_tools = []
+        for tool in filtered_mcp_tools:
+            tool_dict = {
+                "name": getattr(tool, 'name', 'unknown'),
+                "description": getattr(tool, 'description', ''),
+                "annotations": {"read_only": False},
+            }
+            
+            # Add input_schema if available
+            if hasattr(tool, 'inputSchema'):
+                tool_dict["input_schema"] = getattr(tool, 'inputSchema')
+            elif hasattr(tool, 'input_schema'):
+                tool_dict["input_schema"] = getattr(tool, 'input_schema')
+            
+            formatted_tools.append(tool_dict)
+        
+        # Create the output_item.done event with MCP tools list
+        output_item_done_event = OutputItemDoneEvent(
+            type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
+            output_index=0,
+            item={
+                "id": base_item_id,
+                "type": "mcp_list_tools",
+                "server_label": server_label,
+                "tools": formatted_tools
+            }
+        )
+        events.append(output_item_done_event)
+        
+        verbose_logger.debug(f"Created {len(events)} MCP discovery events")
+        
+    except Exception as e:
+        verbose_logger.error(f"Error creating MCP list tools events: {e}")
+        import traceback
+        traceback.print_exc()
+        
+        # Emit failed event on error
+        failed_event = MCPListToolsFailedEvent(
+            type=ResponsesAPIStreamEvents.MCP_LIST_TOOLS_FAILED,
+            sequence_number=2,
+            output_index=0,
+            item_id=base_item_id,
+        )
+        events.append(failed_event)
+        
+        # Still emit output_item.done event even on failure (with empty tools list)
+        from litellm.types.llms.openai import OutputItemDoneEvent
+        
+        output_item_done_event = OutputItemDoneEvent(
+            type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
+            output_index=0,
+            item={
+                "id": base_item_id,
+                "type": "mcp_list_tools",
+                "server_label": "",
+                "tools": []
+            }
+        )
+        events.append(output_item_done_event)
+    
+    return events
+
+
+def create_mcp_call_events(
+    tool_name: str, 
+    tool_call_id: str, 
+    arguments: str,
+    result: Optional[str] = None,
+    base_item_id: Optional[str] = None,
+    sequence_start: int = 1
+) -> List[ResponsesAPIStreamingResponse]:
+    """Create MCP call events following OpenAI's specification"""
+    events: List[ResponsesAPIStreamingResponse] = []
+    item_id = base_item_id or f"mcp_{uuid.uuid4().hex[:8]}"
+    
+    # MCP call in progress event
+    in_progress_event = MCPCallInProgressEvent(
+        type=ResponsesAPIStreamEvents.MCP_CALL_IN_PROGRESS,
+        sequence_number=sequence_start,
+        output_index=0,
+        item_id=item_id,
+    )
+    events.append(in_progress_event)
+    
+    # MCP call arguments delta event (streaming the arguments)
+    arguments_delta_event = MCPCallArgumentsDeltaEvent(
+        type=ResponsesAPIStreamEvents.MCP_CALL_ARGUMENTS_DELTA,
+        output_index=0,
+        item_id=item_id,
+        delta=arguments,  # JSON string with arguments
+        sequence_number=sequence_start + 1,
+    )
+    events.append(arguments_delta_event)
+    
+    # MCP call arguments done event
+    arguments_done_event = MCPCallArgumentsDoneEvent(
+        type=ResponsesAPIStreamEvents.MCP_CALL_ARGUMENTS_DONE,
+        output_index=0,
+        item_id=item_id,
+        arguments=arguments,  # Complete JSON string with finalized arguments
+        sequence_number=sequence_start + 2,
+    )
+    events.append(arguments_done_event)
+    
+    # MCP call completed event (or failed if result indicates failure)
+    if result is not None:
+        completed_event = MCPCallCompletedEvent(
+            type=ResponsesAPIStreamEvents.MCP_CALL_COMPLETED,
+            sequence_number=sequence_start + 3,
+            item_id=item_id,
+            output_index=0,
+        )
+        events.append(completed_event)
+        
+        # Add output_item.done event with the tool call result
+        from litellm.types.llms.openai import OutputItemDoneEvent
+        
+        output_item_done_event = OutputItemDoneEvent(
+            type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
+            output_index=0,
+            item={
+                "id": item_id,
+                "type": "mcp_call",
+                "approval_request_id": f"mcpr_{uuid.uuid4().hex[:8]}",
+                "arguments": arguments,
+                "error": None,
+                "name": tool_name,
+                "output": result,
+                "server_label": "litellm"
+            },
+        )
+        events.append(output_item_done_event)
+    else:
+        failed_event = MCPCallFailedEvent(
+            type=ResponsesAPIStreamEvents.MCP_CALL_FAILED,
+            sequence_number=sequence_start + 3,
+            item_id=item_id,
+            output_index=0,
+        )
+        events.append(failed_event)
+    
+    return events
+
+
+class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator):
+    """
+    A complete MCP streaming iterator that handles the entire flow:
+    1. Immediately emits MCP discovery events
+    2. Makes the first LLM call and streams its response
+    3. Handles tool execution and follow-up calls for auto-execute tools
+    4. Emits tool execution events in the stream
+    """
+    
+    def __init__(
+        self,
+        base_iterator: Any,  # Can be None - will be created internally
+        mcp_events: List[ResponsesAPIStreamingResponse],
+        mcp_tools_with_litellm_proxy: Optional[List[Any]] = None,
+        user_api_key_auth: Any = None,
+        original_request_params: Optional[Dict[str, Any]] = None
+    ):
+        # MCP setup
+        self.mcp_tools_with_litellm_proxy = mcp_tools_with_litellm_proxy or []
+        self.user_api_key_auth = user_api_key_auth
+        self.original_request_params = original_request_params or {}
+        self.should_auto_execute = self._should_auto_execute_tools()
+        
+        # Streaming state management
+        self.phase = "mcp_discovery"  # mcp_discovery -> initial_response -> tool_execution -> follow_up_response -> finished
+        self.finished = False
+        
+        # Event queues and generation flags
+        self.mcp_discovery_events: List[ResponsesAPIStreamingResponse] = mcp_events  # Pre-generated MCP discovery events
+        self.tool_execution_events: List[ResponsesAPIStreamingResponse] = []
+        self.mcp_discovery_generated = True  # Events are already generated
+        self.mcp_events = mcp_events  # Store the initial MCP events for backward compatibility
+        
+        # Iterator references
+        self.base_iterator: Optional[Union[Any, ResponsesAPIResponse]] = base_iterator  # Will be created when needed
+        self.follow_up_iterator: Optional[Any] = None
+        
+        # Response collection for tool execution
+        self.collected_response: Optional[ResponsesAPIResponse] = None
+        
+        # Set up model metadata (will be updated when we get the real iterator)
+        self.model = self.original_request_params.get('model', 'unknown')
+        self.litellm_metadata = {}
+        self.custom_llm_provider = self.original_request_params.get('custom_llm_provider', None)
+        
+        # Mark as async iterator
+        self.is_async = True
+        
+    def _should_auto_execute_tools(self) -> bool:
+        """Check if tools should be auto-executed"""
+        from litellm.responses.mcp.litellm_proxy_mcp_handler import (
+            LiteLLM_Proxy_MCP_Handler,
+        )
+        return LiteLLM_Proxy_MCP_Handler._should_auto_execute_tools(
+            self.mcp_tools_with_litellm_proxy
+        )
+
+    def __aiter__(self):
+        return self
+
+    async def __anext__(self) -> ResponsesAPIStreamingResponse:
+        """
+        Phase-based streaming:
+        1. mcp_discovery - Emit MCP discovery events
+        2. initial_response - Stream the first LLM response  
+        3. tool_execution - Emit tool execution events
+        4. follow_up_response - Stream the follow-up response
+        5. finished - End iteration
+        """
+        
+        # Phase 1: MCP Discovery Events
+        if self.phase == "mcp_discovery":
+            # Generate MCP discovery events if not already done
+            # MCP discovery events are already generated and available
+            
+            # Emit MCP discovery events
+            if self.mcp_discovery_events:
+                return self.mcp_discovery_events.pop(0)
+            
+            # All MCP discovery events emitted, move to next phase
+            verbose_logger.debug("MCP discovery phase complete, transitioning to initial_response")
+            self.phase = "initial_response"
+            await self._create_initial_response_iterator()
+            # Fall through to process the initial response immediately
+        
+        # Phase 2: Initial Response Stream
+        if self.phase == "initial_response":
+            if self.base_iterator:
+                # Check if base_iterator is actually iterable
+                if hasattr(self.base_iterator, '__anext__'):
+                    try:
+                        chunk = await cast(Any, self.base_iterator).__anext__()  # type: ignore[attr-defined]
+                        
+                        # If auto-execution is enabled, check for completed responses
+                        if self.should_auto_execute and self._is_response_completed(chunk):
+                            # Collect the response for tool execution
+                            response_obj = getattr(chunk, 'response', None)
+                            if isinstance(response_obj, ResponsesAPIResponse):
+                                self.collected_response = response_obj
+                            # Move to tool execution phase after emitting this chunk
+                            self.phase = "tool_execution"
+                            await self._generate_tool_execution_events()
+                        
+                        return chunk
+                    except StopAsyncIteration:
+                        # Initial response ended, move to next phase
+                        if self.should_auto_execute and self.collected_response:
+                            self.phase = "tool_execution"
+                            await self._generate_tool_execution_events()
+                        else:
+                            self.phase = "finished"
+                            raise
+                else:
+                    # base_iterator is not async iterable (likely a ResponsesAPIResponse)
+                    # Collect it for tool execution if needed
+                    if self.should_auto_execute and isinstance(self.base_iterator, ResponsesAPIResponse):
+                        self.collected_response = self.base_iterator
+                        self.phase = "tool_execution"
+                        await self._generate_tool_execution_events()
+                    else:
+                        self.phase = "finished"
+                        raise StopAsyncIteration
+        
+        # Phase 3: Tool Execution Events
+        if self.phase == "tool_execution":
+            # Emit any queued tool execution events
+            if self.tool_execution_events:
+                return self.tool_execution_events.pop(0)
+            
+            # Move to follow-up response phase
+            self.phase = "follow_up_response"
+            await self._create_follow_up_iterator()
+        
+        # Phase 4: Follow-up Response Stream
+        if self.phase == "follow_up_response":
+            if self.follow_up_iterator:
+                try:
+                    return await cast(Any, self.follow_up_iterator).__anext__()  # type: ignore[attr-defined]
+                except StopAsyncIteration:
+                    self.phase = "finished"
+                    raise
+            else:
+                self.phase = "finished"
+                raise StopAsyncIteration
+        
+        # Phase 5: Finished
+        if self.phase == "finished":
+            raise StopAsyncIteration
+            
+        # Should not reach here
+        raise StopAsyncIteration
+    
+    def _is_response_completed(self, chunk: ResponsesAPIStreamingResponse) -> bool:
+        """Check if this chunk indicates the response is completed"""
+        from litellm.types.llms.openai import ResponsesAPIStreamEvents
+        return getattr(chunk, 'type', None) == ResponsesAPIStreamEvents.RESPONSE_COMPLETED
+    
+    
+    async def _create_initial_response_iterator(self) -> None:
+        """Create the initial response iterator by making the first LLM call"""
+        try:
+            # Import the core aresponses function that doesn't have MCP logic
+            from litellm.responses.main import aresponses
+
+            # Make the initial response API call - but avoid the MCP wrapper
+            params = self.original_request_params.copy()
+            params['stream'] = True  # Ensure streaming
+            
+            # Use the pre-fetched all_tools from original_request_params (no re-processing needed)
+            params_for_llm = {}
+            for key, value in params.items():
+                params_for_llm[key] = value  # Copy all params as-is since tools are already processed
+            
+            tools_count = len(params_for_llm.get('tools', []))
+            verbose_logger.debug(f"Making LLM call with {tools_count} tools")
+            response = await aresponses(**params_for_llm)
+            
+            # Set the base iterator
+            if hasattr(response, '__aiter__') or hasattr(response, '__iter__'):
+                self.base_iterator = response
+                # Copy metadata from the real iterator
+                self.model = getattr(response, 'model', self.model)
+                self.litellm_metadata = getattr(response, 'litellm_metadata', {})
+                self.custom_llm_provider = getattr(response, 'custom_llm_provider', self.custom_llm_provider)
+                verbose_logger.debug(f"Created base iterator: {type(self.base_iterator)}")
+            else:
+                # Non-streaming response - this shouldn't happen but handle it
+                verbose_logger.warning(f"Got non-streaming response: {type(response)}")
+                self.base_iterator = None
+                self.phase = "finished"
+                
+        except Exception as e:
+            verbose_logger.error(f"Error creating initial response iterator: {e}")
+            import traceback
+            traceback.print_exc()
+            self.base_iterator = None
+            self.phase = "finished"
+    
+    async def _generate_tool_execution_events(self) -> None:
+        """Generate tool execution events and execute tools"""
+        if not self.collected_response:
+            return
+            
+        import uuid
+
+        from litellm.responses.mcp.litellm_proxy_mcp_handler import (
+            LiteLLM_Proxy_MCP_Handler,
+        )
+        
+        try:
+            # Extract tool calls from the response
+            if self.collected_response is not None:
+                tool_calls = LiteLLM_Proxy_MCP_Handler._extract_tool_calls_from_response(self.collected_response)  # type: ignore[arg-type]
+            else:
+                tool_calls = []
+            if not tool_calls:
+                return
+            
+            for tool_call in tool_calls:
+                tool_name, tool_arguments, tool_call_id = LiteLLM_Proxy_MCP_Handler._extract_tool_call_details(tool_call)
+                if tool_name and tool_call_id:
+                    # Create MCP call events for this tool execution
+                    call_events = create_mcp_call_events(
+                        tool_name=tool_name,
+                        tool_call_id=tool_call_id,
+                        arguments=tool_arguments or "{}",  # JSON string with arguments
+                        result=None,  # Will be set after execution
+                        base_item_id=f"mcp_{uuid.uuid4().hex[:8]}",
+                        sequence_start=len(self.tool_execution_events) + 1
+                    )
+                    # Add the in_progress and arguments events (not the completed event yet)
+                    self.tool_execution_events.extend(call_events[:-1])
+            
+            # Execute the tools
+            tool_results = await LiteLLM_Proxy_MCP_Handler._execute_tool_calls(
+                tool_calls=tool_calls,
+                user_api_key_auth=self.user_api_key_auth
+            )
+            
+            # Create completion events and output_item.done events for tool execution
+            for tool_result in tool_results:
+                tool_call_id = tool_result.get("tool_call_id", "unknown")
+                result_text = tool_result.get("result", "")
+                
+                # Find matching tool name and arguments
+                tool_name = "unknown"
+                tool_arguments = "{}"
+                for tool_call in tool_calls:
+                    name, args, call_id = LiteLLM_Proxy_MCP_Handler._extract_tool_call_details(tool_call)
+                    if call_id == tool_call_id:
+                        tool_name = name or "unknown"
+                        tool_arguments = args or "{}"
+                        break
+                
+                item_id = f"mcp_{uuid.uuid4().hex[:8]}"
+                
+                # Create the completion event
+                completed_event = MCPCallCompletedEvent(
+                    type=ResponsesAPIStreamEvents.MCP_CALL_COMPLETED,
+                    sequence_number=len(self.tool_execution_events) + 1,
+                    item_id=item_id,
+                    output_index=0,
+                )
+                self.tool_execution_events.append(completed_event)
+                
+                # Create output_item.done event with the tool call result
+                from litellm.types.llms.openai import OutputItemDoneEvent
+                
+                output_item_done_event = OutputItemDoneEvent(
+                    type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
+                    output_index=0,
+                    item={
+                        "id": item_id,
+                        "type": "mcp_call",
+                        "approval_request_id": f"mcpr_{uuid.uuid4().hex[:8]}",
+                        "arguments": tool_arguments,
+                        "error": None,
+                        "name": tool_name,
+                        "output": result_text,
+                        "server_label": "litellm"  # or extract from tool config
+                    },
+                )
+                self.tool_execution_events.append(output_item_done_event)
+            
+            # Store tool results for follow-up call
+            self.tool_results = tool_results
+            
+        except Exception as e:
+            verbose_logger.error(f"Error in tool execution: {e}")
+            import traceback
+            traceback.print_exc()
+            self.tool_results = []
+    
+    async def _create_follow_up_iterator(self) -> None:
+        """Create the follow-up response iterator with tool results"""
+        if not self.collected_response or not hasattr(self, 'tool_results'):
+            return
+            
+        from litellm.responses.main import aresponses
+        from litellm.responses.mcp.litellm_proxy_mcp_handler import (
+            LiteLLM_Proxy_MCP_Handler,
+        )
+        
+        try:
+            # Create follow-up input
+            if self.collected_response is not None:
+                follow_up_input = LiteLLM_Proxy_MCP_Handler._create_follow_up_input(
+                    response=self.collected_response,  # type: ignore[arg-type]
+                    tool_results=self.tool_results,
+                    original_input=self.original_request_params.get('input')
+                )
+                
+                # Make follow-up call with streaming
+                follow_up_params = self.original_request_params.copy()
+                follow_up_params.update({
+                    'input': follow_up_input,
+                    'previous_response_id': self.collected_response.id,  # type: ignore[attr-defined]
+                    'stream': True
+                })
+            else:
+                return
+            # Remove tool_choice to avoid forcing more tool calls
+            follow_up_params.pop('tool_choice', None)
+            
+            follow_up_response = await aresponses(**follow_up_params)
+            
+            # Set up the follow-up iterator
+            if hasattr(follow_up_response, '__aiter__'):
+                self.follow_up_iterator = follow_up_response
+            
+        except Exception as e:
+            verbose_logger.error(f"Error creating follow-up iterator: {e}")
+            import traceback
+            traceback.print_exc()
+            self.follow_up_iterator = None
+
+
+    def __iter__(self):
+        return self
+
+    def __next__(self) -> ResponsesAPIStreamingResponse:
+        # First, emit any queued MCP events
+        if self.mcp_events:  # type: ignore[attr-defined]
+            return self.mcp_events.pop(0)  # type: ignore[attr-defined]
+        
+        # Then delegate to the base iterator
+        if not self.is_async:
+            try:
+                if self.base_iterator and hasattr(self.base_iterator, '__next__'):
+                    return next(cast(Any, self.base_iterator))  # type: ignore[arg-type]
+                else:
+                    raise StopIteration
+            except StopIteration:
+                self.finished = True
+                raise
+        else:
+            raise RuntimeError("Cannot use sync iteration on async iterator")

litellm/router.py

@@ -3046,7 +3046,7 @@ class Router:
             from litellm.router_utils.common_utils import add_model_file_id_mappings
 
             verbose_router_logger.debug(
-                f"Inside _atext_completion()- model: {model}; kwargs: {kwargs}"
+                f"Inside _acreate_file()- model: {model}; kwargs: {kwargs}"
             )
             parent_otel_span = _get_parent_otel_span_from_kwargs(kwargs)
             healthy_deployments = await self.async_get_healthy_deployments(

litellm/types/guardrails.py

@@ -42,6 +42,7 @@ class SupportedGuardrailIntegrations(Enum):
     OPENAI_MODERATION = "openai_moderation"
     NOMA = "noma"
 
+
 class Role(Enum):
     SYSTEM = "system"
     ASSISTANT = "assistant"
@@ -312,7 +313,6 @@ class BedrockGuardrailConfigModel(BaseModel):
     )
 
 
-
 class LakeraV2GuardrailConfigModel(BaseModel):
     """Configuration parameters for the Lakera AI v2 guardrail"""
 
@@ -375,6 +375,10 @@ class NomaGuardrailConfigModel(BaseModel):
         default=None,
         description="If True, blocks requests on API failures. Defaults to True if not provided",
     )
+    anonymize_input: Optional[bool] = Field(
+        default=None,
+        description="If True, replaces sensitive content with anonymized version when only PII/PCI/secrets are detected. Only applies in blocking mode. Defaults to False if not provided",
+    )
 
 
 class BaseLitellmParams(BaseModel):  # works for new and patch update guardrails
@@ -425,7 +429,8 @@ class BaseLitellmParams(BaseModel):  # works for new and patch update guardrails
     )
 
     model: Optional[str] = Field(
-        default=None, description="Optional field if guardrail requires a 'model' parameter"
+        default=None,
+        description="Optional field if guardrail requires a 'model' parameter",
     )
 
     # Model Armor params
@@ -446,7 +451,7 @@ class BaseLitellmParams(BaseModel):  # works for new and patch update guardrails
         default=True,
         description="Whether to fail the request if Model Armor encounters an error",
     )
-    
+
     model_config = ConfigDict(extra="allow", protected_namespaces=())
 
 

litellm/types/llms/databricks.py

@@ -51,7 +51,7 @@ AllDatabricksContentValues = Union[str, List[AllDatabricksContentListValues]]
 
 class DatabricksFunction(TypedDict, total=False):
     name: Required[str]
-    description: dict
+    description: Union[dict, str]
     parameters: dict
     strict: bool
 

litellm/types/llms/oci.py

@@ -121,7 +121,7 @@ class OCICompletionTokenDetails(BaseModel):
     reasoningTokens: int
 
 
-class OCIPropmtTokensDetails(BaseModel):
+class OCIPromptTokensDetails(BaseModel):
     """Prompt token details in the OCI response."""
 
     cachedTokens: int
@@ -129,12 +129,12 @@ class OCIPropmtTokensDetails(BaseModel):
 
 class OCIResponseUsage(BaseModel):
     """Token usage in the OCI response."""
-
+    
     promptTokens: int
     completionTokens: int
     totalTokens: int
-    completionTokensDetails: OCICompletionTokenDetails
-    promptTokensDetails: OCIPropmtTokensDetails
+    completionTokensDetails: Optional[OCICompletionTokenDetails] = None
+    promptTokensDetails: Optional[OCIPromptTokensDetails] = None
 
 
 class OCIResponseChoice(BaseModel):

litellm/types/llms/openai.py

@@ -1112,6 +1112,16 @@ class ResponsesAPIStreamEvents(str, Enum):
     WEB_SEARCH_CALL_SEARCHING = "response.web_search_call.searching"
     WEB_SEARCH_CALL_COMPLETED = "response.web_search_call.completed"
 
+    # MCP events - matching OpenAI's official specification
+    MCP_LIST_TOOLS_IN_PROGRESS = "response.mcp_list_tools.in_progress"
+    MCP_LIST_TOOLS_COMPLETED = "response.mcp_list_tools.completed"
+    MCP_LIST_TOOLS_FAILED = "response.mcp_list_tools.failed"
+    MCP_CALL_IN_PROGRESS = "response.mcp_call.in_progress"
+    MCP_CALL_ARGUMENTS_DELTA = "response.mcp_call_arguments.delta"
+    MCP_CALL_ARGUMENTS_DONE = "response.mcp_call_arguments.done"
+    MCP_CALL_COMPLETED = "response.mcp_call.completed"
+    MCP_CALL_FAILED = "response.mcp_call.failed"
+
     # Error event
     ERROR = "error"
 
@@ -1275,6 +1285,66 @@ class WebSearchCallCompletedEvent(BaseLiteLLMOpenAIResponseObject):
     item_id: str
 
 
+# MCP List Tools Events
+class MCPListToolsInProgressEvent(BaseLiteLLMOpenAIResponseObject):
+    type: Literal[ResponsesAPIStreamEvents.MCP_LIST_TOOLS_IN_PROGRESS]
+    sequence_number: int
+    output_index: int
+    item_id: str
+
+
+class MCPListToolsCompletedEvent(BaseLiteLLMOpenAIResponseObject):
+    type: Literal[ResponsesAPIStreamEvents.MCP_LIST_TOOLS_COMPLETED]
+    sequence_number: int
+    output_index: int
+    item_id: str
+
+
+class MCPListToolsFailedEvent(BaseLiteLLMOpenAIResponseObject):
+    type: Literal[ResponsesAPIStreamEvents.MCP_LIST_TOOLS_FAILED]
+    sequence_number: int
+    output_index: int
+    item_id: str
+
+
+# MCP Call Events  
+class MCPCallInProgressEvent(BaseLiteLLMOpenAIResponseObject):
+    type: Literal[ResponsesAPIStreamEvents.MCP_CALL_IN_PROGRESS]
+    sequence_number: int
+    output_index: int
+    item_id: str
+
+
+class MCPCallArgumentsDeltaEvent(BaseLiteLLMOpenAIResponseObject):
+    type: Literal[ResponsesAPIStreamEvents.MCP_CALL_ARGUMENTS_DELTA]
+    output_index: int
+    item_id: str
+    delta: str  # JSON string containing partial update to arguments
+    sequence_number: int
+
+
+class MCPCallArgumentsDoneEvent(BaseLiteLLMOpenAIResponseObject):
+    type: Literal[ResponsesAPIStreamEvents.MCP_CALL_ARGUMENTS_DONE]
+    output_index: int
+    item_id: str
+    arguments: str  # JSON string containing finalized arguments
+    sequence_number: int
+
+
+class MCPCallCompletedEvent(BaseLiteLLMOpenAIResponseObject):
+    type: Literal[ResponsesAPIStreamEvents.MCP_CALL_COMPLETED]
+    sequence_number: int
+    item_id: str
+    output_index: int
+
+
+class MCPCallFailedEvent(BaseLiteLLMOpenAIResponseObject):
+    type: Literal[ResponsesAPIStreamEvents.MCP_CALL_FAILED]
+    sequence_number: int
+    item_id: str
+    output_index: int
+
+
 class ErrorEvent(BaseLiteLLMOpenAIResponseObject):
     type: Literal[ResponsesAPIStreamEvents.ERROR]
     code: Optional[str]
@@ -1315,6 +1385,14 @@ ResponsesAPIStreamingResponse = Annotated[
         WebSearchCallInProgressEvent,
         WebSearchCallSearchingEvent,
         WebSearchCallCompletedEvent,
+        MCPListToolsInProgressEvent,
+        MCPListToolsCompletedEvent,
+        MCPListToolsFailedEvent,
+        MCPCallInProgressEvent,
+        MCPCallArgumentsDeltaEvent,
+        MCPCallArgumentsDoneEvent,
+        MCPCallCompletedEvent,
+        MCPCallFailedEvent,
         ErrorEvent,
         GenericEvent,
     ],

litellm/types/llms/vertex_ai.py

@@ -113,7 +113,6 @@ class Schema(TypedDict, total=False):
     pattern: str
     example: Any
     anyOf: List["Schema"]
-    additionalProperties: Any
 
 
 class FunctionDeclaration(TypedDict, total=False):

litellm/types/utils.py

@@ -162,6 +162,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
         SearchContextCostPerQuery
     ]  # Cost for using web search tool
     citation_cost_per_token: Optional[float]  # Cost per citation token for Perplexity
+    tiered_pricing: Optional[List[Dict[str, Any]]]  # Tiered pricing structure for models like Dashscope
     litellm_provider: Required[str]
     mode: Required[
         Literal[
@@ -1995,7 +1996,7 @@ class StandardLoggingGuardrailInformation(TypedDict, total=False):
     ]
     guardrail_request: Optional[dict]
     guardrail_response: Optional[Union[dict, str, List[dict]]]
-    guardrail_status: Literal["success", "failure","blocked"]
+    guardrail_status: Literal["success", "failure", "blocked"]
     start_time: Optional[float]
     end_time: Optional[float]
     duration: Optional[float]
@@ -2123,6 +2124,7 @@ all_litellm_params = [
     "metadata",
     "litellm_metadata",
     "litellm_trace_id",
+    "litellm_request_debug",
     "guardrails",
     "tags",
     "acompletion",

litellm/utils.py

@@ -2437,7 +2437,7 @@ def get_optional_params_transcription(
         "prompt": None,
         "response_format": None,
         "temperature": None,  # openai defaults this to 0
-        "timestamp_granularities": None
+        "timestamp_granularities": None,
     }
 
     non_default_params = {
@@ -2501,6 +2501,25 @@ def get_optional_params_transcription(
     return optional_params
 
 
+def _map_openai_size_to_vertex_ai_aspect_ratio(size: Optional[str]) -> str:
+    """Map OpenAI size parameter to Vertex AI aspectRatio."""
+    if size is None:
+        return "1:1"
+
+    # Map OpenAI size strings to Vertex AI aspect ratio strings
+    # Vertex AI accepts: "1:1", "9:16", "16:9", "4:3", "3:4"
+    size_to_aspect_ratio = {
+        "256x256": "1:1",  # Square
+        "512x512": "1:1",  # Square
+        "1024x1024": "1:1",  # Square (default)
+        "1792x1024": "16:9",  # Landscape
+        "1024x1792": "9:16",  # Portrait
+    }
+    return size_to_aspect_ratio.get(
+        size, "1:1"
+    )  # Default to square if size not recognized
+
+
 def get_optional_params_image_gen(
     model: Optional[str] = None,
     n: Optional[int] = None,
@@ -2614,19 +2633,9 @@ def get_optional_params_image_gen(
 
         # Map OpenAI size parameter to Vertex AI aspectRatio
         if size is not None:
-            # Map OpenAI size strings to Vertex AI aspect ratio strings
-            # Vertex AI accepts: "1:1", "9:16", "16:9", "4:3", "3:4"
-            size_to_aspect_ratio = {
-                "256x256": "1:1",  # Square
-                "512x512": "1:1",  # Square
-                "1024x1024": "1:1",  # Square (default)
-                "1792x1024": "16:9",  # Landscape
-                "1024x1792": "9:16",  # Portrait
-            }
-            aspect_ratio = size_to_aspect_ratio.get(
-                size, "1:1"
-            )  # Default to square if size not recognized
-            optional_params["aspectRatio"] = aspect_ratio
+            optional_params["aspectRatio"] = _map_openai_size_to_vertex_ai_aspect_ratio(
+                size
+            )
 
     openai_params: list[str] = list(default_params.keys())
     if provider_config is not None:
@@ -2642,6 +2651,12 @@ def get_optional_params_image_gen(
         openai_params=openai_params,
         additional_drop_params=additional_drop_params,
     )
+    # remove keys with None or empty dict/list values to avoid sending empty payloads
+    optional_params = {
+        k: v
+        for k, v in optional_params.items()
+        if v is not None and (not isinstance(v, (dict, list)) or len(v) > 0)
+    }
     return optional_params
 
 
@@ -4902,6 +4917,7 @@ def _get_model_info_helper(  # noqa: PLR0915
                 citation_cost_per_token=_model_info.get(
                     "citation_cost_per_token", None
                 ),
+                tiered_pricing=_model_info.get("tiered_pricing", None),
                 litellm_provider=_model_info.get(
                     "litellm_provider", custom_llm_provider
                 ),

package.json

@@ -5,6 +5,9 @@
     "react-copy-to-clipboard": "^5.1.0"
   },
   "devDependencies": {
-    "@types/react-copy-to-clipboard": "^5.0.7"
+    "@testing-library/jest-dom": "^6.8.0",
+    "@testing-library/react": "^14.3.1",
+    "@types/react-copy-to-clipboard": "^5.0.7",
+    "jest": "^29.7.0"
   }
 }

poetry.lock

@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 2.1.2 and should not be changed by hand.
+# This file is automatically @generated by Poetry 2.1.4 and should not be changed by hand.
 
 [[package]]
 name = "aiohappyeyeballs"

tests/batches_tests/bedrock_batch_completions.jsonl

@@ -1,3 +1,128 @@
 {"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
 {"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
-
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}

tests/batches_tests/test_bedrock_files_and_batches.py

@@ -18,6 +18,8 @@ sys.path.insert(
 import pytest
 from typing import Optional
 import litellm
+from unittest.mock import patch, MagicMock
+import httpx
 
 
 @pytest.mark.asyncio()
@@ -64,7 +66,55 @@ async def test_async_file_and_batch():
         input_file_id=file_obj.id,
         metadata={"key1": "value1", "key2": "value2"},
         custom_llm_provider="bedrock",
+        
+        #########################################################
+        # bedrock specific params
+        #########################################################
+        model="us.anthropic.claude-3-5-sonnet-20240620-v1:0",
         aws_batch_role_arn="arn:aws:iam::888602223428:role/service-role/AmazonBedrockExecutionRoleForAgents_BB9HNW6V4CV"
     )
     print("CREATED BATCH RESPONSE=", create_batch_response)
 
+
+@pytest.mark.asyncio()
+async def test_mock_bedrock_file_url_mapping():
+    """
+    Simple test to capture PUT URL and validate mapping to file ID.
+    """
+    print("Testing Bedrock file URL mapping")
+    
+    captured_put_url = None
+    
+    async def mock_async_create_file(transformed_request, **kwargs):
+        nonlocal captured_put_url
+        # Capture PUT URL from transformed request
+        if isinstance(transformed_request, dict) and "url" in transformed_request:
+            captured_put_url = transformed_request["url"]
+        
+        # Call the real method to get actual response
+        from litellm.files.main import base_llm_http_handler
+        return await base_llm_http_handler.__class__.async_create_file(
+            base_llm_http_handler, transformed_request, **kwargs
+        )
+    
+    with patch('litellm.files.main.base_llm_http_handler.async_create_file', side_effect=mock_async_create_file):
+        file_obj = await litellm.acreate_file(
+            file=open(os.path.join(os.path.dirname(__file__), "bedrock_batch_completions.jsonl"), "rb"),
+            purpose="batch",
+            custom_llm_provider="bedrock",
+            s3_bucket_name="litellm-proxy",
+        )
+        
+        print(f"PUT URL: {captured_put_url}")
+        print(f"File ID: {file_obj.id}")
+        
+        # Validate URL was captured and response is correct
+        assert captured_put_url is not None
+        assert file_obj.id.startswith("s3://")
+        
+        # Verify mapping
+        from litellm.llms.bedrock.files.transformation import BedrockFilesConfig
+        bedrock_config = BedrockFilesConfig()
+        expected_s3_uri, _ = bedrock_config._convert_https_url_to_s3_uri(captured_put_url)
+        assert file_obj.id == expected_s3_uri
+

tests/image_gen_tests/request_payloads/azure_gpt_image_1.json

@@ -0,0 +1,4 @@
+{
+  "model": "gpt-image-1",
+  "prompt": "test prompt"
+}

tests/image_gen_tests/test_image_generation.py

@@ -5,6 +5,7 @@ import logging
 import os
 import sys
 import traceback
+from unittest.mock import AsyncMock, patch
 
 
 sys.path.insert(
@@ -329,3 +330,33 @@ async def test_aiml_image_generation_with_dynamic_api_key():
         assert captured_json_data is not None
         assert captured_json_data["prompt"] == "A cute baby sea otter"
         assert captured_json_data["model"] == "flux-pro/v1.1"
+
+@pytest.mark.asyncio
+async def test_azure_image_generation_request_body():
+    from litellm import aimage_generation
+    test_dir = os.path.dirname(__file__)
+    expected_path = os.path.join(
+        test_dir, "request_payloads", "azure_gpt_image_1.json"
+    )
+    with open(expected_path, "r") as f:
+        expected_body = json.load(f)
+
+    with patch(
+        "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post",
+        new_callable=AsyncMock,
+    ) as mock_post:
+        mock_post.side_effect = Exception("test")
+
+        with pytest.raises(Exception):
+            await aimage_generation(
+                    model="azure/gpt-image-1",
+                    prompt="test prompt",
+                    api_base="https://example.azure.com",
+                    api_key="test-key",
+                    api_version="2025-04-01-preview",
+                )
+
+        mock_post.assert_called_once()
+        call_args = mock_post.call_args
+        request_json = call_args.kwargs.get("json", {})
+        assert request_json == expected_body