Merge main into fix-redis-timedelta-serialization

.circleci/config.yml

@@ -671,6 +671,7 @@ jobs:
             pip install mypy
             pip install "google-generativeai==0.3.2"
             pip install "google-cloud-aiplatform==1.43.0"
+            pip install "google-genai==1.22.0"
             pip install pyarrow
             pip install "boto3==1.36.0"
             pip install "aioboto3==13.4.0"
@@ -1457,6 +1458,7 @@ jobs:
       # - run: python ./tests/documentation_tests/test_general_setting_keys.py
       - run: python ./tests/code_coverage_tests/check_licenses.py
       - run: python ./tests/code_coverage_tests/router_code_coverage.py
+      - run: python ./tests/code_coverage_tests/test_chat_completion_imports.py
       - run: python ./tests/code_coverage_tests/info_log_check.py
       - run: python ./tests/code_coverage_tests/test_ban_set_verbose.py
       - run: python ./tests/code_coverage_tests/code_qa_check_tests.py
@@ -2824,8 +2826,8 @@ jobs:
             source "$NVM_DIR/bash_completion"
             
             # Install and use Node version
-            nvm install v18.17.0
-            nvm use v18.17.0
+            nvm install v20
+            nvm use v20
             
             cd ui/litellm-dashboard
             
@@ -2878,7 +2880,26 @@ jobs:
           name: Install Playwright Browsers
           command: |
             npx playwright install
+      - run:
+          name: Run UI unit tests (Vitest)
+          command: |
+            # Use Node 20 (several deps require >=20)
+            export NVM_DIR="/opt/circleci/.nvm"
+            source "$NVM_DIR/nvm.sh"
+            nvm install 20
+            nvm use 20
       
+            cd ui/litellm-dashboard
+            npm ci || npm install
+      
+            # CI run, with both LCOV (Codecov) and HTML (artifact you can click)
+            CI=true npm run test -- --run --coverage \
+              --coverage.provider=v8 \
+              --coverage.reporter=lcov \
+              --coverage.reporter=html \
+              --coverage.reportsDirectory=coverage/html
+
+
       - run:
           name: Build Docker image
           command: docker build -t my-app:latest -f ./docker/Dockerfile.database .

.gitignore

@@ -95,4 +95,5 @@ test.py
 litellm_config.yaml
 .cursor
 .vscode/launch.json
-litellm/proxy/to_delete_loadtest_work/*
+litellm/proxy/to_delete_loadtest_work/*
+update_model_cost_map.py

Dockerfile

@@ -41,9 +41,6 @@ RUN pip uninstall jwt -y
 RUN pip uninstall PyJWT -y
 RUN pip install PyJWT==2.9.0 --no-cache-dir
 
-# Build Admin UI
-RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
-
 # Runtime stage
 FROM $LITELLM_RUNTIME_IMAGE AS runtime
 

README.md

@@ -25,7 +25,7 @@
     <a href="https://discord.gg/wuPM9dRgDw">
         <img src="https://img.shields.io/static/v1?label=Chat%20on&message=Discord&color=blue&logo=Discord&style=flat-square" alt="Discord">
     </a>
-    <a href="https://join.slack.com/share/enQtOTE0ODczMzk2Nzk4NC01YjUxNjY2YjBlYTFmNDRiZTM3NDFiYTM3MzVkODFiMDVjOGRjMmNmZTZkZTMzOWQzZGQyZWIwYjQ0MWExYmE3">
+    <a href="https://www.litellm.ai/support">
         <img src="https://img.shields.io/static/v1?label=Chat%20on&message=Slack&color=black&logo=Slack&style=flat-square" alt="Slack">
     </a>
 </h4>
@@ -37,7 +37,7 @@ LiteLLM manages:
 - Retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - [Router](https://docs.litellm.ai/docs/routing)
 - Set Budgets & Rate limits per project, api key, model [LiteLLM Proxy Server (LLM Gateway)](https://docs.litellm.ai/docs/simple_proxy)
 
-[**Jump to LiteLLM Proxy (LLM Gateway) Docs**](https://github.com/BerriAI/litellm?tab=readme-ov-file#openai-proxy---docs) <br>
+[**Jump to LiteLLM Proxy (LLM Gateway) Docs**](https://github.com/BerriAI/litellm?tab=readme-ov-file#litellm-proxy-server-llm-gateway---docs) <br>
 [**Jump to Supported LLM Providers**](https://github.com/BerriAI/litellm?tab=readme-ov-file#supported-providers-docs)
 
 🚨 **Stable Release:** Use docker images with the `-stable` tag. These have undergone 12 hour load tests, before being published. [More information about the release cycle here](https://docs.litellm.ai/docs/proxy/release_cycle)
@@ -316,6 +316,7 @@ curl 'http://0.0.0.0:4000/key/generate' \
 | [google AI Studio - gemini](https://docs.litellm.ai/docs/providers/gemini)          | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                               |                                                                         |
 | [mistral ai api](https://docs.litellm.ai/docs/providers/mistral)                    | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 | ✅                                                                             |                                                                         |
 | [cloudflare AI Workers](https://docs.litellm.ai/docs/providers/cloudflare_workers)  | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                               |                                                                         |
+| [CompactifAI](https://docs.litellm.ai/docs/providers/compactifai)                   | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                               |                                                                         |
 | [cohere](https://docs.litellm.ai/docs/providers/cohere)                             | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 | ✅                                                                             |                                                                         |
 | [anthropic](https://docs.litellm.ai/docs/providers/anthropic)                       | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                               |                                                                         |
 | [empower](https://docs.litellm.ai/docs/providers/empower)                    | ✅                                                      | ✅                                                                              | ✅                                                                                  | ✅                                                                                |
@@ -345,6 +346,7 @@ curl 'http://0.0.0.0:4000/key/generate' \
 | [Featherless AI](https://docs.litellm.ai/docs/providers/featherless_ai)                              | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                               |                                                                         |
 | [Nebius AI Studio](https://docs.litellm.ai/docs/providers/nebius)                             | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 | ✅                                                                             |                                                                         |
 | [Heroku](https://docs.litellm.ai/docs/providers/heroku)                             | ✅                                                       | ✅                                                                               |                                                                                    |                                                                                  |                                                                              |                                                                         |
+| [OVHCloud AI Endpoints](https://docs.litellm.ai/docs/providers/ovhcloud)                             | ✅                                                       | ✅                                                                               |                                                                                    |                                                                                  |                                                                              |                                                                         |
 
 [**Read the Docs**](https://docs.litellm.ai/docs/)
 
@@ -408,7 +410,7 @@ All these checks must pass before your PR can be merged.
 
 - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version)
 - [Community Discord 💭](https://discord.gg/wuPM9dRgDw)
-- [Community Slack 💭](https://join.slack.com/share/enQtOTE0ODczMzk2Nzk4NC01YjUxNjY2YjBlYTFmNDRiZTM3NDFiYTM3MzVkODFiMDVjOGRjMmNmZTZkZTMzOWQzZGQyZWIwYjQ0MWExYmE3)
+- [Community Slack 💭](https://www.litellm.ai/support)
 - Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬
 - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai
 

cookbook/litellm_proxy_server/batch_api/bedrock/bedrock.py

@@ -0,0 +1,25 @@
+from openai import OpenAI
+
+client = OpenAI(
+    base_url="http://0.0.0.0:4000",
+    api_key="sk-1234",
+)
+
+BEDROCK_BATCH_MODEL = "bedrock/batch-anthropic.claude-3-5-sonnet-20240620-v1:0"
+
+# Upload file
+batch_input_file = client.files.create(
+    file=open("./bedrock_batch_completions.jsonl", "rb"),
+    purpose="batch",
+    extra_body={"target_model_names": BEDROCK_BATCH_MODEL}
+)
+print(batch_input_file)
+
+# Create batch
+batch = client.batches.create( 
+    input_file_id=batch_input_file.id,
+    endpoint="/v1/chat/completions",
+    completion_window="24h",
+    metadata={"description": "Test batch job"},
+)
+print(batch)

cookbook/litellm_proxy_server/batch_api/bedrock/bedrock_batch_completions.jsonl

@@ -0,0 +1,128 @@
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}

cookbook/litellm_proxy_server/mcp/mcp_with_litellm_proxy.py

@@ -0,0 +1,36 @@
+"""
+Use LiteLLM Proxy MCP Gateway to call MCP tools.
+
+When using LiteLLM Proxy, you can use the same MCP tools across all your LLM providers.
+"""
+import openai
+
+client = openai.OpenAI(
+    api_key="sk-1234", # paste your litellm proxy api key here
+    base_url="http://localhost:4000" # paste your litellm proxy base url here
+)
+print("Making API request to Responses API with MCP tools")
+
+response = client.responses.create(
+    model="gpt-5",
+    input=[
+        {
+            "role": "user",
+            "content": "give me TLDR of what BerriAI/litellm repo is about",
+            "type": "message"
+        }
+    ],
+    tools=[
+        {
+            "type": "mcp",
+            "server_label": "litellm",
+            "server_url": "litellm_proxy",
+            "require_approval": "never"
+        }
+    ],
+    stream=True,
+    tool_choice="required"
+)
+
+for chunk in response:
+    print("response chunk: ", chunk)

cookbook/misc/RELEASE_NOTES_GENERATION_INSTRUCTIONS.md

@@ -0,0 +1,256 @@
+# LiteLLM Release Notes Generation Instructions
+
+This document provides comprehensive instructions for AI agents to generate release notes for LiteLLM following the established format and style.
+
+## Required Inputs
+
+1. **Release Version** (e.g., `v1.76.3-stable`)
+2. **PR Diff/Changelog** - List of PRs with titles and contributors
+3. **Previous Version Commit Hash** - To compare model pricing changes
+4. **Reference Release Notes** - Previous release notes to follow style/format
+
+## Step-by-Step Process
+
+### 1. Initial Setup and Analysis
+
+```bash
+# Check git diff for model pricing changes
+git diff <previous_commit_hash> HEAD -- model_prices_and_context_window.json
+```
+
+**Key Analysis Points:**
+- New models added (look for new entries)
+- Deprecated models removed (look for deleted entries)
+- Pricing updates (look for cost changes)
+- Feature support changes (tool calling, reasoning, etc.)
+
+### 2. Release Notes Structure
+
+Follow this exact structure based on `docs/my-website/release_notes/v1.76.1-stable/index.md`:
+
+```markdown
+---
+title: "v1.76.X-stable - [Key Theme]"
+slug: "v1-76-X"
+date: YYYY-MM-DDTHH:mm:ss
+authors: [standard author block]
+hide_table_of_contents: false
+---
+
+## Deploy this version
+[Docker and pip installation tabs]
+
+## Key Highlights
+[3-5 bullet points of major features]
+
+## Major Changes
+[Critical changes users need to know]
+
+## Performance Improvements
+[Performance-related changes]
+
+## New Models / Updated Models
+[Detailed model tables and provider updates]
+
+## LLM API Endpoints
+[API-related features and fixes]
+
+## Management Endpoints / UI
+[Admin interface and management changes]
+
+## Logging / Guardrail Integrations
+[Observability and security features]
+
+## Performance / Loadbalancing / Reliability improvements
+[Infrastructure improvements]
+
+## General Proxy Improvements
+[Other proxy-related changes]
+
+## New Contributors
+[List of first-time contributors]
+
+## Full Changelog
+[Link to GitHub comparison]
+```
+
+### 3. Categorization Rules
+
+**Performance Improvements:**
+- RPS improvements
+- Memory optimizations
+- CPU usage optimizations
+- Timeout controls
+- Worker configuration
+
+**New Models/Updated Models:**
+- Extract from model_prices_and_context_window.json diff
+- Create tables with: Provider, Model, Context Window, Input Cost, Output Cost, Features
+- Group by provider
+- Note pricing corrections
+- Highlight deprecated models
+
+**Provider Features:**
+- Group by provider (Gemini, OpenAI, Anthropic, etc.)
+- Link to provider docs: `../../docs/providers/[provider_name]`
+- Separate features from bug fixes
+
+**API Endpoints:**
+- Images API
+- Video Generation (if applicable)
+- Responses API
+- Passthrough endpoints
+- General chat completions
+
+**UI/Management:**
+- Authentication changes
+- Dashboard improvements
+- Team management
+- Key management
+
+**Integrations:**
+- Logging providers (Datadog, Braintrust, etc.)
+- Guardrails
+- Cost tracking
+- Observability
+
+### 4. Documentation Linking Strategy
+
+**Link to docs when:**
+- New provider support added
+- Significant feature additions
+- API endpoint changes
+- Integration additions
+
+**Link format:** `../../docs/[category]/[specific_doc]`
+
+**Common doc paths:**
+- `../../docs/providers/[provider]` - Provider-specific docs
+- `../../docs/image_generation` - Image generation
+- `../../docs/video_generation` - Video generation (if exists)
+- `../../docs/response_api` - Responses API
+- `../../docs/proxy/logging` - Logging integrations
+- `../../docs/proxy/guardrails` - Guardrails
+- `../../docs/pass_through/[provider]` - Passthrough endpoints
+
+### 5. Model Table Generation
+
+From git diff analysis, create tables like:
+
+```markdown
+| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Features |
+| -------- | ----- | -------------- | ------------------- | -------------------- | -------- |
+| OpenRouter | `openrouter/openai/gpt-4.1` | 1M | $2.00 | $8.00 | Chat completions with vision |
+```
+
+**Extract from JSON:**
+- `max_input_tokens` → Context Window
+- `input_cost_per_token` × 1,000,000 → Input cost
+- `output_cost_per_token` × 1,000,000 → Output cost
+- `supports_*` fields → Features
+- Special pricing fields (per image, per second) for generation models
+
+### 6. PR Categorization Logic
+
+**By Keywords in PR Title:**
+- `[Perf]`, `Performance`, `RPS` → Performance Improvements
+- `[Bug]`, `[Bug Fix]`, `Fix` → Bug Fixes section
+- `[Feat]`, `[Feature]`, `Add support` → Features section
+- `[Docs]` → Documentation (usually exclude from main sections)
+- Provider names (Gemini, OpenAI, etc.) → Group under provider
+
+**By PR Content Analysis:**
+- New model additions → New Models section
+- UI changes → Management Endpoints/UI
+- Logging/observability → Logging/Guardrail Integrations
+- Rate limiting/budgets → Performance/Reliability
+- Authentication → Management Endpoints
+
+### 7. Writing Style Guidelines
+
+**Tone:**
+- Professional but accessible
+- Focus on user impact
+- Highlight breaking changes clearly
+- Use active voice
+
+**Formatting:**
+- Use consistent markdown formatting
+- Include PR links: `[PR #XXXXX](https://github.com/BerriAI/litellm/pull/XXXXX)`
+- Use code blocks for configuration examples
+- Bold important terms and section headers
+
+**Warnings/Notes:**
+- Add warning boxes for breaking changes
+- Include migration instructions when needed
+- Provide override options for default changes
+
+### 8. Quality Checks
+
+**Before finalizing:**
+- Verify all PR links work
+- Check documentation links are valid
+- Ensure model pricing is accurate
+- Confirm provider names are consistent
+- Review for typos and formatting issues
+
+### 9. Common Patterns to Follow
+
+**Performance Changes:**
+```markdown
+- **+400 RPS Performance Boost** - Description - [PR #XXXXX](link)
+```
+
+**New Models:**
+Always include pricing table and feature highlights
+
+**Breaking Changes:**
+```markdown
+:::warning
+This release has a known issue...
+:::
+```
+
+**Provider Features:**
+```markdown
+- **[Provider Name](../../docs/providers/provider)**
+    - Feature description - [PR #XXXXX](link)
+```
+
+### 10. Missing Documentation Check
+
+**Review for missing docs:**
+- New providers without documentation
+- New API endpoints without examples
+- Complex features without guides
+- Integration setup instructions
+
+**Flag for documentation needs:**
+- New provider integrations
+- Significant API changes
+- Complex configuration options
+- Migration requirements
+
+## Example Command Workflow
+
+```bash
+# 1. Get model changes
+git diff <commit> HEAD -- model_prices_and_context_window.json
+
+# 2. Analyze PR list for categorization
+# 3. Create release notes following template
+# 4. Link to appropriate documentation
+# 5. Review for missing documentation needs
+```
+
+## Output Requirements
+
+- Follow exact markdown structure from reference
+- Include all PR links and contributors
+- Provide accurate model pricing tables
+- Link to relevant documentation
+- Highlight breaking changes with warnings
+- Include deployment instructions
+- End with full changelog link
+
+This process ensures consistent, comprehensive release notes that help users understand changes and upgrade smoothly.

docs/my-website/docs/batches.md

@@ -7,7 +7,7 @@ Covers Batches, Files
 
 | Feature | Supported | Notes | 
 |-------|-------|-------|
-| Supported Providers | OpenAI, Azure, Vertex | - |
+| Supported Providers | OpenAI, Azure, Vertex, Bedrock | - |
 | ✨ Cost Tracking | ✅ | LiteLLM Enterprise only |
 | Logging | ✅ | Works across all logging integrations |
 
@@ -178,6 +178,7 @@ print("list_batches_response=", list_batches_response)
 ### [Azure OpenAI](./providers/azure#azure-batches-api)
 ### [OpenAI](#quick-start)
 ### [Vertex AI](./providers/vertex#batch-apis)
+### [Bedrock](./providers/bedrock_batches)
 
 
 ## How Cost Tracking for Batches API Works

docs/my-website/docs/completion/input.md

@@ -65,6 +65,7 @@ Use `litellm.get_supported_openai_params()` for an updated list of params for ea
 | Github | ✅| ✅ | ✅ | ✅| ✅ | ✅ | ✅ | ✅| ✅ | ✅| ✅|| || ✅ | ✅ (model dependent) | ✅ (model dependent) || ||
 | Novita AI| ✅| ✅ || ✅| ✅ | ✅ | ✅ | ✅| ✅ | ✅| || ✅||| |||| ||
 | Bytez | ✅| ✅ || ✅| ✅ | | | ✅|| || || || || || ||
+| OVHCloud AI Endpoints | ✅ | | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | | | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | |
 
 :::note
 

docs/my-website/docs/mcp.md

@@ -114,7 +114,6 @@ mcp_servers:
     description: "My custom MCP server"
     auth_type: "api_key"
     auth_value: "abc123"
-    spec_version: "2025-03-26"
 ```
 
 **Configuration Options:**
@@ -195,70 +194,169 @@ litellm_settings:
 
 ## Using your MCP
 
+### Use on LiteLLM UI 
+
+Follow this walkthrough to use your MCP on LiteLLM UI
+
+<iframe width="840" height="500" src="https://www.loom.com/embed/57e0763267254bc79dbe6658d0b8758c" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>
+
+### Use with Responses API
+
+Replace `http://localhost:4000` with your LiteLLM Proxy base URL.
+
+Demo Video Using Responses API with LiteLLM Proxy: [Demo video here](https://www.loom.com/share/34587e618c5c47c0b0d67b4e4d02718f?sid=2caf3d45-ead4-4490-bcc1-8d6dd6041c02)
+
+
 <Tabs>
-<TabItem value="openai" label="OpenAI API">
-
-#### Connect via OpenAI Responses API
-
-Use the OpenAI Responses API to connect to your LiteLLM MCP server:
+<TabItem value="curl" label="cURL">
 
 ```bash title="cURL Example" showLineNumbers
-curl --location 'https://api.openai.com/v1/responses' \
+curl --location 'http://localhost:4000/v1/responses' \
 --header 'Content-Type: application/json' \
---header "Authorization: Bearer $OPENAI_API_KEY" \
+--header "Authorization: Bearer sk-1234" \
 --data '{
-    "model": "gpt-4o",
+    "model": "gpt-5",
+    "input": [
+    {
+      "role": "user",
+      "content": "give me TLDR of what BerriAI/litellm repo is about",
+      "type": "message"
+    }
+  ],
     "tools": [
         {
             "type": "mcp",
             "server_label": "litellm",
             "server_url": "litellm_proxy",
-            "require_approval": "never",
-            "headers": {
-                "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY"
-            }
+            "require_approval": "never"
         }
     ],
-    "input": "Run available tools",
+    "stream": true,
     "tool_choice": "required"
 }'
 ```
 
 </TabItem>
+<TabItem value="python" label="Python SDK">
 
-<TabItem value="litellm" label="LiteLLM Proxy">
+```python title="Python SDK Example" showLineNumbers
+"""
+Use LiteLLM Proxy MCP Gateway to call MCP tools.
 
-#### Connect via LiteLLM Proxy Responses API
+When using LiteLLM Proxy, you can use the same MCP tools across all your LLM providers.
+"""
+import openai
 
-Use this when calling LiteLLM Proxy for LLM API requests to `/v1/responses` endpoint.
+client = openai.OpenAI(
+    api_key="sk-1234", # paste your litellm proxy api key here
+    base_url="http://localhost:4000" # paste your litellm proxy base url here
+)
+print("Making API request to Responses API with MCP tools")
 
-```bash title="cURL Example" showLineNumbers
-curl --location '<your-litellm-proxy-base-url>/v1/responses' \
---header 'Content-Type: application/json' \
---header "Authorization: Bearer $LITELLM_API_KEY" \
---data '{
-    "model": "gpt-4o",
-    "tools": [
+response = client.responses.create(
+    model="gpt-5",
+    input=[
+        {
+            "role": "user",
+            "content": "give me TLDR of what BerriAI/litellm repo is about",
+            "type": "message"
+        }
+    ],
+    tools=[
         {
             "type": "mcp",
             "server_label": "litellm",
             "server_url": "litellm_proxy",
-            "require_approval": "never",
-            "headers": {
-                "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY"
-            }
+            "require_approval": "never"
         }
     ],
-    "input": "Run available tools",
+    stream=True,
+    tool_choice="required"
+)
+
+for chunk in response:
+    print("response chunk: ", chunk)
+```
+
+</TabItem>
+</Tabs>
+
+#### Specifying MCP Tools
+
+You can specify which MCP tools are available by using the `allowed_tools` parameter. This allows you to restrict access to specific tools within an MCP server.
+
+To get the list of allowed tools when using LiteLLM MCP Gateway, you can naigate to the LiteLLM UI on MCP Servers > MCP Tools > Click the Tool > Copy Tool Name.
+
+<Tabs>
+<TabItem value="curl" label="cURL">
+
+```bash title="cURL Example with allowed_tools" showLineNumbers
+curl --location 'http://localhost:4000/v1/responses' \
+--header 'Content-Type: application/json' \
+--header "Authorization: Bearer sk-1234" \
+--data '{
+    "model": "gpt-5",
+    "input": [
+    {
+      "role": "user",
+      "content": "give me TLDR of what BerriAI/litellm repo is about",
+      "type": "message"
+    }
+  ],
+    "tools": [
+        {
+            "type": "mcp",
+            "server_label": "litellm",
+            "server_url": "litellm_proxy/mcp",
+            "require_approval": "never",
+            "allowed_tools": ["GitMCP-fetch_litellm_documentation"]
+        }
+    ],
+    "stream": true,
     "tool_choice": "required"
 }'
 ```
 
 </TabItem>
+<TabItem value="python" label="Python SDK">
 
-<TabItem value="cursor" label="Cursor IDE">
+```python title="Python SDK Example with allowed_tools" showLineNumbers
+import openai
 
-#### Connect via Cursor IDE
+client = openai.OpenAI(
+    api_key="sk-1234",
+    base_url="http://localhost:4000"
+)
+
+response = client.responses.create(
+    model="gpt-5",
+    input=[
+        {
+            "role": "user",
+            "content": "give me TLDR of what BerriAI/litellm repo is about",
+            "type": "message"
+        }
+    ],
+    tools=[
+        {
+            "type": "mcp",
+            "server_label": "litellm",
+            "server_url": "litellm_proxy/mcp",
+            "require_approval": "never",
+            "allowed_tools": ["GitMCP-fetch_litellm_documentation"]
+        }
+    ],
+    stream=True,
+    tool_choice="required"
+)
+
+print(response)
+```
+
+</TabItem>
+</Tabs>
+
+### Use with Cursor IDE
 
 Use tools directly from Cursor IDE with LiteLLM MCP:
 
@@ -281,9 +379,6 @@ Use tools directly from Cursor IDE with LiteLLM MCP:
 }
 ```
 
-</TabItem>
-</Tabs>
-
 #### How it works when server_url="litellm_proxy"
 
 When server_url="litellm_proxy", LiteLLM bridges non-MCP providers to your MCP tools.
@@ -620,7 +715,6 @@ mcp_servers:
     url: https://mcp.deepwiki.com/mcp
     transport: "http"
     auth_type: "none"
-    spec_version: "2025-03-26"
     access_groups: ["dev_group"]
 ```
 

docs/my-website/docs/observability/helicone_integration.md

@@ -1,3 +1,6 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
 # Helicone - OSS LLM Observability Platform
 
 :::tip
@@ -9,9 +12,68 @@ https://github.com/BerriAI/litellm
 
 [Helicone](https://helicone.ai/) is an open source observability platform that proxies your LLM requests and provides key insights into your usage, spend, latency and more.
 
-## Using Helicone with LiteLLM
+## Quick Start
 
-LiteLLM provides `success_callbacks` and `failure_callbacks`, allowing you to easily log data to Helicone based on the status of your responses.
+<Tabs>
+<TabItem value="sdk" label="Python SDK">
+
+Use just 1 line of code to instantly log your responses **across all providers** with Helicone:
+
+```python
+import os
+from litellm import completion
+
+## Set env variables
+os.environ["HELICONE_API_KEY"] = "your-helicone-key"
+os.environ["OPENAI_API_KEY"] = "your-openai-key"
+
+# Set callbacks
+litellm.success_callback = ["helicone"]
+
+# OpenAI call
+response = completion(
+    model="gpt-4o",
+    messages=[{"role": "user", "content": "Hi 👋 - I'm OpenAI"}],
+)
+
+print(response)
+```
+
+</TabItem>
+<TabItem value="proxy" label="LiteLLM Proxy">
+
+Add Helicone to your LiteLLM proxy configuration:
+
+```yaml title="config.yaml"
+model_list:
+  - model_name: gpt-4
+    litellm_params:
+      model: gpt-4
+      api_key: os.environ/OPENAI_API_KEY
+
+# Add Helicone callback
+litellm_settings:
+  success_callback: ["helicone"]
+  
+# Set Helicone API key
+environment_variables:
+  HELICONE_API_KEY: "your-helicone-key"
+```
+
+Start the proxy:
+```bash
+litellm --config config.yaml
+```
+
+</TabItem>
+</Tabs>
+
+## Integration Methods
+
+There are two main approaches to integrate Helicone with LiteLLM:
+
+1. **Callbacks**: Log to Helicone while using any provider
+2. **Proxy Mode**: Use Helicone as a proxy for advanced features
 
 ### Supported LLM Providers
 
@@ -26,27 +88,16 @@ Helicone can log requests across [various LLM providers](https://docs.helicone.a
 - Replicate
 - And more
 
-### Integration Methods
+## Method 1: Using Callbacks
 
-There are two main approaches to integrate Helicone with LiteLLM:
+Log requests to Helicone while using any LLM provider directly.
 
-1. Using callbacks
-2. Using Helicone as a proxy
-
-Let's explore each method in detail.
-
-### Approach 1: Use Callbacks
-
-Use just 1 line of code to instantly log your responses **across all providers** with Helicone:
-
-```python
-litellm.success_callback = ["helicone"]
-```
-
-Complete Code
+<Tabs>
+<TabItem value="sdk" label="Python SDK">
 
 ```python
 import os
+import litellm
 from litellm import completion
 
 ## Set env variables
@@ -66,28 +117,78 @@ response = completion(
 print(response)
 ```
 
-### Approach 2: Use Helicone as a proxy
+</TabItem>
+<TabItem value="proxy" label="LiteLLM Proxy">
+
+```yaml title="config.yaml"
+model_list:
+  - model_name: gpt-4
+    litellm_params:
+      model: gpt-4
+      api_key: os.environ/OPENAI_API_KEY
+  - model_name: claude-3
+    litellm_params:
+      model: anthropic/claude-3-sonnet-20240229
+      api_key: os.environ/ANTHROPIC_API_KEY
+
+# Add Helicone logging
+litellm_settings:
+  success_callback: ["helicone"]
+  
+# Environment variables
+environment_variables:
+  HELICONE_API_KEY: "your-helicone-key"
+  OPENAI_API_KEY: "your-openai-key"
+  ANTHROPIC_API_KEY: "your-anthropic-key"
+```
+
+Start the proxy:
+```bash
+litellm --config config.yaml
+```
+
+Make requests to your proxy:
+```python
+import openai
+
+client = openai.OpenAI(
+    api_key="anything",  # proxy doesn't require real API key
+    base_url="http://localhost:4000"
+)
+
+response = client.chat.completions.create(
+    model="gpt-4",  # This gets logged to Helicone
+    messages=[{"role": "user", "content": "Hello!"}]
+)
+```
+
+</TabItem>
+</Tabs>
+
+## Method 2: Using Helicone as a Proxy
 
 Helicone's proxy provides [advanced functionality](https://docs.helicone.ai/getting-started/proxy-vs-async) like caching, rate limiting, LLM security through [PromptArmor](https://promptarmor.com/) and more.
 
-To use Helicone as a proxy for your LLM requests:
+<Tabs>
+<TabItem value="sdk" label="Python SDK">
 
-1. Set Helicone as your base URL via: litellm.api_base
-2. Pass in Helicone request headers via: litellm.metadata
-
-Complete Code:
+Set Helicone as your base URL and pass authentication headers:
 
 ```python
 import os
 import litellm
 from litellm import completion
 
+# Configure LiteLLM to use Helicone proxy
 litellm.api_base = "https://oai.hconeai.com/v1"
 litellm.headers = {
-    "Helicone-Auth": f"Bearer {os.getenv('HELICONE_API_KEY')}",  # Authenticate to send requests to Helicone API
+    "Helicone-Auth": f"Bearer {os.getenv('HELICONE_API_KEY')}",
 }
 
-response = litellm.completion(
+# Set your OpenAI API key
+os.environ["OPENAI_API_KEY"] = "your-openai-key"
+
+response = completion(
     model="gpt-3.5-turbo",
     messages=[{"role": "user", "content": "How does a court case get to the Supreme Court?"}]
 )
@@ -136,36 +237,119 @@ litellm.metadata = {
 }
 ```
 
-### Session Tracking and Tracing
+</TabItem>
+</Tabs>
+
+## Session Tracking and Tracing
 
 Track multi-step and agentic LLM interactions using session IDs and paths:
 
-```python
-litellm.metadata = {
-    "Helicone-Auth": f"Bearer {os.getenv('HELICONE_API_KEY')}",  # Authenticate to send requests to Helicone API
-    "Helicone-Session-Id": "session-abc-123",  # The session ID you want to track
-    "Helicone-Session-Path": "parent-trace/child-trace",  # The path of the session
-}
-```
-
-- `Helicone-Session-Id`: Use this to specify the unique identifier for the session you want to track. This allows you to group related requests together.
-- `Helicone-Session-Path`: This header defines the path of the session, allowing you to represent parent and child traces. For example, "parent/child" represents a child trace of a parent trace.
-
-By using these two headers, you can effectively group and visualize multi-step LLM interactions, gaining insights into complex AI workflows.
-
-### Retry and Fallback Mechanisms
-
-Set up retry mechanisms and fallback options:
+<Tabs>
+<TabItem value="sdk" label="Python SDK">
 
 ```python
+import litellm
+
+litellm.api_base = "https://oai.hconeai.com/v1"
 litellm.metadata = {
-    "Helicone-Auth": f"Bearer {os.getenv('HELICONE_API_KEY')}",  # Authenticate to send requests to Helicone API
-    "Helicone-Retry-Enabled": "true",  # Enable retry mechanism
-    "helicone-retry-num": "3",  # Set number of retries
-    "helicone-retry-factor": "2",  # Set exponential backoff factor
-    "Helicone-Fallbacks": '["gpt-3.5-turbo", "gpt-4"]',  # Set fallback models
+    "Helicone-Auth": f"Bearer {os.getenv('HELICONE_API_KEY')}",
+    "Helicone-Session-Id": "session-abc-123",
+    "Helicone-Session-Path": "parent-trace/child-trace",
 }
+
+response = litellm.completion(
+    model="gpt-3.5-turbo",
+    messages=[{"role": "user", "content": "Start a conversation"}]
+)
 ```
 
+</TabItem>
+<TabItem value="proxy" label="LiteLLM Proxy">
+
+```python
+import openai
+
+client = openai.OpenAI(
+    api_key="anything",
+    base_url="http://localhost:4000"
+)
+
+# First request in session
+response1 = client.chat.completions.create(
+    model="gpt-4",
+    messages=[{"role": "user", "content": "Hello"}],
+    extra_headers={
+        "Helicone-Session-Id": "session-abc-123",
+        "Helicone-Session-Path": "conversation/greeting"
+    }
+)
+
+# Follow-up request in same session
+response2 = client.chat.completions.create(
+    model="gpt-4",
+    messages=[{"role": "user", "content": "Tell me more"}],
+    extra_headers={
+        "Helicone-Session-Id": "session-abc-123",
+        "Helicone-Session-Path": "conversation/follow-up"
+    }
+)
+```
+
+</TabItem>
+</Tabs>
+
+- `Helicone-Session-Id`: Unique identifier for the session to group related requests
+- `Helicone-Session-Path`: Hierarchical path to represent parent/child traces (e.g., "parent/child")
+
+## Retry and Fallback Mechanisms
+
+<Tabs>
+<TabItem value="sdk" label="Python SDK">
+
+```python
+import litellm
+
+litellm.api_base = "https://oai.hconeai.com/v1"
+litellm.metadata = {
+    "Helicone-Auth": f"Bearer {os.getenv('HELICONE_API_KEY')}",
+    "Helicone-Retry-Enabled": "true",
+    "helicone-retry-num": "3",
+    "helicone-retry-factor": "2",  # Exponential backoff
+    "Helicone-Fallbacks": '["gpt-3.5-turbo", "gpt-4"]',
+}
+
+response = litellm.completion(
+    model="gpt-4",
+    messages=[{"role": "user", "content": "Hello"}]
+)
+```
+
+</TabItem>
+<TabItem value="proxy" label="LiteLLM Proxy">
+
+```yaml title="config.yaml"
+model_list:
+  - model_name: gpt-4
+    litellm_params:
+      model: gpt-4
+      api_key: os.environ/OPENAI_API_KEY
+      api_base: "https://oai.hconeai.com/v1"
+
+default_litellm_params:
+  headers:
+    Helicone-Auth: "Bearer ${HELICONE_API_KEY}"
+    Helicone-Retry-Enabled: "true"
+    helicone-retry-num: "3"
+    helicone-retry-factor: "2"
+    Helicone-Fallbacks: '["gpt-3.5-turbo", "gpt-4"]'
+
+environment_variables:
+  HELICONE_API_KEY: "your-helicone-key"
+  OPENAI_API_KEY: "your-openai-key"
+```
+
+</TabItem>
+</Tabs>
+
 > **Supported Headers** - For a full list of supported Helicone headers and their descriptions, please refer to the [Helicone documentation](https://docs.helicone.ai/getting-started/quick-start).
 > By utilizing these headers and metadata options, you can gain deeper insights into your LLM usage, optimize performance, and better manage your AI workflows with Helicone and LiteLLM.

docs/my-website/docs/observability/posthog_integration.md

@@ -0,0 +1,216 @@
+# PostHog - Tracking LLM Usage Analytics
+
+## What is PostHog?
+
+PostHog is an open-source product analytics platform that helps you track and analyze how users interact with your product. For LLM applications, PostHog provides specialized AI features to track model usage, performance, and user interactions with your AI features.
+
+## Usage with LiteLLM Proxy (LLM Gateway)
+
+**Step 1**: Create a `config.yaml` file and set `litellm_settings`: `success_callback`
+
+```yaml
+model_list:
+  - model_name: gpt-3.5-turbo
+    litellm_params:
+      model: gpt-3.5-turbo
+
+litellm_settings:
+  success_callback: ["posthog"]
+  failure_callback: ["posthog"]
+```
+
+**Step 2**: Set required environment variables
+
+```shell
+export POSTHOG_API_KEY="your-posthog-api-key"
+# Optional, defaults to https://app.posthog.com
+export POSTHOG_API_URL="https://app.posthog.com" # optional
+```
+
+**Step 3**: Start the proxy, make a test request
+
+Start proxy
+
+```shell
+litellm --config config.yaml --debug
+```
+
+Test Request
+
+```shell
+curl --location 'http://0.0.0.0:4000/chat/completions' \
+    --header 'Content-Type: application/json' \
+    --data '{
+    "model": "gpt-3.5-turbo",
+    "messages": [
+        {
+        "role": "user",
+        "content": "what llm are you"
+        }
+    ],
+    "metadata": {
+        "user_id": "user-123",
+        "custom_field": "custom_value"
+    }
+}'
+```
+
+## Usage with LiteLLM Python SDK
+
+### Quick Start
+
+Use just 2 lines of code, to instantly log your responses **across all providers** with PostHog:
+
+```python
+litellm.success_callback = ["posthog"]
+litellm.failure_callback = ["posthog"] # logs errors to posthog
+```
+```python
+import litellm
+import os
+
+# from PostHog
+os.environ["POSTHOG_API_KEY"] = ""
+# Optional, defaults to https://app.posthog.com
+os.environ["POSTHOG_API_URL"] = "" # optional
+
+# LLM API Keys
+os.environ['OPENAI_API_KEY']=""
+
+# set posthog as a callback, litellm will send the data to posthog
+litellm.success_callback = ["posthog"]
+
+# openai call
+response = litellm.completion(
+    model="gpt-3.5-turbo",
+    messages=[
+        {"role": "user", "content": "Hi - i'm openai"}
+    ],
+    metadata = {
+        "user_id": "user-123", # set posthog user ID
+    }
+)
+```
+
+### Advanced
+
+#### Set User ID and Custom Metadata
+
+Pass `user_id` in `metadata` to associate events with specific users in PostHog:
+
+**With LiteLLM Python SDK:**
+
+```python
+import litellm
+
+litellm.success_callback = ["posthog"]
+
+response = litellm.completion(
+    model="gpt-3.5-turbo",
+    messages=[
+        {"role": "user", "content": "Hello world"}
+    ],
+    metadata={
+        "user_id": "user-123",  # Add user ID for PostHog tracking
+        "custom_field": "custom_value"  # Add custom metadata
+    }
+)
+```
+
+**With LiteLLM Proxy using OpenAI Python SDK:**
+
+```python
+import openai
+
+client = openai.OpenAI(
+    api_key="sk-1234",  # Your LiteLLM Proxy API key
+    base_url="http://0.0.0.0:4000"  # Your LiteLLM Proxy URL
+)
+
+response = client.chat.completions.create(
+    model="gpt-3.5-turbo",
+    messages=[
+        {"role": "user", "content": "Hello world"}
+    ],
+    extra_body={
+        "metadata": {
+            "user_id": "user-123",  # Add user ID for PostHog tracking
+            "project_name": "my-project",  # Add custom metadata
+            "environment": "production"
+        }
+    }
+)
+```
+
+#### Disable Logging for Specific Calls
+
+Use the `no-log` flag to prevent logging for specific calls:
+
+```python
+import litellm
+
+litellm.success_callback = ["posthog"]
+
+response = litellm.completion(
+    model="gpt-3.5-turbo",
+    messages=[
+        {"role": "user", "content": "This won't be logged"}
+    ],
+    metadata={"no-log": True}
+)
+```
+
+## What's Logged to PostHog?
+
+When LiteLLM logs to PostHog, it captures detailed information about your LLM usage:
+
+### For Completion Calls
+- **Model Information**: Provider, model name, model parameters
+- **Usage Metrics**: Input tokens, output tokens, total cost
+- **Performance**: Latency, completion time
+- **Content**: Input messages, model responses (respects privacy settings)
+- **Metadata**: Custom fields, user ID, trace information
+
+### For Embedding Calls
+- **Model Information**: Provider, model name
+- **Usage Metrics**: Input tokens, total cost
+- **Performance**: Latency
+- **Content**: Input text (respects privacy settings)
+- **Metadata**: Custom fields, user ID, trace information
+
+### For Errors
+- **Error Details**: Error type, error message, stack trace
+- **Context**: Model, provider, input that caused the error
+- **Timing**: When the error occurred, request duration
+
+## Environment Variables
+
+| Variable | Required | Description |
+|----------|----------|-------------|
+| `POSTHOG_API_KEY` | Yes | Your PostHog project API key |
+| `POSTHOG_API_URL` | No | PostHog API URL (defaults to https://app.posthog.com) |
+
+## Troubleshooting
+
+### 1. Missing API Key
+```
+Error: POSTHOG_API_KEY is not set
+```
+
+Set your PostHog API key:
+```python
+import os
+os.environ["POSTHOG_API_KEY"] = "your-api-key"
+```
+
+### 2. Custom PostHog Instance
+If you're using a self-hosted PostHog instance:
+```python
+import os
+os.environ["POSTHOG_API_URL"] = "https://your-posthog-instance.com"
+```
+
+### 3. Events Not Appearing
+- Check that your API key is correct
+- Verify network connectivity to PostHog
+- Events may take a few minutes to appear in PostHog dashboard

docs/my-website/docs/providers/bedrock.md

@@ -889,6 +889,19 @@ curl http://0.0.0.0:4000/v1/chat/completions \
 
 Example of using [Bedrock Guardrails with LiteLLM](https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-use-converse-api.html)
 
+### Selective Content Moderation with `guarded_text`
+
+LiteLLM supports selective content moderation using the `guarded_text` content type. This allows you to wrap only specific content that should be moderated by Bedrock Guardrails, rather than evaluating the entire conversation.
+
+**How it works:**
+- Content with `type: "guarded_text"` gets automatically wrapped in `guardrailConverseContent` blocks
+- Only the wrapped content is evaluated by Bedrock Guardrails
+- Regular content with `type: "text"` bypasses guardrail evaluation
+
+:::note
+If `guarded_text` is not used, the entire conversation history will be sent to the guardrail for evaluation, which can increase latency and costs.
+:::
+
 <Tabs>
 <TabItem value="sdk" label="LiteLLM SDK">
 
@@ -915,6 +928,24 @@ response = completion(
         "trace": "disabled",                   # The trace behavior for the guardrail. Can either be "disabled" or "enabled"
     },
 )
+
+# Selective guardrail usage with guarded_text - only specific content is evaluated
+response_guard = completion(
+    model="anthropic.claude-v2",
+    messages=[
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "What is the main topic of this legal document?"},
+                {"type": "guarded_text", "text": "This      document contains sensitive legal information that should be moderated by guardrails."}
+            ]
+        }
+    ],
+    guardrailConfig={
+        "guardrailIdentifier": "gr-abc123",
+        "guardrailVersion": "DRAFT"
+    }
+)
 ```
 </TabItem>
 <TabItem value="proxy" label="Proxy on request">
@@ -993,7 +1024,20 @@ response = client.chat.completions.create(model="bedrock-claude-v1", messages =
 temperature=0.7
 )
 
-print(response)
+# For adding selective guardrail usage with guarded_text
+response_guard = client.chat.completions.create(model="bedrock-claude-v1", messages = [
+   {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "What is the main topic of this legal document?"},
+                {"type": "guarded_text", "text": "This document contains sensitive legal information that should be moderated by guardrails."}
+            ]
+  }
+],
+temperature=0.7
+) 
+
+print(response_guard)
 ```
 </TabItem>
 </Tabs>
@@ -1777,6 +1821,7 @@ Here's an example of using a bedrock model with LiteLLM. For a complete list, re
 | Mistral 7B Instruct        | `completion(model='bedrock/mistral.mistral-7b-instruct-v0:2', messages=messages)`   | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` |
 | Mixtral 8x7B Instruct      | `completion(model='bedrock/mistral.mixtral-8x7b-instruct-v0:1', messages=messages)`   | `os.environ['AWS_ACCESS_KEY_ID']`, `os.environ['AWS_SECRET_ACCESS_KEY']`, `os.environ['AWS_REGION_NAME']` |
 
+
 ## Bedrock Embedding
 
 ### API keys
@@ -1798,11 +1843,29 @@ response = embedding(
 print(response)
 ```
 
+#### Titan V2 - encoding_format support
+```python
+from litellm import embedding
+# Float format (default)
+response = embedding(
+    model="bedrock/amazon.titan-embed-text-v2:0",
+    input=["good morning from litellm"],
+    encoding_format="float"  # Returns float array
+)
+
+# Binary format
+response = embedding(
+    model="bedrock/amazon.titan-embed-text-v2:0",
+    input=["good morning from litellm"],
+    encoding_format="base64"  # Returns base64 encoded binary
+)
+```
+
 ## Supported AWS Bedrock Embedding Models
 
 | Model Name           | Usage                               | Supported Additional OpenAI params |
 |----------------------|---------------------------------------------|-----|
-| Titan Embeddings V2 | `embedding(model="bedrock/amazon.titan-embed-text-v2:0", input=input)` | [here](https://github.com/BerriAI/litellm/blob/f5905e100068e7a4d61441d7453d7cf5609c2121/litellm/llms/bedrock/embed/amazon_titan_v2_transformation.py#L59) |
+| Titan Embeddings V2 | `embedding(model="bedrock/amazon.titan-embed-text-v2:0", input=input)` | `dimensions`, `encoding_format` |
 | Titan Embeddings - V1 | `embedding(model="bedrock/amazon.titan-embed-text-v1", input=input)` | [here](https://github.com/BerriAI/litellm/blob/f5905e100068e7a4d61441d7453d7cf5609c2121/litellm/llms/bedrock/embed/amazon_titan_g1_transformation.py#L53)
 | Titan Multimodal Embeddings | `embedding(model="bedrock/amazon.titan-embed-image-v1", input=input)` | [here](https://github.com/BerriAI/litellm/blob/f5905e100068e7a4d61441d7453d7cf5609c2121/litellm/llms/bedrock/embed/amazon_titan_multimodal_transformation.py#L28) |
 | Cohere Embeddings - English | `embedding(model="bedrock/cohere.embed-english-v3", input=input)` | [here](https://github.com/BerriAI/litellm/blob/f5905e100068e7a4d61441d7453d7cf5609c2121/litellm/llms/bedrock/embed/cohere_transformation.py#L18)
@@ -1891,6 +1954,39 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/images/generations' \
 </TabItem>
 </Tabs>
 
+### Using Inference Profiles with Image Generation
+
+For AWS Bedrock Application Inference Profiles with image generation, use the `model_id` parameter to specify the inference profile ARN:
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+from litellm import image_generation
+
+response = image_generation(
+    model="bedrock/amazon.nova-canvas-v1:0",
+    model_id="arn:aws:bedrock:eu-west-1:000000000000:application-inference-profile/a0a0a0a0a0a0",
+    prompt="A cute baby sea otter"
+)
+print(f"response: {response}")
+```
+
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+```yaml
+model_list:
+  - model_name: nova-canvas-inference-profile
+    litellm_params:
+      model: bedrock/amazon.nova-canvas-v1:0
+      model_id: arn:aws:bedrock:eu-west-1:000000000000:application-inference-profile/a0a0a0a0a0a0
+      aws_region_name: "eu-west-1"
+```
+
+</TabItem>
+</Tabs>
+
 ## Supported AWS Bedrock Image Generation Models
 
 | Model Name           | Function Call                               |

docs/my-website/docs/providers/bedrock_batches.md

@@ -0,0 +1,180 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Bedrock Batches
+
+Use Amazon Bedrock Batch Inference API through LiteLLM.
+
+| Property | Details |
+|----------|---------|
+| Description | Amazon Bedrock Batch Inference allows you to run inference on large datasets asynchronously |
+| Provider Doc | [AWS Bedrock Batch Inference ↗](https://docs.aws.amazon.com/bedrock/latest/userguide/batch-inference.html) |
+
+## Overview
+
+Use this to:
+
+- Run batch inference on large datasets with Bedrock models
+- Control batch model access by key/user/team (same as chat completion models)
+- Manage S3 storage for batch input/output files
+
+## (Proxy Admin) Usage
+
+Here's how to give developers access to your Bedrock Batch models.
+
+### 1. Setup config.yaml
+
+- Specify `mode: batch` for each model: Allows developers to know this is a batch model
+- Configure S3 bucket and AWS credentials for batch operations
+
+```yaml showLineNumbers title="litellm_config.yaml"
+model_list:
+  - model_name: "bedrock-batch-claude"
+    litellm_params:
+      model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0
+      #########################################################
+      ########## batch specific params ########################
+      s3_bucket_name: litellm-proxy
+      s3_region_name: us-west-2
+      s3_access_key_id: os.environ/AWS_ACCESS_KEY_ID
+      s3_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
+      aws_batch_role_arn: arn:aws:iam::888602223428:role/service-role/AmazonBedrockExecutionRoleForAgents_BB9HNW6V4CV
+    model_info: 
+      mode: batch # 👈 SPECIFY MODE AS BATCH, to tell user this is a batch model
+```
+
+**Required Parameters:**
+
+| Parameter | Description |
+|-----------|-------------|
+| `s3_bucket_name` | S3 bucket for batch input/output files |
+| `s3_region_name` | AWS region for S3 bucket |
+| `s3_access_key_id` | AWS access key for S3 bucket |
+| `s3_secret_access_key` | AWS secret key for S3 bucket |
+| `aws_batch_role_arn` | IAM role ARN for Bedrock batch operations. Bedrock Batch APIs require an IAM role ARN to be set. |
+| `mode: batch` | Indicates to LiteLLM this is a batch model |
+
+### 2. Create Virtual Key
+
+```bash showLineNumbers title="create_virtual_key.sh"
+curl -L -X POST 'https://{PROXY_BASE_URL}/key/generate' \
+-H 'Authorization: Bearer ${PROXY_API_KEY}' \
+-H 'Content-Type: application/json' \
+-d '{"models": ["bedrock-batch-claude"]}'
+```
+
+You can now use the virtual key to access the batch models (See Developer flow).
+
+## (Developer) Usage
+
+Here's how to create a LiteLLM managed file and execute Bedrock Batch CRUD operations with the file.
+
+### 1. Create request.jsonl
+
+- Check models available via `/model_group/info`
+- See all models with `mode: batch`
+- Set `model` in .jsonl to the model from `/model_group/info`
+
+```json showLineNumbers title="bedrock_batch_completions.jsonl"
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock-batch-claude", "messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "Hello world!"}], "max_tokens": 1000}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "bedrock-batch-claude", "messages": [{"role": "system", "content": "You are an unhelpful assistant."}, {"role": "user", "content": "Hello world!"}], "max_tokens": 1000}}
+```
+
+Expectation:
+
+- LiteLLM translates this to the bedrock deployment specific value (e.g. `bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0`)
+
+### 2. Upload File
+
+Specify `target_model_names: "<model-name>"` to enable LiteLLM managed files and request validation.
+
+model-name should be the same as the model-name in the request.jsonl
+
+<Tabs>
+<TabItem value="python" label="Python">
+
+```python showLineNumbers title="bedrock_batch.py"
+from openai import OpenAI
+
+client = OpenAI(
+    base_url="http://0.0.0.0:4000",
+    api_key="sk-1234",
+)
+
+# Upload file
+batch_input_file = client.files.create(
+    file=open("./bedrock_batch_completions.jsonl", "rb"), # {"model": "bedrock-batch-claude"} <-> {"model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0"}
+    purpose="batch",
+    extra_body={"target_model_names": "bedrock-batch-claude"}
+)
+print(batch_input_file)
+```
+
+</TabItem>
+<TabItem value="curl" label="Curl">
+
+```bash showLineNumbers title="Upload File"
+curl http://localhost:4000/v1/files \
+    -H "Authorization: Bearer sk-1234" \
+    -F purpose="batch" \
+    -F file="@bedrock_batch_completions.jsonl" \
+    -F extra_body='{"target_model_names": "bedrock-batch-claude"}'
+```
+
+</TabItem>
+</Tabs>
+
+**Where is the file written?**:
+
+The file is written to S3 bucket specified in your config and prepared for Bedrock batch inference.
+
+### 3. Create the batch
+
+<Tabs>
+<TabItem value="python" label="Python">
+
+```python showLineNumbers title="bedrock_batch.py"
+...
+# Create batch
+batch = client.batches.create( 
+    input_file_id=batch_input_file.id,
+    endpoint="/v1/chat/completions",
+    completion_window="24h",
+    metadata={"description": "Test batch job"},
+)
+print(batch)
+```
+
+</TabItem>
+<TabItem value="curl" label="Curl">
+
+```bash showLineNumbers title="Create Batch Request"
+curl http://localhost:4000/v1/batches \
+    -H "Authorization: Bearer sk-1234" \
+    -H "Content-Type: application/json" \
+    -d '{
+        "input_file_id": "file-abc123",
+        "endpoint": "/v1/chat/completions",
+        "completion_window": "24h",
+        "metadata": {"description": "Test batch job"}
+    }'
+```
+
+</TabItem>
+</Tabs>
+
+## FAQ
+
+### Where are my files written?
+
+When a `target_model_names` is specified, the file is written to the S3 bucket configured in your Bedrock batch model configuration.
+
+### What models are supported?
+
+LiteLLM only supports Bedrock Anthropic Models for Batch API. If you want other bedrock models file an issue [here](https://github.com/BerriAI/litellm/issues/new/choose).
+
+## Further Reading
+
+- [AWS Bedrock Batch Inference Documentation](https://docs.aws.amazon.com/bedrock/latest/userguide/batch-inference.html)
+- [LiteLLM Managed Batches](../proxy/managed_batches)
+- [LiteLLM Authentication to Bedrock](https://docs.litellm.ai/docs/providers/bedrock#boto3---authentication)

docs/my-website/docs/providers/bedrock_embedding.md

@@ -0,0 +1,95 @@
+# Bedrock Embedding
+
+## Supported Embedding Models
+
+| Provider | LiteLLM Route | AWS Documentation |
+|----------|---------------|-------------------|
+| Amazon Titan | `bedrock/amazon.*` | [Amazon Titan Embeddings](https://docs.aws.amazon.com/bedrock/latest/userguide/titan-embedding-models.html) |
+| Cohere | `bedrock/cohere.*` | [Cohere Embeddings](https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-cohere-embed.html) |
+| TwelveLabs | `bedrock/us.twelvelabs.*` | [TwelveLabs](https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-twelvelabs.html) |
+
+### API keys
+This can be set as env variables or passed as **params to litellm.embedding()**
+```python
+import os
+os.environ["AWS_ACCESS_KEY_ID"] = ""        # Access key
+os.environ["AWS_SECRET_ACCESS_KEY"] = ""    # Secret access key
+os.environ["AWS_REGION_NAME"] = ""           # us-east-1, us-east-2, us-west-1, us-west-2
+```
+
+## Usage
+### LiteLLM Python SDK
+```python
+from litellm import embedding
+response = embedding(
+    model="bedrock/amazon.titan-embed-text-v1",
+    input=["good morning from litellm"],
+)
+print(response)
+```
+
+### LiteLLM Proxy Server
+
+#### 1. Setup config.yaml
+```yaml
+model_list:
+  - model_name: titan-embed-v1
+    litellm_params:
+      model: bedrock/amazon.titan-embed-text-v1
+      aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
+      aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
+      aws_region_name: us-east-1
+  - model_name: titan-embed-v2
+    litellm_params:
+      model: bedrock/amazon.titan-embed-text-v2:0
+      aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
+      aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
+      aws_region_name: us-east-1
+```
+
+#### 2. Start Proxy 
+```bash
+litellm --config /path/to/config.yaml
+```
+
+#### 3. Use with OpenAI Python SDK
+```python
+import openai
+client = openai.OpenAI(
+    api_key="anything",
+    base_url="http://0.0.0.0:4000"
+)
+
+response = client.embeddings.create(
+    input=["good morning from litellm"],
+    model="titan-embed-v1"
+)
+print(response)
+```
+
+#### 4. Use with LiteLLM Python SDK
+```python
+import litellm
+response = litellm.embedding(
+    model="titan-embed-v1", # model alias from config.yaml
+    input=["good morning from litellm"],
+    api_base="http://0.0.0.0:4000",
+    api_key="anything"
+)
+print(response)
+```
+
+## Supported AWS Bedrock Embedding Models
+
+| Model Name           | Usage                               | Supported Additional OpenAI params |
+|----------------------|---------------------------------------------|-----|
+| Titan Embeddings V2 | `embedding(model="bedrock/amazon.titan-embed-text-v2:0", input=input)` | [here](https://github.com/BerriAI/litellm/blob/f5905e100068e7a4d61441d7453d7cf5609c2121/litellm/llms/bedrock/embed/amazon_titan_v2_transformation.py#L59) |
+| Titan Embeddings - V1 | `embedding(model="bedrock/amazon.titan-embed-text-v1", input=input)` | [here](https://github.com/BerriAI/litellm/blob/f5905e100068e7a4d61441d7453d7cf5609c2121/litellm/llms/bedrock/embed/amazon_titan_g1_transformation.py#L53)
+| Titan Multimodal Embeddings | `embedding(model="bedrock/amazon.titan-embed-image-v1", input=input)` | [here](https://github.com/BerriAI/litellm/blob/f5905e100068e7a4d61441d7453d7cf5609c2121/litellm/llms/bedrock/embed/amazon_titan_multimodal_transformation.py#L28) |
+| TwelveLabs Marengo Embed 2.7 | `embedding(model="bedrock/us.twelvelabs.marengo-embed-2-7-v1:0", input=input)` | Supports multimodal input (text, video, audio, image) |
+| Cohere Embeddings - English | `embedding(model="bedrock/cohere.embed-english-v3", input=input)` | [here](https://github.com/BerriAI/litellm/blob/f5905e100068e7a4d61441d7453d7cf5609c2121/litellm/llms/bedrock/embed/cohere_transformation.py#L18)
+| Cohere Embeddings - Multilingual | `embedding(model="bedrock/cohere.embed-multilingual-v3", input=input)` | [here](https://github.com/BerriAI/litellm/blob/f5905e100068e7a4d61441d7453d7cf5609c2121/litellm/llms/bedrock/embed/cohere_transformation.py#L18)
+
+### Advanced - [Drop Unsupported Params](https://docs.litellm.ai/docs/completion/drop_params#openai-proxy-usage)
+
+### Advanced - [Pass model/provider-specific Params](https://docs.litellm.ai/docs/completion/provider_specific_params#proxy-usage)

docs/my-website/docs/providers/compactifai.md

@@ -0,0 +1,223 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# CompactifAI
+https://docs.compactif.ai/
+
+CompactifAI offers highly compressed versions of leading language models, delivering up to **70% lower inference costs**, **4x throughput gains**, and **low-latency inference** with minimal quality loss (under 5%). CompactifAI's OpenAI-compatible API makes integration straightforward, enabling developers to build ultra-efficient, scalable AI applications with superior concurrency and resource efficiency.
+
+| Property | Details |
+|-------|-------|
+| Description | CompactifAI offers compressed versions of leading language models with up to 70% cost reduction and 4x throughput gains |
+| Provider Route on LiteLLM | `compactifai/` (add this prefix to the model name - e.g. `compactifai/cai-llama-3-1-8b-slim`) |
+| Provider Doc | [CompactifAI ↗](https://docs.compactif.ai/) |
+| API Endpoint for Provider | https://api.compactif.ai/v1 |
+| Supported Endpoints | `/chat/completions`, `/completions` |
+
+## Supported OpenAI Parameters
+
+CompactifAI is fully OpenAI-compatible and supports the following parameters:
+
+```
+"stream",
+"stop",
+"temperature",
+"top_p",
+"max_tokens",
+"presence_penalty",
+"frequency_penalty",
+"logit_bias",
+"user",
+"response_format",
+"seed",
+"tools",
+"tool_choice",
+"parallel_tool_calls",
+"extra_headers"
+```
+
+## API Key Setup
+
+CompactifAI API keys are available through AWS Marketplace subscription:
+
+1. Subscribe via [AWS Marketplace](https://aws.amazon.com/marketplace)
+2. Complete subscription verification (24-hour review process)
+3. Access MultiverseIAM dashboard with provided credentials
+4. Retrieve your API key from the dashboard
+
+```python
+import os
+
+os.environ["COMPACTIFAI_API_KEY"] = "your-api-key"
+```
+
+## Usage
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+from litellm import completion
+import os
+
+os.environ['COMPACTIFAI_API_KEY'] = "your-api-key"
+
+response = completion(
+    model="compactifai/cai-llama-3-1-8b-slim",
+    messages=[
+       {"role": "user", "content": "Hello from LiteLLM!"}
+   ],
+)
+print(response)
+```
+
+</TabItem>
+<TabItem value="proxy" label="Proxy">
+
+```yaml
+model_list:
+  - model_name: llama-2-compressed
+    litellm_params:
+      model: compactifai/cai-llama-3-1-8b-slim
+      api_key: os.environ/COMPACTIFAI_API_KEY
+```
+
+</TabItem>
+</Tabs>
+
+## Streaming
+
+```python
+from litellm import completion
+import os
+
+os.environ['COMPACTIFAI_API_KEY'] = "your-api-key"
+
+response = completion(
+    model="compactifai/cai-llama-3-1-8b-slim",
+    messages=[
+       {"role": "user", "content": "Write a short story"}
+   ],
+    stream=True
+)
+
+for chunk in response:
+    print(chunk)
+```
+
+## Advanced Usage
+
+### Custom Parameters
+
+```python
+from litellm import completion
+
+response = completion(
+    model="compactifai/cai-llama-3-1-8b-slim",
+    messages=[{"role": "user", "content": "Explain quantum computing"}],
+    temperature=0.7,
+    max_tokens=500,
+    top_p=0.9,
+    stop=["Human:", "AI:"]
+)
+```
+
+### Function Calling
+
+CompactifAI supports OpenAI-compatible function calling:
+
+```python
+from litellm import completion
+
+functions = [
+    {
+        "name": "get_weather",
+        "description": "Get current weather information",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "location": {
+                    "type": "string",
+                    "description": "The city and state"
+                }
+            },
+            "required": ["location"]
+        }
+    }
+]
+
+response = completion(
+    model="compactifai/cai-llama-3-1-8b-slim",
+    messages=[{"role": "user", "content": "What's the weather in San Francisco?"}],
+    tools=[{"type": "function", "function": f} for f in functions],
+    tool_choice="auto"
+)
+```
+
+### Async Usage
+
+```python
+import asyncio
+from litellm import acompletion
+
+async def async_call():
+    response = await acompletion(
+        model="compactifai/cai-llama-3-1-8b-slim",
+        messages=[{"role": "user", "content": "Hello async world!"}]
+    )
+    return response
+
+# Run async function
+response = asyncio.run(async_call())
+print(response)
+```
+
+## Available Models
+
+CompactifAI offers compressed versions of popular models. Use the `/models` endpoint to get the latest list:
+
+```python
+import httpx
+
+headers = {"Authorization": f"Bearer {your_api_key}"}
+response = httpx.get("https://api.compactif.ai/v1/models", headers=headers)
+models = response.json()
+```
+
+Common model formats:
+- `compactifai/cai-llama-3-1-8b-slim`
+- `compactifai/mistral-7b-compressed`
+- `compactifai/codellama-7b-compressed`
+
+## Benefits
+
+- **Cost Efficient**: Up to 70% lower inference costs compared to standard models
+- **High Performance**: 4x throughput gains with minimal quality loss (under 5%)
+- **Low Latency**: Optimized for fast response times
+- **Drop-in Replacement**: Full OpenAI API compatibility
+- **Scalable**: Superior concurrency and resource efficiency
+
+## Error Handling
+
+CompactifAI returns standard OpenAI-compatible error responses:
+
+```python
+from litellm import completion
+from litellm.exceptions import AuthenticationError, RateLimitError
+
+try:
+    response = completion(
+        model="compactifai/cai-llama-3-1-8b-slim",
+        messages=[{"role": "user", "content": "Hello"}]
+    )
+except AuthenticationError:
+    print("Invalid API key")
+except RateLimitError:
+    print("Rate limit exceeded")
+```
+
+## Support
+
+- Documentation: https://docs.compactif.ai/
+- LinkedIn: [MultiverseComputing](https://www.linkedin.com/company/multiversecomputing)
+- Analysis: [Artificial Analysis Provider Comparison](https://artificialanalysis.ai/providers/compactifai)

docs/my-website/docs/providers/dashscope.md

@@ -1,4 +1,4 @@
-# Dashscope
+# Dashscope (Qwen API)
 https://dashscope.console.aliyun.com/
 
 **We support ALL Qwen models, just set `dashscope/` as a prefix when sending completion requests**

docs/my-website/docs/providers/ovhcloud.md

@@ -0,0 +1,380 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# 🆕 OVHCloud AI Endpoints
+Leading French Cloud provider in Europe with data sovereignty and privacy.
+
+You can explore the last models we made available in our [catalog](https://endpoints.ai.cloud.ovh.net/catalog).
+
+:::tip
+
+We support ALL OVHCloud AI Endpoints models, just set `model=ovhcloud/<any-model-on-ai-endpoints>` as a prefix when sending litellm requests.
+For the complete models catalog, visit https://endpoints.ai.cloud.ovh.net/catalog. **
+
+:::
+
+## Sample usage
+### Chat completion
+You can define your API key by setting the `OVHCLOUD_API_KEY` environment variable or by overriding the `api_key` parameter. You can generate a key on the [OVHCloud Manager](https://www.ovh.com/manager).
+
+```python
+from litellm import completion
+import os
+
+# Our API is free but ratelimited for calls without an API key.
+os.environ['OVHCLOUD_API_KEY'] = "your-api-key"
+
+response = completion(
+    model = "ovhcloud/Meta-Llama-3_3-70B-Instruct",
+    messages = [
+        {
+            "role": "user",
+            "content": "Hello, how are you?",
+        }
+    ],
+    max_tokens = 10,
+    stop = [],
+    temperature = 0.2,
+    top_p = 0.9,
+    user = "user",
+    api_key = "your-api-key" # Optional if set through the enviromnent variable.
+)
+
+print(response)
+```
+
+### Streaming
+Set the parameter `stream` to `True` to stream a response.
+```python
+from litellm import completion
+import os
+
+os.environ['OVHCLOUD_API_KEY'] = "your-api-key"
+
+response = completion(
+    model = "ovhcloud/Meta-Llama-3_3-70B-Instruct",
+    messages = [
+        {
+            "role": "user",
+            "content": "Hello, how are you?",
+        }
+    ],
+    max_tokens = 10,
+    stop = [],
+    temperature = 0.2,
+    top_p = 0.9,
+    user = "user",
+    api_key = "your-api-key" # Optional if set through the enviromnent variable,
+    stream = True
+)
+
+for part in response:
+    print(response)
+```
+
+### Tool Calling
+
+```python
+from litellm import completion
+import json
+
+def get_current_weather(location, unit="celsius"):
+    if unit == "celsius":
+        return {"location": location, "temperature": "22", "unit": "celsius"}
+    else:
+        return {"location": location, "temperature": "72", "unit": "fahrenheit"}
+
+def print_message(role, content, is_tool_call=False, function_name=None):
+    if role == "user":
+        print(f"🧑 User: {content}")
+    elif role == "assistant":
+        if is_tool_call:
+            print(f"🤖 Assistant: I will call the function '{function_name}' to get some informations.")
+        else:
+            print(f"🤖 Assistant: {content}")
+    elif role == "tool":
+        print(f"🔧 Tool ({function_name}): {content}")
+    print()
+
+messages = [{"role": "user", "content": "What's the weather like in Paris?"}]
+model = "ovhcloud/Meta-Llama-3_3-70B-Instruct"
+
+tools = [
+    {
+        "type": "function",
+        "function": {
+            "name": "get_current_weather",
+            "description": "Get the current weather in a given location",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "location": {
+                        "type": "string",
+                        "description": "The city and country, e.g. Montréal, Canada",
+                    },
+                    "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]},
+                },
+                "required": ["location"],
+            },
+        },
+    }
+]
+
+print("🌟 Beginning of the conversation")
+
+# Initial user message
+print_message("user", messages[0]["content"])
+
+# First request to the model
+print("📡 Sending first request to the model...")
+response = completion(
+    model=model,
+    messages=messages,
+    tools=tools,
+    tool_choice="auto",
+)
+
+response_message = response.choices[0].message
+tool_calls = response_message.tool_calls
+
+if tool_calls:
+    available_functions = {
+        "get_current_weather": get_current_weather,
+    }
+    
+    # Display the tool calls suggested by the model
+    for tool_call in tool_calls:
+        print_message("assistant", "", is_tool_call=True, function_name=tool_call.function.name)
+        print(f"   📋 Arguments: {tool_call.function.arguments}")
+        print()
+    
+    # Add assistant message with tool calls to the conversation history
+    assistant_message = {
+        "role": "assistant",
+        "content": response_message.content,
+        "tool_calls": [
+            {
+                "id": tool_call.id,
+                "type": "function", 
+                "function": {
+                    "name": tool_call.function.name,
+                    "arguments": tool_call.function.arguments
+                }
+            } for tool_call in tool_calls
+        ]
+    }
+    
+    messages.append(assistant_message)
+    
+    # Execute each tool call and add the results to the conversation history
+    for tool_call in tool_calls:
+        function_name = tool_call.function.name
+        function_to_call = available_functions[function_name]
+        function_args = json.loads(tool_call.function.arguments)
+        
+        print(f"🔧 Executing function '{function_name}'...")
+        function_response = function_to_call(
+            location=function_args.get("location"),
+            unit=function_args.get("unit"),
+        )
+        
+        # Display tool response
+        print_message("tool", json.dumps(function_response, indent=2), function_name=function_name)
+        
+        messages.append({
+            "tool_call_id": tool_call.id,
+            "role": "tool",
+            "name": function_name,
+            "content": json.dumps(function_response),
+        })
+    
+    print("📡 Sending second request to the model with results...")
+    
+    # Second request with function results
+    second_response = completion(
+        model=model,
+        messages=messages
+    )
+    
+    # Display final response
+    final_content = second_response.choices[0].message.content
+    print_message("assistant", final_content)
+    
+else:
+    print("❌ No function call detected")
+    print_message("assistant", response_message.content)
+```
+
+### Vision Example
+
+```python
+from base64 import b64encode
+from mimetypes import guess_type
+import litellm
+
+# Auxiliary function to get b64 images
+def data_url_from_image(file_path):
+    mime_type, _ = guess_type(file_path)
+    if mime_type is None:
+        raise ValueError("Could not determine MIME type of the file")
+
+    with open(file_path, "rb") as image_file:
+        encoded_string = b64encode(image_file.read()).decode("utf-8")
+
+    data_url = f"data:{mime_type};base64,{encoded_string}"
+    return data_url
+
+response = litellm.completion(
+    model = "ovhcloud/Mistral-Small-3.2-24B-Instruct-2506", 
+    messages=[
+        {
+            "role": "user",
+            "content": [
+                {
+                    "type": "text",
+                    "text": "What's in this image?"
+                },
+                {
+                    "type": "image_url",
+                    "image_url": {
+                        "url": data_url_from_image("your_image.jpg"),
+                        "format": "image/jpeg"
+                    }
+                }
+            ]
+        }
+    ],
+    stream=False
+)
+
+print(response.choices[0].message.content)
+```
+
+
+### Structured Output
+
+```python
+from litellm import completion
+
+response = completion(
+    model="ovhcloud/Meta-Llama-3_3-70B-Instruct",
+    messages=[
+        {
+            "role": "system",
+            "content": (
+                "You are a specialist in extracting structured data from unstructured text. "
+                "Your task is to identify relevant entities and categories, then format them "
+                "according to the requested structure."
+            ),
+        },
+        {
+            "role": "user",
+            "content": "Room 12 contains books, a desk, and a lamp."
+        },
+    ],
+    response_format={
+        "type": "json_schema",
+        "json_schema": {
+            "title": "data",
+            "name": "data_extraction",
+            "schema": {
+                "type": "object",
+                "properties": {
+                    "section": {"type": "string"},
+                    "products": {
+                        "type": "array",
+                        "items": {"type": "string"}
+                    }
+                },
+                "required": ["section", "products"],
+                "additionalProperties": False
+            },
+            "strict": False
+        }
+    },
+    stream=False
+)
+
+print(response.choices[0].message.content)
+```
+
+### Embeddings
+
+```python
+from litellm import embedding
+
+response = embedding(
+    model="ovhcloud/BGE-M3",
+    input=["sample text to embed", "another sample text to embed"]
+)
+
+print(response.data)
+```
+
+## Usage with LiteLLM Proxy Server
+
+Here's how to call a OVHCloud AI Endpoints model with the LiteLLM Proxy Server
+
+1. Modify the config.yaml 
+
+  ```yaml
+  model_list:
+    - model_name: my-model
+      litellm_params:
+        model: ovhcloud/<your-model-name>  # add ovhcloud/ prefix to route as OVHCloud provider
+        api_key: api-key                   # api key to send your model
+  ```
+
+
+2. Start the proxy 
+
+  ```bash
+  $ litellm --config /path/to/config.yaml
+  ```
+
+3. Send Request to LiteLLM Proxy Server
+
+  <Tabs>
+
+  <TabItem value="openai" label="OpenAI Python v1.0.0+">
+
+  ```python
+  import openai
+  client = openai.OpenAI(
+      api_key="sk-1234",             # pass litellm proxy key, if you're using virtual keys
+      base_url="http://0.0.0.0:4000" # litellm-proxy-base url
+  )
+
+  response = client.chat.completions.create(
+      model="my-model",
+      messages = [
+          {
+              "role": "user",
+              "content": "what llm are you"
+          }
+      ],
+  )
+
+  print(response)
+  ```
+  </TabItem>
+
+  <TabItem value="curl" label="curl">
+
+  ```shell
+  curl --location 'http://0.0.0.0:4000/chat/completions' \
+      --header 'Authorization: Bearer sk-1234' \
+      --header 'Content-Type: application/json' \
+      --data '{
+      "model": "my-model",
+      "messages": [
+          {
+          "role": "user",
+          "content": "what llm are you"
+          }
+      ],
+  }'
+  ```
+  </TabItem>
+
+  </Tabs>

docs/my-website/docs/providers/vertex.md

@@ -2509,150 +2509,6 @@ print("response from proxy", response)
 </TabItem>
 </Tabs>
 
-## **Batch APIs**
-
-Just add the following Vertex env vars to your environment. 
-
-```bash
-# GCS Bucket settings, used to store batch prediction files in
-export GCS_BUCKET_NAME = "litellm-testing-bucket" # the bucket you want to store batch prediction files in
-export GCS_PATH_SERVICE_ACCOUNT="/path/to/service_account.json" # path to your service account json file
-
-# Vertex /batch endpoint settings, used for LLM API requests
-export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service_account.json" # path to your service account json file
-export VERTEXAI_LOCATION="us-central1" # can be any vertex location
-export VERTEXAI_PROJECT="my-test-project" 
-```
-
-### Usage
-
-
-#### 1. Create a file of batch requests for vertex
-
-LiteLLM expects the file to follow the **[OpenAI batches files format](https://platform.openai.com/docs/guides/batch)**
-
-Each `body` in the file should be an **OpenAI API request**
-
-Create a file called `vertex_batch_completions.jsonl` in the current working directory, the `model` should be the Vertex AI model name
-```
-{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gemini-1.5-flash-001", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
-{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gemini-1.5-flash-001", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
-```
-
-
-#### 2. Upload a File of batch requests
-
-For `vertex_ai` litellm will upload the file to the provided `GCS_BUCKET_NAME`
-
-```python
-import os
-oai_client = OpenAI(
-    api_key="sk-1234",               # litellm proxy API key
-    base_url="http://localhost:4000" # litellm proxy base url
-)
-file_name = "vertex_batch_completions.jsonl" # 
-_current_dir = os.path.dirname(os.path.abspath(__file__))
-file_path = os.path.join(_current_dir, file_name)
-file_obj = oai_client.files.create(
-    file=open(file_path, "rb"),
-    purpose="batch",
-    extra_body={"custom_llm_provider": "vertex_ai"}, # tell litellm to use vertex_ai for this file upload
-)
-```
-
-**Expected Response**
-
-```json
-{
-    "id": "gs://litellm-testing-bucket/litellm-vertex-files/publishers/google/models/gemini-1.5-flash-001/d3f198cd-c0d1-436d-9b1e-28e3f282997a",
-    "bytes": 416,
-    "created_at": 1733392026,
-    "filename": "litellm-vertex-files/publishers/google/models/gemini-1.5-flash-001/d3f198cd-c0d1-436d-9b1e-28e3f282997a",
-    "object": "file",
-    "purpose": "batch",
-    "status": "uploaded",
-    "status_details": null
-}
-```
-
-
-
-#### 3. Create a batch 
-
-```python
-batch_input_file_id = file_obj.id # use `file_obj` from step 2
-create_batch_response = oai_client.batches.create(
-    completion_window="24h",
-    endpoint="/v1/chat/completions",
-    input_file_id=batch_input_file_id, # example input_file_id = "gs://litellm-testing-bucket/litellm-vertex-files/publishers/google/models/gemini-1.5-flash-001/c2b1b785-252b-448c-b180-033c4c63b3ce"
-    extra_body={"custom_llm_provider": "vertex_ai"}, # tell litellm to use `vertex_ai` for this batch request
-)
-```
-
-**Expected Response**
-
-```json
-{
-    "id": "3814889423749775360",
-    "completion_window": "24hrs",
-    "created_at": 1733392026,
-    "endpoint": "",
-    "input_file_id": "gs://litellm-testing-bucket/litellm-vertex-files/publishers/google/models/gemini-1.5-flash-001/d3f198cd-c0d1-436d-9b1e-28e3f282997a",
-    "object": "batch",
-    "status": "validating",
-    "cancelled_at": null,
-    "cancelling_at": null,
-    "completed_at": null,
-    "error_file_id": null,
-    "errors": null,
-    "expired_at": null,
-    "expires_at": null,
-    "failed_at": null,
-    "finalizing_at": null,
-    "in_progress_at": null,
-    "metadata": null,
-    "output_file_id": "gs://litellm-testing-bucket/litellm-vertex-files/publishers/google/models/gemini-1.5-flash-001",
-    "request_counts": null
-}
-```
-
-#### 4. Retrieve a batch
-
-```python
-retrieved_batch = oai_client.batches.retrieve(
-    batch_id=create_batch_response.id,
-    extra_body={"custom_llm_provider": "vertex_ai"}, # tell litellm to use `vertex_ai` for this batch request
-)
-```
-
-**Expected Response**
-
-```json
-{
-    "id": "3814889423749775360",
-    "completion_window": "24hrs",
-    "created_at": 1736500100,
-    "endpoint": "",
-    "input_file_id": "gs://example-bucket-1-litellm/litellm-vertex-files/publishers/google/models/gemini-1.5-flash-001/7b2e47f5-3dd4-436d-920f-f9155bbdc952",
-    "object": "batch",
-    "status": "completed",
-    "cancelled_at": null,
-    "cancelling_at": null,
-    "completed_at": null,
-    "error_file_id": null,
-    "errors": null,
-    "expired_at": null,
-    "expires_at": null,
-    "failed_at": null,
-    "finalizing_at": null,
-    "in_progress_at": null,
-    "metadata": null,
-    "output_file_id": "gs://example-bucket-1-litellm/litellm-vertex-files/publishers/google/models/gemini-1.5-flash-001",
-    "request_counts": null
-}
-```
-
-
 ## **Fine Tuning APIs**
 
 
@@ -2758,6 +2614,44 @@ curl http://localhost:4000/v1/fine_tuning/jobs \
 </Tabs>
 
 
+## Labels
+
+
+Google enables you to add custom metadata to its `generateContent` and `streamGenerateContent` calls.
+This mechanism is useful in Vertex AI because it allows costs and usage tracking over multiple
+different applications or users.
+
+
+### Usage
+
+You can use that feature through LiteLLM by sending `labels` or `metadata` field in your requests.
+
+If the client sets the `labels` field in the request to the LiteLLM,
+the LiteLLM will pass the `labels` field to the Vertex AI backend.
+
+If the client sets the `metadata` field in the request to the LiteLLM and the `labels` field is not set,
+the LiteLLM will create the `labels` field filled with `metadata` key/value pairs for all string values and
+pass it to the Vertex AI backend.
+
+
+Here is an example JSON request demonstrating the labels usage:
+
+```json
+{
+    "model": "gemini-2.0-flash-lite",
+    "messages": [
+        { "role": "user", "content": "respond in 20 words. who are you?" }
+    ],
+    "labels": {
+        "client_app": "acme_comp_financial_app",
+        "department": "finance",
+        "project": "acme_ai"
+    }
+}
+```
+
+
+
 ## Extra
 
 ### Using `GOOGLE_APPLICATION_CREDENTIALS`

docs/my-website/docs/providers/vertex_batch.md

@@ -0,0 +1,264 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+## **Batch APIs**
+
+Just add the following Vertex env vars to your environment. 
+
+```bash
+# GCS Bucket settings, used to store batch prediction files in
+export GCS_BUCKET_NAME="my-batch-bucket" # the bucket you want to store batch prediction files in
+export GCS_PATH_SERVICE_ACCOUNT="/path/to/service_account.json" # path to your service account json file
+
+# Vertex /batch endpoint settings, used for LLM API requests
+export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service_account.json" # path to your service account json file
+export VERTEXAI_LOCATION="us-central1" # can be any vertex location
+export VERTEXAI_PROJECT="my-project" 
+```
+
+### Usage
+
+Follow this complete workflow: create JSONL file → upload file → create batch → retrieve batch status → get file content
+
+#### 1. Create a JSONL file of batch requests
+
+LiteLLM expects the file to follow the **[OpenAI batches files format](https://platform.openai.com/docs/guides/batch)**.
+
+Each `body` in the file should be an **OpenAI API request**.
+
+Create a file called `batch_requests.jsonl` with your requests:
+```jsonl
+{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gemini-2.5-flash-lite", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gemini-2.5-flash-lite", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}}
+```
+
+#### 2. Upload the file
+
+Upload your JSONL file. For `vertex_ai`, the file will be stored in your configured GCS bucket provided by `GCS_BUCKET_NAME`.
+
+<Tabs>
+<TabItem value="python" label="Python">
+
+```python showLineNumbers title="upload_file.py"
+from openai import OpenAI
+
+oai_client = OpenAI(
+    api_key="sk-1234",               # litellm proxy API key
+    base_url="http://localhost:4000" # litellm proxy base url
+)
+
+file_obj = oai_client.files.create(
+    file=open("batch_requests.jsonl", "rb"),
+    purpose="batch",
+    extra_body={"custom_llm_provider": "vertex_ai"}
+)
+
+print(f"File uploaded with ID: {file_obj.id}")
+```
+
+</TabItem>
+<TabItem value="curl" label="Curl">
+
+```bash showLineNumbers title="Upload File"
+curl --request POST \
+  --url http://localhost:4000/v1/files \
+  --header 'Content-Type: multipart/form-data' \
+  --form purpose=batch \
+  --form file=@batch_requests.jsonl \
+  --form custom_llm_provider=vertex_ai
+```
+
+</TabItem>
+</Tabs>
+
+**Expected Response:**
+
+```json
+{
+    "id": "gs://my-batch-bucket/litellm-vertex-files/publishers/google/models/gemini-2.5-flash-lite/abc123-def4-5678-9012-34567890abcd",
+    "bytes": 416,
+    "created_at": 1758303684,
+    "filename": "litellm-vertex-files/publishers/google/models/gemini-2.5-flash-lite/abc123-def4-5678-9012-34567890abcd",
+    "object": "file",
+    "purpose": "batch",
+    "status": "uploaded",
+    "expires_at": null,
+    "status_details": null
+}
+```
+
+#### 3. Create a batch
+
+Create a batch job using the uploaded file ID.
+
+<Tabs>
+<TabItem value="python" label="Python">
+
+```python showLineNumbers title="create_batch.py"
+batch_input_file_id = file_obj.id # from step 2
+create_batch_response = oai_client.batches.create(
+    completion_window="24h",
+    endpoint="/v1/chat/completions",
+    input_file_id=batch_input_file_id, # e.g. "gs://my-batch-bucket/litellm-vertex-files/publishers/google/models/gemini-2.5-flash-lite/abc123-def4-5678-9012-34567890abcd"
+    extra_body={"custom_llm_provider": "vertex_ai"}
+)
+
+print(f"Batch created with ID: {create_batch_response.id}")
+```
+
+</TabItem>
+<TabItem value="curl" label="Curl">
+
+```bash showLineNumbers title="Create Batch Request"
+curl --request POST \
+  --url http://localhost:4000/v1/batches \
+  --header 'Content-Type: application/json' \
+  --data '{         
+    "input_file_id": "gs://my-batch-bucket/litellm-vertex-files/publishers/google/models/gemini-2.5-flash-lite/abc123-def4-5678-9012-34567890abcd",
+    "endpoint": "/v1/chat/completions",
+    "completion_window": "24h",
+    "custom_llm_provider": "vertex_ai"
+}'
+```
+
+</TabItem>
+</Tabs>
+
+**Expected Response:**
+
+```json
+{
+    "id": "7814463557919047680",
+    "completion_window": "24hrs",
+    "created_at": 1758328011,
+    "endpoint": "",
+    "input_file_id": "gs://my-batch-bucket/litellm-vertex-files/publishers/google/models/gemini-2.5-flash-lite/abc123-def4-5678-9012-34567890abcd",
+    "object": "batch",
+    "status": "validating",
+    "cancelled_at": null,
+    "cancelling_at": null,
+    "completed_at": null,
+    "error_file_id": null,
+    "errors": null,
+    "expired_at": null,
+    "expires_at": null,
+    "failed_at": null,
+    "finalizing_at": null,
+    "in_progress_at": null,
+    "metadata": null,
+    "output_file_id": "gs://my-batch-bucket/litellm-vertex-files/publishers/google/models/gemini-2.5-flash-lite",
+    "request_counts": null,
+    "usage": null
+}
+```
+
+#### 4. Retrieve batch status
+
+Check the status of your batch job. The batch will progress through states: `validating` → `in_progress` → `completed`.
+
+<Tabs>
+<TabItem value="python" label="Python">
+
+```python showLineNumbers title="retrieve_batch.py"
+retrieved_batch = oai_client.batches.retrieve(
+    batch_id=create_batch_response.id, # Created batch id, e.g. 7814463557919047680
+    extra_body={"custom_llm_provider": "vertex_ai"}
+)
+
+print(f"Batch status: {retrieved_batch.status}")
+if retrieved_batch.status == "completed":
+    print(f"Output file: {retrieved_batch.output_file_id}")
+```
+
+</TabItem>
+<TabItem value="curl" label="Curl">
+
+```bash showLineNumbers title="Retrieve Batch Status"
+curl --request GET \
+  --url 'http://localhost:4000/batches/7814463557919047680?provider=vertex_ai' \
+  --header 'Authorization: Bearer sk-1234'
+```
+
+</TabItem>
+</Tabs>
+
+**Expected Response (when completed):**
+
+```json
+{
+    "id": "7814463557919047680",
+    "completion_window": "24hrs",
+    "created_at": 1758328011,
+    "endpoint": "",
+    "input_file_id": "gs://my-batch-bucket/litellm-vertex-files/publishers/google/models/gemini-2.5-flash-lite/abc123-def4-5678-9012-34567890abcd",
+    "object": "batch",
+    "status": "completed",
+    "cancelled_at": null,
+    "cancelling_at": null,
+    "completed_at": null,
+    "error_file_id": null,
+    "errors": null,
+    "expired_at": null,
+    "expires_at": null,
+    "failed_at": null,
+    "finalizing_at": null,
+    "in_progress_at": null,
+    "metadata": null,
+    "output_file_id": "gs://my-batch-bucket/litellm-vertex-files/publishers/google/models/gemini-2.5-flash-lite/prediction-model-2025-09-19T21:26:51.569037Z/predictions.jsonl",
+    "request_counts": null,
+    "usage": null
+}
+```
+
+#### 5. Get file content
+
+Once the batch is completed, retrieve the results using the `output_file_id` from the batch response.
+
+**Important:** The `output_file_id` must be URL encoded when used in the request path.
+
+<Tabs>
+<TabItem value="python" label="Python">
+
+```python showLineNumbers title="get_file_content.py"
+import urllib.parse
+import json
+
+output_file_id = retrieved_batch.output_file_id
+# URL encode the file ID
+encoded_file_id = urllib.parse.quote_plus(output_file_id)
+
+# Get file content
+file_content = oai_client.files.content(
+    file_id=encoded_file_id,
+    extra_body={"custom_llm_provider": "vertex_ai"}
+)
+
+# Process the results
+for line in file_content.text.strip().split('\n'):
+    result = json.loads(line)
+    print(f"Request: {result['request']}")
+    print(f"Response: {result['response']}")
+    print("---")
+```
+
+</TabItem>
+<TabItem value="curl" label="Curl">
+
+```bash showLineNumbers title="Get File Content"
+# Note: The file ID must be URL encoded
+curl --request GET \
+  --url 'http://localhost:4000/files/gs%253A%252F%252Fmy-batch-bucket%252Flitellm-vertex-files%252Fpublishers%252Fgoogle%252Fmodels%252Fgemini-2.5-flash-lite%252Fprediction-model-2025-09-19T21%253A26%253A51.569037Z%252Fpredictions.jsonl/content?provider=vertex_ai' \
+  --header 'Authorization: Bearer sk-1234'
+```
+
+</TabItem>
+</Tabs>
+
+**Expected Response:**
+
+The response contains JSONL format with one result per line:
+
+```jsonl
+{"status":"","processed_time":"2025-09-19T21:29:47.352+00:00","request":{"contents":[{"parts":[{"text":"Hello world!"}],"role":"user"}],"generationConfig":{"max_output_tokens":10},"system_instruction":{"parts":[{"text":"You are a helpful assistant."}]}},"response":{"candidates":[{"avgLogprobs":-0.48079710006713866,"content":{"parts":[{"text":"Hello there! It's nice to meet you"}],"role":"model"},"finishReason":"MAX_TOKENS"}],"createTime":"2025-09-19T21:29:47.484619Z","modelVersion":"gemini-2.5-flash-lite","responseId":"S8vNaIvKHdvshMIP_aOtuAg","usageMetadata":{"candidatesTokenCount":10,"candidatesTokensDetails":[{"modality":"TEXT","tokenCount":10}],"promptTokenCount":9,"promptTokensDetails":[{"modality":"TEXT","tokenCount":9}],"totalTokenCount":19,"trafficType":"ON_DEMAND"}}}
+{"status":"","processed_time":"2025-09-19T21:29:47.358+00:00","request":{"contents":[{"parts":[{"text":"Hello world!"}],"role":"user"}],"generationConfig":{"max_output_tokens":10},"system_instruction":{"parts":[{"text":"You are an unhelpful assistant."}]}},"response":{"candidates":[{"avgLogprobs":-0.6168075137668185,"content":{"parts":[{"text":"I am unable to assist with this request."}],"role":"model"},"finishReason":"STOP"}],"createTime":"2025-09-19T21:29:47.470889Z","modelVersion":"gemini-2.5-flash-lite","responseId":"S8vNaOneHISShMIP28nA8QQ","usageMetadata":{"candidatesTokenCount":9,"candidatesTokensDetails":[{"modality":"TEXT","tokenCount":9}],"promptTokenCount":9,"promptTokensDetails":[{"modality":"TEXT","tokenCount":9}],"totalTokenCount":18,"trafficType":"ON_DEMAND"}}}
+```

docs/my-website/docs/providers/vllm.md

@@ -8,9 +8,9 @@ LiteLLM supports all models on VLLM.
 | Property | Details |
 |-------|-------|
 | Description | vLLM is a fast and easy-to-use library for LLM inference and serving. [Docs](https://docs.vllm.ai/en/latest/index.html) |
-| Provider Route on LiteLLM | `hosted_vllm/` (for OpenAI compatible server), `vllm/` (for vLLM sdk usage) |
+| Provider Route on LiteLLM | `hosted_vllm/` (for OpenAI compatible server), `vllm/` ([DEPRECATED] for vLLM sdk usage) |
 | Provider Doc | [vLLM ↗](https://docs.vllm.ai/en/latest/index.html) |
-| Supported Endpoints | `/chat/completions`, `/embeddings`, `/completions`, `/rerank` |
+| Supported Endpoints | `/chat/completions`, `/embeddings`, `/completions`, `/rerank`, `/audio/transcriptions` |
 
 
 # Quick Start

docs/my-website/docs/proxy/admin_ui_sso.md

@@ -4,6 +4,10 @@ import TabItem from '@theme/TabItem';
 
 # ✨ SSO for Admin UI
 
+:::info
+From v1.76.0, SSO is now Free for up to 5 users.
+:::
+
 :::info
 
 ✨ SSO is on LiteLLM Enterprise

docs/my-website/docs/proxy/budget_reset_and_tz.md

@@ -29,5 +29,6 @@ Common timezone values:
 - `US/Pacific` - Pacific Time
 - `Europe/London` - UK Time
 - `Asia/Kolkata` - Indian Standard Time (IST)
+- `Asia/Bangkok` - Indochina Time (ICT)
 - `Asia/Tokyo` - Japan Standard Time
 - `Australia/Sydney` - Australian Eastern Time

docs/my-website/docs/proxy/config_settings.md

@@ -93,6 +93,8 @@ callback_settings:
 
 general_settings:
   completion_model: string
+  store_prompts_in_spend_logs: boolean
+  forward_client_headers_to_llm_api: boolean
   disable_spend_logs: boolean  # turn off writing each transaction to the db
   disable_master_key_return: boolean  # turn off returning master key on UI (checked on '/user/info' endpoint)
   disable_retry_on_max_parallel_request_limit_error: boolean  # turn off retries when max parallel request limit is reached
@@ -121,6 +123,35 @@ general_settings:
   alerting: ["slack", "email"]
   alerting_threshold: 0
   use_client_credentials_pass_through_routes: boolean  # use client credentials for all pass through routes like "/vertex-ai", /bedrock/. When this is True Virtual Key auth will not be applied on these endpoints
+
+router_settings:
+  routing_strategy: simple-shuffle # Literal["simple-shuffle", "least-busy", "usage-based-routing","latency-based-routing"], default="simple-shuffle" - RECOMMENDED for best performance
+  redis_host: <your-redis-host>           # string
+  redis_password: <your-redis-password>   # string
+  redis_port: <your-redis-port>           # string
+  enable_pre_call_checks: true            # bool - Before call is made check if a call is within model context window 
+  allowed_fails: 3 # cooldown model if it fails > 1 call in a minute. 
+  cooldown_time: 30 # (in seconds) how long to cooldown model if fails/min > allowed_fails
+  disable_cooldowns: True                  # bool - Disable cooldowns for all models 
+  enable_tag_filtering: True                # bool - Use tag based routing for requests
+  retry_policy: {                          # Dict[str, int]: retry policy for different types of exceptions
+    "AuthenticationErrorRetries": 3,
+    "TimeoutErrorRetries": 3,
+    "RateLimitErrorRetries": 3,
+    "ContentPolicyViolationErrorRetries": 4,
+    "InternalServerErrorRetries": 4
+  }
+  allowed_fails_policy: {
+    "BadRequestErrorAllowedFails": 1000, # Allow 1000 BadRequestErrors before cooling down a deployment
+    "AuthenticationErrorAllowedFails": 10, # int 
+    "TimeoutErrorAllowedFails": 12, # int 
+    "RateLimitErrorAllowedFails": 10000, # int 
+    "ContentPolicyViolationErrorAllowedFails": 15, # int 
+    "InternalServerErrorAllowedFails": 20, # int 
+  }
+  content_policy_fallbacks=[{"claude-2": ["my-fallback-model"]}] # List[Dict[str, List[str]]]: Fallback model for content policy violations
+  fallbacks=[{"claude-2": ["my-fallback-model"]}] # List[Dict[str, List[str]]]: Fallback model for all errors
+
 ```
 
 ### litellm_settings - Reference
@@ -473,6 +504,7 @@ router_settings:
 | EMAIL_SIGNATURE | Custom HTML footer/signature for all emails. Can include HTML tags for formatting and links.
 | EMAIL_SUBJECT_INVITATION | Custom subject template for invitation emails. 
 | EMAIL_SUBJECT_KEY_CREATED | Custom subject template for key creation emails. 
+| EXPERIMENTAL_MULTI_INSTANCE_RATE_LIMITING | Flag to enable new multi-instance rate limiting. **Default is False**
 | FIREWORKS_AI_4_B | Size parameter for Fireworks AI 4B model. Default is 4
 | FIREWORKS_AI_16_B | Size parameter for Fireworks AI 16B model. Default is 16
 | FIREWORKS_AI_56_B_MOE | Size parameter for Fireworks AI 56B MOE model. Default is 56
@@ -658,6 +690,8 @@ router_settings:
 | PILLAR_API_KEY | API key for Pillar API Guardrails
 | PILLAR_ON_FLAGGED_ACTION | Action to take when content is flagged ('block' or 'monitor')
 | POD_NAME | Pod name for the server, this will be [emitted to `datadog` logs](https://docs.litellm.ai/docs/proxy/logging#datadog) as `POD_NAME` 
+| POSTHOG_API_KEY | API key for PostHog analytics integration
+| POSTHOG_API_URL | Base URL for PostHog API (defaults to https://us.i.posthog.com)
 | PREDIBASE_API_BASE | Base URL for Predibase API
 | PRESIDIO_ANALYZER_API_BASE | Base URL for Presidio Analyzer service
 | PRESIDIO_ANONYMIZER_API_BASE | Base URL for Presidio Anonymizer service
@@ -738,3 +772,4 @@ router_settings:
 | WEBHOOK_URL | URL for receiving webhooks from external services
 | SPEND_LOG_RUN_LOOPS | Constant for setting how many runs of 1000 batch deletes should spend_log_cleanup task run |
 | SPEND_LOG_CLEANUP_BATCH_SIZE | Number of logs deleted per batch during cleanup. Default is 1000 |
+| COROUTINE_CHECKER_MAX_SIZE_IN_MEMORY | Maximum size for CoroutineChecker in-memory cache. Default is 1000 |

docs/my-website/docs/proxy/debugging.md

@@ -11,13 +11,13 @@ The proxy also supports json logs. [See here](#json-logs)
 
 **via cli**
 
-```bash
+```bash showLineNumbers
 $ litellm --debug
 ```
 
 **via env**
 
-```python
+```python showLineNumbers
 os.environ["LITELLM_LOG"] = "INFO"
 ```
 
@@ -25,25 +25,25 @@ os.environ["LITELLM_LOG"] = "INFO"
 
 **via cli**
 
-```bash
+```bash showLineNumbers
 $ litellm --detailed_debug
 ```
 
 **via env**
 
-```python
+```python showLineNumbers
 os.environ["LITELLM_LOG"] = "DEBUG"
 ```
 
 ### Debug Logs 
 
 Run the proxy with `--detailed_debug` to view detailed debug logs
-```shell
+```shell showLineNumbers
 litellm --config /path/to/config.yaml --detailed_debug
 ```
 
 When making requests you should see the POST request sent by LiteLLM to the LLM on the Terminal output
-```shell
+```shell showLineNumbers
 POST Request Sent from LiteLLM:
 curl -X POST \
 https://api.openai.com/v1/chat/completions \
@@ -51,25 +51,63 @@ https://api.openai.com/v1/chat/completions \
 -d '{"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "this is a test request, write a short poem"}]}'
 ```
 
+## Debug single request
+
+Pass in `litellm_request_debug=True` in the request body
+
+```bash showLineNumbers
+curl -L -X POST 'http://0.0.0.0:4000/chat/completions' \
+-H 'Content-Type: application/json' \
+-H 'Authorization: Bearer sk-1234' \
+-d '{ 
+    "model":"fake-openai-endpoint",
+    "messages": [{"role": "user","content": "How many r in the word strawberry?"}],
+    "litellm_request_debug": true
+}'
+```
+
+This will emit the raw request sent by LiteLLM to the API Provider and raw response received from the API Provider for **just** this request in the logs. 
+
+
+```bash showLineNumbers
+INFO:     Uvicorn running on http://0.0.0.0:4000 (Press CTRL+C to quit)
+20:14:06 - LiteLLM:WARNING: litellm_logging.py:938 - 
+
+POST Request Sent from LiteLLM:
+curl -X POST \
+https://exampleopenaiendpoint-production.up.railway.app/chat/completions \
+-H 'Authorization: Be****ey' -H 'Content-Type: application/json' \
+-d '{'model': 'fake', 'messages': [{'role': 'user', 'content': 'How many r in the word strawberry?'}], 'stream': False}'
+
+
+20:14:06 - LiteLLM:WARNING: litellm_logging.py:1015 - RAW RESPONSE:
+{"id":"chatcmpl-817fc08f0d6c451485d571dab39b26a1","object":"chat.completion","created":1677652288,"model":"gpt-3.5-turbo-0301","system_fingerprint":"fp_44709d6fcb","choices":[{"index":0,"message":{"role":"assistant","content":"\n\nHello there, how may I assist you today?"},"logprobs":null,"finish_reason":"stop"}],"usage":{"prompt_tokens":9,"completion_tokens":12,"total_tokens":21}}
+
+
+INFO:     127.0.0.1:56155 - "POST /chat/completions HTTP/1.1" 200 OK
+
+```
+
+
 ## JSON LOGS
 
 Set `JSON_LOGS="True"` in your env:
 
-```bash
+```bash showLineNumbers
 export JSON_LOGS="True"
 ```
 **OR**
 
 Set `json_logs: true` in your yaml: 
 
-```yaml
+```yaml showLineNumbers
 litellm_settings:
     json_logs: true
 ```
 
 Start proxy 
 
-```bash
+```bash showLineNumbers
 $ litellm
 ```
 
@@ -80,7 +118,7 @@ The proxy will now all logs in json format.
 Turn off fastapi's default 'INFO' logs 
 
 1. Turn on 'json logs' 
-```yaml
+```yaml showLineNumbers
 litellm_settings:
     json_logs: true
 ```
@@ -89,20 +127,20 @@ litellm_settings:
 
 Only get logs if an error occurs. 
 
-```bash
+```bash showLineNumbers
 LITELLM_LOG="ERROR"
 ```
 
 3. Start proxy 
 
 
-```bash
+```bash showLineNumbers
 $ litellm
 ```
 
 Expected Output: 
 
-```bash
+```bash showLineNumbers
 # no info statements
 ```
 
@@ -119,14 +157,14 @@ This can be caused due to all your models hitting rate limit errors, causing the
 How to control this? 
 - Adjust the cooldown time
 
-```yaml
+```yaml showLineNumbers
 router_settings:
     cooldown_time: 0 # 👈 KEY CHANGE
 ```
 
 - Disable Cooldowns [NOT RECOMMENDED]
 
-```yaml
+```yaml showLineNumbers
 router_settings:
     disable_cooldowns: True
 ```

docs/my-website/docs/proxy/deploy.md

@@ -13,6 +13,7 @@ To start using Litellm, run the following commands in a shell:
 ```bash
 # Get the code
 curl -O https://raw.githubusercontent.com/BerriAI/litellm/main/docker-compose.yml
+curl -O https://raw.githubusercontent.com/BerriAI/litellm/main/prometheus.yml
 
 # Add the master key - you can change this after setup
 echo 'LITELLM_MASTER_KEY="sk-1234"' > .env

docs/my-website/docs/proxy/forward_client_headers.md

@@ -0,0 +1,212 @@
+# Forward Client Headers to LLM API
+
+Control which model groups can forward client headers to the underlying LLM provider APIs.
+
+## Overview
+
+By default, LiteLLM does not forward client headers to LLM provider APIs for security reasons. However, you can selectively enable header forwarding for specific model groups using the `forward_client_headers_to_llm_api` setting.
+
+## Configuration
+
+## Enable Globally
+
+```yaml
+general_settings:
+  forward_client_headers_to_llm_api: true
+```
+
+## Enable for a Model Group
+
+Add the `forward_client_headers_to_llm_api` setting under `model_group_settings` in your configuration:
+
+```yaml
+model_list:
+  - model_name: gpt-4o-mini
+    litellm_params:
+      model: openai/gpt-4o-mini
+      api_key: "your-api-key"
+  - model_name: "wildcard-models/*"
+    litellm_params:
+      model: "openai/*"
+      api_key: "your-api-key"
+
+litellm_settings:
+  model_group_settings:
+    forward_client_headers_to_llm_api:
+      - gpt-4o-mini
+      - wildcard-models/*
+```
+
+## Supported Model Patterns
+
+The configuration supports various model matching patterns:
+
+### 1. Exact Model Names
+```yaml
+forward_client_headers_to_llm_api:
+  - gpt-4o-mini
+  - claude-3-sonnet
+```
+
+### 2. Wildcard Patterns
+```yaml
+forward_client_headers_to_llm_api:
+  - "openai/*"          # All OpenAI models
+  - "anthropic/*"       # All Anthropic models
+  - "wildcard-group/*"  # All models in wildcard-group
+```
+
+### 3. Team Model Aliases
+If your team has model aliases configured, the forwarding will work with both the original model name and the alias.
+
+## Forwarded Headers
+
+When enabled for a model group, LiteLLM forwards the following types of headers:
+
+### Custom Headers (x- prefix)
+- Any header starting with `x-` (except `x-stainless-*` which can cause OpenAI SDK issues)
+- Examples: `x-custom-header`, `x-request-id`, `x-trace-id`
+
+### Provider-Specific Headers
+- **Anthropic**: `anthropic-beta` headers
+- **OpenAI**: `openai-organization` (when enabled via `forward_openai_org_id: true`)
+
+### User Information Headers (Optional)
+When `add_user_information_to_llm_headers` is enabled, LiteLLM adds:
+- `x-litellm-user-id`
+- `x-litellm-org-id`
+- Other user metadata as `x-litellm-*` headers
+
+## Security Considerations
+
+⚠️ **Important Security Notes:**
+
+1. **Sensitive Data**: Only enable header forwarding for trusted model groups, as headers may contain sensitive information
+2. **API Keys**: Never include API keys or secrets in forwarded headers
+3. **PII**: Be cautious about forwarding headers that might contain personally identifiable information
+4. **Provider Limits**: Some providers have restrictions on custom headers
+
+## Example Use Cases
+
+### 1. Request Tracing
+Forward tracing headers to track requests across your system:
+
+```bash
+curl -X POST "https://your-proxy.com/v1/chat/completions" \
+  -H "Authorization: Bearer your-key" \
+  -H "x-trace-id: abc123" \
+  -H "x-request-source: mobile-app" \
+  -d '{
+    "model": "gpt-4o-mini",
+    "messages": [{"role": "user", "content": "Hello"}]
+  }'
+```
+
+### 2. Custom Metadata
+Pass custom metadata to your LLM provider:
+
+```bash
+curl -X POST "https://your-proxy.com/v1/chat/completions" \
+  -H "Authorization: Bearer your-key" \
+  -H "x-customer-id: customer-123" \
+  -H "x-environment: production" \
+  -d '{
+    "model": "gpt-4o-mini", 
+    "messages": [{"role": "user", "content": "Hello"}]
+  }'
+```
+
+### 3. Anthropic Beta Features
+Enable beta features for Anthropic models:
+
+```bash
+curl -X POST "https://your-proxy.com/v1/chat/completions" \
+  -H "Authorization: Bearer your-key" \
+  -H "anthropic-beta: tools-2024-04-04" \
+  -d '{
+    "model": "claude-3-sonnet",
+    "messages": [{"role": "user", "content": "Hello"}]
+  }'
+```
+
+## Complete Configuration Example
+
+```yaml
+model_list:
+  # Fixed model with header forwarding
+  - model_name: byok-fixed-gpt-4o-mini
+    litellm_params:
+      model: openai/gpt-4o-mini
+      api_base: "https://your-openai-endpoint.com"
+      api_key: "your-api-key"
+      
+  # Wildcard model group with header forwarding
+  - model_name: "byok-wildcard/*"
+    litellm_params:
+      model: "openai/*"
+      api_base: "https://your-openai-endpoint.com"
+      api_key: "your-api-key"
+      
+  # Standard model without header forwarding
+  - model_name: standard-gpt-4
+    litellm_params:
+      model: openai/gpt-4
+      api_key: "your-api-key"
+
+litellm_settings:
+  # Enable user info headers globally (optional)
+  add_user_information_to_llm_headers: true
+  
+  model_group_settings:
+    forward_client_headers_to_llm_api:
+      - byok-fixed-gpt-4o-mini
+      - byok-wildcard/*
+      # Note: standard-gpt-4 is NOT included, so no headers forwarded
+
+general_settings:
+  # Enable OpenAI organization header forwarding (optional)
+  forward_openai_org_id: true
+```
+
+## Testing Header Forwarding
+
+To test if headers are being forwarded:
+
+1. **Enable Debug Logging**: Set `set_verbose: true` in your config
+2. **Check Provider Logs**: Monitor your LLM provider's request logs
+3. **Use Webhook Sites**: For testing, you can use webhook.site URLs as api_base to see forwarded headers
+
+## Troubleshooting
+
+### Headers Not Being Forwarded
+
+1. **Check Model Name**: Ensure the model name in your request matches the configuration
+2. **Verify Pattern Matching**: Wildcard patterns must match exactly
+3. **Review Logs**: Enable verbose logging to see header processing
+
+### Provider Errors
+
+1. **Invalid Headers**: Some providers reject unknown headers
+2. **Header Limits**: Providers may have limits on header count/size
+3. **Authentication**: Ensure forwarded headers don't conflict with authentication
+
+## Related Features
+
+- [Request Headers](./request_headers.md) - Complete list of supported request headers
+- [Response Headers](./response_headers.md) - Headers returned by LiteLLM
+- [Team Model Aliases](./team_model_add.md) - Configure model aliases for teams
+- [Model Access Control](./model_access.md) - Control which users can access which models
+
+## API Reference
+
+The header forwarding is controlled by the `ModelGroupSettings` configuration:
+
+```python
+class ModelGroupSettings(BaseModel):
+    forward_client_headers_to_llm_api: Optional[List[str]] = None
+```
+
+Where each string in the list can be:
+- An exact model name (e.g., `"gpt-4o-mini"`)
+- A wildcard pattern (e.g., `"openai/*"`)
+- A model group name (e.g., `"my-model-group/*"`)

docs/my-website/docs/proxy/guardrails/noma_security.md

@@ -135,6 +135,7 @@ guardrails:
       # application_id: "my-app"
       # monitor_mode: false
       # block_failures: true
+      # anonymize_input: false
 ```
 
 ### Required Parameters
@@ -147,6 +148,7 @@ guardrails:
 - **`application_id`**: Your application identifier (defaults to `"litellm"`)
 - **`monitor_mode`**: If `true`, logs violations without blocking (defaults to `false`)
 - **`block_failures`**: If `true`, blocks requests when guardrail API failures occur (defaults to `true`)
+- **`anonymize_input`**: If `true`, replaces sensitive content with anonymized version (defaults to `false`)
 
 ## Environment Variables
 
@@ -158,6 +160,7 @@ export NOMA_API_BASE="https://api.noma.security/"   # Optional
 export NOMA_APPLICATION_ID="my-app"                 # Optional
 export NOMA_MONITOR_MODE="false"                    # Optional
 export NOMA_BLOCK_FAILURES="true"                   # Optional
+export NOMA_ANONYMIZE_INPUT="false"                 # Optional
 ```
 
 ## Advanced Configuration
@@ -190,6 +193,20 @@ guardrails:
       block_failures: false  # Allow requests to proceed if guardrail API fails
 ```
 
+### Content Anonymization
+
+Enable anonymization to replace sensitive content instead of blocking:
+
+```yaml
+guardrails:
+  - guardrail_name: "noma-anonymize"
+    litellm_params:
+      guardrail: noma
+      mode: "pre_call"
+      api_key: os.environ/NOMA_API_KEY
+      anonymize_input: true  # Replace sensitive data with anonymized version
+```
+
 ### Multiple Guardrails
 
 Apply different configurations for input and output:

docs/my-website/docs/proxy/guardrails/tool_permission.md

@@ -0,0 +1,153 @@
+import Image from '@theme/IdealImage';
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Tool Permission Guardrail
+
+LiteLLM provides a Tool Permission Guardrail that lets you control which **tool calls** a model is allowed to invoke, using configurable allow/deny rules. This offers fine-grained, provider-agnostic control over tool execution (e.g., OpenAI Chat Completions `tool_calls`, Anthropic Messages `tool_use`, MCP tools).
+
+## Quick Start
+### 1. Define Guardrails on your LiteLLM config.yaml 
+
+Define your guardrails under the `guardrails` section
+```yaml
+guardrails:
+  - guardrail_name: "tool-permission-guardrail"
+    litellm_params:
+      guardrail: tool_permission
+      mode: "post_call"
+      rules:
+        - id: "allow_bash"
+          tool_name: "Bash"
+          decision: "allow"
+        - id: "allow_github_mcp"
+          tool_name: "mcp__github_*"
+          decision: "allow"
+        - id: "allow_aws_documentation"
+          tool_name: "mcp__aws-documentation_*_documentation"
+          decision: "allow"
+        - id: "deny_read_commands"
+          tool_name: "Read"
+          decision: "Deny"
+      default_action: "deny"  # Fallback when no rule matches: "allow" or "deny"
+      on_disallowed_action: "block"  # How to handle disallowed tools: "block" or "rewrite"
+```
+
+#### Rule Structure
+
+```yaml
+- id: "unique_rule_id"           # Unique identifier for the rule
+  tool_name: "pattern"           # Tool name or pattern to match
+  decision: "allow"              # "allow" or "deny"
+```
+
+#### Supported values for `mode`
+
+- `pre_call` Run **before** LLM call, on **input**
+- `post_call` Run **after** LLM call, on **input & output**
+
+### 2. Start the Proxy
+
+```shell
+litellm --config config.yaml --port 4000
+```
+
+## Examples
+
+<Tabs>
+<TabItem value="block" label="Block Request">
+
+**Block requset**
+
+```bash
+# Test
+curl -X POST "http://localhost:4000/v1/chat/completions" \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer your-master-key-here" \
+  -d '{
+    "model": "gpt-5-mini",
+    "messages": [{"role": "user","content": "What is the weather like in Tokyo today?"}],
+    "tools": [
+      {
+        "type":"function",
+        "function": {
+          "name":"get_current_weather",
+          "description": "Get the current weather in a given location"
+        }
+      }
+    ]
+  }'
+```
+
+**Expected response (Denied):**
+
+```json
+{
+  "error":
+    {
+      "message": "Guardrail raised an exception, Guardrail: tool-permission-guardrail, Message: Tool 'get_current_weather' denied by default action",
+      "type": "None",
+      "param": "None",
+      "code": "500"
+    }
+}
+```
+
+</TabItem>
+<TabItem value="rewrite" label="Rewrite Request">
+
+**Rewrite requset**
+
+```bash
+# Test
+curl -X POST "http://localhost:4000/v1/chat/completions" \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer your-master-key-here" \
+  -d '{
+    "model": "gpt-5-mini",
+    "messages": [{"role": "user","content": "What is the weather like in Tokyo today?"}],
+    "tools": [
+      {
+        "type":"function",
+        "function": {
+          "name":"get_current_weather",
+          "description": "Get the current weather in a given location"
+        }
+      }
+    ]
+  }'
+```
+
+**Expected response:**
+
+```json
+{
+	"id": "chatcmpl-xxxxxxxxxxxxxxx",
+	"created": 1757716050,
+	"model": "gpt-5-mini-2025-08-07",
+	"object": "chat.completion",
+	"choices": [
+		{
+			"finish_reason": "stop",
+			"index": 0,
+			"message": {
+				"content": "I can’t fetch live weather — I don’t have real‑time internet access.",
+				"role": "assistant",
+				"annotations": []
+			},
+			"provider_specific_fields": {}
+		}
+	],
+	"usage": {
+		"prompt_tokens": 112,
+		"total_tokens": 735,
+		"completion_tokens_details": {
+			"reasoning_tokens": 384,
+		},
+	},
+	"service_tier": "default"
+}
+```
+
+</TabItem>
+</Tabs>

docs/my-website/docs/proxy/logging_spec.md

@@ -61,6 +61,11 @@ Inherits from `StandardLoggingUserAPIKeyMetadata` and adds:
 | `requester_metadata` | `Optional[dict]` | Additional requester metadata |
 | `vector_store_request_metadata` | `Optional[List[StandardLoggingVectorStoreRequest]]` | Vector store request metadata |
 | `requester_custom_headers` | Dict[str, str] | Any custom (`x-`) headers sent by the client to the proxy. |
+| `prompt_management_metadata` | `Optional[StandardLoggingPromptManagementMetadata]` | Prompt management and versioning metadata |
+| `mcp_tool_call_metadata` | `Optional[StandardLoggingMCPToolCall]` | MCP (Model Context Protocol) tool call information and cost tracking |
+| `applied_guardrails` | `Optional[List[str]]` | List of applied guardrail names |
+| `usage_object` | `Optional[dict]` | Raw usage object from the LLM provider |
+| `cold_storage_object_key` | `Optional[str]` | S3/GCS object key for cold storage retrieval |
 | `guardrail_information` | `Optional[StandardLoggingGuardrailInformation]` | Guardrail information |
 
 
@@ -145,4 +150,82 @@ A literal type with two possible values:
 | `duration` | `Optional[float]` | Duration of the guardrail in seconds |
 | `masked_entity_count` | `Optional[Dict[str, int]]` | Count of masked entities |
 
+## StandardLoggingPromptManagementMetadata
 
+Used for tracking prompt versioning and management information.
+
+| Field | Type | Description |
+|-------|------|-------------|
+| `prompt_id` | `str` | **Required**. Unique identifier for the prompt template or version |
+| `prompt_variables` | `Optional[dict]` | Variables/parameters used in the prompt template (e.g., `{"user_name": "John", "context": "support"}`) |
+| `prompt_integration` | `str` | **Required**. Integration or system managing the prompt (e.g., `"langfuse"`, `"promptlayer"`, `"custom"`) |
+
+## StandardLoggingMCPToolCall
+
+Used to track Model Context Protocol (MCP) tool calls within LiteLLM requests. This provides detailed logging for external tool integrations.
+
+| Field | Type | Description |
+|-------|------|-------------|
+| `name` | `str` | **Required**. The name of the tool being called (e.g., `"get_weather"`, `"search_database"`) |
+| `arguments` | `dict` | **Required**. Arguments passed to the tool as key-value pairs |
+| `result` | `Optional[dict]` | The response/result returned by the tool execution (populated by custom logging hooks) |
+| `mcp_server_name` | `Optional[str]` | Name of the MCP server that handled the tool call (e.g., `"weather-service"`, `"database-connector"`) |
+| `mcp_server_logo_url` | `Optional[str]` | URL for the MCP server's logo (used for UI display in LiteLLM dashboard) |
+| `namespaced_tool_name` | `Optional[str]` | Fully qualified tool name including server prefix (e.g., `"deepwiki-mcp/get_page_content"`, `"github-mcp/create_issue"`) |
+| `mcp_server_cost_info` | `Optional[MCPServerCostInfo]` | Cost tracking information for the tool call |
+
+### MCPServerCostInfo
+
+Cost tracking structure for MCP server tool calls:
+
+| Field | Type | Description |
+|-------|------|-------------|
+| `default_cost_per_query` | `Optional[float]` | Default cost in USD for any tool call to this MCP server |
+| `tool_name_to_cost_per_query` | `Optional[Dict[str, float]]` | Per-tool cost mapping for granular pricing (e.g., `{"search": 0.01, "create": 0.05}`) |
+
+### Usage
+
+```python
+# Basic MCP tool call metadata
+mcp_tool_call = {
+    "name": "search_documents",
+    "arguments": {
+        "query": "machine learning tutorials",
+        "limit": 10,
+        "filter": "type:pdf"
+    },
+    "mcp_server_name": "document-search-service",
+    "namespaced_tool_name": "docs-mcp/search_documents",
+    "mcp_server_cost_info": {
+        "default_cost_per_query": 0.02,
+        "tool_name_to_cost_per_query": {
+            "search_documents": 0.02,
+            "get_document": 0.01
+        }
+    }
+}
+
+# optional result field (via custom logging hooks)
+mcp_tool_call_with_result = {
+    "name": "search_documents",
+    "arguments": {
+        "query": "machine learning tutorials",
+        "limit": 10,
+        "filter": "type:pdf"
+    },
+    "result": {
+        "documents": [...],
+        "total_found": 42,
+        "search_time_ms": 150
+    },
+    "mcp_server_name": "document-search-service",
+    "namespaced_tool_name": "docs-mcp/search_documents",
+    "mcp_server_cost_info": {
+        "default_cost_per_query": 0.02,
+        "tool_name_to_cost_per_query": {
+            "search_documents": 0.02,
+            "get_document": 0.01
+        }
+    }
+}
+```

docs/my-website/docs/proxy/request_headers.md

@@ -2,6 +2,10 @@
 
 Special headers that are supported by LiteLLM.
 
+## Header Forwarding
+
+By default, LiteLLM does not forward client headers to LLM provider APIs. However, you can selectively enable header forwarding for specific model groups. [Learn more about configuring header forwarding](./forward_client_headers.md).
+
 ## LiteLLM Headers
 
 `x-litellm-timeout` Optional[float]: The timeout for the request in seconds.
@@ -21,11 +25,15 @@ Special headers that are supported by LiteLLM.
 `anthropic-version` Optional[str]: The version of the Anthropic API to use.  
 `anthropic-beta` Optional[str]: The beta version of the Anthropic API to use.
     - For `/v1/messages` endpoint, this will always be forward the header to the underlying model.
-    - For `/chat/completions` endpoint, this will only be forwarded if `forward_client_headers_to_llm_api` is true.
+    - For `/chat/completions` endpoint, this will only be forwarded if the model is configured in `forward_client_headers_to_llm_api`. [Learn more](./forward_client_headers.md)
 
 ## OpenAI Headers
 
 `openai-organization` Optional[str]: The organization to use for the OpenAI API. (currently needs to be enabled via `general_settings::forward_openai_org_id: true`)
 
+## Custom Headers
+
+Custom headers starting with `x-` can be forwarded to LLM provider APIs when the model is configured in `forward_client_headers_to_llm_api`. [Learn more about header forwarding configuration](./forward_client_headers.md).
+
 
 

docs/my-website/docs/proxy/team_budgets.md

@@ -10,8 +10,30 @@ import TabItem from '@theme/TabItem';
 - You must set up a Postgres database (e.g. Supabase, Neon, etc.)
 - To enable team member rate limits, set the environment variable `EXPERIMENTAL_MULTI_INSTANCE_RATE_LIMITING=true` **before starting the proxy server**. Without this, team member rate limits will not be enforced.
 
+
+## Default Budget for Auto-Generated JWT Teams
+
+When using JWT authentication with `team_id_upsert: true`, you can automatically assign a default budget to any newly created team.
+
+This is configured in `default_team_settings` in your `config.yaml`.
+
+**Example:**
+```yaml
+# in your config.yaml
+
+litellm_jwtauth:
+  team_id_upsert: true
+  team_id_jwt_field: "team_id"
+  # ... other jwt settings
+
+litellm_settings:
+  default_team_settings: 
+    - team_id: "default-settings"
+      max_budget: 100.0
+```
 Track spend, set budgets for your Internal Team
 
+
 ## Setting Monthly Team Budgets
 
 ### 1. Create a team 

docs/my-website/docs/troubleshoot.md

@@ -2,7 +2,7 @@
 [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version)
 
 [Community Discord 💭](https://discord.gg/wuPM9dRgDw)
-[Community Slack 💭](https://join.slack.com/share/enQtOTE0ODczMzk2Nzk4NC01YjUxNjY2YjBlYTFmNDRiZTM3NDFiYTM3MzVkODFiMDVjOGRjMmNmZTZkZTMzOWQzZGQyZWIwYjQ0MWExYmE3)
+[Community Slack 💭](https://litellmossslack.slack.com/)
 
 Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬
 

docs/my-website/docs/tutorials/openweb_ui.md

@@ -89,16 +89,20 @@ To track spend and usage for each Open WebUI user, configure both Open WebUI and
 
 2. **Configure LiteLLM to Parse User Headers**
    
-  Add the following to your LiteLLM `config.yaml` to specify a header to use for user tracking:
+  Add the following to your LiteLLM `config.yaml` to specify the request header mapping for user tracking:
 
   ```yaml
   general_settings:
-      user_header_name: X-OpenWebUI-User-Id
+    user_header_mappings:
+      - header_name: X-OpenWebUI-User-Id
+        litellm_user_role: internal_user
+      - header_name: X-OpenWebUI-User-Email
+        litellm_user_role: customer
   ```
 
   ⓘ Available tracking options
 
-  You can use any of the following headers for `user_header_name`:
+  You can use any of the following headers in `header_name` in `user_header_mappings` :
   - `X-OpenWebUI-User-Id`
   - `X-OpenWebUI-User-Email`
   - `X-OpenWebUI-User-Name`
@@ -109,6 +113,12 @@ To track spend and usage for each Open WebUI user, configure both Open WebUI and
   - Users can modify their own usernames
   - Administrators can modify both usernames and emails of any account
 
+This video walks through on how we can map the openweb ui headers to LiteLLM user roles 
+
+<iframe src="https://www.loom.com/embed/a1b6a4635fc0478ba4fd34cae16e2ffd?sid=791c2dcc-7e65-45be-bf7f-27d2601c123e" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen width="840" height="500"></iframe>
+
+<br/>
+<br/>
 
 
 ## Render `thinking` content on Open WebUI

docs/my-website/release_notes/v1.77.2-stable/index.md

@@ -0,0 +1,161 @@
+---
+title: "[Pre-Release] v1.77.2-stable - Bedrock Batches API"
+slug: "v1-77-2"
+date: 2025-09-13T10:00:00
+authors:
+  - name: Krrish Dholakia
+    title: CEO, LiteLLM
+    url: https://www.linkedin.com/in/krish-d/
+    image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
+  - name: Ishaan Jaffer
+    title: CTO, LiteLLM
+    url: https://www.linkedin.com/in/reffajnaahsi/
+    image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+
+hide_table_of_contents: false
+---
+
+import Image from '@theme/IdealImage';
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+## Deploy this version
+
+:::info
+
+This release is not yet live.
+
+:::
+
+<Tabs>
+<TabItem value="docker" label="Docker">
+
+``` showLineNumbers title="docker run litellm"
+docker run \
+-e STORE_MODEL_IN_DB=True \
+-p 4000:4000 \
+ghcr.io/berriai/litellm:main-v1.77.2.rc.2
+```
+
+</TabItem>
+
+<TabItem value="pip" label="Pip">
+
+``` showLineNumbers title="pip install litellm"
+```
+
+</TabItem>
+</Tabs>
+
+---
+
+## Key Highlights
+
+- **Bedrock Batches API** - Support for creating Batch Inference Jobs on Bedrock using LiteLLM's unified batch API (OpenAI compatible)
+- **Qwen API Tiered Pricing** - Cost tracking support for Dashscope (Qwen) models with multiple pricing tiers
+
+## New Models / Updated Models
+
+#### New Model Support
+
+| Provider    | Model                           | Context Window | Pricing ($/1M tokens) | Features |
+| ----------- | ------------------------------- | -------------- | --------------------- | -------- |
+| DeepInfra   | `deepinfra/deepseek-ai/DeepSeek-R1` | 164K | **Input:** $0.70<br/>**Output:** $2.40 | Chat completions, tool calling |
+| Heroku      | `heroku/claude-4-sonnet`        | 8K | Contact provider for pricing | Function calling, tool choice |
+| Heroku      | `heroku/claude-3-7-sonnet`      | 8K | Contact provider for pricing | Function calling, tool choice |
+| Heroku      | `heroku/claude-3-5-sonnet-latest` | 8K | Contact provider for pricing | Function calling, tool choice |
+| Heroku      | `heroku/claude-3-5-haiku`       | 4K | Contact provider for pricing | Function calling, tool choice |
+| Dashscope   | `dashscope/qwen-plus-latest`    | 1M | **Tiered Pricing:**<br/>• 0-256K tokens: $0.40 / $1.20<br/>• 256K-1M tokens: $1.20 / $3.60 | Function calling, reasoning |
+| Dashscope   | `dashscope/qwen3-max-preview`   | 262K | **Tiered Pricing:**<br/>• 0-32K tokens: $1.20 / $6.00<br/>• 32K-128K tokens: $2.40 / $12.00<br/>• 128K-252K tokens: $3.00 / $15.00 | Function calling, reasoning |
+| Dashscope   | `dashscope/qwen-flash`          | 1M | **Tiered Pricing:**<br/>• 0-256K tokens: $0.05 / $0.40<br/>• 256K-1M tokens: $0.25 / $2.00 | Function calling, reasoning |
+| Dashscope   | `dashscope/qwen3-coder-plus`    | 1M | **Tiered Pricing:**<br/>• 0-32K tokens: $1.00 / $5.00<br/>• 32K-128K tokens: $1.80 / $9.00<br/>• 128K-256K tokens: $3.00 / $15.00<br/>• 256K-1M tokens: $6.00 / $60.00 | Function calling, reasoning, caching |
+| Dashscope   | `dashscope/qwen3-coder-flash`   | 1M | **Tiered Pricing:**<br/>• 0-32K tokens: $0.30 / $1.50<br/>• 32K-128K tokens: $0.50 / $2.50<br/>• 128K-256K tokens: $0.80 / $4.00<br/>• 256K-1M tokens: $1.60 / $9.60 | Function calling, reasoning, caching |
+
+---
+
+#### Features
+
+- **[Bedrock](../../docs/providers/bedrock_batches)**
+    - Bedrock Batches API - batch processing support with file upload and request transformation - [PR #14518](https://github.com/BerriAI/litellm/pull/14518), [PR #14522](https://github.com/BerriAI/litellm/pull/14522)
+- **[VLLM](../../docs/providers/vllm)**
+    - Added transcription endpoint support - [PR #14523](https://github.com/BerriAI/litellm/pull/14523)
+- **[Ollama](../../docs/providers/ollama)**
+    - `ollama_chat/` - images, thinking, and content as list handling - [PR #14523](https://github.com/BerriAI/litellm/pull/14523)
+- **General**
+    - New debug flag for detailed request/response logging [PR #14482](https://github.com/BerriAI/litellm/pull/14482)
+
+#### Bug Fixes
+
+- **[Azure OpenAI](../../docs/providers/azure)**
+    - Fixed extra_body injection causing payload rejection in image generation - [PR #14475](https://github.com/BerriAI/litellm/pull/14475)
+- **[LM Studio](../../docs/providers/lm-studio)**
+    - Resolved illegal Bearer header value issue - [PR #14512](https://github.com/BerriAI/litellm/pull/14512)
+
+---
+
+## LLM API Endpoints
+
+#### Bug Fixes
+
+- **[/messages](../../docs/anthropic_unified)**
+    - Don't send content block after message w/ finish reason + usage block - [PR #14477](https://github.com/BerriAI/litellm/pull/14477)
+- **[/generateContent](../../docs/generateContent)**
+    - Gemini CLI Integration - Fixed token count errors - [PR #14451](https://github.com/BerriAI/litellm/pull/14451), [PR #14417](https://github.com/BerriAI/litellm/pull/14417)
+
+---
+
+## Spend Tracking, Budgets and Rate Limiting
+
+#### Features
+
+- **[Qwen API Tiered Pricing](../../docs/providers/dashscope)** - Added comprehensive tiered cost tracking for Dashscope/Qwen models - [PR #14471](https://github.com/BerriAI/litellm/pull/14471), [PR #14479](https://github.com/BerriAI/litellm/pull/14479)
+
+#### Bug Fixes
+
+- **Provider Budgets** - Fixed provider budget calculations - [PR #14459](https://github.com/BerriAI/litellm/pull/14459)
+
+---
+
+## Management Endpoints / UI
+
+#### Features
+
+- **User Headers Mapping** - New X-LiteLLM Users mapping feature for enhanced user tracking - [PR #14485](https://github.com/BerriAI/litellm/pull/14485)
+- **Key Unblocking** - Support for hashed tokens in `/key/unblock` endpoint - [PR #14477](https://github.com/BerriAI/litellm/pull/14477)
+- **Model Group Header Forwarding** - Enhanced wildcard model support with documentation - [PR #14528](https://github.com/BerriAI/litellm/pull/14528)
+
+#### Bug Fixes
+
+- **Log Tab Key Alias** - Fixed filtering inaccuracies for failed logs - [PR #14469](https://github.com/BerriAI/litellm/pull/14469), [PR #14529](https://github.com/BerriAI/litellm/pull/14529)
+
+---
+
+## Logging / Guardrail Integrations
+
+#### Features
+
+- **Noma Integration** - Added non-blocking monitor mode with anonymize input support - [PR #14401](https://github.com/BerriAI/litellm/pull/14401)
+
+---
+
+## Performance / Loadbalancing / Reliability improvements
+
+#### Performance
+- Removed dynamic creation of static values - [PR #14538](https://github.com/BerriAI/litellm/pull/14538)
+- Using `_PROXY_MaxParallelRequestsHandler_v3` by default for optimal throughput - [PR #14450](https://github.com/BerriAI/litellm/pull/14450)
+- Improved execution context propagation into logging tasks - [PR #14455](https://github.com/BerriAI/litellm/pull/14455)
+
+---
+
+
+
+## New Contributors
+* @Sameerlite made their first contribution in [PR #14460](https://github.com/BerriAI/litellm/pull/14460)
+* @holzman made their first contribution in [PR #14459](https://github.com/BerriAI/litellm/pull/14459)
+* @sashank5644 made their first contribution in [PR #14469](https://github.com/BerriAI/litellm/pull/14469)
+* @TomAlon made their first contribution in [PR #14401](https://github.com/BerriAI/litellm/pull/14401)
+* @AlexsanderHamir made their first contribution in [PR #14538](https://github.com/BerriAI/litellm/pull/14538)
+
+---
+
+## **[Full Changelog](https://github.com/BerriAI/litellm/compare/v1.77.1.dev.2...v1.77.2.dev)**

docs/my-website/sidebars.js

@@ -49,6 +49,7 @@ const sidebars = {
           "proxy/guardrails/secret_detection",
           "proxy/guardrails/custom_guardrail",
           "proxy/guardrails/prompt_injection",
+          "proxy/guardrails/tool_permission",
         ].sort(),
       ],
     },
@@ -141,6 +142,7 @@ const sidebars = {
             "proxy/clientside_auth",
             "proxy/request_headers",
             "proxy/response_headers",
+            "proxy/forward_client_headers",
             "proxy/model_discovery",
           ],
         },
@@ -390,6 +392,7 @@ const sidebars = {
             "providers/vertex",
             "providers/vertex_partner",
             "providers/vertex_image",
+            "providers/vertex_batch",
           ]
         },
         {
@@ -409,7 +412,9 @@ const sidebars = {
           label: "Bedrock",
           items: [
             "providers/bedrock",
+            "providers/bedrock_embedding",
             "providers/bedrock_agents",
+            "providers/bedrock_batches",
             "providers/bedrock_vector_store",
           ]
         },
@@ -450,6 +455,7 @@ const sidebars = {
         "providers/elevenlabs",
         "providers/fireworks_ai",
         "providers/clarifai",
+        "providers/compactifai",
         "providers/vllm",
         "providers/llamafile",
         "providers/infinity",
@@ -485,7 +491,8 @@ const sidebars = {
         "providers/bytez",
         "providers/heroku",
         "providers/oci",
-        "providers/datarobot",  
+        "providers/datarobot",
+        "providers/ovhcloud",  
       ],
     },
     {

enterprise/litellm_enterprise/enterprise_callbacks/pagerduty/pagerduty.py

@@ -109,6 +109,9 @@ class PagerDutyAlerting(SlackAlerting):
                 error_llm_provider=error_info.get("llm_provider"),
                 user_api_key_hash=_meta.get("user_api_key_hash"),
                 user_api_key_alias=_meta.get("user_api_key_alias"),
+                user_api_key_spend=_meta.get("user_api_key_spend"),
+                user_api_key_max_budget=_meta.get("user_api_key_max_budget"),
+                user_api_key_budget_reset_at=_meta.get("user_api_key_budget_reset_at"),
                 user_api_key_org_id=_meta.get("user_api_key_org_id"),
                 user_api_key_team_id=_meta.get("user_api_key_team_id"),
                 user_api_key_user_id=_meta.get("user_api_key_user_id"),
@@ -191,6 +194,9 @@ class PagerDutyAlerting(SlackAlerting):
                 error_llm_provider="HangingRequest",
                 user_api_key_hash=user_api_key_dict.api_key,
                 user_api_key_alias=user_api_key_dict.key_alias,
+                user_api_key_spend=user_api_key_dict.spend,
+                user_api_key_max_budget=user_api_key_dict.max_budget,
+                user_api_key_budget_reset_at=user_api_key_dict.budget_reset_at.isoformat() if user_api_key_dict.budget_reset_at else None,
                 user_api_key_org_id=user_api_key_dict.org_id,
                 user_api_key_team_id=user_api_key_dict.team_id,
                 user_api_key_user_id=user_api_key_dict.user_id,

enterprise/litellm_enterprise/integrations/prometheus.py

@@ -1,8 +1,11 @@
 # used for /metrics endpoint on LiteLLM Proxy
 #### What this does ####
 #    On success, log events to Prometheus
+import os
 import sys
+import tempfile
 from datetime import datetime, timedelta
+from pathlib import Path
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -16,6 +19,64 @@ from typing import (
     cast,
 )
 
+
+# CRITICAL: Set up multiprocess mode BEFORE importing prometheus_client
+# This must happen at module import time, not at class instantiation time
+def _setup_early_multiprocess_mode():
+    """Setup multiprocess mode at import time if needed."""
+    try:
+        # Check if we're in a multiprocess environment
+        num_workers = os.environ.get("NUM_WORKERS", "1")
+        is_multiprocess = False
+
+        try:
+            if int(num_workers) > 1:
+                is_multiprocess = True
+        except (ValueError, TypeError):
+            pass
+
+        # Check for gunicorn worker environment variables
+        if os.environ.get("GUNICORN_CMD_ARGS") or os.environ.get("GUNICORN_WORKER_ID"):
+            is_multiprocess = True
+
+        # Check if PROMETHEUS_MULTIPROC_DIR is explicitly set (admin override)
+        if os.environ.get("PROMETHEUS_MULTIPROC_DIR"):
+            is_multiprocess = True
+
+        if is_multiprocess:
+            existing_dir = os.environ.get("PROMETHEUS_MULTIPROC_DIR")
+            if not existing_dir:
+                # Set up multiprocess directory
+                multiproc_dir = os.path.join(
+                    tempfile.gettempdir(), "litellm_prometheus_multiproc"
+                )
+                os.environ["PROMETHEUS_MULTIPROC_DIR"] = multiproc_dir
+
+                # Ensure the directory exists
+                Path(multiproc_dir).mkdir(parents=True, exist_ok=True)
+
+                verbose_logger.info(
+                    f"Prometheus multiprocess mode auto-enabled with directory: {multiproc_dir}"
+                )
+            else:
+                # Directory already set, just ensure it exists
+                Path(existing_dir).mkdir(parents=True, exist_ok=True)
+                verbose_logger.info(
+                    f"Using existing Prometheus multiprocess directory: {existing_dir}"
+                )
+
+    except PermissionError as e:
+        verbose_logger.warning(
+            f"Warning: Unable to create Prometheus multiprocess directory due to permission error. "
+            f"Running in non-root environment. Prometheus metrics may not work correctly in multiprocess mode. Error: {e}"
+        )
+    except Exception as e:
+        verbose_logger.warning(f"Warning: Failed to setup early multiprocess mode: {e}")
+
+
+# Set up multiprocess mode before any prometheus imports
+_setup_early_multiprocess_mode()
+
 import litellm
 from litellm._logging import print_verbose, verbose_logger
 from litellm.integrations.custom_logger import CustomLogger
@@ -44,6 +105,18 @@ class PrometheusLogger(CustomLogger):
             # Always initialize label_filters, even for non-premium users
             self.label_filters = self._parse_prometheus_config()
 
+            # Initialize multiprocess mode for Prometheus metrics to handle multiple workers
+            self._setup_multiprocess_mode()
+
+            # Debug: Check if multiprocess mode is active
+            multiproc_dir = os.environ.get("PROMETHEUS_MULTIPROC_DIR")
+            if multiproc_dir:
+                verbose_logger.info(
+                    f"Prometheus multiprocess mode active with directory: {multiproc_dir}"
+                )
+            else:
+                verbose_logger.info("Prometheus running in single-process mode")
+
             if premium_user is not True:
                 verbose_logger.warning(
                     f"🚨🚨🚨 Prometheus Metrics is on LiteLLM Enterprise\n🚨 {CommonProxyErrors.not_premium_user.value}"
@@ -102,7 +175,9 @@ class PrometheusLogger(CustomLogger):
                 #     "team",
                 #     "team_alias",
                 # ],
-                labelnames=self.get_labels_for_metric("litellm_llm_api_time_to_first_token_metric"),
+                labelnames=self.get_labels_for_metric(
+                    "litellm_llm_api_time_to_first_token_metric"
+                ),
                 buckets=LATENCY_BUCKETS,
             )
 
@@ -132,47 +207,52 @@ class PrometheusLogger(CustomLogger):
                 labelnames=self.get_labels_for_metric("litellm_output_tokens_metric"),
             )
 
-            # Remaining Budget for Team
+            # Remaining Budget for Team (use 'mostrecent' for multiprocess mode)
             self.litellm_remaining_team_budget_metric = self._gauge_factory(
                 "litellm_remaining_team_budget_metric",
                 "Remaining budget for team",
                 labelnames=self.get_labels_for_metric(
                     "litellm_remaining_team_budget_metric"
                 ),
+                multiprocess_mode="mostrecent",
             )
 
-            # Max Budget for Team
+            # Max Budget for Team (use 'mostrecent' for multiprocess mode)
             self.litellm_team_max_budget_metric = self._gauge_factory(
                 "litellm_team_max_budget_metric",
                 "Maximum budget set for team",
                 labelnames=self.get_labels_for_metric("litellm_team_max_budget_metric"),
+                multiprocess_mode="mostrecent",
             )
 
-            # Team Budget Reset At
+            # Team Budget Reset At (use 'mostrecent' for multiprocess mode)
             self.litellm_team_budget_remaining_hours_metric = self._gauge_factory(
                 "litellm_team_budget_remaining_hours_metric",
                 "Remaining days for team budget to be reset",
                 labelnames=self.get_labels_for_metric(
                     "litellm_team_budget_remaining_hours_metric"
                 ),
+                multiprocess_mode="mostrecent",
             )
 
-            # Remaining Budget for API Key
+            # Remaining Budget for API Key (use 'mostrecent' for multiprocess mode)
             self.litellm_remaining_api_key_budget_metric = self._gauge_factory(
                 "litellm_remaining_api_key_budget_metric",
                 "Remaining budget for api key",
                 labelnames=self.get_labels_for_metric(
                     "litellm_remaining_api_key_budget_metric"
                 ),
+                multiprocess_mode="mostrecent",
             )
 
-            # Max Budget for API Key
+            # Max Budget for API Key (use 'mostrecent' for multiprocess mode)
             self.litellm_api_key_max_budget_metric = self._gauge_factory(
                 "litellm_api_key_max_budget_metric",
                 "Maximum budget set for api key",
                 labelnames=self.get_labels_for_metric(
                     "litellm_api_key_max_budget_metric"
                 ),
+                multiprocess_mode="mostrecent",
             )
 
             self.litellm_api_key_budget_remaining_hours_metric = self._gauge_factory(
@@ -181,36 +261,40 @@ class PrometheusLogger(CustomLogger):
                 labelnames=self.get_labels_for_metric(
                     "litellm_api_key_budget_remaining_hours_metric"
                 ),
+                multiprocess_mode="mostrecent",
             )
 
             ########################################
             # LiteLLM Virtual API KEY metrics
             ########################################
-            # Remaining MODEL RPM limit for API Key
+            # Remaining MODEL RPM limit for API Key (use 'mostrecent' for multiprocess mode)
             self.litellm_remaining_api_key_requests_for_model = self._gauge_factory(
                 "litellm_remaining_api_key_requests_for_model",
                 "Remaining Requests API Key can make for model (model based rpm limit on key)",
                 labelnames=["hashed_api_key", "api_key_alias", "model"],
+                multiprocess_mode="mostrecent",
             )
 
-            # Remaining MODEL TPM limit for API Key
+            # Remaining MODEL TPM limit for API Key (use 'mostrecent' for multiprocess mode)
             self.litellm_remaining_api_key_tokens_for_model = self._gauge_factory(
                 "litellm_remaining_api_key_tokens_for_model",
                 "Remaining Tokens API Key can make for model (model based tpm limit on key)",
                 labelnames=["hashed_api_key", "api_key_alias", "model"],
+                multiprocess_mode="mostrecent",
             )
 
             ########################################
             # LLM API Deployment Metrics / analytics
             ########################################
 
-            # Remaining Rate Limit for model
+            # Remaining Rate Limit for model (use 'mostrecent' for multiprocess mode)
             self.litellm_remaining_requests_metric = self._gauge_factory(
                 "litellm_remaining_requests",
                 "LLM Deployment Analytics - remaining requests for model, returned from LLM API Provider",
                 labelnames=self.get_labels_for_metric(
                     "litellm_remaining_requests_metric"
                 ),
+                multiprocess_mode="mostrecent",
             )
 
             self.litellm_remaining_tokens_metric = self._gauge_factory(
@@ -219,6 +303,7 @@ class PrometheusLogger(CustomLogger):
                 labelnames=self.get_labels_for_metric(
                     "litellm_remaining_tokens_metric"
                 ),
+                multiprocess_mode="mostrecent",
             )
 
             self.litellm_overhead_latency_metric = self._histogram_factory(
@@ -229,25 +314,27 @@ class PrometheusLogger(CustomLogger):
                 ),
                 buckets=LATENCY_BUCKETS,
             )
-            # llm api provider budget metrics
+            # llm api provider budget metrics (use 'mostrecent' for multiprocess mode)
             self.litellm_provider_remaining_budget_metric = self._gauge_factory(
                 "litellm_provider_remaining_budget_metric",
                 "Remaining budget for provider - used when you set provider budget limits",
                 labelnames=["api_provider"],
+                multiprocess_mode="mostrecent",
             )
 
-            # Metric for deployment state
+            # Metric for deployment state (use 'mostrecent' for multiprocess mode)
             self.litellm_deployment_state = self._gauge_factory(
                 "litellm_deployment_state",
                 "LLM Deployment Analytics - The state of the deployment: 0 = healthy, 1 = partial outage, 2 = complete outage",
-                labelnames=self.get_labels_for_metric("litellm_deployment_state")
+                labelnames=self.get_labels_for_metric("litellm_deployment_state"),
+                multiprocess_mode="mostrecent",
             )
 
             self.litellm_deployment_cooled_down = self._counter_factory(
                 "litellm_deployment_cooled_down",
                 "LLM Deployment Analytics - Number of times a deployment has been cooled down by LiteLLM load balancing logic. exception_status is the status of the exception that caused the deployment to be cooled down",
                 # labelnames=_logged_llm_labels + [EXCEPTION_STATUS],
-                labelnames=self.get_labels_for_metric("litellm_deployment_cooled_down")
+                labelnames=self.get_labels_for_metric("litellm_deployment_cooled_down"),
             )
 
             self.litellm_deployment_success_responses = self._counter_factory(
@@ -318,6 +405,105 @@ class PrometheusLogger(CustomLogger):
             print_verbose(f"Got exception on init prometheus client {str(e)}")
             raise e
 
+    def _setup_multiprocess_mode(self):
+        """
+        Setup Prometheus multiprocess mode to handle multiple workers properly.
+        This ensures that metrics are aggregated correctly across all worker processes.
+        """
+        import os
+        import tempfile
+        from pathlib import Path
+
+        try:
+            # Check if we're in a multiprocess environment (multiple workers)
+            if not self._is_multiprocess_environment():
+                verbose_logger.debug(
+                    "Single process environment detected, skipping multiprocess setup"
+                )
+                return
+
+            # Set up multiprocess directory if not already configured
+            multiproc_dir = os.environ.get("PROMETHEUS_MULTIPROC_DIR")
+            if not multiproc_dir:
+                # Create a temp directory for multiprocess metrics
+                multiproc_dir = os.path.join(
+                    tempfile.gettempdir(), "litellm_prometheus_multiproc"
+                )
+                os.environ["PROMETHEUS_MULTIPROC_DIR"] = multiproc_dir
+                verbose_logger.debug(f"Set PROMETHEUS_MULTIPROC_DIR to {multiproc_dir}")
+
+            # Ensure the directory exists
+            Path(multiproc_dir).mkdir(parents=True, exist_ok=True)
+
+            # Force the prometheus_client to recognize multiprocess mode
+            # This is important because the environment variable must be set BEFORE importing prometheus_client
+            try:
+                from prometheus_client import multiprocess
+
+                # This will trigger the multiprocess mode if the env var is set
+                verbose_logger.debug(
+                    "Prometheus multiprocess module imported successfully"
+                )
+            except Exception as e:
+                verbose_logger.warning(
+                    f"Failed to import prometheus multiprocess module: {e}"
+                )
+
+            verbose_logger.info(
+                f"Prometheus multiprocess mode enabled with directory: {multiproc_dir}"
+            )
+
+        except Exception as e:
+            verbose_logger.warning(f"Failed to setup Prometheus multiprocess mode: {e}")
+
+    def _is_multiprocess_environment(self) -> bool:
+        """
+        Detect if we're running in a multiprocess environment (uvicorn/gunicorn with multiple workers).
+        """
+        import os
+
+        # Check for common environment variables that indicate multiple workers
+        num_workers = os.environ.get("NUM_WORKERS", "1")
+        try:
+            if int(num_workers) > 1:
+                return True
+        except (ValueError, TypeError):
+            pass
+
+        # Check for gunicorn worker environment variables
+        if os.environ.get("GUNICORN_CMD_ARGS") or os.environ.get("GUNICORN_WORKER_ID"):
+            return True
+
+        # Check if PROMETHEUS_MULTIPROC_DIR is explicitly set (admin override)
+        if os.environ.get("PROMETHEUS_MULTIPROC_DIR"):
+            return True
+
+        return False
+
+    @staticmethod
+    def cleanup_multiprocess_metrics():
+        """
+        Clean up multiprocess metrics directory on startup.
+        This should be called once during application startup to prevent stale metrics.
+        """
+        import os
+        from pathlib import Path
+
+        multiproc_dir = os.environ.get("PROMETHEUS_MULTIPROC_DIR")
+        if multiproc_dir and os.path.exists(multiproc_dir):
+            try:
+                # Remove all files in the directory but keep the directory itself
+                for file_path in Path(multiproc_dir).glob("*"):
+                    if file_path.is_file():
+                        file_path.unlink()
+                verbose_logger.info(
+                    f"Cleaned up Prometheus multiprocess metrics directory: {multiproc_dir}"
+                )
+            except Exception as e:
+                verbose_logger.warning(
+                    f"Failed to cleanup Prometheus multiprocess directory: {e}"
+                )
+
     def _parse_prometheus_config(self) -> Dict[str, List[str]]:
         """Parse prometheus metrics configuration for label filtering and enabled metrics"""
         import litellm
@@ -727,7 +913,16 @@ class PrometheusLogger(CustomLogger):
             metric_name = args[0] if args else kwargs.get("name", "")
 
             if self._is_metric_enabled(metric_name):
-                return metric_class(*args, **kwargs)
+                # Handle multiprocess_mode parameter for Gauge metrics
+                if metric_class.__name__ == "Gauge" and "multiprocess_mode" in kwargs:
+                    # Pass through multiprocess_mode to the Gauge constructor
+                    return metric_class(*args, **kwargs)
+                else:
+                    # For Counter and Histogram, remove multiprocess_mode if present
+                    filtered_kwargs = {
+                        k: v for k, v in kwargs.items() if k != "multiprocess_mode"
+                    }
+                    return metric_class(*args, **filtered_kwargs)
             else:
                 return NoOpMetric()
 
@@ -845,13 +1040,6 @@ class PrometheusLogger(CustomLogger):
 
         # increment total LLM requests and spend metric
         self._increment_top_level_request_and_spend_metrics(
-            end_user_id=end_user_id,
-            user_api_key=user_api_key,
-            user_api_key_alias=user_api_key_alias,
-            model=model,
-            user_api_team=user_api_team,
-            user_api_team_alias=user_api_team_alias,
-            user_id=user_id,
             response_cost=response_cost,
             enum_values=enum_values,
         )
@@ -1018,13 +1206,6 @@ class PrometheusLogger(CustomLogger):
 
     def _increment_top_level_request_and_spend_metrics(
         self,
-        end_user_id: Optional[str],
-        user_api_key: Optional[str],
-        user_api_key_alias: Optional[str],
-        model: Optional[str],
-        user_api_team: Optional[str],
-        user_api_team_alias: Optional[str],
-        user_id: Optional[str],
         response_cost: float,
         enum_values: UserAPIKeyLabelValues,
     ):
@@ -1039,20 +1220,11 @@ class PrometheusLogger(CustomLogger):
 
         _labels = prometheus_label_factory(
             supported_enum_labels=self.get_labels_for_metric(
-                metric_name="litellm_proxy_total_requests_metric"
+                metric_name="litellm_spend_metric"
             ),
             enum_values=enum_values,
         )
-
-        self.litellm_spend_metric.labels(
-            end_user_id,
-            user_api_key,
-            user_api_key_alias,
-            model,
-            user_api_team,
-            user_api_team_alias,
-            user_id,
-        ).inc(response_cost)
+        self.litellm_spend_metric.labels(**_labels).inc(response_cost)
 
     def _set_virtual_key_rate_limit_metrics(
         self,
@@ -2179,13 +2351,14 @@ class PrometheusLogger(CustomLogger):
     def _mount_metrics_endpoint(premium_user: bool):
         """
         Mount the Prometheus metrics endpoint with optional authentication.
+        Uses multiprocess collector when running with multiple workers.
 
         Args:
             premium_user (bool): Whether the user is a premium user
-            require_auth (bool, optional): Whether to require authentication for the metrics endpoint.
-                                        Defaults to False.
         """
-        from prometheus_client import make_asgi_app
+        import os
+
+        from prometheus_client import CollectorRegistry, make_asgi_app
 
         from litellm._logging import verbose_proxy_logger
         from litellm.proxy._types import CommonProxyErrors
@@ -2196,14 +2369,34 @@ class PrometheusLogger(CustomLogger):
                 f"Prometheus metrics are only available for premium users. {CommonProxyErrors.not_premium_user.value}"
             )
 
-        # Create metrics ASGI app
-        metrics_app = make_asgi_app()
+        # Check if we're in multiprocess mode
+        multiproc_dir = os.environ.get("PROMETHEUS_MULTIPROC_DIR")
+
+        if multiproc_dir:
+            # Use multiprocess collector for worker aggregation
+            try:
+                from prometheus_client import multiprocess
+
+                registry = CollectorRegistry()
+                multiprocess.MultiProcessCollector(registry)
+                metrics_app = make_asgi_app(registry)
+                verbose_proxy_logger.info(
+                    f"Starting Prometheus Metrics on /metrics with multiprocess collector (directory: {multiproc_dir})"
+                )
+            except Exception as e:
+                verbose_proxy_logger.warning(
+                    f"Failed to setup multiprocess collector, falling back to default: {e}"
+                )
+                metrics_app = make_asgi_app()
+        else:
+            # Use default single-process collector
+            metrics_app = make_asgi_app()
+            verbose_proxy_logger.debug(
+                "Starting Prometheus Metrics on /metrics (single process mode)"
+            )
 
         # Mount the metrics app to the app
         app.mount("/metrics", metrics_app)
-        verbose_proxy_logger.debug(
-            "Starting Prometheus Metrics on /metrics (no authentication)"
-        )
 
 
 def prometheus_label_factory(
@@ -2280,7 +2473,9 @@ def get_custom_labels_from_metadata(metadata: dict) -> Dict[str, str]:
     return result
 
 
-def _tag_matches_wildcard_configured_pattern(tags: List[str], configured_tag: str) -> bool:
+def _tag_matches_wildcard_configured_pattern(
+    tags: List[str], configured_tag: str
+) -> bool:
     """
     Check if any of the request tags matches a wildcard configured pattern
 
@@ -2305,6 +2500,7 @@ def _tag_matches_wildcard_configured_pattern(tags: List[str], configured_tag: st
     import re
 
     from litellm.router_utils.pattern_match_deployments import PatternMatchRouter
+
     pattern_router = PatternMatchRouter()
     regex_pattern = pattern_router._pattern_to_regex(configured_tag)
     return any(re.match(pattern=regex_pattern, string=tag) for tag in tags)
@@ -2313,11 +2509,11 @@ def _tag_matches_wildcard_configured_pattern(tags: List[str], configured_tag: st
 def get_custom_labels_from_tags(tags: List[str]) -> Dict[str, str]:
     """
     Get custom labels from tags based on admin configuration.
-    
+
     Supports both exact matches and wildcard patterns:
     - Exact match: "prod" matches "prod" exactly
-    - Wildcard pattern: "User-Agent: curl/*" matches "User-Agent: curl/7.68.0" 
-    
+    - Wildcard pattern: "User-Agent: curl/*" matches "User-Agent: curl/7.68.0"
+
     Reuses PatternMatchRouter for wildcard pattern matching.
 
     Returns dict of label_name: "true" if the tag matches the configured tag, "false" otherwise
@@ -2331,9 +2527,6 @@ def get_custom_labels_from_tags(tags: List[str]) -> Dict[str, str]:
         "tag_Service_web_app_v1": "false",
     }
     """
-    import re
-
-    from litellm.router_utils.pattern_match_deployments import PatternMatchRouter
     from litellm.types.integrations.prometheus import _sanitize_prometheus_label_name
 
     configured_tags = litellm.custom_prometheus_tags
@@ -2341,21 +2534,22 @@ def get_custom_labels_from_tags(tags: List[str]) -> Dict[str, str]:
         return {}
 
     result: Dict[str, str] = {}
-    pattern_router = PatternMatchRouter()
 
     for configured_tag in configured_tags:
         label_name = _sanitize_prometheus_label_name(f"tag_{configured_tag}")
-        
+
         # Check for exact match first (backwards compatibility)
         if configured_tag in tags:
             result[label_name] = "true"
             continue
-            
+
         # Use PatternMatchRouter for wildcard pattern matching
-        if "*" in configured_tag and _tag_matches_wildcard_configured_pattern(tags=tags, configured_tag=configured_tag):
+        if "*" in configured_tag and _tag_matches_wildcard_configured_pattern(
+            tags=tags, configured_tag=configured_tag
+        ):
             result[label_name] = "true"
             continue
-        
+
         # No match found
         result[label_name] = "false"
 

enterprise/litellm_enterprise/types/proxy/proxy_server.py

@@ -1,4 +1,6 @@
-from typing import Literal, TypedDict
+from typing import Literal
+
+from typing_extensions import TypedDict
 
 
 class CustomAuthSettings(TypedDict):

enterprise/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "litellm-enterprise"
-version = "0.1.19"
+version = "0.1.20"
 description = "Package for LiteLLM Enterprise features"
 authors = ["BerriAI"]
 readme = "README.md"
@@ -22,7 +22,7 @@ requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"
 
 [tool.commitizen]
-version = "0.1.19"
+version = "0.1.20"
 version_files = [
     "pyproject.toml:version",
     "../requirements.txt:litellm-enterprise==",

litellm-js/spend-logs/package-lock.json

@@ -6,7 +6,7 @@
     "": {
       "dependencies": {
         "@hono/node-server": "^1.10.1",
-        "hono": "^4.6.5"
+        "hono": "^4.9.7"
       },
       "devDependencies": {
         "@types/node": "^20.11.17",
@@ -463,9 +463,10 @@
       }
     },
     "node_modules/hono": {
-      "version": "4.6.5",
-      "resolved": "https://registry.npmjs.org/hono/-/hono-4.6.5.tgz",
-      "integrity": "sha512-qsmN3V5fgtwdKARGLgwwHvcdLKursMd+YOt69eGpl1dUCJb8mCd7hZfyZnBYjxCegBG7qkJRQRUy2oO25yHcyQ==",
+      "version": "4.9.7",
+      "resolved": "https://registry.npmjs.org/hono/-/hono-4.9.7.tgz",
+      "integrity": "sha512-t4Te6ERzIaC48W3x4hJmBwgNlLhmiEdEE5ViYb02ffw4ignHNHa5IBtPjmbKstmtKa8X6C35iWwK4HaqvrzG9w==",
+      "license": "MIT",
       "engines": {
         "node": ">=16.9.0"
       }

litellm-js/spend-logs/package.json

@@ -4,7 +4,7 @@
   },
   "dependencies": {
     "@hono/node-server": "^1.10.1",
-    "hono": "^4.6.5"
+    "hono": "^4.9.7"
   },
   "devDependencies": {
     "@types/node": "^20.11.17",

litellm-proxy-extras/litellm_proxy_extras/migrations/20250918083359_drop_spec_version_column_from_mcp_table/migration.sql

@@ -0,0 +1,8 @@
+/*
+  Warnings:
+
+  - You are about to drop the column `spec_version` on the `LiteLLM_MCPServerTable` table. All the data in the column will be lost.
+
+*/
+-- AlterTable
+ALTER TABLE "public"."LiteLLM_MCPServerTable" DROP COLUMN "spec_version";

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -171,7 +171,6 @@ model LiteLLM_MCPServerTable {
   description  String?
   url          String?
   transport    String         @default("sse")
-  spec_version String         @default("2025-03-26")
   auth_type    String?        
   created_at   DateTime?      @default(now()) @map("created_at")
   created_by   String?

litellm-proxy-extras/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "litellm-proxy-extras"
-version = "0.2.18"
+version = "0.2.19"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 authors = ["BerriAI"]
 readme = "README.md"
@@ -22,7 +22,7 @@ requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"
 
 [tool.commitizen]
-version = "0.2.18"
+version = "0.2.19"
 version_files = [
     "pyproject.toml:version",
     "../requirements.txt:litellm-proxy-extras==",

litellm/__init__.py

@@ -67,6 +67,7 @@ from litellm.constants import (
     bedrock_embedding_models,
     known_tokenizer_config,
     BEDROCK_INVOKE_PROVIDERS_LITERAL,
+    BEDROCK_EMBEDDING_PROVIDERS_LITERAL,
     BEDROCK_CONVERSE_MODELS,
     DEFAULT_MAX_TOKENS,
     DEFAULT_SOFT_BUDGET,
@@ -116,6 +117,7 @@ _custom_logger_compatible_callbacks_literal = Literal[
     "logfire",
     "literalai",
     "dynamic_rate_limiter",
+    "dynamic_rate_limiter_v3",
     "langsmith",
     "prometheus",
     "otel",
@@ -147,6 +149,7 @@ _custom_logger_compatible_callbacks_literal = Literal[
     "vector_store_pre_call_hook",
     "dotprompt",
     "cloudzero",
+    "posthog",
 ]
 configured_cold_storage_logger: Optional[
     _custom_logger_compatible_callbacks_literal
@@ -241,6 +244,7 @@ gradient_ai_api_key: Optional[str] = None
 nebius_key: Optional[str] = None
 heroku_key: Optional[str] = None
 cometapi_key: Optional[str] = None
+ovhcloud_key: Optional[str] = None
 common_cloud_provider_auth_params: dict = {
     "params": ["project", "region_name", "token"],
     "providers": ["vertex_ai", "bedrock", "watsonx", "azure", "vertex_ai_beta"],
@@ -520,6 +524,8 @@ cometapi_models: Set = set()
 oci_models: Set = set()
 vercel_ai_gateway_models: Set = set()
 volcengine_models: Set = set()
+ovhcloud_models: Set = set()
+ovhcloud_embedding_models: Set = set()
 
 
 def is_bedrock_pricing_only_model(key: str) -> bool:
@@ -734,6 +740,10 @@ def add_known_models():
             oci_models.add(key)
         elif value.get("litellm_provider") == "volcengine":
             volcengine_models.add(key)
+        elif value.get("litellm_provider") == "ovhcloud":
+            ovhcloud_models.add(key)
+        elif value.get("litellm_provider") == "ovhcloud-embedding-models":
+            ovhcloud_embedding_models.add(key)
 
 
 add_known_models()
@@ -828,6 +838,7 @@ model_list = list(
     | heroku_models
     | vercel_ai_gateway_models
     | volcengine_models
+    | ovhcloud_models
 )
 
 model_list_set = set(model_list)
@@ -909,6 +920,7 @@ models_by_provider: dict = {
     "cometapi": cometapi_models,
     "oci": oci_models,
     "volcengine": volcengine_models,
+    "ovhcloud": ovhcloud_models | ovhcloud_embedding_models,
 }
 
 # mapping for those models which have larger equivalents
@@ -943,6 +955,7 @@ all_embedding_models = (
     | fireworks_ai_embedding_models
     | nebius_embedding_models
     | sambanova_embedding_models
+    | ovhcloud_embedding_models
 )
 
 ####### IMAGE GENERATION MODELS ###################
@@ -1013,6 +1026,7 @@ from .llms.openai_like.chat.handler import OpenAILikeChatConfig
 from .llms.aiohttp_openai.chat.transformation import AiohttpOpenAIChatConfig
 from .llms.galadriel.chat.transformation import GaladrielChatConfig
 from .llms.github.chat.transformation import GithubChatConfig
+from .llms.compactifai.chat.transformation import CompactifAIChatConfig
 from .llms.empower.chat.transformation import EmpowerChatConfig
 from .llms.huggingface.chat.transformation import HuggingFaceChatConfig
 from .llms.huggingface.embedding.transformation import HuggingFaceEmbeddingConfig
@@ -1033,7 +1047,6 @@ from .llms.databricks.chat.transformation import DatabricksConfig
 from .llms.databricks.embed.transformation import DatabricksEmbeddingConfig
 from .llms.predibase.chat.transformation import PredibaseConfig
 from .llms.replicate.chat.transformation import ReplicateConfig
-from .llms.cohere.completion.transformation import CohereTextConfig as CohereConfig
 from .llms.snowflake.chat.transformation import SnowflakeConfig
 from .llms.cohere.rerank.transformation import CohereRerankConfig
 from .llms.cohere.rerank_v2.transformation import CohereRerankV2Config
@@ -1254,6 +1267,8 @@ from .llms.morph.chat.transformation import MorphChatConfig
 from .llms.lambda_ai.chat.transformation import LambdaAIChatConfig
 from .llms.hyperbolic.chat.transformation import HyperbolicChatConfig
 from .llms.vercel_ai_gateway.chat.transformation import VercelAIGatewayConfig
+from .llms.ovhcloud.chat.transformation import OVHCloudChatConfig
+from .llms.ovhcloud.embedding.transformation import OVHCloudEmbeddingConfig
 from .main import *  # type: ignore
 from .integrations import *
 from .llms.custom_httpx.async_client_cleanup import close_litellm_async_clients

litellm/batches/main.py

@@ -19,6 +19,7 @@ from typing import Any, Coroutine, Dict, Literal, Optional, Union, cast
 import httpx
 
 import litellm
+from litellm._logging import verbose_logger
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.llms.azure.batches.handler import AzureBatchesAPI
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
@@ -38,6 +39,7 @@ from litellm.utils import (
     ProviderConfigManager,
     client,
     get_litellm_params,
+    get_llm_provider,
     supports_httpx_timeout,
 )
 
@@ -49,6 +51,45 @@ base_llm_http_handler = BaseLLMHTTPHandler()
 #################################################
 
 
+def _resolve_timeout(
+    optional_params: GenericLiteLLMParams,
+    kwargs: Dict[str, Any],
+    custom_llm_provider: str,
+    default_timeout: float = 600.0,
+) -> float:
+    """
+    Resolve timeout value from various sources and handle httpx.Timeout objects.
+    
+    Args:
+        optional_params: GenericLiteLLMParams object containing timeout
+        kwargs: Additional kwargs that may contain request_timeout
+        custom_llm_provider: Provider name for httpx timeout support check
+        default_timeout: Default timeout value to use
+        
+    Returns:
+        Resolved timeout as float
+    """
+    timeout = optional_params.timeout or kwargs.get("request_timeout", default_timeout) or default_timeout
+    
+    # Handle httpx.Timeout objects
+    if isinstance(timeout, httpx.Timeout):
+        if supports_httpx_timeout(custom_llm_provider) is False:
+            # Extract read timeout for providers that don't support httpx.Timeout
+            read_timeout = timeout.read or default_timeout
+            return float(read_timeout)
+        else:
+            # For providers that support httpx.Timeout, we still need to return a float
+            # This case might need to be handled differently based on the actual use case
+            return float(timeout.read or default_timeout)
+    
+    # Handle None case
+    if timeout is None:
+        return float(default_timeout)
+    
+    # Handle numeric values (int, float, string representations)
+    return float(timeout)
+
+
 @client
 async def acreate_batch(
     completion_window: Literal["24h"],
@@ -118,13 +159,23 @@ def create_batch(
         litellm_call_id = kwargs.get("litellm_call_id", None)
         proxy_server_request = kwargs.get("proxy_server_request", None)
         model_info = kwargs.get("model_info", None)
+        model: Optional[str] = kwargs.get("model", None)
+        try:
+            if model is not None:
+                model, _, _, _ = get_llm_provider(
+                                model=model, 
+                                custom_llm_provider=None,
+                        )
+        except Exception as e:
+            verbose_logger.exception(f"litellm.batches.main.py::create_batch() - Error inferring custom_llm_provider - {str(e)}")
+            
         _is_async = kwargs.pop("acreate_batch", False) is True
         litellm_params = dict(GenericLiteLLMParams(**kwargs))
         litellm_logging_obj: LiteLLMLoggingObj = cast(LiteLLMLoggingObj, kwargs.get("litellm_logging_obj", None))
         ### TIMEOUT LOGIC ###
-        timeout = optional_params.timeout or kwargs.get("request_timeout", 600) or 600
+        timeout = _resolve_timeout(optional_params, kwargs, custom_llm_provider)
         litellm_logging_obj.update_environment_variables(
-            model=None,
+            model=model,
             user=None,
             optional_params=optional_params.model_dump(),
             litellm_params={
@@ -138,18 +189,6 @@ def create_batch(
             },
             custom_llm_provider=custom_llm_provider,
         )
-
-        if (
-            timeout is not None
-            and isinstance(timeout, httpx.Timeout)
-            and supports_httpx_timeout(custom_llm_provider) is False
-        ):
-            read_timeout = timeout.read or 600
-            timeout = read_timeout  # default 10 min timeout
-        elif timeout is not None and not isinstance(timeout, httpx.Timeout):
-            timeout = float(timeout)  # type: ignore
-        elif timeout is None:
-            timeout = 600.0
         
 
         _create_batch_request = CreateBatchRequest(
@@ -160,10 +199,13 @@ def create_batch(
             extra_headers=extra_headers,
             extra_body=extra_body,
         )
-        provider_config = ProviderConfigManager.get_provider_batches_config(
-            model="",
-            provider=LlmProviders(custom_llm_provider),
-        )
+        if model is not None:
+            provider_config = ProviderConfigManager.get_provider_batches_config(
+                model=model,
+                provider=LlmProviders(custom_llm_provider),
+            )
+        else:
+            provider_config = None
         if provider_config is not None:
             response = base_llm_http_handler.create_batch(
                 provider_config=provider_config,
@@ -179,6 +221,7 @@ def create_batch(
                 and isinstance(client, (HTTPHandler, AsyncHTTPHandler))
                 else None,
                 timeout=timeout,
+                model=model,
             )
             return response
         api_base: Optional[str] = None
@@ -297,7 +340,7 @@ def create_batch(
 @client
 async def aretrieve_batch(
     batch_id: str,
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock"] = "openai",
     metadata: Optional[Dict[str, str]] = None,
     extra_headers: Optional[Dict[str, str]] = None,
     extra_body: Optional[Dict[str, str]] = None,
@@ -335,11 +378,129 @@ async def aretrieve_batch(
     except Exception as e:
         raise e
 
+def _handle_retrieve_batch_providers_without_provider_config(
+    batch_id: str,
+    optional_params: GenericLiteLLMParams,
+    timeout: Union[float, httpx.Timeout],
+    litellm_params: dict,
+    _retrieve_batch_request: RetrieveBatchRequest,
+    _is_async: bool,
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock"] = "openai",
+):
+    api_base: Optional[str] = None
+    if custom_llm_provider == "openai":
+        # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
+        api_base = (
+            optional_params.api_base
+            or litellm.api_base
+            or os.getenv("OPENAI_BASE_URL")
+            or os.getenv("OPENAI_API_BASE")
+            or "https://api.openai.com/v1"
+        )
+        organization = (
+            optional_params.organization
+            or litellm.organization
+            or os.getenv("OPENAI_ORGANIZATION", None)
+            or None  # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105
+        )
+        # set API KEY
+        api_key = (
+            optional_params.api_key
+            or litellm.api_key  # for deepinfra/perplexity/anyscale we check in get_llm_provider and pass in the api key from there
+            or litellm.openai_key
+            or os.getenv("OPENAI_API_KEY")
+        )
+
+        response = openai_batches_instance.retrieve_batch(
+            _is_async=_is_async,
+            retrieve_batch_data=_retrieve_batch_request,
+            api_base=api_base,
+            api_key=api_key,
+            organization=organization,
+            timeout=timeout,
+            max_retries=optional_params.max_retries,
+        )
+    elif custom_llm_provider == "azure":
+        api_base = (
+            optional_params.api_base
+            or litellm.api_base
+            or get_secret_str("AZURE_API_BASE")
+        )
+        api_version = (
+            optional_params.api_version
+            or litellm.api_version
+            or get_secret_str("AZURE_API_VERSION")
+        )
+
+        api_key = (
+            optional_params.api_key
+            or litellm.api_key
+            or litellm.azure_key
+            or get_secret_str("AZURE_OPENAI_API_KEY")
+            or get_secret_str("AZURE_API_KEY")
+        )
+
+        extra_body = optional_params.get("extra_body", {})
+        if extra_body is not None:
+            extra_body.pop("azure_ad_token", None)
+        else:
+            get_secret_str("AZURE_AD_TOKEN")  # type: ignore
+
+        response = azure_batches_instance.retrieve_batch(
+            _is_async=_is_async,
+            api_base=api_base,
+            api_key=api_key,
+            api_version=api_version,
+            timeout=timeout,
+            max_retries=optional_params.max_retries,
+            retrieve_batch_data=_retrieve_batch_request,
+            litellm_params=litellm_params,
+        )
+    elif custom_llm_provider == "vertex_ai":
+        api_base = optional_params.api_base or ""
+        vertex_ai_project = (
+            optional_params.vertex_project
+            or litellm.vertex_project
+            or get_secret_str("VERTEXAI_PROJECT")
+        )
+        vertex_ai_location = (
+            optional_params.vertex_location
+            or litellm.vertex_location
+            or get_secret_str("VERTEXAI_LOCATION")
+        )
+        vertex_credentials = optional_params.vertex_credentials or get_secret_str(
+            "VERTEXAI_CREDENTIALS"
+        )
+
+        response = vertex_ai_batches_instance.retrieve_batch(
+            _is_async=_is_async,
+            batch_id=batch_id,
+            api_base=api_base,
+            vertex_project=vertex_ai_project,
+            vertex_location=vertex_ai_location,
+            vertex_credentials=vertex_credentials,
+            timeout=timeout,
+            max_retries=optional_params.max_retries,
+        )
+    else:
+        raise litellm.exceptions.BadRequestError(
+            message="LiteLLM doesn't support {} for 'create_batch'. Only 'openai' is supported.".format(
+                custom_llm_provider
+            ),
+            model="n/a",
+            llm_provider=custom_llm_provider,
+            response=httpx.Response(
+                status_code=400,
+                content="Unsupported provider",
+                request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"),  # type: ignore
+            ),
+        )
+    return response
 
 @client
 def retrieve_batch(
     batch_id: str,
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock"] = "openai",
     metadata: Optional[Dict[str, str]] = None,
     extra_headers: Optional[Dict[str, str]] = None,
     extra_body: Optional[Dict[str, str]] = None,
@@ -387,115 +548,59 @@ def retrieve_batch(
         )
 
         _is_async = kwargs.pop("aretrieve_batch", False) is True
-        api_base: Optional[str] = None
-        if custom_llm_provider == "openai":
-            # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
-            api_base = (
-                optional_params.api_base
-                or litellm.api_base
-                or os.getenv("OPENAI_BASE_URL")
-                or os.getenv("OPENAI_API_BASE")
-                or "https://api.openai.com/v1"
-            )
-            organization = (
-                optional_params.organization
-                or litellm.organization
-                or os.getenv("OPENAI_ORGANIZATION", None)
-                or None  # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105
-            )
-            # set API KEY
-            api_key = (
-                optional_params.api_key
-                or litellm.api_key  # for deepinfra/perplexity/anyscale we check in get_llm_provider and pass in the api key from there
-                or litellm.openai_key
-                or os.getenv("OPENAI_API_KEY")
-            )
-
-            response = openai_batches_instance.retrieve_batch(
-                _is_async=_is_async,
-                retrieve_batch_data=_retrieve_batch_request,
-                api_base=api_base,
-                api_key=api_key,
-                organization=organization,
-                timeout=timeout,
-                max_retries=optional_params.max_retries,
-            )
-        elif custom_llm_provider == "azure":
-            api_base = (
-                optional_params.api_base
-                or litellm.api_base
-                or get_secret_str("AZURE_API_BASE")
-            )
-            api_version = (
-                optional_params.api_version
-                or litellm.api_version
-                or get_secret_str("AZURE_API_VERSION")
-            )
-
-            api_key = (
-                optional_params.api_key
-                or litellm.api_key
-                or litellm.azure_key
-                or get_secret_str("AZURE_OPENAI_API_KEY")
-                or get_secret_str("AZURE_API_KEY")
-            )
-
-            extra_body = optional_params.get("extra_body", {})
-            if extra_body is not None:
-                extra_body.pop("azure_ad_token", None)
-            else:
-                get_secret_str("AZURE_AD_TOKEN")  # type: ignore
-
-            response = azure_batches_instance.retrieve_batch(
-                _is_async=_is_async,
-                api_base=api_base,
-                api_key=api_key,
-                api_version=api_version,
-                timeout=timeout,
-                max_retries=optional_params.max_retries,
-                retrieve_batch_data=_retrieve_batch_request,
-                litellm_params=litellm_params,
-            )
-        elif custom_llm_provider == "vertex_ai":
-            api_base = optional_params.api_base or ""
-            vertex_ai_project = (
-                optional_params.vertex_project
-                or litellm.vertex_project
-                or get_secret_str("VERTEXAI_PROJECT")
-            )
-            vertex_ai_location = (
-                optional_params.vertex_location
-                or litellm.vertex_location
-                or get_secret_str("VERTEXAI_LOCATION")
-            )
-            vertex_credentials = optional_params.vertex_credentials or get_secret_str(
-                "VERTEXAI_CREDENTIALS"
-            )
-
-            response = vertex_ai_batches_instance.retrieve_batch(
-                _is_async=_is_async,
-                batch_id=batch_id,
-                api_base=api_base,
-                vertex_project=vertex_ai_project,
-                vertex_location=vertex_ai_location,
-                vertex_credentials=vertex_credentials,
-                timeout=timeout,
-                max_retries=optional_params.max_retries,
+        client = kwargs.get("client", None)
+        
+        # Try to use provider config first (for providers like bedrock)
+        model: Optional[str] = kwargs.get("model", None)
+        if model is not None:
+            provider_config = ProviderConfigManager.get_provider_batches_config(
+                model=model,
+                provider=LlmProviders(custom_llm_provider),
             )
         else:
-            raise litellm.exceptions.BadRequestError(
-                message="LiteLLM doesn't support {} for 'create_batch'. Only 'openai' is supported.".format(
-                    custom_llm_provider
-                ),
-                model="n/a",
-                llm_provider=custom_llm_provider,
-                response=httpx.Response(
-                    status_code=400,
-                    content="Unsupported provider",
-                    request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"),  # type: ignore
+            provider_config = None
+            
+        if provider_config is not None:
+            response = base_llm_http_handler.retrieve_batch(
+                batch_id=batch_id,
+                provider_config=provider_config,
+                litellm_params=litellm_params,
+                headers=extra_headers or {},
+                api_base=optional_params.api_base,
+                api_key=optional_params.api_key,
+                logging_obj=litellm_logging_obj or LiteLLMLoggingObj(
+                    model=model or "bedrock/unknown",
+                    messages=[],
+                    stream=False,
+                    call_type="batch_retrieve",
+                    start_time=None,
+                    litellm_call_id="batch_retrieve_" + batch_id,
+                    function_id="batch_retrieve",
                 ),
+                _is_async=_is_async,
+                client=client
+                if client is not None
+                and isinstance(client, (HTTPHandler, AsyncHTTPHandler))
+                else None,
+                timeout=timeout,
+                model=model,
             )
-        return response
+            return response
+        
+
+        #########################################################
+        # Handle providers without provider config
+        #########################################################
+        return _handle_retrieve_batch_providers_without_provider_config(
+            batch_id=batch_id,
+            custom_llm_provider=custom_llm_provider,
+            optional_params=optional_params,
+            litellm_params=litellm_params,
+            _retrieve_batch_request=_retrieve_batch_request,
+            _is_async=_is_async,
+            timeout=timeout,
+        )
+        
     except Exception as e:
         raise e
 

litellm/caching/redis_cache.py

@@ -19,6 +19,7 @@ from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Union, cast
 import litellm
 from litellm._logging import print_verbose, verbose_logger
 from litellm.litellm_core_utils.core_helpers import _get_parent_otel_span_from_kwargs
+from litellm.litellm_core_utils.coroutine_checker import coroutine_checker
 from litellm.types.caching import RedisPipelineIncrementOperation
 from litellm.types.services import ServiceTypes
 
@@ -140,7 +141,7 @@ class RedisCache(BaseCache):
             self.redis_flush_size = redis_flush_size
         self.redis_version = "Unknown"
         try:
-            if not inspect.iscoroutinefunction(self.redis_client):
+            if not coroutine_checker.is_async_callable(self.redis_client):
                 self.redis_version = self.redis_client.info()["redis_version"]  # type: ignore
         except Exception:
             pass

litellm/completion_extras/litellm_responses_transformation/handler.py

@@ -2,7 +2,9 @@
 Handler for transforming /chat/completions api requests to litellm.responses requests
 """
 
-from typing import TYPE_CHECKING, Any, Coroutine, TypedDict, Union
+from typing import TYPE_CHECKING, Any, Coroutine, Union
+
+from typing_extensions import TypedDict
 
 if TYPE_CHECKING:
     from litellm import CustomStreamWrapper, LiteLLMLoggingObj, ModelResponse

litellm/constants.py

@@ -15,7 +15,7 @@ DEFAULT_SQS_FLUSH_INTERVAL_SECONDS = int(
     os.getenv("DEFAULT_SQS_FLUSH_INTERVAL_SECONDS", 10)
 )
 DEFAULT_NUM_WORKERS_LITELLM_PROXY = int(
-    os.getenv("DEFAULT_NUM_WORKERS_LITELLM_PROXY", os.cpu_count() or 4)
+    os.getenv("DEFAULT_NUM_WORKERS_LITELLM_PROXY", 1)
 )
 DEFAULT_SQS_BATCH_SIZE = int(os.getenv("DEFAULT_SQS_BATCH_SIZE", 512))
 SQS_SEND_MESSAGE_ACTION = "SendMessage"
@@ -60,7 +60,9 @@ DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_PRO = int(
     os.getenv("DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_PRO", 128)
 )
 DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE = int(
-    os.getenv("DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE", 512)
+    os.getenv(
+        "DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE", 512
+    )
 )
 
 # Generic fallback for unknown models
@@ -177,7 +179,7 @@ NON_LLM_CONNECTION_TIMEOUT = int(
     os.getenv("NON_LLM_CONNECTION_TIMEOUT", 15)
 )  # timeout for adjacent services (e.g. jwt auth)
 MAX_EXCEPTION_MESSAGE_LENGTH = int(os.getenv("MAX_EXCEPTION_MESSAGE_LENGTH", 2000))
-MAX_STRING_LENGTH_PROMPT_IN_DB = int(os.getenv("MAX_STRING_LENGTH_PROMPT_IN_DB", 1000))
+MAX_STRING_LENGTH_PROMPT_IN_DB = int(os.getenv("MAX_STRING_LENGTH_PROMPT_IN_DB", 2048))
 BEDROCK_MAX_POLICY_SIZE = int(os.getenv("BEDROCK_MAX_POLICY_SIZE", 75))
 REPLICATE_POLLING_DELAY_SECONDS = float(
     os.getenv("REPLICATE_POLLING_DELAY_SECONDS", 0.5)
@@ -311,6 +313,7 @@ LITELLM_CHAT_PROVIDERS = [
     "morph",
     "lambda_ai",
     "vercel_ai_gateway",
+    "ovhcloud",
 ]
 
 LITELLM_EMBEDDING_PROVIDERS_SUPPORTING_INPUT_ARRAY_OF_TOKENS = [
@@ -766,6 +769,12 @@ BEDROCK_INVOKE_PROVIDERS_LITERAL = Literal[
     "deepseek_r1",
 ]
 
+BEDROCK_EMBEDDING_PROVIDERS_LITERAL = Literal[
+    "cohere",
+    "amazon",
+    "twelvelabs",
+]
+
 BEDROCK_CONVERSE_MODELS = [
     "openai.gpt-oss-20b-1:0",
     "openai.gpt-oss-120b-1:0",
@@ -819,6 +828,7 @@ bedrock_embedding_models: set = set(
         "amazon.titan-embed-text-v1",
         "cohere.embed-english-v3",
         "cohere.embed-multilingual-v3",
+        "twelvelabs.marengo-embed-2-7-v1:0",
     ]
 )
 
@@ -949,7 +959,9 @@ LITELLM_CLI_SESSION_TOKEN_PREFIX = "litellm-session-token"
 DB_SPEND_UPDATE_JOB_NAME = "db_spend_update_job"
 PROMETHEUS_EMIT_BUDGET_METRICS_JOB_NAME = "prometheus_emit_budget_metrics"
 CLOUDZERO_EXPORT_USAGE_DATA_JOB_NAME = "cloudzero_export_usage_data"
-CLOUDZERO_MAX_FETCHED_DATA_RECORDS = int(os.getenv("CLOUDZERO_MAX_FETCHED_DATA_RECORDS", 50000))
+CLOUDZERO_MAX_FETCHED_DATA_RECORDS = int(
+    os.getenv("CLOUDZERO_MAX_FETCHED_DATA_RECORDS", 50000)
+)
 SPEND_LOG_CLEANUP_JOB_NAME = "spend_log_cleanup"
 SPEND_LOG_RUN_LOOPS = int(os.getenv("SPEND_LOG_RUN_LOOPS", 500))
 SPEND_LOG_CLEANUP_BATCH_SIZE = int(os.getenv("SPEND_LOG_CLEANUP_BATCH_SIZE", 1000))
@@ -1019,6 +1031,7 @@ SENTRY_DENYLIST = [
     "FIREWORKS_API_KEY",
     "FIREWORKS_AI_API_KEY",
     "FIREWORKSAI_API_KEY",
+    "OVHCLOUD_API_KEY",
     # Database and Connection Strings
     "database_url",
     "redis_url",
@@ -1057,3 +1070,8 @@ SENTRY_PII_DENYLIST = [
     "SMTP_SENDER_EMAIL",
     "TEST_EMAIL_ADDRESS",
 ]
+
+# CoroutineChecker cache configuration
+COROUTINE_CHECKER_MAX_SIZE_IN_MEMORY = int(
+    os.getenv("COROUTINE_CHECKER_MAX_SIZE_IN_MEMORY", 1000)
+)

litellm/cost_calculator.py

@@ -344,6 +344,11 @@ def cost_per_token(  # noqa: PLR0915
         return perplexity_cost_per_token(model=model, usage=usage_block)
     elif custom_llm_provider == "xai":
         return xai_cost_per_token(model=model, usage=usage_block)
+    elif custom_llm_provider == "dashscope":
+        from litellm.llms.dashscope.cost_calculator import (
+            cost_per_token as dashscope_cost_per_token,
+        )
+        return dashscope_cost_per_token(model=model, usage=usage_block)
     else:
         model_info = _cached_get_model_info_helper(
             model=model, custom_llm_provider=custom_llm_provider

litellm/endpoints/speech/speech_to_completion_bridge/handler.py

@@ -2,7 +2,9 @@
 Handler for transforming /chat/completions api requests to litellm.responses requests
 """
 
-from typing import TYPE_CHECKING, Optional, TypedDict, Union
+from typing import TYPE_CHECKING, Optional, Union
+
+from typing_extensions import TypedDict
 
 if TYPE_CHECKING:
     from litellm import LiteLLMLoggingObj

litellm/experimental_mcp_client/client.py

@@ -19,8 +19,6 @@ from litellm._logging import verbose_logger
 from litellm.types.mcp import (
     MCPAuth,
     MCPAuthType,
-    MCPSpecVersion,
-    MCPSpecVersionType,
     MCPStdioConfig,
     MCPTransport,
     MCPTransportType,
@@ -48,7 +46,6 @@ class MCPClient:
         auth_value: Optional[str] = None,
         timeout: float = 60.0,
         stdio_config: Optional[MCPStdioConfig] = None,
-        protocol_version: MCPSpecVersionType = MCPSpecVersion.jun_2025,
     ):
         self.server_url: str = server_url
         self.transport_type: MCPTransport = transport_type
@@ -62,7 +59,6 @@ class MCPClient:
         self._session_ctx = None
         self._task: Optional[asyncio.Task] = None
         self.stdio_config: Optional[MCPStdioConfig] = stdio_config
-        self.protocol_version: MCPSpecVersionType = protocol_version
 
         # handle the basic auth value if provided
         if auth_value:
@@ -84,22 +80,24 @@ class MCPClient:
         """Initialize the transport and session."""
         if self._session:
             return  # Already connected
-            
+
         try:
             if self.transport_type == MCPTransport.stdio:
                 # For stdio transport, use stdio_client with command-line parameters
                 if not self.stdio_config:
                     raise ValueError("stdio_config is required for stdio transport")
-                    
+
                 server_params = StdioServerParameters(
                     command=self.stdio_config.get("command", ""),
                     args=self.stdio_config.get("args", []),
-                    env=self.stdio_config.get("env", {})
+                    env=self.stdio_config.get("env", {}),
                 )
-                
+
                 self._transport_ctx = stdio_client(server_params)
                 self._transport = await self._transport_ctx.__aenter__()
-                self._session_ctx = ClientSession(self._transport[0], self._transport[1])
+                self._session_ctx = ClientSession(
+                    self._transport[0], self._transport[1]
+                )
                 self._session = await self._session_ctx.__aenter__()
                 await self._session.initialize()
             elif self.transport_type == MCPTransport.sse:
@@ -110,7 +108,9 @@ class MCPClient:
                     headers=headers,
                 )
                 self._transport = await self._transport_ctx.__aenter__()
-                self._session_ctx = ClientSession(self._transport[0], self._transport[1])
+                self._session_ctx = ClientSession(
+                    self._transport[0], self._transport[1]
+                )
                 self._session = await self._session_ctx.__aenter__()
                 await self._session.initialize()
             else:  # http
@@ -121,7 +121,9 @@ class MCPClient:
                     headers=headers,
                 )
                 self._transport = await self._transport_ctx.__aenter__()
-                self._session_ctx = ClientSession(self._transport[0], self._transport[1])
+                self._session_ctx = ClientSession(
+                    self._transport[0], self._transport[1]
+                )
                 self._session = await self._session_ctx.__aenter__()
                 await self._session.initialize()
         except ValueError as e:
@@ -184,8 +186,10 @@ class MCPClient:
 
     def _get_auth_headers(self) -> dict:
         """Generate authentication headers based on auth type."""
-        headers = {}
-        
+        headers = {
+            "MCP-Protocol-Version": "2025-06-18"
+        }
+
         if self._mcp_auth_value:
             if self.auth_type == MCPAuth.bearer_token:
                 headers["Authorization"] = f"Bearer {self._mcp_auth_value}"
@@ -196,18 +200,8 @@ class MCPClient:
             elif self.auth_type == MCPAuth.authorization:
                 headers["Authorization"] = self._mcp_auth_value
 
-        # Handle protocol version - it might be a string or enum
-        if hasattr(self.protocol_version, 'value'):
-            # It's an enum
-            protocol_version_str = self.protocol_version.value
-        else:
-            # It's a string
-            protocol_version_str = str(self.protocol_version)
-        
-        headers["MCP-Protocol-Version"] = protocol_version_str
         return headers
 
-
     async def list_tools(self) -> List[MCPTool]:
         """List available tools from the server."""
         if not self._session:
@@ -216,7 +210,7 @@ class MCPClient:
             except Exception as e:
                 verbose_logger.warning(f"MCP client connection failed: {str(e)}")
                 return []
-        
+
         if self._session is None:
             verbose_logger.warning("MCP client session is not initialized")
             return []
@@ -245,17 +239,20 @@ class MCPClient:
             except Exception as e:
                 verbose_logger.warning(f"MCP client connection failed: {str(e)}")
                 return MCPCallToolResult(
-                    content=[TextContent(type="text", text=f"{str(e)}")],
-                    isError=True
+                    content=[TextContent(type="text", text=f"{str(e)}")], isError=True
                 )
 
         if self._session is None:
             verbose_logger.warning("MCP client session is not initialized")
             return MCPCallToolResult(
-                content=[TextContent(type="text", text="MCP client session is not initialized")],
+                content=[
+                    TextContent(
+                        type="text", text="MCP client session is not initialized"
+                    )
+                ],
                 isError=True,
             )
-        
+
         try:
             tool_result = await self._session.call_tool(
                 name=call_tool_request_params.name,
@@ -270,8 +267,8 @@ class MCPClient:
             await self.disconnect()
             # Return a default error result instead of raising
             return MCPCallToolResult(
-                content=[TextContent(type="text", text=f"{str(e)}")],  # Empty content for error case
+                content=[
+                    TextContent(type="text", text=f"{str(e)}")
+                ],  # Empty content for error case
                 isError=True,
             )
-        
-

litellm/experimental_mcp_client/tools.py

@@ -17,22 +17,60 @@ from litellm.types.utils import ChatCompletionMessageToolCall
 ########################################################
 def transform_mcp_tool_to_openai_tool(mcp_tool: MCPTool) -> ChatCompletionToolParam:
     """Convert an MCP tool to an OpenAI tool."""
+    normalized_parameters = _normalize_mcp_input_schema(mcp_tool.inputSchema)
+    
     return ChatCompletionToolParam(
         type="function",
         function=FunctionDefinition(
             name=mcp_tool.name,
             description=mcp_tool.description or "",
-            parameters=mcp_tool.inputSchema,
+            parameters=normalized_parameters,
             strict=False,
         ),
     )
 
 
+def _normalize_mcp_input_schema(input_schema: dict) -> dict:
+    """
+    Normalize MCP input schema to ensure it's valid for OpenAI function calling.
+    
+    OpenAI requires that function parameters have:
+    - type: 'object'
+    - properties: dict (can be empty)
+    - additionalProperties: false (recommended)
+    """
+    if not input_schema:
+        return {
+            "type": "object",
+            "properties": {},
+            "additionalProperties": False
+        }
+    
+    # Make a copy to avoid modifying the original
+    normalized_schema = dict(input_schema)
+    
+    # Ensure type is 'object'
+    if "type" not in normalized_schema:
+        normalized_schema["type"] = "object"
+    
+    # Ensure properties exists (can be empty)
+    if "properties" not in normalized_schema:
+        normalized_schema["properties"] = {}
+    
+    # Add additionalProperties if not present (recommended by OpenAI)
+    if "additionalProperties" not in normalized_schema:
+        normalized_schema["additionalProperties"] = False
+    
+    return normalized_schema
+
+
 def transform_mcp_tool_to_openai_responses_api_tool(mcp_tool: MCPTool) -> FunctionToolParam:
     """Convert an MCP tool to an OpenAI Responses API tool."""
+    normalized_parameters = _normalize_mcp_input_schema(mcp_tool.inputSchema)
+    
     return FunctionToolParam(
         name=mcp_tool.name,
-        parameters=mcp_tool.inputSchema,
+        parameters=normalized_parameters,
         strict=False,
         type="function",
         description=mcp_tool.description or "",

litellm/files/main.py

@@ -731,7 +731,7 @@ def file_list(
 
 async def afile_content(
     file_id: str,
-    custom_llm_provider: Literal["openai", "azure"] = "openai",
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
     extra_headers: Optional[Dict[str, str]] = None,
     extra_body: Optional[Dict[str, str]] = None,
     **kwargs,
@@ -887,6 +887,32 @@ def file_content(
                 client=client,
                 litellm_params=litellm_params_dict,
             )
+        elif custom_llm_provider == "vertex_ai":
+            api_base = optional_params.api_base or ""
+            vertex_ai_project = (
+                optional_params.vertex_project
+                or litellm.vertex_project
+                or get_secret_str("VERTEXAI_PROJECT")
+            )
+            vertex_ai_location = (
+                optional_params.vertex_location
+                or litellm.vertex_location
+                or get_secret_str("VERTEXAI_LOCATION")
+            )
+            vertex_credentials = optional_params.vertex_credentials or get_secret_str(
+                "VERTEXAI_CREDENTIALS"
+            )
+
+            response = vertex_ai_files_instance.file_content(
+                _is_async=_is_async,
+                file_content_request=_file_content_request,
+                api_base=api_base,
+                vertex_credentials=vertex_credentials,
+                vertex_project=vertex_ai_project,
+                vertex_location=vertex_ai_location,
+                timeout=timeout,
+                max_retries=optional_params.max_retries,
+            )
         else:
             raise litellm.exceptions.BadRequestError(
                 message="LiteLLM doesn't support {} for 'custom_llm_provider'. Supported providers are 'openai', 'azure', 'vertex_ai'.".format(

litellm/files/utils.py

@@ -0,0 +1,27 @@
+from typing import Optional
+
+from litellm.types.llms.openai import CreateFileRequest
+from litellm.types.utils import ExtractedFileData
+
+
+class FilesAPIUtils:
+    """
+    Utils for files API interface on litellm
+    """
+    @staticmethod
+    def is_batch_jsonl_file(create_file_data: CreateFileRequest, extracted_file_data: ExtractedFileData) -> bool:
+        """
+        Check if the file is a batch jsonl file
+        """
+        return (
+            create_file_data.get("purpose") == "batch"
+            and FilesAPIUtils.valid_content_type(extracted_file_data.get("content_type"))
+            and extracted_file_data.get("content") is not None
+        )
+    
+    @staticmethod
+    def valid_content_type(content_type: Optional[str]) -> bool:
+        """
+        Check if the content type is valid
+        """
+        return content_type in set(["application/jsonl", "application/octet-stream"])

litellm/google_genai/main.py

@@ -224,6 +224,9 @@ async def agenerate_content(
         loop = asyncio.get_event_loop()
         kwargs["agenerate_content"] = True
 
+        # Handle generationConfig parameter from kwargs for backward compatibility
+        if "generationConfig" in kwargs and config is None:
+            config = kwargs.pop("generationConfig")
         # get custom llm provider so we can use this for mapping exceptions
         if custom_llm_provider is None:
             _, custom_llm_provider, _, _ = litellm.get_llm_provider(
@@ -288,6 +291,9 @@ def generate_content(
     try:
         _is_async = kwargs.pop("agenerate_content", False) is True
 
+        # Handle generationConfig parameter from kwargs for backward compatibility
+        if "generationConfig" in kwargs and config is None:
+            config = kwargs.pop("generationConfig")
         # Check for mock response first
         litellm_params = GenericLiteLLMParams(**kwargs)
         if litellm_params.mock_response and isinstance(
@@ -374,6 +380,9 @@ async def agenerate_content_stream(
     try:
         kwargs["agenerate_content_stream"] = True
 
+        # Handle generationConfig parameter from kwargs for backward compatibility
+        if "generationConfig" in kwargs and config is None:
+            config = kwargs.pop("generationConfig")
         # get custom llm provider so we can use this for mapping exceptions
         if custom_llm_provider is None:
             _, custom_llm_provider, _, _ = litellm.get_llm_provider(
@@ -461,6 +470,9 @@ def generate_content_stream(
         # Remove any async-related flags since this is the sync function
         _is_async = kwargs.pop("agenerate_content_stream", False)
 
+        # Handle generationConfig parameter from kwargs for backward compatibility
+        if "generationConfig" in kwargs and config is None:
+            config = kwargs.pop("generationConfig")
         # Setup the call
         setup_result = GenerateContentHelper.setup_generate_content_call(
             model=model,

litellm/integrations/SlackAlerting/budget_alert_types.py

@@ -31,7 +31,7 @@ class SoftBudgetAlert(BaseBudgetAlertType):
         return "Soft Budget Crossed: "
 
     def get_id(self, user_info: CallInfo) -> str:
-        return "default_id"
+        return user_info.token or "default_id"
 
 
 class UserBudgetAlert(BaseBudgetAlertType):

litellm/integrations/custom_guardrail.py

@@ -357,6 +357,7 @@ class CustomGuardrail(CustomLogger):
         end_time: Optional[float] = None,
         duration: Optional[float] = None,
         masked_entity_count: Optional[Dict[str, int]] = None,
+        guardrail_provider: Optional[str] = None,
     ) -> None:
         """
         Builds `StandardLoggingGuardrailInformation` and adds it to the request metadata so it can be used for logging to DataDog, Langfuse, etc.
@@ -367,6 +368,7 @@ class CustomGuardrail(CustomLogger):
 
         slg = StandardLoggingGuardrailInformation(
             guardrail_name=self.guardrail_name,
+            guardrail_provider=guardrail_provider,
             guardrail_mode=(
                 GuardrailMode(**self.event_hook.model_dump())  # type: ignore
                 if isinstance(self.event_hook, Mode)
@@ -487,7 +489,8 @@ class CustomGuardrail(CustomLogger):
         """
         Update the guardrails litellm params in memory
         """
-        pass
+        for key, value in vars(litellm_params).items():
+            setattr(self, key, value)
 
 
 def log_guardrail_information(func):

litellm/integrations/datadog/datadog_llm_obs.py

@@ -64,7 +64,7 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
             asyncio.create_task(self.periodic_flush())
             self.flush_lock = asyncio.Lock()
             self.log_queue: List[LLMObsPayload] = []
-            
+
             #########################################################
             # Handle datadog_llm_observability_params set as litellm.datadog_llm_observability_params
             #########################################################
@@ -83,22 +83,25 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
         """
         dict_datadog_llm_obs_params: Dict = {}
         if litellm.datadog_llm_observability_params is not None:
-            if isinstance(litellm.datadog_llm_observability_params, DatadogLLMObsInitParams):
-                dict_datadog_llm_obs_params = litellm.datadog_llm_observability_params.model_dump()
+            if isinstance(
+                litellm.datadog_llm_observability_params, DatadogLLMObsInitParams
+            ):
+                dict_datadog_llm_obs_params = (
+                    litellm.datadog_llm_observability_params.model_dump()
+                )
             elif isinstance(litellm.datadog_llm_observability_params, Dict):
                 # only allow params that are of DatadogLLMObsInitParams
-                dict_datadog_llm_obs_params = DatadogLLMObsInitParams(**litellm.datadog_llm_observability_params).model_dump()
+                dict_datadog_llm_obs_params = DatadogLLMObsInitParams(
+                    **litellm.datadog_llm_observability_params
+                ).model_dump()
         return dict_datadog_llm_obs_params
-            
 
     async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
         try:
             verbose_logger.debug(
                 f"DataDogLLMObs: Logging success event for model {kwargs.get('model', 'unknown')}"
             )
-            payload = self.create_llm_obs_payload(
-                kwargs, start_time, end_time
-            )
+            payload = self.create_llm_obs_payload(kwargs, start_time, end_time)
             verbose_logger.debug(f"DataDogLLMObs: Payload: {payload}")
             self.log_queue.append(payload)
 
@@ -108,15 +111,13 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
             verbose_logger.exception(
                 f"DataDogLLMObs: Error logging success event - {str(e)}"
             )
-    
+
     async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
         try:
             verbose_logger.debug(
                 f"DataDogLLMObs: Logging failure event for model {kwargs.get('model', 'unknown')}"
             )
-            payload = self.create_llm_obs_payload(
-                kwargs, start_time, end_time
-            )
+            payload = self.create_llm_obs_payload(kwargs, start_time, end_time)
             verbose_logger.debug(f"DataDogLLMObs: Payload: {payload}")
             self.log_queue.append(payload)
 
@@ -147,10 +148,22 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
                     ),
                 ),
             }
-            verbose_logger.debug("payload %s", json.dumps(payload, indent=4))
+
+            # serialize datetime objects - for budget reset time in spend metrics
+            from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
+
+            try:
+                verbose_logger.debug("payload %s", safe_dumps(payload))
+            except Exception as debug_error:
+                verbose_logger.debug(
+                    "payload serialization failed: %s", str(debug_error)
+                )
+
+            json_payload = safe_dumps(payload)
+
             response = await self.async_client.post(
                 url=self.intake_url,
-                json=payload,
+                content=json_payload,
                 headers={
                     "DD-API-KEY": self.DD_API_KEY,
                     "Content-Type": "application/json",
@@ -184,7 +197,6 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
 
         messages = standard_logging_payload["messages"]
         messages = self._ensure_string_content(messages=messages)
-        response_obj = standard_logging_payload.get("response")
 
         metadata = kwargs.get("litellm_params", {}).get("metadata", {})
 
@@ -193,10 +205,12 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
                 messages
             )
         )
-        output_meta = OutputMeta(messages=self._get_response_messages(
-            response_obj=response_obj,
-            call_type=standard_logging_payload.get("call_type")
-        ))
+        output_meta = OutputMeta(
+            messages=self._get_response_messages(
+                standard_logging_payload=standard_logging_payload,
+                call_type=standard_logging_payload.get("call_type"),
+            )
+        )
 
         error_info = self._assemble_error_info(standard_logging_payload)
 
@@ -214,7 +228,9 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
             output_tokens=float(standard_logging_payload.get("completion_tokens", 0)),
             total_tokens=float(standard_logging_payload.get("total_tokens", 0)),
             total_cost=float(standard_logging_payload.get("response_cost", 0)),
-            time_to_first_token=self._get_time_to_first_token_seconds(standard_logging_payload),
+            time_to_first_token=self._get_time_to_first_token_seconds(
+                standard_logging_payload
+            ),
         )
 
         payload: LLMObsPayload = LLMObsPayload(
@@ -251,27 +267,35 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
         except Exception:
             pass
         return None
-    
-    def _assemble_error_info(self, standard_logging_payload: StandardLoggingPayload) -> Optional[DDLLMObsError]:
+
+    def _assemble_error_info(
+        self, standard_logging_payload: StandardLoggingPayload
+    ) -> Optional[DDLLMObsError]:
         """
         Assemble error information for failure cases according to DD LLM Obs API spec
         """
         # Handle error information for failure cases according to DD LLM Obs API spec
         error_info: Optional[DDLLMObsError] = None
-        
+
         if standard_logging_payload.get("status") == "failure":
             # Try to get structured error information first
-            error_information: Optional[StandardLoggingPayloadErrorInformation] = standard_logging_payload.get("error_information")
-            
+            error_information: Optional[
+                StandardLoggingPayloadErrorInformation
+            ] = standard_logging_payload.get("error_information")
+
             if error_information:
                 error_info = DDLLMObsError(
-                    message=error_information.get("error_message") or standard_logging_payload.get("error_str") or "Unknown error",
+                    message=error_information.get("error_message")
+                    or standard_logging_payload.get("error_str")
+                    or "Unknown error",
                     type=error_information.get("error_class"),
-                    stack=error_information.get("traceback")
+                    stack=error_information.get("traceback"),
                 )
         return error_info
 
-    def _get_time_to_first_token_seconds(self, standard_logging_payload: StandardLoggingPayload) -> float:
+    def _get_time_to_first_token_seconds(
+        self, standard_logging_payload: StandardLoggingPayload
+    ) -> float:
         """
         Get the time to first token in seconds
 
@@ -280,7 +304,9 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
         For non streaming calls, CompletionStartTime is time we get the response back
         """
         start_time: Optional[float] = standard_logging_payload.get("startTime")
-        completion_start_time: Optional[float] = standard_logging_payload.get("completionStartTime")
+        completion_start_time: Optional[float] = standard_logging_payload.get(
+            "completionStartTime"
+        )
         end_time: Optional[float] = standard_logging_payload.get("endTime")
 
         if completion_start_time is not None and start_time is not None:
@@ -290,19 +316,43 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
         else:
             return 0.0
 
-
     def _get_response_messages(
-        self, response_obj: Any, call_type: Optional[str]
+        self, standard_logging_payload: StandardLoggingPayload, call_type: Optional[str]
     ) -> List[Any]:
         """
         Get the messages from the response object
 
         for now this handles logging /chat/completions responses
         """
+
+        response_obj = standard_logging_payload.get("response")
         if response_obj is None:
             return []
-        
-        if call_type in [CallTypes.completion.value, CallTypes.acompletion.value]:
+
+        # edge case: handle response_obj is a string representation of a dict
+        if isinstance(response_obj, str):
+            try:
+                import ast
+
+                response_obj = ast.literal_eval(response_obj)
+            except (ValueError, SyntaxError):
+                try:
+                    # fallback to json parsing
+                    response_obj = json.loads(str(response_obj))
+                except json.JSONDecodeError:
+                    return []
+
+        if call_type in [
+            CallTypes.completion.value,
+            CallTypes.acompletion.value,
+            CallTypes.text_completion.value,
+            CallTypes.atext_completion.value,
+            CallTypes.generate_content.value,
+            CallTypes.agenerate_content.value,
+            CallTypes.generate_content_stream.value,
+            CallTypes.agenerate_content_stream.value,
+            CallTypes.anthropic_messages.value,
+        ]:
             try:
                 # Safely extract message from response_obj, handle failure cases
                 if isinstance(response_obj, dict) and "choices" in response_obj:
@@ -315,102 +365,104 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
                 return []
         return []
 
-    def _get_datadog_span_kind(self, call_type: Optional[str]) -> Literal["llm", "tool", "task", "embedding", "retrieval"]:
+    def _get_datadog_span_kind(
+        self, call_type: Optional[str]
+    ) -> Literal["llm", "tool", "task", "embedding", "retrieval"]:
         """
         Map liteLLM call_type to appropriate DataDog LLM Observability span kind.
-        
+
         Available DataDog span kinds: "llm", "tool", "task", "embedding", "retrieval"
         """
         if call_type is None:
             return "llm"
-        
+
         # Embedding operations
         if call_type in [CallTypes.embedding.value, CallTypes.aembedding.value]:
             return "embedding"
-        
-        # LLM completion operations  
+
+        # LLM completion operations
         if call_type in [
-            CallTypes.completion.value, 
+            CallTypes.completion.value,
             CallTypes.acompletion.value,
-            CallTypes.text_completion.value, 
+            CallTypes.text_completion.value,
             CallTypes.atext_completion.value,
-            CallTypes.generate_content.value, 
+            CallTypes.generate_content.value,
             CallTypes.agenerate_content.value,
-            CallTypes.generate_content_stream.value, 
+            CallTypes.generate_content_stream.value,
             CallTypes.agenerate_content_stream.value,
-            CallTypes.anthropic_messages.value
+            CallTypes.anthropic_messages.value,
         ]:
             return "llm"
-        
+
         # Tool operations
         if call_type in [CallTypes.call_mcp_tool.value]:
             return "tool"
-            
+
         # Retrieval operations
         if call_type in [
-            CallTypes.get_assistants.value, 
+            CallTypes.get_assistants.value,
             CallTypes.aget_assistants.value,
-            CallTypes.get_thread.value, 
+            CallTypes.get_thread.value,
             CallTypes.aget_thread.value,
-            CallTypes.get_messages.value, 
+            CallTypes.get_messages.value,
             CallTypes.aget_messages.value,
-            CallTypes.afile_retrieve.value, 
+            CallTypes.afile_retrieve.value,
             CallTypes.file_retrieve.value,
-            CallTypes.afile_list.value, 
+            CallTypes.afile_list.value,
             CallTypes.file_list.value,
-            CallTypes.afile_content.value, 
+            CallTypes.afile_content.value,
             CallTypes.file_content.value,
-            CallTypes.retrieve_batch.value, 
+            CallTypes.retrieve_batch.value,
             CallTypes.aretrieve_batch.value,
-            CallTypes.retrieve_fine_tuning_job.value, 
+            CallTypes.retrieve_fine_tuning_job.value,
             CallTypes.aretrieve_fine_tuning_job.value,
-            CallTypes.responses.value, 
+            CallTypes.responses.value,
             CallTypes.aresponses.value,
-            CallTypes.alist_input_items.value
+            CallTypes.alist_input_items.value,
         ]:
             return "retrieval"
-            
+
         # Task operations (batch, fine-tuning, file operations, etc.)
         if call_type in [
-            CallTypes.create_batch.value, 
+            CallTypes.create_batch.value,
             CallTypes.acreate_batch.value,
-            CallTypes.create_fine_tuning_job.value, 
+            CallTypes.create_fine_tuning_job.value,
             CallTypes.acreate_fine_tuning_job.value,
-            CallTypes.cancel_fine_tuning_job.value, 
+            CallTypes.cancel_fine_tuning_job.value,
             CallTypes.acancel_fine_tuning_job.value,
-            CallTypes.list_fine_tuning_jobs.value, 
+            CallTypes.list_fine_tuning_jobs.value,
             CallTypes.alist_fine_tuning_jobs.value,
-            CallTypes.create_assistants.value, 
+            CallTypes.create_assistants.value,
             CallTypes.acreate_assistants.value,
-            CallTypes.delete_assistant.value, 
+            CallTypes.delete_assistant.value,
             CallTypes.adelete_assistant.value,
-            CallTypes.create_thread.value, 
+            CallTypes.create_thread.value,
             CallTypes.acreate_thread.value,
-            CallTypes.add_message.value, 
+            CallTypes.add_message.value,
             CallTypes.a_add_message.value,
-            CallTypes.run_thread.value, 
+            CallTypes.run_thread.value,
             CallTypes.arun_thread.value,
-            CallTypes.run_thread_stream.value, 
+            CallTypes.run_thread_stream.value,
             CallTypes.arun_thread_stream.value,
-            CallTypes.file_delete.value, 
+            CallTypes.file_delete.value,
             CallTypes.afile_delete.value,
-            CallTypes.create_file.value, 
+            CallTypes.create_file.value,
             CallTypes.acreate_file.value,
-            CallTypes.image_generation.value, 
+            CallTypes.image_generation.value,
             CallTypes.aimage_generation.value,
-            CallTypes.image_edit.value, 
+            CallTypes.image_edit.value,
             CallTypes.aimage_edit.value,
-            CallTypes.moderation.value, 
+            CallTypes.moderation.value,
             CallTypes.amoderation.value,
-            CallTypes.transcription.value, 
+            CallTypes.transcription.value,
             CallTypes.atranscription.value,
-            CallTypes.speech.value, 
+            CallTypes.speech.value,
             CallTypes.aspeech.value,
-            CallTypes.rerank.value, 
-            CallTypes.arerank.value
+            CallTypes.rerank.value,
+            CallTypes.arerank.value,
         ]:
             return "task"
-            
+
         # Default fallback for unknown or passthrough operations
         return "llm"
 
@@ -443,7 +495,10 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
             "cache_hit": standard_logging_payload.get("cache_hit", "unknown"),
             "cache_key": standard_logging_payload.get("cache_key", "unknown"),
             "saved_cache_cost": standard_logging_payload.get("saved_cache_cost", 0),
-            "guardrail_information": standard_logging_payload.get("guardrail_information", None),
+            "guardrail_information": standard_logging_payload.get(
+                "guardrail_information", None
+            ),
+            "is_streamed_request": self._get_stream_value_from_payload(standard_logging_payload),
         }
 
         #########################################################
@@ -452,22 +507,38 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
         latency_metrics = self._get_latency_metrics(standard_logging_payload)
         _metadata.update({"latency_metrics": dict(latency_metrics)})
 
+        #########################################################
+        # Add spend metrics to metadata
+        #########################################################
+        spend_metrics = self._get_spend_metrics(standard_logging_payload)
+        _metadata.update({"spend_metrics": dict(spend_metrics)})
+
+        ## extract tool calls and add to metadata
+        tool_call_metadata = self._extract_tool_call_metadata(standard_logging_payload)
+        _metadata.update(tool_call_metadata)
+
         _standard_logging_metadata: dict = (
             dict(standard_logging_payload.get("metadata", {})) or {}
         )
         _metadata.update(_standard_logging_metadata)
         return _metadata
 
-    def _get_latency_metrics(self, standard_logging_payload: StandardLoggingPayload) -> DDLLMObsLatencyMetrics:
+    def _get_latency_metrics(
+        self, standard_logging_payload: StandardLoggingPayload
+    ) -> DDLLMObsLatencyMetrics:
         """
         Get the latency metrics from the standard logging payload
         """
         latency_metrics: DDLLMObsLatencyMetrics = DDLLMObsLatencyMetrics()
         # Add latency metrics to metadata
         # Time to first token (convert from seconds to milliseconds for consistency)
-        time_to_first_token_seconds = self._get_time_to_first_token_seconds(standard_logging_payload)
+        time_to_first_token_seconds = self._get_time_to_first_token_seconds(
+            standard_logging_payload
+        )
         if time_to_first_token_seconds > 0:
-            latency_metrics["time_to_first_token_ms"] = time_to_first_token_seconds * 1000
+            latency_metrics["time_to_first_token_ms"] = (
+                time_to_first_token_seconds * 1000
+            )
 
         # LiteLLM overhead time
         hidden_params = standard_logging_payload.get("hidden_params", {})
@@ -476,11 +547,233 @@ class DataDogLLMObsLogger(DataDogLogger, CustomBatchLogger):
             latency_metrics["litellm_overhead_time_ms"] = litellm_overhead_ms
 
         # Guardrail overhead latency
-        guardrail_info: Optional[StandardLoggingGuardrailInformation] = standard_logging_payload.get("guardrail_information")
+        guardrail_info: Optional[
+            StandardLoggingGuardrailInformation
+        ] = standard_logging_payload.get("guardrail_information")
         if guardrail_info is not None:
-            _guardrail_duration_seconds: Optional[float] = guardrail_info.get("duration")
+            _guardrail_duration_seconds: Optional[float] = guardrail_info.get(
+                "duration"
+            )
             if _guardrail_duration_seconds is not None:
                 # Convert from seconds to milliseconds for consistency
-                latency_metrics["guardrail_overhead_time_ms"] = _guardrail_duration_seconds * 1000
-            
-        return latency_metrics
+                latency_metrics["guardrail_overhead_time_ms"] = (
+                    _guardrail_duration_seconds * 1000
+                )
+
+        return latency_metrics
+
+    def _get_stream_value_from_payload(self, standard_logging_payload: StandardLoggingPayload) -> bool:
+        """
+        Extract the stream value from standard logging payload.
+
+        The stream field in StandardLoggingPayload is only set to True for completed streaming responses.
+        For non-streaming requests, it's None. The original stream parameter is in model_parameters.
+
+        Returns:
+            bool: True if this was a streaming request, False otherwise
+        """
+        # Check top-level stream field first (only True for completed streaming)
+        stream_value = standard_logging_payload.get("stream")
+        if stream_value is True:
+            return True
+
+        # Fallback to model_parameters.stream for original request parameters
+        model_params = standard_logging_payload.get("model_parameters", {})
+        if isinstance(model_params, dict):
+            stream_value = model_params.get("stream")
+            if stream_value is True:
+                return True
+
+        # Default to False for non-streaming requests
+        return False
+
+    def _get_spend_metrics(
+        self, standard_logging_payload: StandardLoggingPayload
+    ) -> DDLLMObsSpendMetrics:
+        """
+        Get the spend metrics from the standard logging payload
+        """
+        spend_metrics: DDLLMObsSpendMetrics = DDLLMObsSpendMetrics()
+
+        # send response cost
+        spend_metrics["response_cost"] = standard_logging_payload.get(
+            "response_cost", 0.0
+        )
+
+        # Get budget information from metadata
+        metadata = standard_logging_payload.get("metadata", {})
+
+        # API key max budget
+        user_api_key_max_budget = metadata.get("user_api_key_max_budget")
+        if user_api_key_max_budget is not None:
+            spend_metrics["user_api_key_max_budget"] = float(user_api_key_max_budget)
+
+        # API key spend
+        user_api_key_spend = metadata.get("user_api_key_spend")
+        if user_api_key_spend is not None:
+            try:
+                spend_metrics["user_api_key_spend"] = float(user_api_key_spend)
+            except (ValueError, TypeError):
+                verbose_logger.debug(
+                    f"Invalid user_api_key_spend value: {user_api_key_spend}"
+                )
+
+        # API key budget reset datetime
+        user_api_key_budget_reset_at = metadata.get("user_api_key_budget_reset_at")
+        if user_api_key_budget_reset_at is not None:
+            try:
+                from datetime import datetime, timezone
+
+                budget_reset_at = None
+                if isinstance(user_api_key_budget_reset_at, str):
+                    # Handle ISO format strings that might have 'Z' suffix
+                    iso_string = user_api_key_budget_reset_at.replace("Z", "+00:00")
+                    budget_reset_at = datetime.fromisoformat(iso_string)
+                elif isinstance(user_api_key_budget_reset_at, datetime):
+                    budget_reset_at = user_api_key_budget_reset_at
+
+                if budget_reset_at is not None:
+                    # Preserve timezone info if already present
+                    if budget_reset_at.tzinfo is None:
+                        budget_reset_at = budget_reset_at.replace(tzinfo=timezone.utc)
+
+                    # Convert to ISO string format for JSON serialization
+                    # This prevents circular reference issues and ensures proper timezone representation
+                    iso_string = budget_reset_at.isoformat()
+                    spend_metrics["user_api_key_budget_reset_at"] = iso_string
+
+                    # Debug logging to verify the conversion
+                    verbose_logger.debug(
+                        f"Converted budget_reset_at to ISO format: {iso_string}"
+                    )
+            except Exception as e:
+                verbose_logger.debug(f"Error processing budget reset datetime: {e}")
+                verbose_logger.debug(f"Original value: {user_api_key_budget_reset_at}")
+
+        return spend_metrics
+
+    def _process_input_messages_preserving_tool_calls(
+        self, messages: List[Any]
+    ) -> List[Dict[str, Any]]:
+        """
+        Process input messages while preserving tool_calls and tool message types.
+
+        This bypasses the lossy string conversion when tool calls are present,
+        allowing complex nested tool_calls objects to be preserved for Datadog.
+        """
+        processed = []
+        for msg in messages:
+            if isinstance(msg, dict):
+                # Preserve messages with tool_calls or tool role as-is
+                if "tool_calls" in msg or msg.get("role") == "tool":
+                    processed.append(msg)
+                else:
+                    # For regular messages, still apply string conversion
+                    converted = (
+                        handle_any_messages_to_chat_completion_str_messages_conversion(
+                            [msg]
+                        )
+                    )
+                    processed.extend(converted)
+            else:
+                # For non-dict messages, apply string conversion
+                converted = (
+                    handle_any_messages_to_chat_completion_str_messages_conversion(
+                        [msg]
+                    )
+                )
+                processed.extend(converted)
+        return processed
+
+    @staticmethod
+    def _tool_calls_kv_pair(tool_calls: List[Dict[str, Any]]) -> Dict[str, Any]:
+        """
+        Extract tool call information into key-value pairs for Datadog metadata.
+
+        Similar to OpenTelemetry's implementation but adapted for Datadog's format.
+        """
+        kv_pairs: Dict[str, Any] = {}
+        for idx, tool_call in enumerate(tool_calls):
+            try:
+                # Extract tool call ID
+                tool_id = tool_call.get("id")
+                if tool_id:
+                    kv_pairs[f"tool_calls.{idx}.id"] = tool_id
+
+                # Extract tool call type
+                tool_type = tool_call.get("type")
+                if tool_type:
+                    kv_pairs[f"tool_calls.{idx}.type"] = tool_type
+
+                # Extract function information
+                function = tool_call.get("function")
+                if function:
+                    function_name = function.get("name")
+                    if function_name:
+                        kv_pairs[f"tool_calls.{idx}.function.name"] = function_name
+
+                    function_arguments = function.get("arguments")
+                    if function_arguments:
+                        # Store arguments as JSON string for Datadog
+                        if isinstance(function_arguments, str):
+                            kv_pairs[
+                                f"tool_calls.{idx}.function.arguments"
+                            ] = function_arguments
+                        else:
+                            import json
+
+                            kv_pairs[
+                                f"tool_calls.{idx}.function.arguments"
+                            ] = json.dumps(function_arguments)
+            except (KeyError, TypeError, ValueError) as e:
+                verbose_logger.debug(
+                    f"DataDogLLMObs: Error processing tool call {idx}: {str(e)}"
+                )
+                continue
+
+        return kv_pairs
+
+    def _extract_tool_call_metadata(
+        self, standard_logging_payload: StandardLoggingPayload
+    ) -> Dict[str, Any]:
+        """
+        Extract tool call information from both input messages and response for Datadog metadata.
+        """
+        tool_call_metadata: Dict[str, Any] = {}
+
+        try:
+            # Extract tool calls from input messages
+            messages = standard_logging_payload.get("messages", [])
+            if messages and isinstance(messages, list):
+                for message in messages:
+                    if isinstance(message, dict) and "tool_calls" in message:
+                        tool_calls = message.get("tool_calls")
+                        if tool_calls:
+                            input_tool_calls_kv = self._tool_calls_kv_pair(tool_calls)
+                            # Prefix with "input_" to distinguish from response tool calls
+                            for key, value in input_tool_calls_kv.items():
+                                tool_call_metadata[f"input_{key}"] = value
+
+            # Extract tool calls from response
+            response_obj = standard_logging_payload.get("response")
+            if response_obj and isinstance(response_obj, dict):
+                choices = response_obj.get("choices", [])
+                for choice in choices:
+                    if isinstance(choice, dict):
+                        message = choice.get("message")
+                        if message and isinstance(message, dict):
+                            tool_calls = message.get("tool_calls")
+                            if tool_calls:
+                                response_tool_calls_kv = self._tool_calls_kv_pair(
+                                    tool_calls
+                                )
+                                # Prefix with "output_" to distinguish from input tool calls
+                                for key, value in response_tool_calls_kv.items():
+                                    tool_call_metadata[f"output_{key}"] = value
+
+        except Exception as e:
+            verbose_logger.debug(
+                f"DataDogLLMObs: Error extracting tool call metadata: {str(e)}"
+            )
+
+        return tool_call_metadata

litellm/integrations/humanloop.py

@@ -4,9 +4,10 @@ Humanloop integration
 https://humanloop.com/
 """
 
-from typing import Any, Dict, List, Optional, Tuple, TypedDict, Union, cast
+from typing import Any, Dict, List, Optional, Tuple, Union, cast
 
 import httpx
+from typing_extensions import TypedDict
 
 import litellm
 from litellm.caching import DualCache

litellm/integrations/langfuse/langfuse.py

@@ -15,7 +15,7 @@ from litellm.litellm_core_utils.redact_messages import redact_user_api_key_info
 from litellm.llms.custom_httpx.http_handler import _get_httpx_client
 from litellm.secret_managers.main import str_to_bool
 from litellm.types.integrations.langfuse import *
-from litellm.types.llms.openai import HttpxBinaryResponseContent
+from litellm.types.llms.openai import HttpxBinaryResponseContent, ResponsesAPIResponse
 from litellm.types.utils import (
     EmbeddingResponse,
     ImageResponse,
@@ -196,6 +196,7 @@ class LangFuseLogger:
             TranscriptionResponse,
             RerankResponse,
             HttpxBinaryResponseContent,
+            ResponsesAPIResponse,
         ],
         start_time: Optional[datetime] = None,
         end_time: Optional[datetime] = None,
@@ -305,6 +306,7 @@ class LangFuseLogger:
             TranscriptionResponse,
             RerankResponse,
             HttpxBinaryResponseContent,
+            ResponsesAPIResponse,
         ],
         prompt: dict,
         level: str,
@@ -369,6 +371,11 @@ class LangFuseLogger:
         ):
             input = prompt
             output = response_obj.results
+        elif response_obj is not None and isinstance(
+            response_obj, litellm.ResponsesAPIResponse
+        ):
+            input = prompt
+            output = self._get_responses_api_content_for_langfuse(response_obj)
         elif (
             kwargs.get("call_type") is not None
             and kwargs.get("call_type") == "_arealtime"
@@ -768,6 +775,19 @@ class LangFuseLogger:
         else:
             return None
 
+    @staticmethod
+    def _get_responses_api_content_for_langfuse(
+        response_obj: ResponsesAPIResponse,
+    ):
+        """
+        Get the responses API content for Langfuse logging
+        """
+        if hasattr(response_obj, 'output') and response_obj.output:
+            # ResponsesAPIResponse.output is a list of strings
+            return response_obj.output
+        else:
+            return None
+
     @staticmethod
     def _get_langfuse_tags(
         standard_logging_object: Optional[StandardLoggingPayload],

litellm/integrations/langsmith.py

@@ -39,6 +39,7 @@ class LangsmithLogger(CustomBatchLogger):
         langsmith_api_key: Optional[str] = None,
         langsmith_project: Optional[str] = None,
         langsmith_base_url: Optional[str] = None,
+        langsmith_sampling_rate: Optional[float] = None,
         **kwargs,
     ):
         self.flush_lock = asyncio.Lock()
@@ -49,7 +50,8 @@ class LangsmithLogger(CustomBatchLogger):
             langsmith_base_url=langsmith_base_url,
         )
         self.sampling_rate: float = (
-            float(os.getenv("LANGSMITH_SAMPLING_RATE"))  # type: ignore
+            langsmith_sampling_rate
+            or float(os.getenv("LANGSMITH_SAMPLING_RATE"))  # type: ignore
             if os.getenv("LANGSMITH_SAMPLING_RATE") is not None
             and os.getenv("LANGSMITH_SAMPLING_RATE").strip().isdigit()  # type: ignore
             else 1.0
@@ -76,26 +78,14 @@ class LangsmithLogger(CustomBatchLogger):
         langsmith_base_url: Optional[str] = None,
     ) -> LangsmithCredentialsObject:
         _credentials_api_key = langsmith_api_key or os.getenv("LANGSMITH_API_KEY")
-        if _credentials_api_key is None:
-            raise Exception(
-                "Invalid Langsmith API Key given. _credentials_api_key=None."
-            )
         _credentials_project = (
             langsmith_project or os.getenv("LANGSMITH_PROJECT") or "litellm-completion"
         )
-        if _credentials_project is None:
-            raise Exception(
-                "Invalid Langsmith API Key given. _credentials_project=None."
-            )
         _credentials_base_url = (
             langsmith_base_url
             or os.getenv("LANGSMITH_BASE_URL")
             or "https://api.smith.langchain.com"
         )
-        if _credentials_base_url is None:
-            raise Exception(
-                "Invalid Langsmith API Key given. _credentials_base_url=None."
-            )
 
         return LangsmithCredentialsObject(
             LANGSMITH_API_KEY=_credentials_api_key,
@@ -200,12 +190,7 @@ class LangsmithLogger(CustomBatchLogger):
 
     def log_success_event(self, kwargs, response_obj, start_time, end_time):
         try:
-            sampling_rate = (
-                float(os.getenv("LANGSMITH_SAMPLING_RATE"))  # type: ignore
-                if os.getenv("LANGSMITH_SAMPLING_RATE") is not None
-                and os.getenv("LANGSMITH_SAMPLING_RATE").strip().isdigit()  # type: ignore
-                else 1.0
-            )
+            sampling_rate = self._get_sampling_rate_to_use_for_request(kwargs=kwargs)
             random_sample = random.random()
             if random_sample > sampling_rate:
                 verbose_logger.info(
@@ -219,6 +204,7 @@ class LangsmithLogger(CustomBatchLogger):
                 kwargs,
                 response_obj,
             )
+
             credentials = self._get_credentials_to_use_for_request(kwargs=kwargs)
             data = self._prepare_log_data(
                 kwargs=kwargs,
@@ -245,7 +231,7 @@ class LangsmithLogger(CustomBatchLogger):
 
     async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
         try:
-            sampling_rate = self.sampling_rate
+            sampling_rate = self._get_sampling_rate_to_use_for_request(kwargs=kwargs)
             random_sample = random.random()
             if random_sample > sampling_rate:
                 verbose_logger.info(
@@ -286,7 +272,7 @@ class LangsmithLogger(CustomBatchLogger):
             )
 
     async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
-        sampling_rate = self.sampling_rate
+        sampling_rate = self._get_sampling_rate_to_use_for_request(kwargs=kwargs)
         random_sample = random.random()
         if random_sample > sampling_rate:
             verbose_logger.info(
@@ -417,6 +403,17 @@ class LangsmithLogger(CustomBatchLogger):
 
         for queue_object in self.log_queue:
             credentials = queue_object["credentials"]
+            # if credential missing, skip - log warning
+            if (
+                credentials["LANGSMITH_API_KEY"] is None
+                or credentials["LANGSMITH_PROJECT"] is None
+            ):
+                verbose_logger.warning(
+                    "Langsmith Logging - credentials missing - api_key: %s, project: %s",
+                    credentials["LANGSMITH_API_KEY"],
+                    credentials["LANGSMITH_PROJECT"],
+                )
+                continue
             key = CredentialsKey(
                 api_key=credentials["LANGSMITH_API_KEY"],
                 project=credentials["LANGSMITH_PROJECT"],
@@ -432,6 +429,19 @@ class LangsmithLogger(CustomBatchLogger):
 
         return log_queue_by_credentials
 
+    def _get_sampling_rate_to_use_for_request(self, kwargs: Dict[str, Any]) -> float:
+        standard_callback_dynamic_params: Optional[StandardCallbackDynamicParams] = (
+            kwargs.get("standard_callback_dynamic_params", None)
+        )
+        sampling_rate: float = self.sampling_rate
+        if standard_callback_dynamic_params is not None:
+            _sampling_rate = standard_callback_dynamic_params.get(
+                "langsmith_sampling_rate"
+            )
+            if _sampling_rate is not None:
+                sampling_rate = float(_sampling_rate)
+        return sampling_rate
+
     def _get_credentials_to_use_for_request(
         self, kwargs: Dict[str, Any]
     ) -> LangsmithCredentialsObject:
@@ -442,9 +452,9 @@ class LangsmithLogger(CustomBatchLogger):
 
         Otherwise, use the default credentials.
         """
-        standard_callback_dynamic_params: Optional[
-            StandardCallbackDynamicParams
-        ] = kwargs.get("standard_callback_dynamic_params", None)
+        standard_callback_dynamic_params: Optional[StandardCallbackDynamicParams] = (
+            kwargs.get("standard_callback_dynamic_params", None)
+        )
         if standard_callback_dynamic_params is not None:
             credentials = self.get_credentials_from_env(
                 langsmith_api_key=standard_callback_dynamic_params.get(

litellm/integrations/opik/opik.py

@@ -3,6 +3,7 @@ Opik Logger that logs LLM events to an Opik server
 """
 
 import asyncio
+from datetime import timezone
 import json
 import traceback
 from typing import Dict, List
@@ -291,8 +292,8 @@ class OpikLogger(CustomBatchLogger):
                     "project_name": project_name,
                     "id": trace_id,
                     "name": trace_name,
-                    "start_time": start_time.isoformat() + "Z",
-                    "end_time": end_time.isoformat() + "Z",
+                    "start_time": start_time.astimezone(timezone.utc).isoformat().replace("+00:00", "Z"),
+                    "end_time": end_time.astimezone(timezone.utc).isoformat().replace("+00:00", "Z"),
                     "input": input_data,
                     "output": output_data,
                     "metadata": metadata,
@@ -312,8 +313,8 @@ class OpikLogger(CustomBatchLogger):
                 "parent_span_id": parent_span_id,
                 "name": span_name,
                 "type": "llm",
-                "start_time": start_time.isoformat() + "Z",
-                "end_time": end_time.isoformat() + "Z",
+                "start_time": start_time.astimezone(timezone.utc).isoformat().replace("+00:00", "Z"),
+                "end_time": end_time.astimezone(timezone.utc).isoformat().replace("+00:00", "Z"),
                 "input": input_data,
                 "output": output_data,
                 "metadata": metadata,

litellm/integrations/posthog.py

@@ -0,0 +1,333 @@
+"""
+PostHog Integration - sends LLM analytics events to PostHog
+
+Follows PostHog's LLM Analytics format: https://posthog.com/docs/llm-analytics/manual-capture
+
+async_log_success_event: stores batch of events in memory and flushes to PostHog
+async_log_failure_event: logs failed LLM calls with error information
+
+For batching specific details see CustomBatchLogger class
+"""
+
+import asyncio
+import os
+import uuid
+from typing import Any, Dict, Optional
+
+
+from litellm._logging import verbose_logger
+from litellm.integrations.custom_batch_logger import CustomBatchLogger
+from litellm.llms.custom_httpx.http_handler import (
+    _get_httpx_client,
+    get_async_httpx_client,
+    httpxSpecialProvider,
+)
+from litellm.types.integrations.posthog import (
+    POSTHOG_MAX_BATCH_SIZE,
+    PostHogEventPayload,
+)
+from litellm.types.utils import StandardLoggingPayload
+
+
+class PostHogLogger(CustomBatchLogger):
+    def __init__(self, **kwargs):
+        """
+        Initializes the PostHog logger, checks if the correct env variables are set
+
+        Required environment variables:
+        `POSTHOG_API_KEY` - your PostHog API key
+        `POSTHOG_API_URL` - your PostHog API URL (defaults to https://app.posthog.com)
+        """
+        try:
+            verbose_logger.debug("PostHog: in init posthog logger")
+            if os.getenv("POSTHOG_API_KEY", None) is None:
+                raise Exception("POSTHOG_API_KEY is not set, set 'POSTHOG_API_KEY=<>'")
+
+            self.async_client = get_async_httpx_client(
+                llm_provider=httpxSpecialProvider.LoggingCallback
+            )
+            self.sync_client = _get_httpx_client()
+            
+            self.POSTHOG_API_KEY = os.getenv("POSTHOG_API_KEY")
+            posthog_api_url = os.getenv("POSTHOG_API_URL", "https://us.i.posthog.com")
+            self.posthog_host = posthog_api_url.rstrip('/')
+            self.capture_url = f"{self.posthog_host}/batch/"
+
+            self._async_initialized = False
+            self.flush_lock = None
+            self.log_queue = []
+            
+            super().__init__(
+                **kwargs, flush_lock=None, batch_size=POSTHOG_MAX_BATCH_SIZE
+            )
+
+        except Exception as e:
+            verbose_logger.exception(
+                f"PostHog: Got exception on init PostHog client {str(e)}"
+            )
+            raise e
+
+    def log_success_event(self, kwargs, response_obj, start_time, end_time):
+        try:
+            verbose_logger.debug(
+                "PostHog: Sync logging - Enters logging function for model %s", kwargs
+            )
+            
+            event_payload = self.create_posthog_event_payload(kwargs)
+
+            headers = {
+                "Content-Type": "application/json",
+            }
+
+            payload = self._create_posthog_payload([event_payload])
+
+            response = self.sync_client.post(
+                url=self.capture_url,
+                json=payload,
+                headers=headers,
+            )
+            response.raise_for_status()
+
+            if response.status_code != 200:
+                raise Exception(
+                    f"Response from PostHog API status_code: {response.status_code}, text: {response.text}"
+                )
+            
+            verbose_logger.debug("PostHog: Sync event successfully sent")
+            
+        except Exception as e:
+            verbose_logger.exception(f"PostHog Sync Layer Error - {str(e)}")
+
+    async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
+        try:
+            verbose_logger.debug(
+                "PostHog: Async logging - Enters logging function for model %s", kwargs
+            )
+            self._ensure_async_setup()  # Lazy initialization
+            await self._log_async_event(kwargs, response_obj, start_time, end_time)
+        except Exception as e:
+            verbose_logger.exception(f"PostHog Layer Error - {str(e)}")
+            pass
+
+    async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
+        try:
+            verbose_logger.debug(
+                "PostHog: Async logging - Enters logging function for model %s", kwargs
+            )
+            self._ensure_async_setup()  # Lazy initialization
+            await self._log_async_event(kwargs, response_obj, start_time, end_time)
+        except Exception as e:
+            verbose_logger.exception(f"PostHog Layer Error - {str(e)}")
+            pass
+
+    async def _log_async_event(self, kwargs, response_obj=None, start_time=0.0, end_time=0.0):
+        # Note: response_obj, start_time, end_time not used - all data comes from kwargs
+        event_payload = self.create_posthog_event_payload(kwargs)
+
+        self.log_queue.append(event_payload)
+        verbose_logger.debug(
+            f"PostHog, event added to queue. Will flush in {self.flush_interval} seconds..."
+        )
+
+        if len(self.log_queue) >= self.batch_size:
+            await self.flush_queue()
+
+    def create_posthog_event_payload(self, kwargs: Dict[str, Any]) -> PostHogEventPayload:
+        """
+        Helper function to create a PostHog event payload for logging
+
+        Args:
+            kwargs (Dict[str, Any]): request kwargs containing standard_logging_object
+
+        Returns:
+            PostHogEventPayload: defined in types.py
+        """
+        standard_logging_object: Optional[StandardLoggingPayload] = kwargs.get(
+            "standard_logging_object", None
+        )
+        if standard_logging_object is None:
+            raise ValueError("standard_logging_object not found in kwargs")
+
+        call_type = standard_logging_object.get("call_type", "")
+        event_name = "$ai_embedding" if call_type == "embedding" else "$ai_generation"
+
+        properties = self._create_posthog_properties(
+            standard_logging_object=standard_logging_object,
+            kwargs=kwargs,
+            event_name=event_name,
+        )
+
+        distinct_id = self._get_distinct_id(standard_logging_object, kwargs)
+
+        return PostHogEventPayload(
+            event=event_name,
+            properties=properties,
+            distinct_id=distinct_id,
+        )
+
+    def _create_posthog_properties(
+        self,
+        standard_logging_object: StandardLoggingPayload,
+        kwargs: Dict[str, Any],
+        event_name: str,
+    ) -> Dict[str, Any]:
+        """Create PostHog properties following LLM Analytics spec"""
+        properties = {}
+
+        # Core model information
+        properties["$ai_model"] = self._safe_get(standard_logging_object, "model", "")
+        properties["$ai_provider"] = self._safe_get(standard_logging_object, "custom_llm_provider", "")
+
+        # Input/Output data
+        messages = self._safe_get(standard_logging_object, "messages")
+        if messages is not None:
+            properties["$ai_input"] = messages
+
+        if event_name == "$ai_generation":
+            response = self._safe_get(standard_logging_object, "response")
+            if response is not None:
+                properties["$ai_output_choices"] = response
+
+        # Token information
+        properties["$ai_input_tokens"] = self._safe_get(standard_logging_object, "prompt_tokens", 0)
+        if event_name == "$ai_generation":
+            properties["$ai_output_tokens"] = self._safe_get(standard_logging_object, "completion_tokens", 0)
+
+        # Cost and performance
+        response_cost = self._safe_get(standard_logging_object, "response_cost")
+        if response_cost is not None:
+            properties["$ai_total_cost_usd"] = response_cost
+
+        properties["$ai_latency"] = self._safe_get(standard_logging_object, "response_time", 0.0)
+
+        # Error handling
+        if self._safe_get(standard_logging_object, "status") == "failure":
+            properties["$ai_is_error"] = True
+            error_str = self._safe_get(standard_logging_object, "error_str")
+            if error_str is not None:
+                properties["$ai_error"] = error_str
+
+        # Add trace properties
+        self._add_trace_properties(properties, kwargs)
+
+        # Add custom metadata fields
+        self._add_custom_metadata_properties(properties, kwargs)
+
+        return properties
+
+    def _add_trace_properties(self, properties: Dict[str, Any], kwargs: Dict[str, Any]):
+        standard_logging_object = self._safe_get(kwargs, "standard_logging_object", {})
+
+        trace_id = self._safe_get(standard_logging_object, "trace_id", self._safe_uuid())
+        properties["$ai_trace_id"] = trace_id
+
+        span_id = self._safe_get(standard_logging_object, "id", self._safe_uuid())
+        properties["$ai_span_id"] = span_id
+
+        metadata = self._extract_metadata(kwargs)
+        parent_id = metadata.get("parent_run_id") or metadata.get("parent_id")
+        if parent_id:
+            properties["$ai_parent_id"] = parent_id
+
+    def _add_custom_metadata_properties(self, properties: Dict[str, Any], kwargs: Dict[str, Any]):
+        """Add custom metadata fields to PostHog properties"""
+        metadata = self._extract_metadata(kwargs)
+        if not isinstance(metadata, dict):
+            return
+
+        litellm_internal_fields = {
+            "endpoint", "caching_groups", "user_api_key_hash", "user_api_key_alias",
+            "user_api_key_team_id", "user_api_key_user_id", "user_api_key_org_id",
+            "user_api_key_team_alias", "user_api_key_end_user_id", "user_api_key_user_email",
+            "user_api_key", "user_api_end_user_max_budget", "litellm_api_version",
+            "global_max_parallel_requests", "user_api_key_team_max_budget", "user_api_key_team_spend",
+            "user_api_key_spend", "user_api_key_max_budget", "user_api_key_model_max_budget",
+            "user_api_key_metadata", "headers", "litellm_parent_otel_span", "requester_ip_address",
+            "model_group", "model_group_size", "deployment", "model_info", "api_base",
+            "caching_groups", "hidden_params", "parent_run_id", "parent_id", "user_id"
+        }
+
+        for key, value in metadata.items():
+            if key not in litellm_internal_fields:
+                properties[key] = value
+
+    def _get_distinct_id(
+        self, standard_logging_object: StandardLoggingPayload, kwargs: Dict[str, Any]
+    ) -> str:
+        metadata = self._extract_metadata(kwargs)
+        user_id = self._safe_get(metadata, "user_id")
+        if user_id:
+            return str(user_id)            
+        end_user = self._safe_get(standard_logging_object, "end_user")
+        if end_user:
+            return str(end_user)
+        trace_id = self._safe_get(standard_logging_object, "trace_id")
+        if trace_id:
+            return str(trace_id)            
+        
+        return self._safe_uuid()
+
+    async def async_send_batch(self):
+        """
+        Sends the in memory logs queue to PostHog API
+
+        Raises:
+            Raises a NON Blocking verbose_logger.exception if an error occurs
+        """
+        try:
+            if not self.log_queue:
+                return
+
+            verbose_logger.debug(
+                f"PostHog: Sending batch of {len(self.log_queue)} events"
+            )
+
+            headers = {
+                "Content-Type": "application/json",
+            }
+
+            payload = self._create_posthog_payload(list(self.log_queue))
+
+            response = await self.async_client.post(
+                url=self.capture_url,
+                json=payload,
+                headers=headers,
+            )
+            response.raise_for_status()
+
+            if response.status_code != 200:
+                raise Exception(
+                    f"Response from PostHog API status_code: {response.status_code}, text: {response.text}"
+                )
+
+            verbose_logger.debug(
+                f"PostHog: Batch of {len(self.log_queue)} events successfully sent"
+            )
+        except Exception as e:
+            verbose_logger.exception(f"PostHog Error sending batch API - {str(e)}")
+
+    def _ensure_async_setup(self):
+        if not self._async_initialized:
+            try:
+                self.flush_lock = asyncio.Lock()
+                asyncio.create_task(self.periodic_flush())
+                self._async_initialized = True
+                verbose_logger.debug("PostHog: Async components initialized")
+            except Exception as e:
+                verbose_logger.error(f"PostHog: Failed to initialize async components: {str(e)}")
+                raise
+
+    def _extract_metadata(self, kwargs: Dict[str, Any]) -> Dict[str, Any]:
+        litellm_params = kwargs.get("litellm_params", {}) or {}
+        return litellm_params.get("metadata", {}) or {}
+
+    def _safe_uuid(self) -> str:
+        return str(uuid.uuid4())
+
+    def _create_posthog_payload(self, events: list) -> Dict[str, Any]:
+        return {"api_key": self.POSTHOG_API_KEY, "batch": events}
+
+    def _safe_get(self, obj: Any, key: str, default: Any = None) -> Any:
+        if obj is None or not hasattr(obj, 'get'):
+            return default
+        return obj.get(key, default)

litellm/integrations/prompt_management_base.py

@@ -1,5 +1,7 @@
 from abc import ABC, abstractmethod
-from typing import Any, Dict, List, Optional, Tuple, TypedDict
+from typing import Any, Dict, List, Optional, Tuple
+
+from typing_extensions import TypedDict
 
 from litellm.types.llms.openai import AllMessageValues
 from litellm.types.utils import StandardCallbackDynamicParams

litellm/integrations/s3_v2.py

@@ -203,7 +203,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
             start_time=start_time,
             end_time=end_time,
         )
-    
+
     async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
         await self._async_log_event_base(
             kwargs=kwargs,
@@ -212,7 +212,6 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
             end_time=end_time,
         )
         pass
-    
 
     async def _async_log_event_base(self, kwargs, response_obj, start_time, end_time):
         try:
@@ -242,7 +241,6 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
             verbose_logger.exception(f"s3 Layer Error - {str(e)}")
             pass
 
-
     async def async_upload_data_to_s3(
         self, batch_logging_element: s3BatchLoggingElement
     ):
@@ -277,8 +275,14 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
             # Prepare the URL
             url = f"https://{self.s3_bucket_name}.s3.{self.s3_region_name}.amazonaws.com/{batch_logging_element.s3_object_key}"
 
-            if self.s3_endpoint_url:
-                url = self.s3_endpoint_url + "/" + batch_logging_element.s3_object_key
+            if self.s3_endpoint_url and self.s3_bucket_name:
+                url = (
+                    self.s3_endpoint_url
+                    + "/"
+                    + self.s3_bucket_name
+                    + "/"
+                    + batch_logging_element.s3_object_key
+                )
 
             # Convert JSON to string
             json_string = safe_dumps(batch_logging_element.payload)
@@ -420,8 +424,14 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
             # Prepare the URL
             url = f"https://{self.s3_bucket_name}.s3.{self.s3_region_name}.amazonaws.com/{batch_logging_element.s3_object_key}"
 
-            if self.s3_endpoint_url:
-                url = self.s3_endpoint_url + "/" + batch_logging_element.s3_object_key
+            if self.s3_endpoint_url and self.s3_bucket_name:
+                url = (
+                    self.s3_endpoint_url
+                    + "/"
+                    + self.s3_bucket_name
+                    + "/"
+                    + batch_logging_element.s3_object_key
+                )
 
             # Convert JSON to string
             json_string = safe_dumps(batch_logging_element.payload)
@@ -462,14 +472,13 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
         except Exception as e:
             verbose_logger.exception(f"Error uploading to s3: {str(e)}")
 
-
     async def _download_object_from_s3(self, s3_object_key: str) -> Optional[dict]:
         """
         Download and parse JSON object from S3.
-        
+
         Args:
             s3_object_key: The S3 object key to download
-            
+
         Returns:
             Optional[dict]: The parsed JSON object or None if not found/error
         """
@@ -481,7 +490,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
             from botocore.awsrequest import AWSRequest
         except ImportError:
             raise ImportError("Missing boto3 to call S3. Run 'pip install boto3'.")
-            
+
         try:
             from litellm.litellm_core_utils.asyncify import asyncify
 
@@ -506,8 +515,14 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
             # Prepare the URL
             url = f"https://{self.s3_bucket_name}.s3.{self.s3_region_name}.amazonaws.com/{s3_object_key}"
 
-            if self.s3_endpoint_url:
-                url = self.s3_endpoint_url + "/" + s3_object_key
+            if self.s3_endpoint_url and self.s3_bucket_name:
+                url = (
+                    self.s3_endpoint_url
+                    + "/"
+                    + self.s3_bucket_name
+                    + "/"
+                    + s3_object_key
+                )
 
             # Prepare the request for GET operation
             # For GET requests, we need x-amz-content-sha256 with hash of empty string
@@ -533,12 +548,14 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
             response = await self.async_httpx_client.get(url, headers=signed_headers)
 
             if response.status_code != 200:
-                verbose_logger.exception("S3 object not found, saw response=", response.text)
+                verbose_logger.exception(
+                    "S3 object not found, saw response=", response.text
+                )
                 return None
-            
+
             # Parse JSON response
             return response.json()
-            
+
         except Exception as e:
             verbose_logger.exception(f"Error downloading from S3: {str(e)}")
             return None
@@ -551,11 +568,11 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
         Get the proxy server request from cold storage
 
         Allows fetching a dict of the proxy server request from s3 or GCS bucket.
-        
+
         Args:
             request_id: The unique request ID to search for
             start_time: The start time of the request (datetime or ISO string)
-            
+
         Returns:
             Optional[dict]: The request data dictionary or None if not found
         """
@@ -564,5 +581,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
             downloaded_object = await self._download_object_from_s3(object_key)
             return downloaded_object
         except Exception as e:
-            verbose_logger.exception(f"Error retrieving object {object_key} from cold storage: {str(e)}")
-            return None
+            verbose_logger.exception(
+                f"Error retrieving object {object_key} from cold storage: {str(e)}"
+            )
+            return None

litellm/litellm_core_utils/cached_imports.py

@@ -0,0 +1,56 @@
+"""
+Cached imports module for LiteLLM.
+
+This module provides cached import functionality to avoid repeated imports
+inside functions that are critical to performance.
+"""
+
+from typing import TYPE_CHECKING, Callable, Optional, Type
+
+# Type annotations for cached imports
+if TYPE_CHECKING:
+    from litellm.litellm_core_utils.litellm_logging import Logging
+    from litellm.litellm_core_utils.coroutine_checker import CoroutineChecker
+
+# Global cache variables
+_LiteLLMLogging: Optional[Type["Logging"]] = None
+_coroutine_checker: Optional["CoroutineChecker"] = None
+_set_callbacks: Optional[Callable] = None
+
+
+def get_litellm_logging_class() -> Type["Logging"]:
+    """Get the cached LiteLLM Logging class, initializing if needed."""
+    global _LiteLLMLogging
+    if _LiteLLMLogging is not None:
+        return _LiteLLMLogging
+    from litellm.litellm_core_utils.litellm_logging import Logging
+    _LiteLLMLogging = Logging
+    return _LiteLLMLogging
+
+
+def get_coroutine_checker() -> "CoroutineChecker":
+    """Get the cached coroutine checker instance, initializing if needed."""
+    global _coroutine_checker
+    if _coroutine_checker is not None:
+        return _coroutine_checker
+    from litellm.litellm_core_utils.coroutine_checker import coroutine_checker
+    _coroutine_checker = coroutine_checker
+    return _coroutine_checker
+
+
+def get_set_callbacks() -> Callable:
+    """Get the cached set_callbacks function, initializing if needed."""
+    global _set_callbacks
+    if _set_callbacks is not None:
+        return _set_callbacks
+    from litellm.litellm_core_utils.litellm_logging import set_callbacks
+    _set_callbacks = set_callbacks
+    return _set_callbacks
+
+
+def clear_cached_imports() -> None:
+    """Clear all cached imports. Useful for testing or memory management."""
+    global _LiteLLMLogging, _coroutine_checker, _set_callbacks
+    _LiteLLMLogging = None
+    _coroutine_checker = None
+    _set_callbacks = None

litellm/litellm_core_utils/core_helpers.py

@@ -228,9 +228,11 @@ def safe_deep_copy(data):
     """
     Safe Deep Copy
 
-    The LiteLLM Request has some object that can-not be pickled / deep copied
-
-    Use this function to safely deep copy the LiteLLM Request
+    The LiteLLM request may contain objects that cannot be pickled/deep-copied
+    (e.g., tracing spans, locks, clients). 
+    
+    This helper deep-copies each top-level key independently; on failure keeps
+    original ref
     """
     import copy
 
@@ -255,9 +257,22 @@ def safe_deep_copy(data):
                 "litellm_parent_otel_span"
             )
             data["litellm_metadata"]["litellm_parent_otel_span"] = "placeholder"
-    new_data = copy.deepcopy(data)
 
-    # Step 2: re-add the litellm_parent_otel_span after doing a deep copy
+    # Step 2: Per-key deepcopy with fallback
+    if isinstance(data, dict):
+        new_data = {}
+        for k, v in data.items():
+            try:
+                new_data[k] = copy.deepcopy(v)
+            except Exception:
+                new_data[k] = v
+    else:
+        try:
+            new_data = copy.deepcopy(data)
+        except Exception:
+            new_data = data
+
+    # Step 3: re-add the litellm_parent_otel_span after doing a deep copy
     if isinstance(data, dict) and litellm_parent_otel_span is not None:
         if "metadata" in data and "litellm_parent_otel_span" in data["metadata"]:
             data["metadata"]["litellm_parent_otel_span"] = litellm_parent_otel_span
@@ -268,4 +283,4 @@ def safe_deep_copy(data):
             data["litellm_metadata"][
                 "litellm_parent_otel_span"
             ] = litellm_parent_otel_span
-    return new_data
+    return new_data

litellm/litellm_core_utils/coroutine_checker.py

@@ -0,0 +1,63 @@
+# CoroutineChecker utility for checking if functions/callables are coroutines or coroutine functions
+
+import inspect
+from typing import Any
+from weakref import WeakKeyDictionary
+from litellm.constants import (
+    COROUTINE_CHECKER_MAX_SIZE_IN_MEMORY,
+)
+
+
+class CoroutineChecker:
+    """Utility class for checking coroutine status of functions and callables.
+    
+    Simple bounded cache using WeakKeyDictionary to avoid memory leaks.
+    """
+    
+    def __init__(self):
+        self._cache = WeakKeyDictionary()
+        self._max_size = COROUTINE_CHECKER_MAX_SIZE_IN_MEMORY
+    
+    def is_async_callable(self, callback: Any) -> bool:
+        """Fast, cached check for whether a callback is an async function.
+        Falls back gracefully if the object cannot be weak-referenced or cached.
+        2.59x speedup.
+        """
+        # Fast path: check cache first (most common case)
+        try:
+            cached = self._cache.get(callback)
+            if cached is not None:
+                return cached
+        except Exception:
+            pass
+
+        # Determine target - optimized path for common cases
+        target = callback
+        if not inspect.isfunction(target) and not inspect.ismethod(target):
+            try:
+                call_attr = getattr(target, "__call__", None)
+                if call_attr is not None:
+                    target = call_attr
+            except Exception:
+                pass
+
+        # Compute result
+        try:
+            result = inspect.iscoroutinefunction(target)
+        except Exception:
+            result = False
+
+        # Cache the result with size enforcement
+        try:
+            # Simple size enforcement: clear cache if it gets too large
+            if len(self._cache) >= self._max_size:
+                self._cache.clear()
+            
+            self._cache[callback] = result
+        except Exception:
+            pass
+
+        return result
+
+# Global instance for backward compatibility and convenience
+coroutine_checker = CoroutineChecker()

litellm/litellm_core_utils/custom_logger_registry.py

@@ -33,6 +33,7 @@ from litellm.integrations.mlflow import MlflowLogger
 from litellm.integrations.openmeter import OpenMeterLogger
 from litellm.integrations.opentelemetry import OpenTelemetry
 from litellm.integrations.opik.opik import OpikLogger
+from litellm.integrations.posthog import PostHogLogger
 
 try:
     from litellm_enterprise.integrations.prometheus import PrometheusLogger
@@ -46,6 +47,7 @@ from litellm.integrations.vector_store_integrations.vector_store_pre_call_hook i
     VectorStorePreCallHook,
 )
 from litellm.proxy.hooks.dynamic_rate_limiter import _PROXY_DynamicRateLimitHandler
+from litellm.proxy.hooks.dynamic_rate_limiter_v3 import _PROXY_DynamicRateLimitHandlerV3
 
 
 class CustomLoggerRegistry:
@@ -85,9 +87,11 @@ class CustomLoggerRegistry:
         "s3_v2": S3Logger,
         "aws_sqs": SQSLogger,
         "dynamic_rate_limiter": _PROXY_DynamicRateLimitHandler,
+        "dynamic_rate_limiter_v3": _PROXY_DynamicRateLimitHandlerV3,
         "vector_store_pre_call_hook": VectorStorePreCallHook,
         "dotprompt": DotpromptManager,
         "cloudzero": CloudZeroLogger,
+        "posthog": PostHogLogger,
     }
 
     try:

litellm/litellm_core_utils/duration_parser.py

@@ -158,6 +158,7 @@ def _setup_timezone(
                 "US/Eastern": timezone(timedelta(hours=-4)),  # EDT
                 "US/Pacific": timezone(timedelta(hours=-7)),  # PDT
                 "Asia/Kolkata": timezone(timedelta(hours=5, minutes=30)),  # IST
+                "Asia/Bangkok": timezone(timedelta(hours=7)),  # ICT (Indochina Time)
                 "Europe/London": timezone(timedelta(hours=1)),  # BST
                 "UTC": timezone.utc,
             }

litellm/litellm_core_utils/exception_mapping_utils.py

@@ -6,6 +6,7 @@ import httpx
 
 import litellm
 from litellm._logging import verbose_logger
+from litellm.types.utils import LlmProviders
 
 from ..exceptions import (
     APIConnectionError,
@@ -556,7 +557,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         model=model,
                         llm_provider="anthropic",
                     )
-                elif "overloaded_error" in error_str:
+                elif "overloaded_error" in error_str or "Overloaded" in error_str:
                     exception_mapping_worked = True
                     raise InternalServerError(
                         message="AnthropicError - {}".format(error_str),
@@ -762,7 +763,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         error_str += "XXXXXXX" + '"'
 
                     raise AuthenticationError(
-                        message=f"{custom_llm_provider}Exception: Authentication Error - {error_str}",
+                        message=f"{custom_llm_provider.capitalize()}Exception: Authentication Error - {error_str}",
                         llm_provider=custom_llm_provider,
                         model=model,
                         response=getattr(original_exception, "response", None),
@@ -771,14 +772,14 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                 elif "model's maximum context limit" in error_str:
                     exception_mapping_worked = True
                     raise ContextWindowExceededError(
-                        message=f"{custom_llm_provider}Exception: Context Window Error - {error_str}",
+                        message=f"{custom_llm_provider.capitalize()}Exception: Context Window Error - {error_str}",
                         model=model,
                         llm_provider=custom_llm_provider,
                     )
                 elif "token_quota_reached" in error_str:
                     exception_mapping_worked = True
                     raise RateLimitError(
-                        message=f"{custom_llm_provider}Exception: Rate Limit Errror - {error_str}",
+                        message=f"{custom_llm_provider.capitalize()}Exception: Rate Limit Errror - {error_str}",
                         llm_provider=custom_llm_provider,
                         model=model,
                         response=getattr(original_exception, "response", None),
@@ -789,14 +790,14 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                 ):
                     exception_mapping_worked = True
                     raise litellm.InternalServerError(
-                        message=f"{custom_llm_provider}Exception - {original_exception.message}",
+                        message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
                         llm_provider=custom_llm_provider,
                         model=model,
                     )
                 elif "model_no_support_for_function" in error_str:
                     exception_mapping_worked = True
                     raise BadRequestError(
-                        message=f"{custom_llm_provider}Exception - Use 'watsonx_text' route instead. IBM WatsonX does not support `/text/chat` endpoint. - {error_str}",
+                        message=f"{custom_llm_provider.capitalize()}Exception - Use 'watsonx_text' route instead. IBM WatsonX does not support `/text/chat` endpoint. - {error_str}",
                         llm_provider=custom_llm_provider,
                         model=model,
                     )
@@ -804,7 +805,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                     if original_exception.status_code == 500:
                         exception_mapping_worked = True
                         raise litellm.InternalServerError(
-                            message=f"{custom_llm_provider}Exception - {original_exception.message}",
+                            message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
                             llm_provider=custom_llm_provider,
                             model=model,
                         )
@@ -814,28 +815,28 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                     ):
                         exception_mapping_worked = True
                         raise AuthenticationError(
-                            message=f"{custom_llm_provider}Exception - {original_exception.message}",
+                            message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
                             llm_provider=custom_llm_provider,
                             model=model,
                         )
                     elif original_exception.status_code == 400:
                         exception_mapping_worked = True
                         raise BadRequestError(
-                            message=f"{custom_llm_provider}Exception - {original_exception.message}",
+                            message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
                             llm_provider=custom_llm_provider,
                             model=model,
                         )
                     elif original_exception.status_code == 404:
                         exception_mapping_worked = True
                         raise NotFoundError(
-                            message=f"{custom_llm_provider}Exception - {original_exception.message}",
+                            message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
                             llm_provider=custom_llm_provider,
                             model=model,
                         )
                     elif original_exception.status_code == 408:
                         exception_mapping_worked = True
                         raise Timeout(
-                            message=f"{custom_llm_provider}Exception - {original_exception.message}",
+                            message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
                             model=model,
                             llm_provider=custom_llm_provider,
                             litellm_debug_info=extra_information,
@@ -846,7 +847,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                     ):
                         exception_mapping_worked = True
                         raise BadRequestError(
-                            message=f"{custom_llm_provider}Exception - {original_exception.message}",
+                            message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
                             model=model,
                             llm_provider=custom_llm_provider,
                             litellm_debug_info=extra_information,
@@ -854,7 +855,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                     elif original_exception.status_code == 429:
                         exception_mapping_worked = True
                         raise RateLimitError(
-                            message=f"{custom_llm_provider}Exception - {original_exception.message}",
+                            message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
                             model=model,
                             llm_provider=custom_llm_provider,
                             litellm_debug_info=extra_information,
@@ -862,7 +863,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                     elif original_exception.status_code == 503:
                         exception_mapping_worked = True
                         raise ServiceUnavailableError(
-                            message=f"{custom_llm_provider}Exception - {original_exception.message}",
+                            message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
                             model=model,
                             llm_provider=custom_llm_provider,
                             litellm_debug_info=extra_information,
@@ -870,7 +871,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                     elif original_exception.status_code == 504:  # gateway timeout error
                         exception_mapping_worked = True
                         raise Timeout(
-                            message=f"{custom_llm_provider}Exception - {original_exception.message}",
+                            message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
                             model=model,
                             llm_provider=custom_llm_provider,
                             litellm_debug_info=extra_information,
@@ -1168,9 +1169,9 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                             exception_status_code=original_exception.status_code,
                         )
             elif (
-                custom_llm_provider == "vertex_ai"
-                or custom_llm_provider == "vertex_ai_beta"
-                or custom_llm_provider == "gemini"
+                custom_llm_provider == LlmProviders.VERTEX_AI
+                or custom_llm_provider == LlmProviders.VERTEX_AI_BETA
+                or custom_llm_provider == LlmProviders.GEMINI
             ):
                 if (
                     "Vertex AI API has not been used in project" in error_str
@@ -1178,9 +1179,9 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                 ):
                     exception_mapping_worked = True
                     raise BadRequestError(
-                        message=f"litellm.BadRequestError: VertexAIException - {error_str}",
+                        message=f"litellm.BadRequestError: {custom_llm_provider}Exception - {error_str}",
                         model=model,
-                        llm_provider="vertex_ai",
+                        llm_provider=custom_llm_provider,
                         response=httpx.Response(
                             status_code=400,
                             request=httpx.Request(
@@ -1193,7 +1194,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                 if "400 Request payload size exceeds" in error_str:
                     exception_mapping_worked = True
                     raise ContextWindowExceededError(
-                        message=f"VertexException - {error_str}",
+                        message=f"{custom_llm_provider.capitalize()}Exception - {error_str}",
                         model=model,
                         llm_provider=custom_llm_provider,
                     )
@@ -1203,9 +1204,9 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                 ):
                     exception_mapping_worked = True
                     raise litellm.InternalServerError(
-                        message=f"litellm.InternalServerError: VertexAIException - {error_str}",
+                        message=f"litellm.InternalServerError: {custom_llm_provider}Exception - {error_str}",
                         model=model,
-                        llm_provider="vertex_ai",
+                        llm_provider=custom_llm_provider,
                         response=httpx.Response(
                             status_code=500,
                             content=str(original_exception),
@@ -1216,7 +1217,7 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                 elif "API key not valid." in error_str:
                     exception_mapping_worked = True
                     raise AuthenticationError(
-                        message=f"{custom_llm_provider}Exception - {error_str}",
+                        message=f"{custom_llm_provider.capitalize()}Exception - {error_str}",
                         model=model,
                         llm_provider=custom_llm_provider,
                         litellm_debug_info=extra_information,
@@ -1224,9 +1225,9 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                 elif "403" in error_str:
                     exception_mapping_worked = True
                     raise BadRequestError(
-                        message=f"VertexAIException BadRequestError - {error_str}",
+                        message=f"{custom_llm_provider.capitalize()}Exception BadRequestError - {error_str}",
                         model=model,
-                        llm_provider="vertex_ai",
+                        llm_provider=custom_llm_provider,
                         response=httpx.Response(
                             status_code=403,
                             request=httpx.Request(
@@ -1243,9 +1244,9 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                 ):
                     exception_mapping_worked = True
                     raise ContentPolicyViolationError(
-                        message=f"VertexAIException ContentPolicyViolationError - {error_str}",
+                        message=f"{custom_llm_provider.capitalize()}Exception ContentPolicyViolationError - {error_str}",
                         model=model,
-                        llm_provider="vertex_ai",
+                        llm_provider=custom_llm_provider,
                         litellm_debug_info=extra_information,
                         response=httpx.Response(
                             status_code=400,
@@ -1264,9 +1265,9 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                 ):
                     exception_mapping_worked = True
                     raise RateLimitError(
-                        message=f"litellm.RateLimitError: VertexAIException - {error_str}",
+                        message=f"litellm.RateLimitError: {custom_llm_provider}Exception - {error_str}",
                         model=model,
-                        llm_provider="vertex_ai",
+                        llm_provider=custom_llm_provider,
                         litellm_debug_info=extra_information,
                         response=httpx.Response(
                             status_code=429,
@@ -1282,18 +1283,18 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                 ):
                     exception_mapping_worked = True
                     raise litellm.InternalServerError(
-                        message=f"litellm.InternalServerError: VertexAIException - {error_str}",
+                        message=f"litellm.InternalServerError: {custom_llm_provider}Exception - {error_str}",
                         model=model,
-                        llm_provider="vertex_ai",
+                        llm_provider=custom_llm_provider,
                         litellm_debug_info=extra_information,
                     )
                 if hasattr(original_exception, "status_code"):
                     if original_exception.status_code == 400:
                         exception_mapping_worked = True
                         raise BadRequestError(
-                            message=f"VertexAIException BadRequestError - {error_str}",
+                            message=f"{custom_llm_provider.capitalize()}Exception BadRequestError - {error_str}",
                             model=model,
-                            llm_provider="vertex_ai",
+                            llm_provider=custom_llm_provider,
                             litellm_debug_info=extra_information,
                             response=httpx.Response(
                                 status_code=400,
@@ -1306,21 +1307,21 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                     if original_exception.status_code == 401:
                         exception_mapping_worked = True
                         raise AuthenticationError(
-                            message=f"VertexAIException - {original_exception.message}",
+                            message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
                             llm_provider=custom_llm_provider,
                             model=model,
                         )
                     if original_exception.status_code == 404:
                         exception_mapping_worked = True
                         raise NotFoundError(
-                            message=f"VertexAIException - {original_exception.message}",
+                            message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
                             llm_provider=custom_llm_provider,
                             model=model,
                         )
                     if original_exception.status_code == 408:
                         exception_mapping_worked = True
                         raise Timeout(
-                            message=f"VertexAIException - {original_exception.message}",
+                            message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
                             llm_provider=custom_llm_provider,
                             model=model,
                         )
@@ -1328,9 +1329,9 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                     if original_exception.status_code == 429:
                         exception_mapping_worked = True
                         raise RateLimitError(
-                            message=f"litellm.RateLimitError: VertexAIException - {error_str}",
+                            message=f"litellm.RateLimitError: {custom_llm_provider}Exception - {error_str}",
                             model=model,
-                            llm_provider="vertex_ai",
+                            llm_provider=custom_llm_provider,
                             litellm_debug_info=extra_information,
                             response=httpx.Response(
                                 status_code=429,
@@ -1343,9 +1344,9 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                     if original_exception.status_code == 500:
                         exception_mapping_worked = True
                         raise litellm.InternalServerError(
-                            message=f"VertexAIException InternalServerError - {error_str}",
+                            message=f"{custom_llm_provider.capitalize()}Exception InternalServerError - {error_str}",
                             model=model,
-                            llm_provider="vertex_ai",
+                            llm_provider=custom_llm_provider,
                             litellm_debug_info=extra_information,
                             response=httpx.Response(
                                 status_code=500,
@@ -1356,68 +1357,10 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                     if original_exception.status_code == 503:
                         exception_mapping_worked = True
                         raise ServiceUnavailableError(
-                            message=f"VertexAIException - {original_exception.message}",
+                            message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
                             llm_provider=custom_llm_provider,
                             model=model,
                         )
-            elif custom_llm_provider == "palm" or custom_llm_provider == "gemini":
-                if "503 Getting metadata" in error_str:
-                    # auth errors look like this
-                    # 503 Getting metadata from plugin failed with error: Reauthentication is needed. Please run `gcloud auth application-default login` to reauthenticate.
-                    exception_mapping_worked = True
-                    raise BadRequestError(
-                        message="GeminiException - Invalid api key",
-                        model=model,
-                        llm_provider="palm",
-                        response=getattr(original_exception, "response", None),
-                    )
-                if (
-                    "504 Deadline expired before operation could complete." in error_str
-                    or "504 Deadline Exceeded" in error_str
-                ):
-                    exception_mapping_worked = True
-                    raise Timeout(
-                        message=f"GeminiException - {original_exception.message}",
-                        model=model,
-                        llm_provider="palm",
-                        exception_status_code=original_exception.status_code,
-                    )
-                if "400 Request payload size exceeds" in error_str:
-                    exception_mapping_worked = True
-                    raise ContextWindowExceededError(
-                        message=f"GeminiException - {error_str}",
-                        model=model,
-                        llm_provider="palm",
-                        response=getattr(original_exception, "response", None),
-                    )
-                if (
-                    "500 An internal error has occurred." in error_str
-                    or "list index out of range" in error_str
-                ):
-                    exception_mapping_worked = True
-                    raise APIError(
-                        status_code=getattr(original_exception, "status_code", 500),
-                        message=f"GeminiException - {original_exception.message}",
-                        llm_provider="palm",
-                        model=model,
-                        request=httpx.Response(
-                            status_code=429,
-                            request=httpx.Request(
-                                method="POST",
-                                url=" https://cloud.google.com/vertex-ai/",
-                            ),
-                        ),
-                    )
-                if hasattr(original_exception, "status_code"):
-                    if original_exception.status_code == 400:
-                        exception_mapping_worked = True
-                        raise BadRequestError(
-                            message=f"GeminiException - {error_str}",
-                            model=model,
-                            llm_provider="palm",
-                            response=getattr(original_exception, "response", None),
-                        )
-                # Dailed: Error occurred: 400 Request payload size exceeds the limit: 20000 bytes
             elif custom_llm_provider == "cloudflare":
                 if "Authentication error" in error_str:
                     exception_mapping_worked = True
@@ -1449,6 +1392,14 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                         model=model,
                         response=getattr(original_exception, "response", None),
                     )
+                elif "invalid type: parameter" in error_str:
+                    exception_mapping_worked = True
+                    raise BadRequestError(
+                        message=f"CohereException - {original_exception.message}",
+                        llm_provider="cohere",
+                        model=model,
+                        response=getattr(original_exception, "response", None),
+                    )
                 elif "too many tokens" in error_str:
                     exception_mapping_worked = True
                     raise ContextWindowExceededError(

litellm/litellm_core_utils/get_litellm_params.py

@@ -62,6 +62,7 @@ def get_litellm_params(
     use_litellm_proxy: Optional[bool] = None,
     api_version: Optional[str] = None,
     max_retries: Optional[int] = None,
+    litellm_request_debug: Optional[bool] = None,
     **kwargs,
 ) -> dict:
     litellm_params = {
@@ -118,5 +119,6 @@ def get_litellm_params(
         "vertex_credentials": kwargs.get("vertex_credentials"),
         "vertex_project": kwargs.get("vertex_project"),
         "use_litellm_proxy": use_litellm_proxy,
+        "litellm_request_debug": litellm_request_debug,
     }
     return litellm_params

litellm/litellm_core_utils/get_llm_provider_logic.py

@@ -372,6 +372,10 @@ def get_llm_provider(  # noqa: PLR0915
             custom_llm_provider = "cometapi"
         elif model.startswith("oci/"):
             custom_llm_provider = "oci"
+        elif model.startswith("compactifai/"):
+            custom_llm_provider = "compactifai"
+        elif model.startswith("ovhcloud/"):
+            custom_llm_provider = "ovhcloud"
         if not custom_llm_provider:
             if litellm.suppress_debug_info is False:
                 print()  # noqa

litellm/litellm_core_utils/get_supported_openai_params.py

@@ -94,9 +94,7 @@ def get_supported_openai_params(  # noqa: PLR0915
         return litellm.VLLMConfig().get_supported_openai_params(model=model)
     elif custom_llm_provider == "deepseek":
         return litellm.DeepSeekChatConfig().get_supported_openai_params(model=model)
-    elif custom_llm_provider == "cohere":
-        return litellm.CohereConfig().get_supported_openai_params(model=model)
-    elif custom_llm_provider == "cohere_chat":
+    elif custom_llm_provider == "cohere_chat" or custom_llm_provider == "cohere":
         return litellm.CohereChatConfig().get_supported_openai_params(model=model)
     elif custom_llm_provider == "maritalk":
         return litellm.MaritalkConfig().get_supported_openai_params(model=model)

litellm/litellm_core_utils/litellm_logging.py

@@ -138,6 +138,7 @@ from ..integrations.logfire_logger import LogfireLevel, LogfireLogger
 from ..integrations.lunary import LunaryLogger
 from ..integrations.openmeter import OpenMeterLogger
 from ..integrations.opik.opik import OpikLogger
+from ..integrations.posthog import PostHogLogger
 from ..integrations.prompt_layer import PromptLayerLogger
 from ..integrations.s3 import S3Logger
 from ..integrations.s3_v2 import S3Logger as S3V2Logger
@@ -193,7 +194,6 @@ _in_memory_loggers: List[Any] = []
 sentry_sdk_instance = None
 capture_exception = None
 add_breadcrumb = None
-posthog = None
 slack_app = None
 alerts_channel = None
 heliconeLogger = None
@@ -245,6 +245,7 @@ class Logging(LiteLLMLoggingBaseClass):
     global supabaseClient, promptLayerLogger, weightsBiasesLogger, logfireLogger, capture_exception, add_breadcrumb, lunaryLogger, logfireLogger, prometheusLogger, slack_app
     custom_pricing: bool = False
     stream_options = None
+    litellm_request_debug: bool = False
 
     def __init__(
         self,
@@ -299,9 +300,9 @@ class Logging(LiteLLMLoggingBaseClass):
         self.litellm_trace_id: str = litellm_trace_id or str(uuid.uuid4())
         self.function_id = function_id
         self.streaming_chunks: List[Any] = []  # for generating complete stream response
-        self.sync_streaming_chunks: List[Any] = (
-            []
-        )  # for generating complete stream response
+        self.sync_streaming_chunks: List[
+            Any
+        ] = []  # for generating complete stream response
         self.log_raw_request_response = log_raw_request_response
 
         # Initialize dynamic callbacks
@@ -470,6 +471,7 @@ class Logging(LiteLLMLoggingBaseClass):
             **self.litellm_params,
             **scrub_sensitive_keys_in_metadata(litellm_params),
         }
+        self.litellm_request_debug = litellm_params.get("litellm_request_debug", False)
         self.logger_fn = litellm_params.get("logger_fn", None)
         verbose_logger.debug(f"self.optional_params: {self.optional_params}")
 
@@ -670,24 +672,23 @@ class Logging(LiteLLMLoggingBaseClass):
         if anthropic_cache_control_logger := AnthropicCacheControlHook.get_custom_logger_for_anthropic_cache_control_hook(
             non_default_params
         ):
-            self.model_call_details["prompt_integration"] = (
-                anthropic_cache_control_logger.__class__.__name__
-            )
+            self.model_call_details[
+                "prompt_integration"
+            ] = anthropic_cache_control_logger.__class__.__name__
             return anthropic_cache_control_logger
 
         #########################################################
         # Vector Store / Knowledge Base hooks
         #########################################################
         if litellm.vector_store_registry is not None:
-
             vector_store_custom_logger = _init_custom_logger_compatible_class(
                 logging_integration="vector_store_pre_call_hook",
                 internal_usage_cache=None,
                 llm_router=None,
             )
-            self.model_call_details["prompt_integration"] = (
-                vector_store_custom_logger.__class__.__name__
-            )
+            self.model_call_details[
+                "prompt_integration"
+            ] = vector_store_custom_logger.__class__.__name__
             return vector_store_custom_logger
 
         return None
@@ -739,9 +740,9 @@ class Logging(LiteLLMLoggingBaseClass):
             model
         ):  # if model name was changes pre-call, overwrite the initial model call name with the new one
             self.model_call_details["model"] = model
-        self.model_call_details["litellm_params"]["api_base"] = (
-            self._get_masked_api_base(additional_args.get("api_base", ""))
-        )
+        self.model_call_details["litellm_params"][
+            "api_base"
+        ] = self._get_masked_api_base(additional_args.get("api_base", ""))
 
     def pre_call(self, input, api_key, model=None, additional_args={}):  # noqa: PLR0915
         # Log the exact input to the LLM API
@@ -770,10 +771,10 @@ class Logging(LiteLLMLoggingBaseClass):
                 try:
                     # [Non-blocking Extra Debug Information in metadata]
                     if turn_off_message_logging is True:
-                        _metadata["raw_request"] = (
-                            "redacted by litellm. \
+                        _metadata[
+                            "raw_request"
+                        ] = "redacted by litellm. \
                             'litellm.turn_off_message_logging=True'"
-                        )
                     else:
                         curl_command = self._get_request_curl_command(
                             api_base=additional_args.get("api_base", ""),
@@ -784,32 +785,32 @@ class Logging(LiteLLMLoggingBaseClass):
 
                         _metadata["raw_request"] = str(curl_command)
                         # split up, so it's easier to parse in the UI
-                        self.model_call_details["raw_request_typed_dict"] = (
-                            RawRequestTypedDict(
-                                raw_request_api_base=str(
-                                    additional_args.get("api_base") or ""
-                                ),
-                                raw_request_body=self._get_raw_request_body(
-                                    additional_args.get("complete_input_dict", {})
-                                ),
-                                raw_request_headers=self._get_masked_headers(
-                                    additional_args.get("headers", {}) or {},
-                                    ignore_sensitive_headers=True,
-                                ),
-                                error=None,
-                            )
+                        self.model_call_details[
+                            "raw_request_typed_dict"
+                        ] = RawRequestTypedDict(
+                            raw_request_api_base=str(
+                                additional_args.get("api_base") or ""
+                            ),
+                            raw_request_body=self._get_raw_request_body(
+                                additional_args.get("complete_input_dict", {})
+                            ),
+                            raw_request_headers=self._get_masked_headers(
+                                additional_args.get("headers", {}) or {},
+                                ignore_sensitive_headers=True,
+                            ),
+                            error=None,
                         )
                 except Exception as e:
-                    self.model_call_details["raw_request_typed_dict"] = (
-                        RawRequestTypedDict(
-                            error=str(e),
-                        )
+                    self.model_call_details[
+                        "raw_request_typed_dict"
+                    ] = RawRequestTypedDict(
+                        error=str(e),
                     )
-                    _metadata["raw_request"] = (
-                        "Unable to Log \
+                    _metadata[
+                        "raw_request"
+                    ] = "Unable to Log \
                         raw request: {}".format(
-                            str(e)
-                        )
+                        str(e)
                     )
             if getattr(self, "logger_fn", None) and callable(self.logger_fn):
                 try:
@@ -907,13 +908,19 @@ class Logging(LiteLLMLoggingBaseClass):
 
         Prints the RAW curl command sent from LiteLLM
         """
-        if _is_debugging_on():
+        if _is_debugging_on() or self.litellm_request_debug:
             if json_logs:
                 masked_headers = self._get_masked_headers(headers)
-                verbose_logger.debug(
-                    "POST Request Sent from LiteLLM",
-                    extra={"api_base": {api_base}, **masked_headers},
-                )
+                if self.litellm_request_debug:
+                    verbose_logger.warning(  # .warning ensures this shows up in all environments
+                        "POST Request Sent from LiteLLM",
+                        extra={"api_base": {api_base}, **masked_headers},
+                    )
+                else:
+                    verbose_logger.debug(
+                        "POST Request Sent from LiteLLM",
+                        extra={"api_base": {api_base}, **masked_headers},
+                    )
             else:
                 headers = additional_args.get("headers", {})
                 if headers is None:
@@ -926,7 +933,12 @@ class Logging(LiteLLMLoggingBaseClass):
                     additional_args=additional_args,
                     data=data,
                 )
-                verbose_logger.debug(f"\033[92m{curl_command}\033[0m\n")
+                if self.litellm_request_debug:
+                    verbose_logger.warning(
+                        f"\033[92m{curl_command}\033[0m\n"
+                    )  # .warning ensures this shows up in all environments
+                else:
+                    verbose_logger.debug(f"\033[92m{curl_command}\033[0m\n")
 
     def _get_request_body(self, data: dict) -> str:
         return str(data)
@@ -983,8 +995,14 @@ class Logging(LiteLLMLoggingBaseClass):
             self.model_call_details["additional_args"] = additional_args
             self.model_call_details["log_event_type"] = "post_api_call"
 
+            if self.litellm_request_debug:
+                attr = "warning"
+            else:
+                attr = "debug"
+
             if json_logs:
-                verbose_logger.debug(
+                callattr = getattr(verbose_logger, attr)
+                callattr(
                     "RAW RESPONSE:\n{}\n\n".format(
                         self.model_call_details.get(
                             "original_response", self.model_call_details
@@ -992,7 +1010,8 @@ class Logging(LiteLLMLoggingBaseClass):
                     ),
                 )
             else:
-                print_verbose(
+                callattr = getattr(verbose_logger, attr)
+                callattr(
                     "RAW RESPONSE:\n{}\n\n".format(
                         self.model_call_details.get(
                             "original_response", self.model_call_details
@@ -1092,13 +1111,13 @@ class Logging(LiteLLMLoggingBaseClass):
         for callback in callbacks:
             try:
                 if isinstance(callback, CustomLogger):
-                    response: Optional[MCPPostCallResponseObject] = (
-                        await callback.async_post_mcp_tool_call_hook(
-                            kwargs=kwargs,
-                            response_obj=post_mcp_tool_call_response_obj,
-                            start_time=start_time,
-                            end_time=end_time,
-                        )
+                    response: Optional[
+                        MCPPostCallResponseObject
+                    ] = await callback.async_post_mcp_tool_call_hook(
+                        kwargs=kwargs,
+                        response_obj=post_mcp_tool_call_response_obj,
+                        start_time=start_time,
+                        end_time=end_time,
                     )
                     ######################################################################
                     # if any of the callbacks modify the response, use the modified response
@@ -1218,9 +1237,9 @@ class Logging(LiteLLMLoggingBaseClass):
             verbose_logger.debug(
                 f"response_cost_failure_debug_information: {debug_info}"
             )
-            self.model_call_details["response_cost_failure_debug_information"] = (
-                debug_info
-            )
+            self.model_call_details[
+                "response_cost_failure_debug_information"
+            ] = debug_info
             return None
 
         try:
@@ -1245,9 +1264,9 @@ class Logging(LiteLLMLoggingBaseClass):
             verbose_logger.debug(
                 f"response_cost_failure_debug_information: {debug_info}"
             )
-            self.model_call_details["response_cost_failure_debug_information"] = (
-                debug_info
-            )
+            self.model_call_details[
+                "response_cost_failure_debug_information"
+            ] = debug_info
 
         return None
 
@@ -1391,9 +1410,9 @@ class Logging(LiteLLMLoggingBaseClass):
                 end_time = datetime.datetime.now()
             if self.completion_start_time is None:
                 self.completion_start_time = end_time
-                self.model_call_details["completion_start_time"] = (
-                    self.completion_start_time
-                )
+                self.model_call_details[
+                    "completion_start_time"
+                ] = self.completion_start_time
             self.model_call_details["log_event_type"] = "successful_api_call"
             self.model_call_details["end_time"] = end_time
             self.model_call_details["cache_hit"] = cache_hit
@@ -1446,39 +1465,39 @@ class Logging(LiteLLMLoggingBaseClass):
                             "response_cost"
                         ]
                     else:
-                        self.model_call_details["response_cost"] = (
-                            self._response_cost_calculator(result=logging_result)
-                        )
+                        self.model_call_details[
+                            "response_cost"
+                        ] = self._response_cost_calculator(result=logging_result)
                     ## STANDARDIZED LOGGING PAYLOAD
 
-                    self.model_call_details["standard_logging_object"] = (
-                        get_standard_logging_object_payload(
-                            kwargs=self.model_call_details,
-                            init_response_obj=logging_result,
-                            start_time=start_time,
-                            end_time=end_time,
-                            logging_obj=self,
-                            status="success",
-                            standard_built_in_tools_params=self.standard_built_in_tools_params,
-                        )
+                    self.model_call_details[
+                        "standard_logging_object"
+                    ] = get_standard_logging_object_payload(
+                        kwargs=self.model_call_details,
+                        init_response_obj=logging_result,
+                        start_time=start_time,
+                        end_time=end_time,
+                        logging_obj=self,
+                        status="success",
+                        standard_built_in_tools_params=self.standard_built_in_tools_params,
                     )
                 elif isinstance(result, dict) or isinstance(result, list):
                     ## STANDARDIZED LOGGING PAYLOAD
-                    self.model_call_details["standard_logging_object"] = (
-                        get_standard_logging_object_payload(
-                            kwargs=self.model_call_details,
-                            init_response_obj=result,
-                            start_time=start_time,
-                            end_time=end_time,
-                            logging_obj=self,
-                            status="success",
-                            standard_built_in_tools_params=self.standard_built_in_tools_params,
-                        )
+                    self.model_call_details[
+                        "standard_logging_object"
+                    ] = get_standard_logging_object_payload(
+                        kwargs=self.model_call_details,
+                        init_response_obj=result,
+                        start_time=start_time,
+                        end_time=end_time,
+                        logging_obj=self,
+                        status="success",
+                        standard_built_in_tools_params=self.standard_built_in_tools_params,
                     )
             elif standard_logging_object is not None:
-                self.model_call_details["standard_logging_object"] = (
-                    standard_logging_object
-                )
+                self.model_call_details[
+                    "standard_logging_object"
+                ] = standard_logging_object
             else:  # streaming chunks + image gen.
                 self.model_call_details["response_cost"] = None
 
@@ -1577,7 +1596,6 @@ class Logging(LiteLLMLoggingBaseClass):
         )
 
         if complete_streaming_response is not None:
-
             self.success_handler(result=complete_streaming_response)
         return
 
@@ -1630,23 +1648,23 @@ class Logging(LiteLLMLoggingBaseClass):
                 verbose_logger.debug(
                     "Logging Details LiteLLM-Success Call streaming complete"
                 )
-                self.model_call_details["complete_streaming_response"] = (
-                    complete_streaming_response
-                )
-                self.model_call_details["response_cost"] = (
-                    self._response_cost_calculator(result=complete_streaming_response)
-                )
+                self.model_call_details[
+                    "complete_streaming_response"
+                ] = complete_streaming_response
+                self.model_call_details[
+                    "response_cost"
+                ] = self._response_cost_calculator(result=complete_streaming_response)
                 ## STANDARDIZED LOGGING PAYLOAD
-                self.model_call_details["standard_logging_object"] = (
-                    get_standard_logging_object_payload(
-                        kwargs=self.model_call_details,
-                        init_response_obj=complete_streaming_response,
-                        start_time=start_time,
-                        end_time=end_time,
-                        logging_obj=self,
-                        status="success",
-                        standard_built_in_tools_params=self.standard_built_in_tools_params,
-                    )
+                self.model_call_details[
+                    "standard_logging_object"
+                ] = get_standard_logging_object_payload(
+                    kwargs=self.model_call_details,
+                    init_response_obj=complete_streaming_response,
+                    start_time=start_time,
+                    end_time=end_time,
+                    logging_obj=self,
+                    status="success",
+                    standard_built_in_tools_params=self.standard_built_in_tools_params,
                 )
             callbacks = self.get_combined_callback_list(
                 dynamic_success_callbacks=self.dynamic_success_callbacks,
@@ -1714,12 +1732,16 @@ class Logging(LiteLLMLoggingBaseClass):
                             response_obj=result,
                             start_time=start_time,
                             end_time=end_time,
-                            litellm_call_id=current_call_id
-                            if (
-                                current_call_id := litellm_params.get("litellm_call_id")
-                            )
-                            is not None
-                            else str(uuid.uuid4()),
+                            litellm_call_id=(
+                                current_call_id
+                                if (
+                                    current_call_id := litellm_params.get(
+                                        "litellm_call_id"
+                                    )
+                                )
+                                is not None
+                                else str(uuid.uuid4())
+                            ),
                             print_verbose=print_verbose,
                         )
                     if callback == "wandb" and weightsBiasesLogger is not None:
@@ -1970,10 +1992,10 @@ class Logging(LiteLLMLoggingBaseClass):
                             )
                         else:
                             if self.stream and complete_streaming_response:
-                                self.model_call_details["complete_response"] = (
-                                    self.model_call_details.get(
-                                        "complete_streaming_response", {}
-                                    )
+                                self.model_call_details[
+                                    "complete_response"
+                                ] = self.model_call_details.get(
+                                    "complete_streaming_response", {}
                                 )
                                 result = self.model_call_details["complete_response"]
                             openMeterLogger.log_success_event(
@@ -2012,10 +2034,10 @@ class Logging(LiteLLMLoggingBaseClass):
                             )
                         else:
                             if self.stream and complete_streaming_response:
-                                self.model_call_details["complete_response"] = (
-                                    self.model_call_details.get(
-                                        "complete_streaming_response", {}
-                                    )
+                                self.model_call_details[
+                                    "complete_response"
+                                ] = self.model_call_details.get(
+                                    "complete_streaming_response", {}
                                 )
                                 result = self.model_call_details["complete_response"]
 
@@ -2117,10 +2139,12 @@ class Logging(LiteLLMLoggingBaseClass):
                 result.usage = batch_usage
 
             elif not is_base64_unified_file_id:  # only run for non-unified file ids
-                response_cost, batch_usage, batch_models = (
-                    await _handle_completed_batch(
-                        batch=result, custom_llm_provider=self.custom_llm_provider
-                    )
+                (
+                    response_cost,
+                    batch_usage,
+                    batch_models,
+                ) = await _handle_completed_batch(
+                    batch=result, custom_llm_provider=self.custom_llm_provider
                 )
 
                 result._hidden_params["response_cost"] = response_cost
@@ -2151,9 +2175,9 @@ class Logging(LiteLLMLoggingBaseClass):
         if complete_streaming_response is not None:
             print_verbose("Async success callbacks: Got a complete streaming response")
 
-            self.model_call_details["async_complete_streaming_response"] = (
-                complete_streaming_response
-            )
+            self.model_call_details[
+                "async_complete_streaming_response"
+            ] = complete_streaming_response
 
             try:
                 if self.model_call_details.get("cache_hit", False) is True:
@@ -2164,10 +2188,10 @@ class Logging(LiteLLMLoggingBaseClass):
                         model_call_details=self.model_call_details
                     )
                     # base_model defaults to None if not set on model_info
-                    self.model_call_details["response_cost"] = (
-                        self._response_cost_calculator(
-                            result=complete_streaming_response
-                        )
+                    self.model_call_details[
+                        "response_cost"
+                    ] = self._response_cost_calculator(
+                        result=complete_streaming_response
                     )
 
                 verbose_logger.debug(
@@ -2180,16 +2204,16 @@ class Logging(LiteLLMLoggingBaseClass):
                 self.model_call_details["response_cost"] = None
 
             ## STANDARDIZED LOGGING PAYLOAD
-            self.model_call_details["standard_logging_object"] = (
-                get_standard_logging_object_payload(
-                    kwargs=self.model_call_details,
-                    init_response_obj=complete_streaming_response,
-                    start_time=start_time,
-                    end_time=end_time,
-                    logging_obj=self,
-                    status="success",
-                    standard_built_in_tools_params=self.standard_built_in_tools_params,
-                )
+            self.model_call_details[
+                "standard_logging_object"
+            ] = get_standard_logging_object_payload(
+                kwargs=self.model_call_details,
+                init_response_obj=complete_streaming_response,
+                start_time=start_time,
+                end_time=end_time,
+                logging_obj=self,
+                status="success",
+                standard_built_in_tools_params=self.standard_built_in_tools_params,
             )
         callbacks = self.get_combined_callback_list(
             dynamic_success_callbacks=self.dynamic_async_success_callbacks,
@@ -2402,18 +2426,18 @@ class Logging(LiteLLMLoggingBaseClass):
 
         ## STANDARDIZED LOGGING PAYLOAD
 
-        self.model_call_details["standard_logging_object"] = (
-            get_standard_logging_object_payload(
-                kwargs=self.model_call_details,
-                init_response_obj={},
-                start_time=start_time,
-                end_time=end_time,
-                logging_obj=self,
-                status="failure",
-                error_str=str(exception),
-                original_exception=exception,
-                standard_built_in_tools_params=self.standard_built_in_tools_params,
-            )
+        self.model_call_details[
+            "standard_logging_object"
+        ] = get_standard_logging_object_payload(
+            kwargs=self.model_call_details,
+            init_response_obj={},
+            start_time=start_time,
+            end_time=end_time,
+            logging_obj=self,
+            status="failure",
+            error_str=str(exception),
+            original_exception=exception,
+            standard_built_in_tools_params=self.standard_built_in_tools_params,
         )
         return start_time, end_time
 
@@ -3044,7 +3068,7 @@ def set_callbacks(callback_list, function_id=None):  # noqa: PLR0915
     """
     Globally sets the callback client
     """
-    global sentry_sdk_instance, capture_exception, add_breadcrumb, posthog, slack_app, alerts_channel, traceloopLogger, athinaLogger, heliconeLogger, supabaseClient, lunaryLogger, promptLayerLogger, langFuseLogger, customLogger, weightsBiasesLogger, logfireLogger, dynamoLogger, s3Logger, dataDogLogger, prometheusLogger, greenscaleLogger, openMeterLogger, deepevalLogger
+    global sentry_sdk_instance, capture_exception, add_breadcrumb, slack_app, alerts_channel, traceloopLogger, athinaLogger, heliconeLogger, supabaseClient, lunaryLogger, promptLayerLogger, langFuseLogger, customLogger, weightsBiasesLogger, logfireLogger, dynamoLogger, s3Logger, dataDogLogger, prometheusLogger, greenscaleLogger, openMeterLogger, deepevalLogger
 
     try:
         for callback in callback_list:
@@ -3083,19 +3107,6 @@ def set_callbacks(callback_list, function_id=None):  # noqa: PLR0915
                 )
                 capture_exception = sentry_sdk_instance.capture_exception
                 add_breadcrumb = sentry_sdk_instance.add_breadcrumb
-            elif callback == "posthog":
-                try:
-                    from posthog import Posthog
-                except ImportError:
-                    print_verbose("Package 'posthog' is missing. Installing it...")
-                    subprocess.check_call(
-                        [sys.executable, "-m", "pip", "install", "posthog"]
-                    )
-                    from posthog import Posthog
-                posthog = Posthog(
-                    project_api_key=os.environ.get("POSTHOG_API_KEY"),
-                    host=os.environ.get("POSTHOG_API_URL"),
-                )
             elif callback == "slack":
                 try:
                     from slack_bolt import App
@@ -3190,6 +3201,14 @@ def _init_custom_logger_compatible_class(  # noqa: PLR0915
             _openmeter_logger = OpenMeterLogger()
             _in_memory_loggers.append(_openmeter_logger)
             return _openmeter_logger  # type: ignore
+        elif logging_integration == "posthog":
+            for callback in _in_memory_loggers:
+                if isinstance(callback, PostHogLogger):
+                    return callback  # type: ignore
+
+            _posthog_logger = PostHogLogger()
+            _in_memory_loggers.append(_posthog_logger)
+            return _posthog_logger  # type: ignore
         elif logging_integration == "braintrust":
             from litellm.integrations.braintrust_logging import BraintrustLogger
 
@@ -3302,9 +3321,9 @@ def _init_custom_logger_compatible_class(  # noqa: PLR0915
                 endpoint=arize_config.endpoint,
             )
 
-            os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = (
-                f"space_id={arize_config.space_key},api_key={arize_config.api_key}"
-            )
+            os.environ[
+                "OTEL_EXPORTER_OTLP_TRACES_HEADERS"
+            ] = f"space_id={arize_config.space_key},api_key={arize_config.api_key}"
             for callback in _in_memory_loggers:
                 if (
                     isinstance(callback, ArizeLogger)
@@ -3328,9 +3347,9 @@ def _init_custom_logger_compatible_class(  # noqa: PLR0915
 
             # auth can be disabled on local deployments of arize phoenix
             if arize_phoenix_config.otlp_auth_headers is not None:
-                os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = (
-                    arize_phoenix_config.otlp_auth_headers
-                )
+                os.environ[
+                    "OTEL_EXPORTER_OTLP_TRACES_HEADERS"
+                ] = arize_phoenix_config.otlp_auth_headers
 
             for callback in _in_memory_loggers:
                 if (
@@ -3367,6 +3386,7 @@ def _init_custom_logger_compatible_class(  # noqa: PLR0915
             return galileo_logger  # type: ignore
         elif logging_integration == "cloudzero":
             from litellm.integrations.cloudzero.cloudzero import CloudZeroLogger
+
             for callback in _in_memory_loggers:
                 if isinstance(callback, CloudZeroLogger):
                     return callback  # type: ignore
@@ -3424,6 +3444,30 @@ def _init_custom_logger_compatible_class(  # noqa: PLR0915
                 dynamic_rate_limiter_obj.update_variables(llm_router=llm_router)
             _in_memory_loggers.append(dynamic_rate_limiter_obj)
             return dynamic_rate_limiter_obj  # type: ignore
+        elif logging_integration == "dynamic_rate_limiter_v3":
+            from litellm.proxy.hooks.dynamic_rate_limiter_v3 import (
+                _PROXY_DynamicRateLimitHandlerV3,
+            )
+
+            for callback in _in_memory_loggers:
+                if isinstance(callback, _PROXY_DynamicRateLimitHandlerV3):
+                    return callback  # type: ignore
+
+            if internal_usage_cache is None:
+                raise Exception(
+                    "Internal Error: Cache cannot be empty - internal_usage_cache={}".format(
+                        internal_usage_cache
+                    )
+                )
+
+            dynamic_rate_limiter_obj_v3 = _PROXY_DynamicRateLimitHandlerV3(
+                internal_usage_cache=internal_usage_cache
+            )
+
+            if llm_router is not None and isinstance(llm_router, litellm.Router):
+                dynamic_rate_limiter_obj_v3.update_variables(llm_router=llm_router)
+            _in_memory_loggers.append(dynamic_rate_limiter_obj_v3)
+            return dynamic_rate_limiter_obj_v3  # type: ignore
         elif logging_integration == "langtrace":
             if "LANGTRACE_API_KEY" not in os.environ:
                 raise ValueError("LANGTRACE_API_KEY not found in environment variables")
@@ -3437,9 +3481,9 @@ def _init_custom_logger_compatible_class(  # noqa: PLR0915
                 exporter="otlp_http",
                 endpoint="https://langtrace.ai/api/trace",
             )
-            os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = (
-                f"api_key={os.getenv('LANGTRACE_API_KEY')}"
-            )
+            os.environ[
+                "OTEL_EXPORTER_OTLP_TRACES_HEADERS"
+            ] = f"api_key={os.getenv('LANGTRACE_API_KEY')}"
             for callback in _in_memory_loggers:
                 if (
                     isinstance(callback, OpenTelemetry)
@@ -3594,6 +3638,7 @@ def get_custom_logger_compatible_class(  # noqa: PLR0915
                     return callback
         elif logging_integration == "cloudzero":
             from litellm.integrations.cloudzero.cloudzero import CloudZeroLogger
+
             for callback in _in_memory_loggers:
                 if isinstance(callback, CloudZeroLogger):
                     return callback
@@ -3686,6 +3731,14 @@ def get_custom_logger_compatible_class(  # noqa: PLR0915
             for callback in _in_memory_loggers:
                 if isinstance(callback, _PROXY_DynamicRateLimitHandler):
                     return callback  # type: ignore
+        elif logging_integration == "dynamic_rate_limiter_v3":
+            from litellm.proxy.hooks.dynamic_rate_limiter_v3 import (
+                _PROXY_DynamicRateLimitHandlerV3,
+            )
+
+            for callback in _in_memory_loggers:
+                if isinstance(callback, _PROXY_DynamicRateLimitHandlerV3):
+                    return callback  # type: ignore
 
         elif logging_integration == "langtrace":
             from litellm.integrations.opentelemetry import OpenTelemetry
@@ -3884,22 +3937,25 @@ class StandardLoggingPayloadSetup:
         clean_metadata = StandardLoggingMetadata(
             user_api_key_hash=None,
             user_api_key_alias=None,
+            user_api_key_spend=None,
+            user_api_key_max_budget=None,
+            user_api_key_budget_reset_at=None,
             user_api_key_team_id=None,
             user_api_key_org_id=None,
             user_api_key_user_id=None,
             user_api_key_team_alias=None,
             user_api_key_user_email=None,
+            user_api_key_end_user_id=None,
+            user_api_key_request_route=None,
             spend_logs_metadata=None,
             requester_ip_address=None,
             requester_metadata=None,
-            user_api_key_end_user_id=None,
             prompt_management_metadata=prompt_management_metadata,
             applied_guardrails=applied_guardrails,
             mcp_tool_call_metadata=mcp_tool_call_metadata,
             vector_store_request_metadata=vector_store_request_metadata,
             usage_object=usage_object,
             requester_custom_headers=None,
-            user_api_key_request_route=None,
             cold_storage_object_key=None,
         )
         if isinstance(metadata, dict):
@@ -4088,10 +4144,10 @@ class StandardLoggingPayloadSetup:
             for key in StandardLoggingHiddenParams.__annotations__.keys():
                 if key in hidden_params:
                     if key == "additional_headers":
-                        clean_hidden_params["additional_headers"] = (
-                            StandardLoggingPayloadSetup.get_additional_headers(
-                                hidden_params[key]
-                            )
+                        clean_hidden_params[
+                            "additional_headers"
+                        ] = StandardLoggingPayloadSetup.get_additional_headers(
+                            hidden_params[key]
                         )
                     else:
                         clean_hidden_params[key] = hidden_params[key]  # type: ignore
@@ -4124,15 +4180,28 @@ class StandardLoggingPayloadSetup:
         from litellm.integrations.s3 import get_s3_object_key
 
         # Only generate object key if cold storage is configured
-        if litellm.configured_cold_storage_logger is None:
+        configured_cold_storage_logger = litellm.configured_cold_storage_logger
+        if configured_cold_storage_logger is None:
             return None
 
         try:
             # Generate file name in same format as litellm.utils.get_logging_id
             s3_file_name = f"time-{start_time.strftime('%H-%M-%S-%f')}_{response_id}"
 
+            # Get the actual s3_path from the configured cold storage logger instance
+            s3_path = ""  # default value
+            
+            # Try to get the actual logger instance from the logger name
+            try:
+                custom_logger = litellm.logging_callback_manager.get_active_custom_logger_for_callback_name(configured_cold_storage_logger)
+                if custom_logger and hasattr(custom_logger, 's3_path') and custom_logger.s3_path:
+                    s3_path = custom_logger.s3_path
+            except Exception:
+                # If any error occurs in getting the logger instance, use default empty s3_path
+                pass
+            
             s3_object_key = get_s3_object_key(
-                s3_path="",  # Use empty path as default
+                s3_path=s3_path,  # Use actual s3_path from logger configuration
                 team_alias_prefix="",  # Don't split by team alias for cold storage
                 start_time=start_time,
                 s3_file_name=s3_file_name,
@@ -4504,7 +4573,7 @@ def get_standard_logging_object_payload(
 
 def emit_standard_logging_payload(payload: StandardLoggingPayload):
     if os.getenv("LITELLM_PRINT_STANDARD_LOGGING_PAYLOAD"):
-        print(json.dumps(payload, indent=4)) # noqa
+        print(json.dumps(payload, indent=4))  # noqa
 
 
 def get_standard_logging_metadata(
@@ -4527,6 +4596,9 @@ def get_standard_logging_metadata(
     clean_metadata = StandardLoggingMetadata(
         user_api_key_hash=None,
         user_api_key_alias=None,
+        user_api_key_spend=None,
+        user_api_key_max_budget=None,
+        user_api_key_budget_reset_at=None,
         user_api_key_team_id=None,
         user_api_key_org_id=None,
         user_api_key_user_id=None,
@@ -4546,14 +4618,10 @@ def get_standard_logging_metadata(
         cold_storage_object_key=None,
     )
     if isinstance(metadata, dict):
-        # Filter the metadata dictionary to include only the specified keys
-        clean_metadata = StandardLoggingMetadata(
-            **{  # type: ignore
-                key: metadata[key]
-                for key in StandardLoggingMetadata.__annotations__.keys()
-                if key in metadata
-            }
-        )
+        # Update the clean_metadata with values from input metadata that match StandardLoggingMetadata fields
+        for key in StandardLoggingMetadata.__annotations__.keys():
+            if key in metadata:
+                clean_metadata[key] = metadata[key]  # type: ignore
 
         if metadata.get("user_api_key") is not None:
             if is_valid_sha256_hash(str(metadata.get("user_api_key"))):
@@ -4576,9 +4644,9 @@ def scrub_sensitive_keys_in_metadata(litellm_params: Optional[dict]):
     ):
         for k, v in metadata["user_api_key_metadata"].items():
             if k == "logging":  # prevent logging user logging keys
-                cleaned_user_api_key_metadata[k] = (
-                    "scrubbed_by_litellm_for_sensitive_keys"
-                )
+                cleaned_user_api_key_metadata[
+                    k
+                ] = "scrubbed_by_litellm_for_sensitive_keys"
             else:
                 cleaned_user_api_key_metadata[k] = v
 

litellm/litellm_core_utils/llm_cost_calc/utils.py

@@ -1,11 +1,12 @@
 # What is this?
 ## Helper utilities for cost_per_token()
 
-from typing import Any, Literal, Optional, Tuple, cast
+from typing import Any, Literal, Optional, Tuple, TypedDict, cast
 
 import litellm
 from litellm._logging import verbose_logger
 from litellm.types.utils import (
+    CacheCreationTokenDetails,
     CallTypes,
     ImageResponse,
     ModelInfo,
@@ -113,20 +114,34 @@ def _generic_cost_per_character(
     return prompt_cost, completion_cost
 
 
-def _get_token_base_cost(model_info: ModelInfo, usage: Usage) -> Tuple[float, float, float, float]:
+def _get_token_base_cost(
+    model_info: ModelInfo, usage: Usage
+) -> Tuple[float, float, float, float, float]:
     """
     Return prompt cost, completion cost, and cache costs for a given model and usage.
 
     If input_tokens > threshold and `input_cost_per_token_above_[x]k_tokens` or `input_cost_per_token_above_[x]_tokens` is set,
     then we use the corresponding threshold cost for all token types.
-    
+
     Returns:
         Tuple[float, float, float, float] - (prompt_cost, completion_cost, cache_creation_cost, cache_read_cost)
     """
-    prompt_base_cost = cast(float, _get_cost_per_unit(model_info, "input_cost_per_token"))
-    completion_base_cost = cast(float, _get_cost_per_unit(model_info, "output_cost_per_token"))
-    cache_creation_cost = cast(float, _get_cost_per_unit(model_info, "cache_creation_input_token_cost"))
-    cache_read_cost = cast(float, _get_cost_per_unit(model_info, "cache_read_input_token_cost"))
+    prompt_base_cost = cast(
+        float, _get_cost_per_unit(model_info, "input_cost_per_token")
+    )
+    completion_base_cost = cast(
+        float, _get_cost_per_unit(model_info, "output_cost_per_token")
+    )
+    cache_creation_cost = cast(
+        float, _get_cost_per_unit(model_info, "cache_creation_input_token_cost")
+    )
+    cache_creation_cost_above_1hr = cast(
+        float,
+        _get_cost_per_unit(model_info, "cache_creation_input_token_cost_above_1hr"),
+    )
+    cache_read_cost = cast(
+        float, _get_cost_per_unit(model_info, "cache_read_input_token_cost")
+    )
 
     ## CHECK IF ABOVE THRESHOLD
     threshold: Optional[float] = None
@@ -140,34 +155,57 @@ def _get_token_base_cost(model_info: ModelInfo, usage: Usage) -> Tuple[float, fl
                 )
                 if usage.prompt_tokens > threshold:
 
-                    prompt_base_cost = cast(float, _get_cost_per_unit(model_info, key, prompt_base_cost))
-                    completion_base_cost = cast(float, _get_cost_per_unit(
-                        model_info,
-                        f"output_cost_per_token_above_{threshold_str}_tokens",
-                        completion_base_cost,
-                    ))
-                    
+                    prompt_base_cost = cast(
+                        float, _get_cost_per_unit(model_info, key, prompt_base_cost)
+                    )
+                    completion_base_cost = cast(
+                        float,
+                        _get_cost_per_unit(
+                            model_info,
+                            f"output_cost_per_token_above_{threshold_str}_tokens",
+                            completion_base_cost,
+                        ),
+                    )
+
                     # Apply tiered pricing to cache costs
-                    cache_creation_tiered_key = f"cache_creation_input_token_cost_above_{threshold_str}_tokens"
-                    cache_read_tiered_key = f"cache_read_input_token_cost_above_{threshold_str}_tokens"
-                    
+                    cache_creation_tiered_key = (
+                        f"cache_creation_input_token_cost_above_{threshold_str}_tokens"
+                    )
+                    cache_read_tiered_key = (
+                        f"cache_read_input_token_cost_above_{threshold_str}_tokens"
+                    )
+
                     if cache_creation_tiered_key in model_info:
-                        cache_creation_cost = cast(float, _get_cost_per_unit(
-                            model_info, cache_creation_tiered_key, cache_creation_cost
-                        ))
-                    
+                        cache_creation_cost = cast(
+                            float,
+                            _get_cost_per_unit(
+                                model_info,
+                                cache_creation_tiered_key,
+                                cache_creation_cost,
+                            ),
+                        )
+
                     if cache_read_tiered_key in model_info:
-                        cache_read_cost = cast(float, _get_cost_per_unit(
-                            model_info, cache_read_tiered_key, cache_read_cost
-                        ))
-                    
+                        cache_read_cost = cast(
+                            float,
+                            _get_cost_per_unit(
+                                model_info, cache_read_tiered_key, cache_read_cost
+                            ),
+                        )
+
                     break
             except (IndexError, ValueError):
                 continue
             except Exception:
                 continue
 
-    return prompt_base_cost, completion_base_cost, cache_creation_cost, cache_read_cost
+    return (
+        prompt_base_cost,
+        completion_base_cost,
+        cache_creation_cost,
+        cache_creation_cost_above_1hr,
+        cache_read_cost,
+    )
 
 
 def calculate_cost_component(
@@ -195,7 +233,9 @@ def calculate_cost_component(
     return 0.0
 
 
-def _get_cost_per_unit(model_info: ModelInfo, cost_key: str, default_value: Optional[float] = 0.0) -> Optional[float]:
+def _get_cost_per_unit(
+    model_info: ModelInfo, cost_key: str, default_value: Optional[float] = 0.0
+) -> Optional[float]:
     # Sometimes the cost per unit is a string (e.g.: If a value like "3e-7" was read from the config.yaml)
     cost_per_unit = model_info.get(cost_key)
     if isinstance(cost_per_unit, float):
@@ -210,7 +250,196 @@ def _get_cost_per_unit(model_info: ModelInfo, cost_key: str, default_value: Opti
                 f"litellm.litellm_core_utils.llm_cost_calc.utils.py::calculate_cost_per_component(): Exception occured - {cost_per_unit}\nDefaulting to 0.0"
             )
     return default_value
-    
+
+
+def calculate_cache_writing_cost(
+    cache_creation_tokens: int,
+    cache_creation_token_details: Optional[CacheCreationTokenDetails],
+    cache_creation_cost_above_1hr: float,
+    cache_creation_cost: float,
+) -> float:
+    """
+    Adjust cost of cache creation tokens based on the cache creation token details.
+    """
+    total_cost: float = 0.0
+    if cache_creation_token_details is not None:
+        # get the number of 5m and 1h cache creation tokens
+        cache_creation_tokens_5m = (
+            cache_creation_token_details.ephemeral_5m_input_tokens
+        )
+        cache_creation_tokens_1h = (
+            cache_creation_token_details.ephemeral_1h_input_tokens
+        )
+        # add the number of 5m and 1h cache creation tokens to the cache creation tokens
+        total_cost += (
+            cache_creation_tokens_5m * cache_creation_cost
+            if cache_creation_tokens_5m is not None
+            else 0.0
+        )
+        total_cost += (
+            cache_creation_tokens_1h * cache_creation_cost_above_1hr
+            if cache_creation_tokens_1h is not None
+            else 0.0
+        )
+    else:
+        total_cost += cache_creation_tokens * cache_creation_cost
+    return total_cost
+
+
+class PromptTokensDetailsResult(TypedDict):
+    cache_hit_tokens: int
+    cache_creation_tokens: int
+    cache_creation_token_details: Optional[CacheCreationTokenDetails]
+    text_tokens: int
+    audio_tokens: int
+    character_count: int
+    image_count: int
+    video_length_seconds: int
+
+
+def _parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult:
+    cache_hit_tokens = (
+        cast(Optional[int], getattr(usage.prompt_tokens_details, "cached_tokens", 0))
+        or 0
+    )
+    cache_creation_tokens = (
+        cast(
+            Optional[int],
+            getattr(usage.prompt_tokens_details, "cache_creation_tokens", 0),
+        )
+        or 0
+    )
+    cache_creation_token_details = (
+        cast(
+            Optional[CacheCreationTokenDetails],
+            getattr(usage.prompt_tokens_details, "cache_creation_token_details", None),
+        )
+        or None
+    )
+    text_tokens = (
+        cast(Optional[int], getattr(usage.prompt_tokens_details, "text_tokens", None))
+        or 0  # default to prompt tokens, if this field is not set
+    )
+    audio_tokens = (
+        cast(Optional[int], getattr(usage.prompt_tokens_details, "audio_tokens", 0))
+        or 0
+    )
+    character_count = (
+        cast(
+            Optional[int],
+            getattr(usage.prompt_tokens_details, "character_count", 0),
+        )
+        or 0
+    )
+    image_count = (
+        cast(Optional[int], getattr(usage.prompt_tokens_details, "image_count", 0)) or 0
+    )
+    video_length_seconds = (
+        cast(
+            Optional[int],
+            getattr(usage.prompt_tokens_details, "video_length_seconds", 0),
+        )
+        or 0
+    )
+
+    return PromptTokensDetailsResult(
+        cache_hit_tokens=cache_hit_tokens,
+        cache_creation_tokens=cache_creation_tokens,
+        cache_creation_token_details=cache_creation_token_details,
+        text_tokens=text_tokens,
+        audio_tokens=audio_tokens,
+        character_count=character_count,
+        image_count=image_count,
+        video_length_seconds=video_length_seconds,
+    )
+
+
+class CompletionTokensDetailsResult(TypedDict):
+    audio_tokens: int
+    text_tokens: int
+    reasoning_tokens: int
+
+
+def _parse_completion_tokens_details(usage: Usage) -> CompletionTokensDetailsResult:
+    audio_tokens = (
+        cast(
+            Optional[int],
+            getattr(usage.completion_tokens_details, "audio_tokens", 0),
+        )
+        or 0
+    )
+    text_tokens = (
+        cast(
+            Optional[int],
+            getattr(usage.completion_tokens_details, "text_tokens", None),
+        )
+        or 0  # default to completion tokens, if this field is not set
+    )
+    reasoning_tokens = (
+        cast(
+            Optional[int],
+            getattr(usage.completion_tokens_details, "reasoning_tokens", 0),
+        )
+        or 0
+    )
+
+    return CompletionTokensDetailsResult(
+        audio_tokens=audio_tokens,
+        text_tokens=text_tokens,
+        reasoning_tokens=reasoning_tokens,
+    )
+
+
+def _calculate_input_cost(
+    prompt_tokens_details: PromptTokensDetailsResult,
+    model_info: ModelInfo,
+    prompt_base_cost: float,
+    cache_read_cost: float,
+    cache_creation_cost: float,
+    cache_creation_cost_above_1hr: float,
+) -> float:
+    """
+    Calculates the input cost for a given model, prompt tokens, and completion tokens.
+    """
+    prompt_cost = float(prompt_tokens_details["text_tokens"]) * prompt_base_cost
+
+    ### CACHE READ COST - Now uses tiered pricing
+    prompt_cost += float(prompt_tokens_details["cache_hit_tokens"]) * cache_read_cost
+
+    ### AUDIO COST
+    prompt_cost += calculate_cost_component(
+        model_info, "input_cost_per_audio_token", prompt_tokens_details["audio_tokens"]
+    )
+
+    ### CACHE WRITING COST - Now uses tiered pricing
+    prompt_cost += calculate_cache_writing_cost(
+        cache_creation_tokens=prompt_tokens_details["cache_creation_tokens"],
+        cache_creation_token_details=prompt_tokens_details[
+            "cache_creation_token_details"
+        ],
+        cache_creation_cost_above_1hr=cache_creation_cost_above_1hr,
+        cache_creation_cost=cache_creation_cost,
+    )
+
+    ### CHARACTER COST
+
+    prompt_cost += calculate_cost_component(
+        model_info, "input_cost_per_character", prompt_tokens_details["character_count"]
+    )
+
+    ### IMAGE COUNT COST
+    prompt_cost += calculate_cost_component(
+        model_info, "input_cost_per_image", prompt_tokens_details["image_count"]
+    )
+
+    ### VIDEO LENGTH COST
+    prompt_cost += calculate_cost_component(
+        model_info,
+        "input_cost_per_video_per_second",
+        prompt_tokens_details["video_length_seconds"],
+    )
+
+    return prompt_cost
 
 
 def generic_cost_per_token(
@@ -236,83 +465,45 @@ def generic_cost_per_token(
     ### Cost of processing (non-cache hit + cache hit) + Cost of cache-writing (cache writing)
     prompt_cost = 0.0
     ### PROCESSING COST
-    text_tokens = usage.prompt_tokens
-    cache_hit_tokens = 0
-    audio_tokens = 0
-    character_count = 0
-    image_count = 0
-    video_length_seconds = 0
+    prompt_tokens_details = PromptTokensDetailsResult(
+        cache_hit_tokens=0,
+        cache_creation_tokens=0,
+        cache_creation_token_details=None,
+        text_tokens=usage.prompt_tokens,
+        audio_tokens=0,
+        character_count=0,
+        image_count=0,
+        video_length_seconds=0,
+    )
     if usage.prompt_tokens_details:
-        cache_hit_tokens = (
-            cast(
-                Optional[int], getattr(usage.prompt_tokens_details, "cached_tokens", 0)
-            )
-            or 0
-        )
-        text_tokens = (
-            cast(
-                Optional[int], getattr(usage.prompt_tokens_details, "text_tokens", None)
-            )
-            or 0  # default to prompt tokens, if this field is not set
-        )
-        audio_tokens = (
-            cast(Optional[int], getattr(usage.prompt_tokens_details, "audio_tokens", 0))
-            or 0
-        )
-        character_count = (
-            cast(
-                Optional[int],
-                getattr(usage.prompt_tokens_details, "character_count", 0),
-            )
-            or 0
-        )
-        image_count = (
-            cast(Optional[int], getattr(usage.prompt_tokens_details, "image_count", 0))
-            or 0
-        )
-        video_length_seconds = (
-            cast(
-                Optional[int],
-                getattr(usage.prompt_tokens_details, "video_length_seconds", 0),
-            )
-            or 0
-        )
+        prompt_tokens_details = _parse_prompt_tokens_details(usage)
 
     ## EDGE CASE - text tokens not set inside PromptTokensDetails
-    if text_tokens == 0:
-        text_tokens = usage.prompt_tokens - cache_hit_tokens - audio_tokens
 
-    prompt_base_cost, completion_base_cost, cache_creation_cost, cache_read_cost = _get_token_base_cost(
-        model_info=model_info, usage=usage
-    )
+    if prompt_tokens_details["text_tokens"] == 0:
+        text_tokens = (
+            usage.prompt_tokens
+            - prompt_tokens_details["cache_hit_tokens"]
+            - prompt_tokens_details["audio_tokens"]
+            - prompt_tokens_details["cache_creation_tokens"]
+        )
+        prompt_tokens_details["text_tokens"] = text_tokens
 
-    prompt_cost = float(text_tokens) * prompt_base_cost
+    (
+        prompt_base_cost,
+        completion_base_cost,
+        cache_creation_cost,
+        cache_creation_cost_above_1hr,
+        cache_read_cost,
+    ) = _get_token_base_cost(model_info=model_info, usage=usage)
 
-    ### CACHE READ COST - Now uses tiered pricing
-    prompt_cost += float(cache_hit_tokens) * cache_read_cost
-
-    ### AUDIO COST
-    prompt_cost += calculate_cost_component(
-        model_info, "input_cost_per_audio_token", audio_tokens
-    )
-
-    ### CACHE WRITING COST - Now uses tiered pricing
-    prompt_cost += float(usage._cache_creation_input_tokens or 0) * cache_creation_cost
-
-    ### CHARACTER COST
-
-    prompt_cost += calculate_cost_component(
-        model_info, "input_cost_per_character", character_count
-    )
-
-    ### IMAGE COUNT COST
-    prompt_cost += calculate_cost_component(
-        model_info, "input_cost_per_image", image_count
-    )
-
-    ### VIDEO LENGTH COST
-    prompt_cost += calculate_cost_component(
-        model_info, "input_cost_per_video_per_second", video_length_seconds
+    prompt_cost = _calculate_input_cost(
+        prompt_tokens_details=prompt_tokens_details,
+        model_info=model_info,
+        prompt_base_cost=prompt_base_cost,
+        cache_read_cost=cache_read_cost,
+        cache_creation_cost=cache_creation_cost,
+        cache_creation_cost_above_1hr=cache_creation_cost_above_1hr,
     )
 
     ## CALCULATE OUTPUT COST
@@ -321,27 +512,10 @@ def generic_cost_per_token(
     reasoning_tokens = 0
     is_text_tokens_total = False
     if usage.completion_tokens_details is not None:
-        audio_tokens = (
-            cast(
-                Optional[int],
-                getattr(usage.completion_tokens_details, "audio_tokens", 0),
-            )
-            or 0
-        )
-        text_tokens = (
-            cast(
-                Optional[int],
-                getattr(usage.completion_tokens_details, "text_tokens", None),
-            )
-            or 0  # default to completion tokens, if this field is not set
-        )
-        reasoning_tokens = (
-            cast(
-                Optional[int],
-                getattr(usage.completion_tokens_details, "reasoning_tokens", 0),
-            )
-            or 0
-        )
+        completion_tokens_details = _parse_completion_tokens_details(usage)
+        audio_tokens = completion_tokens_details["audio_tokens"]
+        text_tokens = completion_tokens_details["text_tokens"]
+        reasoning_tokens = completion_tokens_details["reasoning_tokens"]
 
     if text_tokens == 0:
         text_tokens = usage.completion_tokens
@@ -350,8 +524,12 @@ def generic_cost_per_token(
     ## TEXT COST
     completion_cost = float(text_tokens) * completion_base_cost
 
-    _output_cost_per_audio_token = _get_cost_per_unit(model_info, "output_cost_per_audio_token", None)
-    _output_cost_per_reasoning_token = _get_cost_per_unit(model_info, "output_cost_per_reasoning_token", None)
+    _output_cost_per_audio_token = _get_cost_per_unit(
+        model_info, "output_cost_per_audio_token", None
+    )
+    _output_cost_per_reasoning_token = _get_cost_per_unit(
+        model_info, "output_cost_per_reasoning_token", None
+    )
 
     ## AUDIO COST
     if not is_text_tokens_total and audio_tokens is not None and audio_tokens > 0:
@@ -397,7 +575,7 @@ class CostCalculatorUtils:
         ]:
             return True
         return False
-    
+
     @staticmethod
     def route_image_generation_cost_calculator(
         model: str,

litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py

@@ -1,6 +1,5 @@
 import asyncio
 import json
-import re
 import time
 import traceback
 import uuid
@@ -9,6 +8,9 @@ from typing import Dict, Iterable, List, Literal, Optional, Tuple, Union
 import litellm
 from litellm._logging import verbose_logger
 from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
+from litellm.litellm_core_utils.prompt_templates.common_utils import (
+    _extract_reasoning_content,
+)
 from litellm.types.llms.databricks import DatabricksTool
 from litellm.types.llms.openai import (
     ChatCompletionThinkingBlock,
@@ -274,49 +276,6 @@ def _handle_invalid_parallel_tool_calls(
         return tool_calls
 
 
-def _parse_content_for_reasoning(
-    message_text: Optional[str],
-) -> Tuple[Optional[str], Optional[str]]:
-    """
-    Parse the content for reasoning
-
-    Returns:
-    - reasoning_content: The content of the reasoning
-    - content: The content of the message
-    """
-    if not message_text:
-        return None, message_text
-
-    reasoning_match = re.match(
-        r"<(?:think|thinking)>(.*?)</(?:think|thinking)>(.*)", message_text, re.DOTALL
-    )
-
-    if reasoning_match:
-        return reasoning_match.group(1), reasoning_match.group(2)
-
-    return None, message_text
-
-
-def _extract_reasoning_content(message: dict) -> Tuple[Optional[str], Optional[str]]:
-    """
-    Extract reasoning content and main content from a message.
-
-    Args:
-        message (dict): The message dictionary that may contain reasoning_content
-
-    Returns:
-        tuple[Optional[str], Optional[str]]: A tuple of (reasoning_content, content)
-    """
-    message_content = message.get("content")
-    if "reasoning_content" in message:
-        return message["reasoning_content"], message["content"]
-    elif "reasoning" in message:
-        return message["reasoning"], message["content"]
-    elif isinstance(message_content, str):
-        return _parse_content_for_reasoning(message_content)
-    return None, message_content
-
-
 class LiteLLMResponseObjectHandler:
     @staticmethod
     def convert_to_image_response(

litellm/litellm_core_utils/logging_worker.py

@@ -1,10 +1,23 @@
 import asyncio
 import contextlib
+import contextvars
 from typing import Coroutine, Optional
 
+from typing_extensions import TypedDict
+
 from litellm._logging import verbose_logger
 
 
+class LoggingTask(TypedDict):
+    """
+    A logging task with its associated context to ensure logging is executed in
+    the original task's context.
+    """
+
+    coroutine: Coroutine
+    context: contextvars.Context
+
+
 class LoggingWorker:
     """
     A simple, async logging worker that processes log coroutines in the background.
@@ -13,77 +26,84 @@ class LoggingWorker:
     This leads to a +200 RPS performance improvement when using LiteLLM Python SDK or Proxy Server.
     - Use this to queue coroutine tasks that are not critical to the main flow of the application. e.g Success/Error callbacks, logging, etc.
     """
+
     LOGGING_WORKER_MAX_QUEUE_SIZE = 50_000
     LOGGING_WORKER_MAX_TIME_PER_COROUTINE = 20.0
 
     MAX_ITERATIONS_TO_CLEAR_QUEUE = 200
     MAX_TIME_TO_CLEAR_QUEUE = 5.0
-    
+
     def __init__(
-        self, 
-        timeout: float = LOGGING_WORKER_MAX_TIME_PER_COROUTINE, 
+        self,
+        timeout: float = LOGGING_WORKER_MAX_TIME_PER_COROUTINE,
         max_queue_size: int = LOGGING_WORKER_MAX_QUEUE_SIZE,
     ):
         self.timeout = timeout
         self.max_queue_size = max_queue_size
-        self._queue: Optional[asyncio.Queue] = None
+        self._queue: Optional[asyncio.Queue[LoggingTask]] = None
         self._worker_task: Optional[asyncio.Task] = None
-    
+
     def _ensure_queue(self) -> None:
         """Initialize the queue if it doesn't exist."""
         if self._queue is None:
             self._queue = asyncio.Queue(maxsize=self.max_queue_size)
-    
+
     def start(self) -> None:
         """Start the logging worker. Idempotent - safe to call multiple times."""
         self._ensure_queue()
         if self._worker_task is None or self._worker_task.done():
             self._worker_task = asyncio.create_task(self._worker_loop())
-    
+
     async def _worker_loop(self) -> None:
         """Main worker loop that processes log coroutines sequentially."""
         try:
             if self._queue is None:
                 return
-            
+
             while True:
                 # Process one coroutine at a time to keep event loop load predictable
-                coroutine = await self._queue.get()
+                task = await self._queue.get()
                 try:
-                    await asyncio.wait_for(coroutine, timeout=self.timeout)
+                    # Run the coroutine in its original context
+                    await asyncio.wait_for(
+                        task["context"].run(asyncio.create_task, task["coroutine"]),
+                        timeout=self.timeout,
+                    )
                 except Exception as e:
                     verbose_logger.exception(f"LoggingWorker error: {e}")
                     pass
                 finally:
                     self._queue.task_done()
-                    
+
         except asyncio.CancelledError:
             verbose_logger.debug("LoggingWorker cancelled during shutdown")
             # Attempt to clear remaining items to prevent "never awaited" warnings
             await self.clear_queue()
-    
+
     def enqueue(self, coroutine: Coroutine) -> None:
         """
-        Add a coroutine to the logging queue. 
+        Add a coroutine to the logging queue.
         Hot path: never blocks, drops logs if queue is full.
         """
         if self._queue is None:
             return
-        
+
         try:
-            self._queue.put_nowait(coroutine)
+            # Capture the current context when enqueueing
+            task = LoggingTask(coroutine=coroutine, context=contextvars.copy_context())
+            self._queue.put_nowait(task)
         except asyncio.QueueFull as e:
             verbose_logger.exception(f"LoggingWorker queue is full: {e}")
             # Drop logs on overload to protect request throughput
             pass
-    
+
     def ensure_initialized_and_enqueue(self, async_coroutine: Coroutine):
         """
         Ensure the logging worker is initialized and enqueue the coroutine.
         """
         self.start()
         self.enqueue(async_coroutine)
-    
+
     async def stop(self) -> None:
         """Stop the logging worker and clean up resources."""
         if self._worker_task:
@@ -91,34 +111,42 @@ class LoggingWorker:
             with contextlib.suppress(Exception):
                 await self._worker_task
             self._worker_task = None
-    
+
     async def flush(self) -> None:
         """Flush the logging queue."""
         if self._queue is None:
             return
         while not self._queue.empty():
             await self._queue.join()
-    
+
     async def clear_queue(self):
         """
         Clear the queue with a maximum time limit.
         """
         if self._queue is None:
             return
-        
+
         start_time = asyncio.get_event_loop().time()
-        
+
         for _ in range(self.MAX_ITERATIONS_TO_CLEAR_QUEUE):
             # Check if we've exceeded the maximum time
-            if asyncio.get_event_loop().time() - start_time >= self.MAX_TIME_TO_CLEAR_QUEUE:
-                verbose_logger.warning(f"clear_queue exceeded max_time of {self.MAX_TIME_TO_CLEAR_QUEUE}s, stopping early")
+            if (
+                asyncio.get_event_loop().time() - start_time
+                >= self.MAX_TIME_TO_CLEAR_QUEUE
+            ):
+                verbose_logger.warning(
+                    f"clear_queue exceeded max_time of {self.MAX_TIME_TO_CLEAR_QUEUE}s, stopping early"
+                )
                 break
-                
+
             try:
-                coroutine = self._queue.get_nowait()
+                task = self._queue.get_nowait()
                 # Await the coroutine to properly execute and avoid "never awaited" warnings
                 try:
-                    await asyncio.wait_for(coroutine, timeout=self.timeout)
+                    await asyncio.wait_for(
+                        task["context"].run(asyncio.create_task, task["coroutine"]),
+                        timeout=self.timeout,
+                    )
                 except Exception:
                     # Suppress errors during cleanup
                     pass
@@ -129,4 +157,3 @@ class LoggingWorker:
 
 # Global instance for backward compatibility
 GLOBAL_LOGGING_WORKER = LoggingWorker()
-

litellm/litellm_core_utils/object_pooling.py

@@ -0,0 +1,137 @@
+"""
+Generic object pooling utilities for LiteLLM.
+
+This module provides a flexible object pooling system that can be used
+to pool any type of object, reducing memory allocation overhead and
+improving performance for frequently created/destroyed objects.
+
+Memory Management Strategy:
+- Balanced eviction-based memory control to optimize reuse ratio
+- Moderate eviction frequency (300s) to maintain high object reuse
+- Conservative eviction weight (0.3) to avoid destroying useful objects
+- Lower pre-warm count (5) to reduce initial memory footprint
+- Always keeps at least one object available for high availability
+- Unlimited pools when maxsize is not specified (eviction controls actual usage)
+"""
+
+from typing import Any, Callable, Optional, Type, TypeVar
+
+from pond import Pond, PooledObject, PooledObjectFactory
+
+T = TypeVar('T')
+
+class GenericPooledObjectFactory(PooledObjectFactory):
+    """Generic factory class for creating pooled objects of any type."""
+    
+    def __init__(
+        self, 
+        object_class: Type[T], 
+        pooled_maxsize: Optional[int] = None,  # None = unlimited pool with eviction-based memory control
+        least_one: bool = True,  # Always keep at least one for high concurrency
+        initializer: Optional[Callable[[T], None]] = None
+    ):
+        # Only pass maxsize to Pond if user specified it - otherwise let Pond handle unlimited pools
+        if pooled_maxsize is not None:
+            super().__init__(pooled_maxsize=pooled_maxsize, least_one=least_one)
+        else:
+            super().__init__(least_one=least_one)
+        self.object_class = object_class
+        self.initializer = initializer
+        self._user_maxsize = pooled_maxsize  # Store original user preference
+    
+    def createInstance(self) -> PooledObject:
+        """Create a new instance wrapped in a PooledObject."""
+        # Create a properly initialized instance
+        obj = self.object_class()
+        return PooledObject(obj)
+    
+    def destroy(self, pooled_object: PooledObject):
+        """Destroy the pooled object."""
+        if hasattr(pooled_object.keeped_object, '__dict__'):
+            pooled_object.keeped_object.__dict__.clear()
+        del pooled_object
+    
+    def reset(self, pooled_object: PooledObject, **kwargs: Any) -> PooledObject:
+        """Reset the pooled object to a clean state."""
+        obj = pooled_object.keeped_object
+        # Reset the object by calling its reset method if it exists
+        if hasattr(obj, 'reset') and callable(getattr(obj, 'reset')):
+            obj.reset()
+        else:
+            # Fallback: clear all attributes to reset the object
+            if hasattr(obj, '__dict__'):
+                obj.__dict__.clear()
+        return pooled_object
+    
+    def validate(self, pooled_object: PooledObject) -> bool:
+        """Validate if the pooled object is still usable."""
+        return pooled_object.keeped_object is not None
+
+# Global pond instances
+_pools: dict[str, Pond] = {}
+
+def get_object_pool(
+    pool_name: str,
+    object_class: Type[T],
+    pooled_maxsize: Optional[int] = None,  # None = unlimited pool with eviction-based memory control
+    least_one: bool = True,  # Always keep at least one
+    borrowed_timeout: int = 10,  # Longer timeout for high concurrency
+    time_between_eviction_runs: int = 300,  # Less frequent eviction to maintain high reuse ratio
+    eviction_weight: float = 0.3,  # Less aggressive eviction for better reuse
+    prewarm_count: int = 5  # Lower pre-warm count to reduce initial memory usage
+) -> Pond:
+    """Get or create a global object pool instance with balanced eviction-based memory control.
+    
+    Memory is controlled through moderate eviction to balance reuse ratio and memory usage:
+    - Moderate eviction frequency (300s) to maintain high object reuse ratio
+    - Conservative eviction weight (0.3) to avoid destroying useful objects
+    - Lower pre-warm count (5) to reduce initial memory footprint
+    
+    Args:
+        pool_name: Unique name for the pool
+        object_class: The class type to pool
+        pooled_maxsize: Maximum number of objects in the pool (None = truly unlimited)
+        least_one: Whether to keep at least one object in the pool (default: True)
+        borrowed_timeout: Timeout for borrowing objects (seconds, default: 10)
+        time_between_eviction_runs: Time between eviction runs (seconds, default: 300)
+        eviction_weight: Weight for eviction algorithm (default: 0.3, conservative)
+        prewarm_count: Number of objects to pre-warm the pool with (default: 5)
+    
+    Returns:
+        Pond instance for the specified object type
+    """
+    
+    if pool_name in _pools:
+        return _pools[pool_name]
+    
+    # Create new pond
+    pond = Pond(
+        borrowed_timeout=borrowed_timeout,
+        time_between_eviction_runs=time_between_eviction_runs,
+        thread_daemon=True,
+        eviction_weight=eviction_weight
+    )
+    
+    # Register the factory with user's maxsize preference
+    factory = GenericPooledObjectFactory(
+        object_class=object_class,
+        pooled_maxsize=pooled_maxsize,
+        least_one=least_one
+    )
+    pond.register(factory, name=f"{pool_name}Factory")
+    
+    # Pre-warm the pool
+    _prewarm_pool(pond, pool_name, prewarm_count)
+    
+    _pools[pool_name] = pond
+    return pond
+
+def _prewarm_pool(pond: Pond, pool_name: str, prewarm_count: int = 20) -> None:
+    """Pre-warm the pool with initial objects for high concurrency."""
+    for _ in range(prewarm_count):
+        try:
+            pooled_obj = pond.borrow(name=f"{pool_name}Factory")
+            pond.recycle(pooled_obj, name=f"{pool_name}Factory")
+        except Exception:
+            # If pre-warming fails, just continue
+            break

litellm/litellm_core_utils/prompt_templates/common_utils.py

@@ -14,6 +14,7 @@ from typing import (
     Literal,
     Mapping,
     Optional,
+    Tuple,
     Union,
     cast,
 )
@@ -869,3 +870,63 @@ def convert_prefix_message_to_non_prefix_messages(
         else:
             new_messages.append(message)
     return new_messages
+
+
+def _extract_reasoning_content(message: dict) -> Tuple[Optional[str], Optional[str]]:
+    """
+    Extract reasoning content and main content from a message.
+
+    Args:
+        message (dict): The message dictionary that may contain reasoning_content
+
+    Returns:
+        tuple[Optional[str], Optional[str]]: A tuple of (reasoning_content, content)
+    """
+    message_content = message.get("content")
+    if "reasoning_content" in message:
+        return message["reasoning_content"], message["content"]
+    elif "reasoning" in message:
+        return message["reasoning"], message["content"]
+    elif isinstance(message_content, str):
+        return _parse_content_for_reasoning(message_content)
+    return None, message_content
+
+
+def _parse_content_for_reasoning(
+    message_text: Optional[str],
+) -> Tuple[Optional[str], Optional[str]]:
+    """
+    Parse the content for reasoning
+
+    Returns:
+    - reasoning_content: The content of the reasoning
+    - content: The content of the message
+    """
+    if not message_text:
+        return None, message_text
+
+    reasoning_match = re.match(
+        r"<(?:think|thinking)>(.*?)</(?:think|thinking)>(.*)", message_text, re.DOTALL
+    )
+
+    if reasoning_match:
+        return reasoning_match.group(1), reasoning_match.group(2)
+
+    return None, message_text
+
+
+def extract_images_from_message(message: AllMessageValues) -> List[str]:
+    """
+    Extract images from a message
+    """
+    images = []
+    message_content = message.get("content")
+    if isinstance(message_content, list):
+        for m in message_content:
+            image_url = m.get("image_url")
+            if image_url:
+                if isinstance(image_url, str):
+                    images.append(image_url)
+                elif isinstance(image_url, dict) and "url" in image_url:
+                    images.append(image_url["url"])
+    return images

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -16,8 +16,8 @@ from litellm import verbose_logger
 from litellm.llms.custom_httpx.http_handler import HTTPHandler, get_async_httpx_client
 from litellm.types.files import get_file_extension_from_mime_type
 from litellm.types.llms.anthropic import *
-from litellm.types.llms.bedrock import MessageBlock as BedrockMessageBlock
 from litellm.types.llms.bedrock import CachePointBlock
+from litellm.types.llms.bedrock import MessageBlock as BedrockMessageBlock
 from litellm.types.llms.custom_http import httpxSpecialProvider
 from litellm.types.llms.ollama import OllamaVisionModelObject
 from litellm.types.llms.openai import (
@@ -1067,10 +1067,10 @@ def convert_to_gemini_tool_call_invoke(
         if tool_calls is not None:
             for tool in tool_calls:
                 if "function" in tool:
-                    gemini_function_call: Optional[VertexFunctionCall] = (
-                        _gemini_tool_call_invoke_helper(
-                            function_call_params=tool["function"]
-                        )
+                    gemini_function_call: Optional[
+                        VertexFunctionCall
+                    ] = _gemini_tool_call_invoke_helper(
+                        function_call_params=tool["function"]
                     )
                     if gemini_function_call is not None:
                         _parts_list.append(
@@ -1589,9 +1589,9 @@ def anthropic_messages_pt(  # noqa: PLR0915
                             )
 
                             if "cache_control" in _content_element:
-                                _anthropic_content_element["cache_control"] = (
-                                    _content_element["cache_control"]
-                                )
+                                _anthropic_content_element[
+                                    "cache_control"
+                                ] = _content_element["cache_control"]
                             user_content.append(_anthropic_content_element)
                         elif m.get("type", "") == "text":
                             m = cast(ChatCompletionTextObject, m)
@@ -1629,9 +1629,9 @@ def anthropic_messages_pt(  # noqa: PLR0915
                     )
 
                     if "cache_control" in _content_element:
-                        _anthropic_content_text_element["cache_control"] = (
-                            _content_element["cache_control"]
-                        )
+                        _anthropic_content_text_element[
+                            "cache_control"
+                        ] = _content_element["cache_control"]
 
                     user_content.append(_anthropic_content_text_element)
 
@@ -2482,8 +2482,7 @@ class BedrockImageProcessor:
 
         if is_document:
             return BedrockImageProcessor._get_document_format(
-                mime_type=mime_type,
-                supported_doc_formats=supported_doc_formats
+                mime_type=mime_type, supported_doc_formats=supported_doc_formats
             )
 
         else:
@@ -2495,12 +2494,9 @@ class BedrockImageProcessor:
                     f"Unsupported image format: {image_format}. Supported formats: {supported_image_and_video_formats}"
                 )
             return image_format
-    
+
     @staticmethod
-    def _get_document_format(
-        mime_type: str,
-        supported_doc_formats: List[str]
-        ) -> str:
+    def _get_document_format(mime_type: str, supported_doc_formats: List[str]) -> str:
         """
         Get the document format from the mime type
 
@@ -2519,13 +2515,9 @@ class BedrockImageProcessor:
             The document format
         """
         valid_extensions: Optional[List[str]] = None
-        potential_extensions = mimetypes.guess_all_extensions(
-            mime_type, strict=False
-        )
+        potential_extensions = mimetypes.guess_all_extensions(mime_type, strict=False)
         valid_extensions = [
-            ext[1:]
-            for ext in potential_extensions
-            if ext[1:] in supported_doc_formats
+            ext[1:] for ext in potential_extensions if ext[1:] in supported_doc_formats
         ]
 
         # Fallback to types/files.py if mimetypes doesn't return valid extensions
@@ -2680,16 +2672,21 @@ def _convert_to_bedrock_tool_call_invoke(
                 id = tool["id"]
                 name = tool["function"].get("name", "")
                 arguments = tool["function"].get("arguments", "")
-                arguments_dict = json.loads(arguments) if arguments else {}
+                if not arguments or not arguments.strip():
+                    arguments_dict = {}
+                else:
+                    arguments_dict = json.loads(arguments)
                 bedrock_tool = BedrockToolUseBlock(
                     input=arguments_dict, name=name, toolUseId=id
                 )
                 bedrock_content_block = BedrockContentBlock(toolUse=bedrock_tool)
                 _parts_list.append(bedrock_content_block)
-                
+
                 # Check for cache_control and add a separate cachePoint block
                 if tool.get("cache_control", None) is not None:
-                    cache_point_block = BedrockContentBlock(cachePoint=CachePointBlock(type="default"))
+                    cache_point_block = BedrockContentBlock(
+                        cachePoint=CachePointBlock(type="default")
+                    )
                     _parts_list.append(cache_point_block)
         return _parts_list
     except Exception as e:
@@ -2751,7 +2748,7 @@ def _convert_to_bedrock_tool_call_result(
         for content in content_list:
             if content["type"] == "text":
                 content_str += content["text"]
-    
+
     message.get("name", "")
     id = str(message.get("tool_call_id", str(uuid.uuid4())))
 
@@ -2760,7 +2757,7 @@ def _convert_to_bedrock_tool_call_result(
         content=[tool_result_content_block],
         toolUseId=id,
     )
-    
+
     content_block = BedrockContentBlock(toolResult=tool_result)
 
     return content_block
@@ -3125,6 +3122,12 @@ class BedrockConverseMessagesProcessor:
                             if element["type"] == "text":
                                 _part = BedrockContentBlock(text=element["text"])
                                 _parts.append(_part)
+                            elif element["type"] == "guarded_text":
+                                # Wrap guarded_text in guardContent block
+                                _part = BedrockContentBlock(
+                                    guardContent={"text": {"text": element["text"]}}
+                                )
+                                _parts.append(_part)
                             elif element["type"] == "image_url":
                                 format: Optional[str] = None
                                 if isinstance(element["image_url"], dict):
@@ -3196,26 +3199,29 @@ class BedrockConverseMessagesProcessor:
                 current_message = messages[msg_i]
                 tool_call_result = _convert_to_bedrock_tool_call_result(current_message)
                 tool_content.append(tool_call_result)
-                            
+
                 # Check if we need to add a separate cachePoint block
                 has_cache_control = False
-                
+
                 # Check for message-level cache_control
                 if current_message.get("cache_control", None) is not None:
                     has_cache_control = True
                 # Check for content-level cache_control in list content
                 elif isinstance(current_message.get("content"), list):
                     for content_element in current_message["content"]:
-                        if (isinstance(content_element, dict) and 
-                            content_element.get("cache_control", None) is not None):
+                        if (
+                            isinstance(content_element, dict)
+                            and content_element.get("cache_control", None) is not None
+                        ):
                             has_cache_control = True
                             break
-                
+
                 # Add a separate cachePoint block if cache_control is present
                 if has_cache_control:
-                    cache_point_block = BedrockContentBlock(cachePoint=CachePointBlock(type="default"))
+                    cache_point_block = BedrockContentBlock(
+                        cachePoint=CachePointBlock(type="default")
+                    )
                     tool_content.append(cache_point_block)
-            
 
                 msg_i += 1
             if tool_content:
@@ -3296,7 +3302,7 @@ class BedrockConverseMessagesProcessor:
                                     image_url=image_url
                                 )
                                 assistants_parts.append(assistants_part)
-                                                        # Add cache point block for assistant content elements
+                                # Add cache point block for assistant content elements
                         _cache_point_block = (
                             litellm.AmazonConverseConfig()._get_cache_point_block(
                                 message_block=cast(
@@ -3308,8 +3314,12 @@ class BedrockConverseMessagesProcessor:
                         if _cache_point_block is not None:
                             assistants_parts.append(_cache_point_block)
                     assistant_content.extend(assistants_parts)
-                elif _assistant_content is not None and isinstance(_assistant_content, str):
-                    assistant_content.append(BedrockContentBlock(text=_assistant_content))
+                elif _assistant_content is not None and isinstance(
+                    _assistant_content, str
+                ):
+                    assistant_content.append(
+                        BedrockContentBlock(text=_assistant_content)
+                    )
                     # Add cache point block for assistant string content
                     _cache_point_block = (
                         litellm.AmazonConverseConfig()._get_cache_point_block(
@@ -3493,6 +3503,12 @@ def _bedrock_converse_messages_pt(  # noqa: PLR0915
                         if element["type"] == "text":
                             _part = BedrockContentBlock(text=element["text"])
                             _parts.append(_part)
+                        elif element["type"] == "guarded_text":
+                            # Wrap guarded_text in guardContent block
+                            _part = BedrockContentBlock(
+                                guardContent={"text": {"text": element["text"]}}
+                            )
+                            _parts.append(_part)
                         elif element["type"] == "image_url":
                             format: Optional[str] = None
                             if isinstance(element["image_url"], dict):
@@ -3562,29 +3578,33 @@ def _bedrock_converse_messages_pt(  # noqa: PLR0915
         while msg_i < len(messages) and messages[msg_i]["role"] == "tool":
             tool_call_result = _convert_to_bedrock_tool_call_result(messages[msg_i])
             current_message = messages[msg_i]
-            
+
             # Add the tool result first
             tool_content.append(tool_call_result)
-            
+
             # Check if we need to add a separate cachePoint block
             has_cache_control = False
-            
+
             # Check for message-level cache_control
             if current_message.get("cache_control", None) is not None:
                 has_cache_control = True
             # Check for content-level cache_control in list content
             elif isinstance(current_message.get("content"), list):
                 for content_element in current_message["content"]:
-                    if (isinstance(content_element, dict) and 
-                        content_element.get("cache_control", None) is not None):
+                    if (
+                        isinstance(content_element, dict)
+                        and content_element.get("cache_control", None) is not None
+                    ):
                         has_cache_control = True
                         break
-            
+
             # Add a separate cachePoint block if cache_control is present
             if has_cache_control:
-                cache_point_block = BedrockContentBlock(cachePoint=CachePointBlock(type="default"))
+                cache_point_block = BedrockContentBlock(
+                    cachePoint=CachePointBlock(type="default")
+                )
                 tool_content.append(cache_point_block)
-            
+
             msg_i += 1
         if tool_content:
             # if last message was a 'user' message, then add a blank assistant message (bedrock requires alternating roles)
@@ -3849,10 +3869,9 @@ def function_call_prompt(messages: list, functions: list):
             if isinstance(message["content"], str):
                 message["content"] += f""" {function_prompt}"""
             else:
-                message["content"].append({
-                    "type": "text",
-                    "text": f""" {function_prompt}"""
-                })
+                message["content"].append(
+                    {"type": "text", "text": f""" {function_prompt}"""}
+                )
             function_added_to_prompt = True
 
     if function_added_to_prompt is False:

litellm/litellm_core_utils/streaming_handler.py

@@ -1024,6 +1024,8 @@ class CustomStreamWrapper:
         return
 
     def chunk_creator(self, chunk: Any):  # type: ignore  # noqa: PLR0915
+        if hasattr(chunk, 'id'):
+            self.response_id = chunk.id
         model_response = self.model_response_creator()
         response_obj: Dict[str, Any] = {}
         try:

litellm/llms/anthropic/chat/transformation.py

@@ -45,7 +45,10 @@ from litellm.types.llms.openai import (
     OpenAIMcpServerTool,
     OpenAIWebSearchOptions,
 )
-from litellm.types.utils import CompletionTokensDetailsWrapper
+from litellm.types.utils import (
+    CacheCreationTokenDetails,
+    CompletionTokensDetailsWrapper,
+)
 from litellm.types.utils import Message as LitellmMessage
 from litellm.types.utils import PromptTokensDetailsWrapper, ServerToolUse
 from litellm.utils import (
@@ -200,8 +203,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
             )
 
             _allowed_properties = set(AnthropicInputSchema.__annotations__.keys())
-            input_schema_filtered = {k: v for k, v in _input_schema.items() if k in _allowed_properties}
-            input_anthropic_schema: AnthropicInputSchema = AnthropicInputSchema(**input_schema_filtered)
+            input_schema_filtered = {
+                k: v for k, v in _input_schema.items() if k in _allowed_properties
+            }
+            input_anthropic_schema: AnthropicInputSchema = AnthropicInputSchema(
+                **input_schema_filtered
+            )
 
             _tool = AnthropicMessagesTool(
                 name=tool["function"]["name"],
@@ -816,12 +823,14 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
         _usage = usage_object
         cache_creation_input_tokens: int = 0
         cache_read_input_tokens: int = 0
+        cache_creation_token_details: Optional[CacheCreationTokenDetails] = None
         web_search_requests: Optional[int] = None
         if (
             "cache_creation_input_tokens" in _usage
             and _usage["cache_creation_input_tokens"] is not None
         ):
             cache_creation_input_tokens = _usage["cache_creation_input_tokens"]
+            prompt_tokens += cache_creation_input_tokens
         if (
             "cache_read_input_tokens" in _usage
             and _usage["cache_read_input_tokens"] is not None
@@ -837,8 +846,20 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
                     int, _usage["server_tool_use"]["web_search_requests"]
                 )
 
+        if "cache_creation" in _usage and _usage["cache_creation"] is not None:
+            cache_creation_token_details = CacheCreationTokenDetails(
+                ephemeral_5m_input_tokens=_usage["cache_creation"].get(
+                    "ephemeral_5m_input_tokens"
+                ),
+                ephemeral_1h_input_tokens=_usage["cache_creation"].get(
+                    "ephemeral_1h_input_tokens"
+                ),
+            )
+
         prompt_tokens_details = PromptTokensDetailsWrapper(
             cached_tokens=cache_read_input_tokens,
+            cache_creation_tokens=cache_read_input_tokens,
+            cache_creation_token_details=cache_creation_token_details,
         )
         completion_token_details = (
             CompletionTokensDetailsWrapper(

litellm/llms/anthropic/common_utils.py

@@ -107,10 +107,8 @@ class AnthropicModelInfo(BaseLLMModelInfo):
         user_anthropic_beta_headers: Optional[List[str]] = None,
     ) -> dict:
         betas = set()
-        # Note: prompt-caching-2024-07-31 header is no longer required for prompt caching
-        # as per current Anthropic documentation. It's now generally available.
-        # if prompt_caching_set:
-        #     betas.add("prompt-caching-2024-07-31")
+        if prompt_caching_set:
+            betas.add("prompt-caching-2024-07-31")
         if computer_tool_used:
             betas.add("computer-use-2024-10-22")
         # if pdf_used:
@@ -178,11 +176,6 @@ class AnthropicModelInfo(BaseLLMModelInfo):
             mcp_server_used=mcp_server_used,
         )
 
-        # For Vertex AI requests, remove any user-provided anthropic-beta headers
-        # since Vertex AI rejects them and they're no longer required for prompt caching
-        if optional_params.get("is_vertex_request", False):
-            headers = {k: v for k, v in headers.items() if k != "anthropic-beta"}
-        
         headers = {**headers, **anthropic_headers}
 
         return headers

litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py

@@ -28,10 +28,6 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
         TextBlock,
     )
 
-    def __init__(self, completion_stream: Any, model: str):
-        super().__init__(completion_stream)
-        self.model = model
-
     sent_first_chunk: bool = False
     sent_content_block_start: bool = False
     sent_content_block_finish: bool = False
@@ -39,6 +35,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
     sent_last_message: bool = False
     holding_chunk: Optional[Any] = None
     holding_stop_reason_chunk: Optional[Any] = None
+    queued_usage_chunk: bool = False
     current_content_block_index: int = 0
     current_content_block_start: ContentBlockContentBlockDict = TextBlock(
         type="text",
@@ -47,6 +44,10 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
     pending_new_content_block: bool = False
     chunk_queue: deque = deque()  # Queue for buffering multiple chunks
 
+    def __init__(self, completion_stream: Any, model: str):
+        super().__init__(completion_stream)
+        self.model = model
+
     def __next__(self):
         from .transformation import LiteLLMAnthropicMessagesAdapter
 
@@ -217,77 +218,83 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
 
                     # Queue the merged chunk and reset
                     self.chunk_queue.append(merged_chunk)
+                    self.queued_usage_chunk = True
                     self.holding_stop_reason_chunk = None
                     return self.chunk_queue.popleft()
 
                 # Check if this processed chunk has a stop_reason - hold it for next chunk
 
-                if should_start_new_block and not self.sent_content_block_finish:
-                    # Queue the sequence: content_block_stop -> content_block_start -> current_chunk
+                if not self.queued_usage_chunk:
+                    if should_start_new_block and not self.sent_content_block_finish:
+                        # Queue the sequence: content_block_stop -> content_block_start -> current_chunk
 
-                    # 1. Stop current content block
-                    self.chunk_queue.append(
-                        {
-                            "type": "content_block_stop",
-                            "index": max(self.current_content_block_index - 1, 0),
-                        }
-                    )
+                        # 1. Stop current content block
+                        self.chunk_queue.append(
+                            {
+                                "type": "content_block_stop",
+                                "index": max(self.current_content_block_index - 1, 0),
+                            }
+                        )
 
-                    # 2. Start new content block
-                    self.chunk_queue.append(
-                        {
-                            "type": "content_block_start",
-                            "index": self.current_content_block_index,
-                            "content_block": self.current_content_block_start,
-                        }
-                    )
+                        # 2. Start new content block
+                        self.chunk_queue.append(
+                            {
+                                "type": "content_block_start",
+                                "index": self.current_content_block_index,
+                                "content_block": self.current_content_block_start,
+                            }
+                        )
 
-                    # 3. Queue the current chunk (don't lose it!)
-                    self.chunk_queue.append(processed_chunk)
-
-                    # Reset state for new block
-                    self.sent_content_block_finish = False
-
-                    # Return the first queued item
-                    return self.chunk_queue.popleft()
-
-                if (
-                    processed_chunk["type"] == "message_delta"
-                    and self.sent_content_block_finish is False
-                ):
-                    # Queue both the content_block_stop and the holding chunk
-                    self.chunk_queue.append(
-                        {
-                            "type": "content_block_stop",
-                            "index": self.current_content_block_index,
-                        }
-                    )
-                    self.sent_content_block_finish = True
-                    if processed_chunk.get("delta", {}).get("stop_reason") is not None:
-
-                        self.holding_stop_reason_chunk = processed_chunk
-                    else:
+                        # 3. Queue the current chunk (don't lose it!)
                         self.chunk_queue.append(processed_chunk)
-                    return self.chunk_queue.popleft()
-                elif self.holding_chunk is not None:
-                    # Queue both chunks
-                    self.chunk_queue.append(self.holding_chunk)
-                    self.chunk_queue.append(processed_chunk)
-                    self.holding_chunk = None
-                    return self.chunk_queue.popleft()
-                else:
-                    # Queue the current chunk
-                    self.chunk_queue.append(processed_chunk)
-                    return self.chunk_queue.popleft()
+
+                        # Reset state for new block
+                        self.sent_content_block_finish = False
+
+                        # Return the first queued item
+                        return self.chunk_queue.popleft()
+
+                    if (
+                        processed_chunk["type"] == "message_delta"
+                        and self.sent_content_block_finish is False
+                    ):
+                        # Queue both the content_block_stop and the holding chunk
+                        self.chunk_queue.append(
+                            {
+                                "type": "content_block_stop",
+                                "index": self.current_content_block_index,
+                            }
+                        )
+                        self.sent_content_block_finish = True
+                        if (
+                            processed_chunk.get("delta", {}).get("stop_reason")
+                            is not None
+                        ):
+
+                            self.holding_stop_reason_chunk = processed_chunk
+                        else:
+                            self.chunk_queue.append(processed_chunk)
+                        return self.chunk_queue.popleft()
+                    elif self.holding_chunk is not None:
+                        # Queue both chunks
+                        self.chunk_queue.append(self.holding_chunk)
+                        self.chunk_queue.append(processed_chunk)
+                        self.holding_chunk = None
+                        return self.chunk_queue.popleft()
+                    else:
+                        # Queue the current chunk
+                        self.chunk_queue.append(processed_chunk)
+                        return self.chunk_queue.popleft()
 
             # Handle any remaining held chunks after stream ends
-            if self.holding_stop_reason_chunk is not None:
-                self.chunk_queue.append(self.holding_stop_reason_chunk)
-                self.holding_stop_reason_chunk = None
+            if not self.queued_usage_chunk:
+                if self.holding_stop_reason_chunk is not None:
+                    self.chunk_queue.append(self.holding_stop_reason_chunk)
+                    self.holding_stop_reason_chunk = None
 
-            if self.holding_chunk is not None:
-                self.chunk_queue.append(self.holding_chunk)
-                self.holding_chunk = None
+                if self.holding_chunk is not None:
+                    self.chunk_queue.append(self.holding_chunk)
+                    self.holding_chunk = None
 
             if not self.sent_last_message:
                 self.sent_last_message = True

litellm/llms/azure/responses/transformation.py

@@ -1,5 +1,7 @@
 from typing import TYPE_CHECKING, Any, Dict, Literal, Optional, Tuple
 
+import httpx
+
 from litellm._logging import verbose_logger
 from litellm.llms.azure.common_utils import BaseAzureLLM
 from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
@@ -194,3 +196,66 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
             params["order"] = order
         verbose_logger.debug(f"list input items url={url}")
         return url, params
+
+    #########################################################
+    ########## CANCEL RESPONSE API TRANSFORMATION ##########
+    #########################################################
+    def transform_cancel_response_api_request(
+        self,
+        response_id: str,
+        api_base: str,
+        litellm_params: GenericLiteLLMParams,
+        headers: dict,
+    ) -> Tuple[str, Dict]:
+        """
+        Transform the cancel response API request into a URL and data
+
+        Azure OpenAI API expects the following request:
+        - POST /openai/responses/{response_id}/cancel?api-version=xxx
+
+        This function handles URLs with query parameters by inserting the response_id
+        at the correct location (before any query parameters).
+        """
+        from urllib.parse import urlparse, urlunparse
+        
+        # Parse the URL to separate its components
+        parsed_url = urlparse(api_base)
+        
+        # Insert the response_id and /cancel at the end of the path component
+        # Remove trailing slash if present to avoid double slashes
+        path = parsed_url.path.rstrip("/")
+        new_path = f"{path}/{response_id}/cancel"
+        
+        # Reconstruct the URL with all original components but with the modified path
+        cancel_url = urlunparse(
+            (
+                parsed_url.scheme,  # http, https
+                parsed_url.netloc,  # domain name, port
+                new_path,  # path with response_id and /cancel added
+                parsed_url.params,  # parameters
+                parsed_url.query,  # query string
+                parsed_url.fragment,  # fragment
+            )
+        )
+
+        data: Dict = {}
+        verbose_logger.debug(f"cancel response url={cancel_url}")
+        return cancel_url, data
+
+    def transform_cancel_response_api_response(
+        self,
+        raw_response: httpx.Response,
+        logging_obj: LiteLLMLoggingObj,
+    ) -> ResponsesAPIResponse:
+        """
+        Transform the cancel response API response into a ResponsesAPIResponse
+        """
+        try:
+            raw_response_json = raw_response.json()
+        except Exception:
+            from litellm.llms.azure.chat.gpt_transformation import AzureOpenAIError
+
+            raise AzureOpenAIError(
+                message=raw_response.text, status_code=raw_response.status_code
+            )
+        return ResponsesAPIResponse(**raw_response_json)

litellm/llms/base_llm/audio_transcription/transformation.py

@@ -1,6 +1,6 @@
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
+from typing import TYPE_CHECKING, Any, List, Optional, Union
 
 import httpx
 
@@ -23,12 +23,13 @@ else:
 class AudioTranscriptionRequestData:
     """
     Structured data for audio transcription requests.
-    
+
     Attributes:
         data: The request data (form data for multipart, json data for regular requests)
         files: Optional files dict for multipart form data
         content_type: Optional content type override
     """
+
     data: Union[dict, bytes]
     files: Optional[dict] = None
     content_type: Optional[str] = None
@@ -66,13 +67,11 @@ class BaseAudioTranscriptionConfig(BaseConfig, ABC):
         audio_file: FileTypes,
         optional_params: dict,
         litellm_params: dict,
-    ) -> Union[AudioTranscriptionRequestData, Dict]:
+    ) -> AudioTranscriptionRequestData:
         raise NotImplementedError(
             "AudioTranscriptionConfig needs a request transformation for audio transcription models"
         )
 
-
-    
     def transform_audio_transcription_response(
         self,
         raw_response: httpx.Response,
@@ -110,7 +109,6 @@ class BaseAudioTranscriptionConfig(BaseConfig, ABC):
         raise NotImplementedError(
             "AudioTranscriptionConfig does not need a response transformation for audio transcription models"
         )
-    
 
     def get_provider_specific_params(
         self,
@@ -141,7 +139,7 @@ class BaseAudioTranscriptionConfig(BaseConfig, ABC):
             provider_specific_params[key] = value
 
         return provider_specific_params
-    
+
     def _should_exclude_param(
         self,
         param_name: str,

litellm/llms/base_llm/batches/transformation.py

@@ -158,6 +158,48 @@ class BaseBatchesConfig(ABC):
         """
         pass
 
+    @abstractmethod
+    def transform_retrieve_batch_request(
+        self,
+        batch_id: str,
+        optional_params: dict,
+        litellm_params: dict,
+    ) -> Union[bytes, str, Dict[str, Any]]:
+        """
+        Transform the batch retrieval request to provider-specific format.
+        
+        Args:
+            batch_id: Batch ID to retrieve
+            optional_params: Optional parameters
+            litellm_params: LiteLLM parameters
+            
+        Returns:
+            Transformed request data
+        """
+        pass
+
+    @abstractmethod
+    def transform_retrieve_batch_response(
+        self,
+        model: Optional[str],
+        raw_response: httpx.Response,
+        logging_obj: LiteLLMLoggingObj,
+        litellm_params: dict,
+    ) -> LiteLLMBatch:
+        """
+        Transform provider-specific batch retrieval response to LiteLLM format.
+        
+        Args:
+            model: Model name
+            raw_response: Raw HTTP response
+            logging_obj: Logging object
+            litellm_params: LiteLLM parameters
+            
+        Returns:
+            LiteLLM batch object
+        """
+        pass
+
     @abstractmethod
     def get_error_class(
         self, error_message: str, status_code: int, headers: Union[Dict, Headers]

litellm/llms/base_llm/responses/transformation.py

@@ -217,3 +217,28 @@ class BaseResponsesAPIConfig(ABC):
     ) -> bool:
         """Returns True if litellm should fake a stream for the given model and stream value"""
         return False
+
+    #########################################################
+    ########## CANCEL RESPONSE API TRANSFORMATION ##########
+    #########################################################
+    @abstractmethod
+    def transform_cancel_response_api_request(
+        self,
+        response_id: str,
+        api_base: str,
+        litellm_params: GenericLiteLLMParams,
+        headers: dict,
+    ) -> Tuple[str, Dict]:
+        pass
+
+    @abstractmethod
+    def transform_cancel_response_api_response(
+        self,
+        raw_response: httpx.Response,
+        logging_obj: LiteLLMLoggingObj,
+    ) -> ResponsesAPIResponse:
+        pass
+
+    #########################################################
+    ########## END CANCEL RESPONSE API TRANSFORMATION #######
+    #########################################################

litellm/llms/bedrock/base_aws_llm.py

@@ -20,7 +20,11 @@ from pydantic import BaseModel
 
 from litellm._logging import verbose_logger
 from litellm.caching.caching import DualCache
-from litellm.constants import BEDROCK_INVOKE_PROVIDERS_LITERAL, BEDROCK_MAX_POLICY_SIZE
+from litellm.constants import (
+    BEDROCK_EMBEDDING_PROVIDERS_LITERAL,
+    BEDROCK_INVOKE_PROVIDERS_LITERAL,
+    BEDROCK_MAX_POLICY_SIZE,
+)
 from litellm.litellm_core_utils.dd_tracing import tracer
 from litellm.secret_managers.main import get_secret, get_secret_str
 
@@ -66,6 +70,7 @@ class BaseAWSLLM:
             "aws_web_identity_token",
             "aws_sts_endpoint",
             "aws_bedrock_runtime_endpoint",
+            "aws_external_id",
         ]
 
     def get_cache_key(self, credential_args: Dict[str, Optional[str]]) -> str:
@@ -88,6 +93,7 @@ class BaseAWSLLM:
         aws_role_name: Optional[str] = None,
         aws_web_identity_token: Optional[str] = None,
         aws_sts_endpoint: Optional[str] = None,
+        aws_external_id: Optional[str] = None,
     ):
         """
         Return a boto3.Credentials object
@@ -103,6 +109,7 @@ class BaseAWSLLM:
             aws_role_name,
             aws_web_identity_token,
             aws_sts_endpoint,
+            aws_external_id,
         ]
 
         # Iterate over parameters and update if needed
@@ -127,6 +134,7 @@ class BaseAWSLLM:
             aws_role_name,
             aws_web_identity_token,
             aws_sts_endpoint,
+            aws_external_id,
         ) = params_to_check
 
         verbose_logger.debug(
@@ -139,7 +147,8 @@ class BaseAWSLLM:
             "aws_profile_name=%s\n"
             "aws_role_name=%s\n"
             "aws_web_identity_token=%s\n"
-            "aws_sts_endpoint=%s",
+            "aws_sts_endpoint=%s\n"
+            "aws_external_id=%s",
             aws_access_key_id,
             aws_secret_access_key,
             aws_session_token,
@@ -149,6 +158,7 @@ class BaseAWSLLM:
             aws_role_name,
             aws_web_identity_token,
             aws_sts_endpoint,
+            aws_external_id,
         )
 
         # create cache key for non-expiring auth flows
@@ -177,34 +187,45 @@ class BaseAWSLLM:
                 aws_session_name=aws_session_name,
                 aws_region_name=aws_region_name,
                 aws_sts_endpoint=aws_sts_endpoint,
+                aws_external_id=aws_external_id,
             )
         elif aws_role_name is not None:
             # Check if we're in IRSA and trying to assume the same role we already have
             current_role_arn = os.getenv("AWS_ROLE_ARN")
             web_identity_token_file = os.getenv("AWS_WEB_IDENTITY_TOKEN_FILE")
-            
+
             # In IRSA environments, we should skip role assumption if we're already running as the target role
             # This is true when:
             # 1. We have AWS_ROLE_ARN set (current role)
             # 2. We have AWS_WEB_IDENTITY_TOKEN_FILE set (IRSA environment)
             # 3. The current role matches the requested role
-            if (current_role_arn and web_identity_token_file and 
-                current_role_arn == aws_role_name):
-                verbose_logger.debug("Using IRSA same-role optimization: calling _auth_with_env_vars")
+            if (
+                current_role_arn
+                and web_identity_token_file
+                and current_role_arn == aws_role_name
+            ):
+                verbose_logger.debug(
+                    "Using IRSA same-role optimization: calling _auth_with_env_vars"
+                )
                 # We're already running as this role via IRSA, no need to assume it again
                 # Use the default boto3 credentials (which will use the IRSA credentials)
                 credentials, _cache_ttl = self._auth_with_env_vars()
             else:
-                verbose_logger.debug("Using role assumption: calling _auth_with_aws_role")
+                verbose_logger.debug(
+                    "Using role assumption: calling _auth_with_aws_role"
+                )
                 # If aws_session_name is not provided, generate a default one
                 if aws_session_name is None:
-                    aws_session_name = f"litellm-session-{int(datetime.now().timestamp())}"
+                    aws_session_name = (
+                        f"litellm-session-{int(datetime.now().timestamp())}"
+                    )
                 credentials, _cache_ttl = self._auth_with_aws_role(
                     aws_access_key_id=aws_access_key_id,
                     aws_secret_access_key=aws_secret_access_key,
                     aws_session_token=aws_session_token,
                     aws_role_name=aws_role_name,
                     aws_session_name=aws_session_name,
+                    aws_external_id=aws_external_id,
                 )
 
         elif aws_profile_name is not None:  ### CHECK SESSION ###
@@ -310,6 +331,40 @@ class BaseAWSLLM:
                     return provider
         return None
 
+    @staticmethod
+    def get_bedrock_embedding_provider(
+        model: str,
+    ) -> Optional[BEDROCK_EMBEDDING_PROVIDERS_LITERAL]:
+        """
+        Helper function to get the bedrock embedding provider from the model
+        
+        Handles scenarios like:
+        1. model=cohere.embed-english-v3:0 -> Returns `cohere`
+        2. model=amazon.titan-embed-text-v1 -> Returns `amazon`  
+        3. model=us.twelvelabs.marengo-embed-2-7-v1:0 -> Returns `twelvelabs`
+        4. model=twelvelabs.marengo-embed-2-7-v1:0 -> Returns `twelvelabs`
+        """
+        # Handle regional models like us.twelvelabs.marengo-embed-2-7-v1:0
+        if "." in model:
+            parts = model.split(".")
+            # Check if the second part (after potential region) is a known provider
+            if len(parts) >= 2:
+                potential_provider = parts[1]  # e.g., "twelvelabs" from "us.twelvelabs.marengo-embed-2-7-v1:0"
+                if potential_provider in get_args(BEDROCK_EMBEDDING_PROVIDERS_LITERAL):
+                    return cast(BEDROCK_EMBEDDING_PROVIDERS_LITERAL, potential_provider)
+            
+            # Check if the first part is a known provider (standard format)
+            potential_provider = parts[0]  # e.g., "cohere" from "cohere.embed-english-v3:0"
+            if potential_provider in get_args(BEDROCK_EMBEDDING_PROVIDERS_LITERAL):
+                return cast(BEDROCK_EMBEDDING_PROVIDERS_LITERAL, potential_provider)
+        
+        # Fallback: check if any provider name appears in the model string
+        for provider in get_args(BEDROCK_EMBEDDING_PROVIDERS_LITERAL):
+            if provider in model:
+                return cast(BEDROCK_EMBEDDING_PROVIDERS_LITERAL, provider)
+        
+        return None
+
     def _get_aws_region_name(
         self,
         optional_params: dict,
@@ -406,6 +461,7 @@ class BaseAWSLLM:
         aws_session_name: str,
         aws_region_name: Optional[str],
         aws_sts_endpoint: Optional[str],
+        aws_external_id: Optional[str] = None,
     ) -> Tuple[Credentials, Optional[int]]:
         """
         Authenticate with AWS Web Identity Token
@@ -438,13 +494,19 @@ class BaseAWSLLM:
 
         # https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html
         # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts/client/assume_role_with_web_identity.html
-        sts_response = sts_client.assume_role_with_web_identity(
-            RoleArn=aws_role_name,
-            RoleSessionName=aws_session_name,
-            WebIdentityToken=oidc_token,
-            DurationSeconds=3600,
-            Policy='{"Version":"2012-10-17","Statement":[{"Sid":"BedrockLiteLLM","Effect":"Allow","Action":["bedrock:InvokeModel","bedrock:InvokeModelWithResponseStream"],"Resource":"*","Condition":{"Bool":{"aws:SecureTransport":"true"},"StringLike":{"aws:UserAgent":"litellm/*"}}}]}',
-        )
+        assume_role_params = {
+            "RoleArn": aws_role_name,
+            "RoleSessionName": aws_session_name,
+            "WebIdentityToken": oidc_token,
+            "DurationSeconds": 3600,
+            "Policy": '{"Version":"2012-10-17","Statement":[{"Sid":"BedrockLiteLLM","Effect":"Allow","Action":["bedrock:InvokeModel","bedrock:InvokeModelWithResponseStream"],"Resource":"*","Condition":{"Bool":{"aws:SecureTransport":"true"},"StringLike":{"aws:UserAgent":"litellm/*"}}}]}',
+        }
+
+        # Add ExternalId parameter if provided
+        if aws_external_id is not None:
+            assume_role_params["ExternalId"] = aws_external_id
+
+        sts_response = sts_client.assume_role_with_web_identity(**assume_role_params)
 
         iam_creds_dict = {
             "aws_access_key_id": sts_response["Credentials"]["AccessKeyId"],
@@ -464,90 +526,131 @@ class BaseAWSLLM:
         iam_creds = session.get_credentials()
         return iam_creds, self._get_default_ttl_for_boto3_credentials()
 
-    def _handle_irsa_cross_account(self, irsa_role_arn: str, aws_role_name: str, 
-                                   aws_session_name: str, region: str, web_identity_token_file: str) -> dict:
+    def _handle_irsa_cross_account(
+        self,
+        irsa_role_arn: str,
+        aws_role_name: str,
+        aws_session_name: str,
+        region: str,
+        web_identity_token_file: str,
+        aws_external_id: Optional[str] = None,
+    ) -> dict:
         """Handle cross-account role assumption for IRSA."""
         import boto3
-        
+
         verbose_logger.debug("Cross-account role assumption detected")
-        
+
         # Read the web identity token
-        with open(web_identity_token_file, 'r') as f:
+        with open(web_identity_token_file, "r") as f:
             web_identity_token = f.read().strip()
-        
+
         # Create an STS client without credentials
         with tracer.trace("boto3.client(sts) for manual IRSA"):
-            sts_client = boto3.client('sts', region_name=region)
-        
+            sts_client = boto3.client("sts", region_name=region)
+
         # Manually assume the IRSA role with the session name
-        verbose_logger.debug(f"Manually assuming IRSA role {irsa_role_arn} with session {aws_session_name}")
+        verbose_logger.debug(
+            f"Manually assuming IRSA role {irsa_role_arn} with session {aws_session_name}"
+        )
         irsa_response = sts_client.assume_role_with_web_identity(
             RoleArn=irsa_role_arn,
             RoleSessionName=aws_session_name,
-            WebIdentityToken=web_identity_token
+            WebIdentityToken=web_identity_token,
         )
-        
+
         # Extract the credentials from the IRSA assumption
         irsa_creds = irsa_response["Credentials"]
-        
+
         # Create a new STS client with the IRSA credentials
         with tracer.trace("boto3.client(sts) with manual IRSA credentials"):
             sts_client_with_creds = boto3.client(
-                'sts',
+                "sts",
                 region_name=region,
                 aws_access_key_id=irsa_creds["AccessKeyId"],
                 aws_secret_access_key=irsa_creds["SecretAccessKey"],
-                aws_session_token=irsa_creds["SessionToken"]
+                aws_session_token=irsa_creds["SessionToken"],
             )
-        
+
         # Get current caller identity for debugging
         try:
             caller_identity = sts_client_with_creds.get_caller_identity()
-            verbose_logger.debug(f"Current identity after manual IRSA assumption: {caller_identity.get('Arn', 'unknown')}")
+            verbose_logger.debug(
+                f"Current identity after manual IRSA assumption: {caller_identity.get('Arn', 'unknown')}"
+            )
         except Exception as e:
             verbose_logger.debug(f"Failed to get caller identity: {e}")
-        
-        # Now assume the target role
-        verbose_logger.debug(f"Attempting to assume target role: {aws_role_name} with session: {aws_session_name}")
-        return sts_client_with_creds.assume_role(
-            RoleArn=aws_role_name, RoleSessionName=aws_session_name
-        )
 
-    def _handle_irsa_same_account(self, aws_role_name: str, aws_session_name: str, region: str) -> dict:
+        # Now assume the target role
+        verbose_logger.debug(
+            f"Attempting to assume target role: {aws_role_name} with session: {aws_session_name}"
+        )
+        assume_role_params = {
+            "RoleArn": aws_role_name,
+            "RoleSessionName": aws_session_name,
+        }
+
+        # Add ExternalId parameter if provided
+        if aws_external_id is not None:
+            assume_role_params["ExternalId"] = aws_external_id
+
+        return sts_client_with_creds.assume_role(**assume_role_params)
+
+    def _handle_irsa_same_account(
+        self,
+        aws_role_name: str,
+        aws_session_name: str,
+        region: str,
+        aws_external_id: Optional[str] = None,
+    ) -> dict:
         """Handle same-account role assumption for IRSA."""
         import boto3
-        
+
         verbose_logger.debug("Same account role assumption, using automatic IRSA")
         with tracer.trace("boto3.client(sts) with automatic IRSA"):
             sts_client = boto3.client("sts", region_name=region)
-        
+
         # Get current caller identity for debugging
         try:
             caller_identity = sts_client.get_caller_identity()
-            verbose_logger.debug(f"Current IRSA identity: {caller_identity.get('Arn', 'unknown')}")
+            verbose_logger.debug(
+                f"Current IRSA identity: {caller_identity.get('Arn', 'unknown')}"
+            )
         except Exception as e:
             verbose_logger.debug(f"Failed to get caller identity: {e}")
-        
-        # Assume the role
-        verbose_logger.debug(f"Attempting to assume role: {aws_role_name} with session: {aws_session_name}")
-        return sts_client.assume_role(
-            RoleArn=aws_role_name, RoleSessionName=aws_session_name
-        )
 
-    def _extract_credentials_and_ttl(self, sts_response: dict) -> Tuple[Credentials, Optional[int]]:
+        # Assume the role
+        verbose_logger.debug(
+            f"Attempting to assume role: {aws_role_name} with session: {aws_session_name}"
+        )
+        assume_role_params = {
+            "RoleArn": aws_role_name,
+            "RoleSessionName": aws_session_name,
+        }
+
+        # Add ExternalId parameter if provided
+        if aws_external_id is not None:
+            assume_role_params["ExternalId"] = aws_external_id
+
+        return sts_client.assume_role(**assume_role_params)
+
+    def _extract_credentials_and_ttl(
+        self, sts_response: dict
+    ) -> Tuple[Credentials, Optional[int]]:
         """Extract credentials and TTL from STS response."""
         from botocore.credentials import Credentials
-        
+
         sts_credentials = sts_response["Credentials"]
         credentials = Credentials(
             access_key=sts_credentials["AccessKeyId"],
             secret_key=sts_credentials["SecretAccessKey"],
             token=sts_credentials["SessionToken"],
         )
-        
+
         expiration_time = sts_credentials["Expiration"]
-        ttl = int((expiration_time - datetime.now(expiration_time.tzinfo)).total_seconds())
-        
+        ttl = int(
+            (expiration_time - datetime.now(expiration_time.tzinfo)).total_seconds()
+        )
+
         return credentials, ttl
 
     @tracer.wrap()
@@ -558,6 +661,7 @@ class BaseAWSLLM:
         aws_session_token: Optional[str],
         aws_role_name: str,
         aws_session_name: str,
+        aws_external_id: Optional[str] = None,
     ) -> Tuple[Credentials, Optional[int]]:
         """
         Authenticate with AWS Role
@@ -568,34 +672,51 @@ class BaseAWSLLM:
         # Check if we're in an EKS/IRSA environment
         web_identity_token_file = os.getenv("AWS_WEB_IDENTITY_TOKEN_FILE")
         irsa_role_arn = os.getenv("AWS_ROLE_ARN")
-        
+
         # If we have IRSA environment variables and no explicit credentials,
         # we need to use the web identity token flow
-        if (web_identity_token_file and irsa_role_arn and 
-            aws_access_key_id is None and aws_secret_access_key is None):
+        if (
+            web_identity_token_file
+            and irsa_role_arn
+            and aws_access_key_id is None
+            and aws_secret_access_key is None
+        ):
             # For cross-account role assumption with specific session names,
             # we need to manually assume the IRSA role first with the correct session name
-            verbose_logger.debug(f"IRSA detected: using web identity token from {web_identity_token_file}")
-            
+            verbose_logger.debug(
+                f"IRSA detected: using web identity token from {web_identity_token_file}"
+            )
+
             try:
                 # Get region from environment
-                region = os.getenv("AWS_REGION") or os.getenv("AWS_DEFAULT_REGION") or "us-east-1"
-                
+                region = (
+                    os.getenv("AWS_REGION")
+                    or os.getenv("AWS_DEFAULT_REGION")
+                    or "us-east-1"
+                )
+
                 # Check if we need to do cross-account role assumption
                 if aws_role_name != irsa_role_arn:
                     sts_response = self._handle_irsa_cross_account(
-                        irsa_role_arn, aws_role_name, aws_session_name, region, web_identity_token_file
+                        irsa_role_arn,
+                        aws_role_name,
+                        aws_session_name,
+                        region,
+                        web_identity_token_file,
+                        aws_external_id,
                     )
                 else:
                     sts_response = self._handle_irsa_same_account(
-                        aws_role_name, aws_session_name, region
+                        aws_role_name, aws_session_name, region, aws_external_id
                     )
-                
+
                 return self._extract_credentials_and_ttl(sts_response)
-                
+
             except Exception as e:
                 verbose_logger.debug(f"Failed to assume role via IRSA: {e}")
-                if "AccessDenied" in str(e) and "is not authorized to perform: sts:AssumeRole" in str(e):
+                if "AccessDenied" in str(
+                    e
+                ) and "is not authorized to perform: sts:AssumeRole" in str(e):
                     # Provide a more helpful error message for trust policy issues
                     verbose_logger.error(
                         f"Access denied when trying to assume role {aws_role_name}. "
@@ -604,7 +725,7 @@ class BaseAWSLLM:
                     )
                 # Re-raise the exception instead of falling through
                 raise
-        
+
         # In EKS/IRSA environments, use ambient credentials (no explicit keys needed)
         # This allows the web identity token to work automatically
         if aws_access_key_id is None and aws_secret_access_key is None:
@@ -619,9 +740,16 @@ class BaseAWSLLM:
                     aws_session_token=aws_session_token,
                 )
 
-        sts_response = sts_client.assume_role(
-            RoleArn=aws_role_name, RoleSessionName=aws_session_name
-        )
+        assume_role_params = {
+            "RoleArn": aws_role_name,
+            "RoleSessionName": aws_session_name,
+        }
+
+        # Add ExternalId parameter if provided
+        if aws_external_id is not None:
+            assume_role_params["ExternalId"] = aws_external_id
+
+        sts_response = sts_client.assume_role(**assume_role_params)
 
         # Extract the credentials from the response and convert to Session Credentials
         sts_credentials = sts_response["Credentials"]
@@ -743,14 +871,14 @@ class BaseAWSLLM:
             )
 
         # Determine proxy_endpoint_url
-        if env_aws_bedrock_runtime_endpoint and isinstance(
-            env_aws_bedrock_runtime_endpoint, str
-        ):
-            proxy_endpoint_url = env_aws_bedrock_runtime_endpoint
-        elif aws_bedrock_runtime_endpoint is not None and isinstance(
+        if aws_bedrock_runtime_endpoint is not None and isinstance(
             aws_bedrock_runtime_endpoint, str
         ):
             proxy_endpoint_url = aws_bedrock_runtime_endpoint
+        elif env_aws_bedrock_runtime_endpoint and isinstance(
+            env_aws_bedrock_runtime_endpoint, str
+        ):
+            proxy_endpoint_url = env_aws_bedrock_runtime_endpoint
         else:
             proxy_endpoint_url = endpoint_url
 
@@ -800,6 +928,7 @@ class BaseAWSLLM:
         aws_bedrock_runtime_endpoint = optional_params.pop(
             "aws_bedrock_runtime_endpoint", None
         )  # https://bedrock-runtime.{region_name}.amazonaws.com
+        aws_external_id = optional_params.pop("aws_external_id", None)
 
         credentials: Credentials = self.get_credentials(
             aws_access_key_id=aws_access_key_id,
@@ -811,6 +940,7 @@ class BaseAWSLLM:
             aws_role_name=aws_role_name,
             aws_web_identity_token=aws_web_identity_token,
             aws_sts_endpoint=aws_sts_endpoint,
+            aws_external_id=aws_external_id,
         )
 
         return Boto3CredentialsInfo(
@@ -915,6 +1045,7 @@ class BaseAWSLLM:
         aws_profile_name = optional_params.get("aws_profile_name", None)
         aws_web_identity_token = optional_params.get("aws_web_identity_token", None)
         aws_sts_endpoint = optional_params.get("aws_sts_endpoint", None)
+        aws_external_id = optional_params.get("aws_external_id", None)
         aws_region_name = self._get_aws_region_name(
             optional_params=optional_params, model=model
         )
@@ -929,6 +1060,7 @@ class BaseAWSLLM:
             aws_role_name=aws_role_name,
             aws_web_identity_token=aws_web_identity_token,
             aws_sts_endpoint=aws_sts_endpoint,
+            aws_external_id=aws_external_id,
         )
 
         sigv4 = SigV4Auth(credentials, service_name, aws_region_name)

litellm/llms/bedrock/batches/transformation.py

@@ -7,7 +7,6 @@ from httpx import Headers, Response
 from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.types.llms.bedrock import (
-    BedrockBatchJobStatus,
     BedrockCreateBatchRequest,
     BedrockCreateBatchResponse,
     BedrockInputDataConfig,
@@ -124,15 +123,13 @@ class BedrockBatchesConfig(BaseAWSLLM, BaseBatchesConfig):
                 "AWS IAM role ARN is required for Bedrock batch jobs. "
                 "Set 'aws_batch_role_arn' in litellm_params or AWS_BATCH_ROLE_ARN env var"
             )
+
         
-        # Get the actual Bedrock model ID using common utility
-        bedrock_model_id = self.common_utils.extract_model_from_s3_file_path(input_file_id, optional_params)
-        
-        if not bedrock_model_id:
-            raise ValueError("Could not determine Bedrock model ID. Ensure the model is specified in the input file or passed as a parameter.")
+        if not model:
+            raise ValueError("Could not determine Bedrock model ID. Please pass `model` in your request body.")
         
         # Generate job name with the correct model ID using common utility
-        job_name = self.common_utils.generate_unique_job_name(bedrock_model_id, prefix="litellm")
+        job_name = self.common_utils.generate_unique_job_name(model, prefix="litellm")
         output_key = f"litellm-batch-outputs/{job_name}/"
         
         # Build input data config
@@ -151,7 +148,7 @@ class BedrockBatchesConfig(BaseAWSLLM, BaseBatchesConfig):
         
         # Create Bedrock batch request with proper typing
         bedrock_request: BedrockCreateBatchRequest = {
-            "modelId": bedrock_model_id,
+            "modelId": model,
             "jobName": job_name,
             "inputDataConfig": input_data_config,
             "outputDataConfig": output_data_config,
@@ -202,19 +199,23 @@ class BedrockBatchesConfig(BaseAWSLLM, BaseBatchesConfig):
         
         # Extract information from typed Bedrock response
         job_arn = response_data.get("jobArn", "")
-        status: BedrockBatchJobStatus = response_data.get("status", "Submitted")
+        status_str: str = str(response_data.get("status", "Submitted"))
         
         # Map Bedrock status to OpenAI-compatible status
-        status_mapping: Dict[BedrockBatchJobStatus, str] = {
+        status_mapping: Dict[str, str] = {
             "Submitted": "validating",
+            "Validating": "validating",
+            "Scheduled": "in_progress",
             "InProgress": "in_progress", 
+            "PartiallyCompleted": "completed",
             "Completed": "completed",
             "Failed": "failed",
             "Stopping": "cancelling",
-            "Stopped": "cancelled"
+            "Stopped": "cancelled",
+            "Expired": "expired",
         }
         
-        openai_status = cast(Literal["validating", "failed", "in_progress", "finalizing", "completed", "expired", "cancelling", "cancelled"], status_mapping.get(status, "validating"))
+        openai_status = cast(Literal["validating", "failed", "in_progress", "finalizing", "completed", "expired", "cancelling", "cancelled"], status_mapping.get(status_str, "validating"))
         
         # Get original request data from litellm_params if available
         original_request = litellm_params.get("original_batch_request", {})
@@ -231,7 +232,7 @@ class BedrockBatchesConfig(BaseAWSLLM, BaseBatchesConfig):
             output_file_id=None,  # Will be populated when job completes
             error_file_id=None,
             created_at=int(time.time()),
-            in_progress_at=int(time.time()) if status == "InProgress" else None,
+            in_progress_at=int(time.time()) if status_str == "InProgress" else None,
             expires_at=None,
             finalizing_at=None,
             completed_at=None,
@@ -243,6 +244,203 @@ class BedrockBatchesConfig(BaseAWSLLM, BaseBatchesConfig):
             metadata=original_request.get("metadata", {}),
         )
 
+    def transform_retrieve_batch_request(
+        self,
+        batch_id: str,
+        optional_params: dict,
+        litellm_params: dict,
+    ) -> Dict[str, Any]:
+        """
+        Transform batch retrieval request for Bedrock.
+        
+        Args:
+            batch_id: Bedrock job ARN
+            optional_params: Optional parameters
+            litellm_params: LiteLLM parameters
+            
+        Returns:
+            Transformed request data for Bedrock GetModelInvocationJob API
+        """
+        # For Bedrock, batch_id should be the full job ARN
+        # The GetModelInvocationJob API expects the full ARN as the identifier
+        if not batch_id.startswith("arn:aws:bedrock:"):
+            raise ValueError(f"Invalid batch_id format. Expected ARN, got: {batch_id}")
+        
+        # Extract the job identifier from the ARN - use the full ARN path part
+        # ARN format: arn:aws:bedrock:region:account:model-invocation-job/job-name
+        arn_parts = batch_id.split(":")
+        if len(arn_parts) < 6:
+            raise ValueError(f"Invalid ARN format: {batch_id}")
+        
+        region = arn_parts[3]
+        # arn_parts[5] contains "model-invocation-job/{jobId}"
+        
+        # Build the endpoint URL for GetModelInvocationJob
+        # AWS API format: GET /model-invocation-job/{jobIdentifier}
+        # Use the FULL ARN as jobIdentifier and URL-encode it (includes ':' and '/')
+        import urllib.parse as _ul
+        encoded_arn = _ul.quote(batch_id, safe="")
+        endpoint_url = f"https://bedrock.{region}.amazonaws.com/model-invocation-job/{encoded_arn}"
+        
+        # Use common utility for AWS signing
+        signed_headers, _ = self.common_utils.sign_aws_request(
+            service_name="bedrock",
+            data={},  # GET request has no body
+            endpoint_url=endpoint_url,
+            optional_params=optional_params,
+            method="GET"
+        )
+        
+        # Return pre-signed request format
+        return {
+            "method": "GET",
+            "url": endpoint_url,
+            "headers": signed_headers,
+            "data": None
+        }
+
+    def _parse_timestamps_and_status(self, response_data, status_str: str):
+        """Helper to parse timestamps based on status."""
+        import datetime
+        def parse_timestamp(ts_str: Optional[str]) -> Optional[int]:
+            if not ts_str:
+                return None
+            try:
+                dt = datetime.datetime.fromisoformat(ts_str.replace('Z', '+00:00'))
+                return int(dt.timestamp())
+            except Exception:
+                return None
+        
+        created_at = parse_timestamp(str(response_data.get("submitTime")) if response_data.get("submitTime") is not None else None)
+        in_progress_states = {"InProgress", "Validating", "Scheduled"}
+        in_progress_at = (
+            parse_timestamp(str(response_data.get("lastModifiedTime")) if response_data.get("lastModifiedTime") is not None else None)
+            if status_str in in_progress_states
+            else None
+        )
+        completed_at = parse_timestamp(str(response_data.get("endTime")) if response_data.get("endTime") is not None else None) if status_str in {"Completed", "PartiallyCompleted"} else None
+        failed_at = parse_timestamp(str(response_data.get("endTime")) if response_data.get("endTime") is not None else None) if status_str == "Failed" else None
+        cancelled_at = parse_timestamp(str(response_data.get("endTime")) if response_data.get("endTime") is not None else None) if status_str == "Stopped" else None
+        expires_at = parse_timestamp(str(response_data.get("jobExpirationTime")) if response_data.get("jobExpirationTime") is not None else None)
+        
+        return created_at, in_progress_at, completed_at, failed_at, cancelled_at, expires_at
+    
+    def _extract_file_configs(self, response_data):
+        """Helper to extract input and output file configurations."""
+        # Extract input file ID
+        input_file_id = ""
+        input_data_config = response_data.get("inputDataConfig", {})
+        if isinstance(input_data_config, dict):
+            s3_input_config = input_data_config.get("s3InputDataConfig", {})
+            if isinstance(s3_input_config, dict):
+                input_file_id = s3_input_config.get("s3Uri", "")
+        
+        # Extract output file ID
+        output_file_id = None
+        output_data_config = response_data.get("outputDataConfig", {})
+        if isinstance(output_data_config, dict):
+            s3_output_config = output_data_config.get("s3OutputDataConfig", {})
+            if isinstance(s3_output_config, dict):
+                output_file_id = s3_output_config.get("s3Uri", "")
+        
+        return input_file_id, output_file_id
+    
+    def _extract_errors_and_metadata(self, response_data, raw_response):
+        """Helper to extract errors and enriched metadata."""
+        # Extract errors
+        message = response_data.get("message")
+        errors = None
+        if message:
+            from openai.types.batch import Errors
+            from openai.types.batch_error import BatchError
+            errors = Errors(
+                data=[BatchError(message=message, code=str(raw_response.status_code))],
+                object="list"
+            )
+        
+        # Enrich metadata with useful Bedrock fields
+        enriched_metadata_raw: Dict[str, Any] = {
+            "jobName": response_data.get("jobName"),
+            "clientRequestToken": response_data.get("clientRequestToken"),
+            "modelId": response_data.get("modelId"),
+            "roleArn": response_data.get("roleArn"),
+            "timeoutDurationInHours": response_data.get("timeoutDurationInHours"),
+            "vpcConfig": response_data.get("vpcConfig"),
+        }
+        import json as _json
+        enriched_metadata: Dict[str, str] = {}
+        for _k, _v in enriched_metadata_raw.items():
+            if _v is None:
+                continue
+            if isinstance(_v, (dict, list)):
+                try:
+                    enriched_metadata[_k] = _json.dumps(_v)
+                except Exception:
+                    enriched_metadata[_k] = str(_v)
+            else:
+                enriched_metadata[_k] = str(_v)
+        
+        return errors, enriched_metadata
+
+    def transform_retrieve_batch_response(
+        self,
+        model: Optional[str],
+        raw_response: Response,
+        logging_obj: Any,
+        litellm_params: dict,
+    ) -> LiteLLMBatch:
+        """
+        Transform Bedrock batch retrieval response to LiteLLM format.
+        """
+        from litellm.types.llms.bedrock import BedrockGetBatchResponse
+        try:
+            response_data: BedrockGetBatchResponse = raw_response.json()
+        except Exception as e:
+            raise ValueError(f"Failed to parse Bedrock batch response: {e}")
+        
+        job_arn = response_data.get("jobArn", "")
+        status_str: str = str(response_data.get("status", "Submitted"))
+        
+        # Map Bedrock status to OpenAI-compatible status
+        status_mapping: Dict[str, str] = {
+            "Submitted": "validating", "Validating": "validating", "Scheduled": "in_progress",
+            "InProgress": "in_progress", "PartiallyCompleted": "completed", "Completed": "completed",
+            "Failed": "failed", "Stopping": "cancelling", "Stopped": "cancelled", "Expired": "expired"
+        }
+        openai_status = cast(Literal["validating", "failed", "in_progress", "finalizing", "completed", "expired", "cancelling", "cancelled"], status_mapping.get(status_str, "validating"))
+        
+        # Parse timestamps
+        created_at, in_progress_at, completed_at, failed_at, cancelled_at, expires_at = self._parse_timestamps_and_status(response_data, status_str)
+        
+        # Extract file configurations
+        input_file_id, output_file_id = self._extract_file_configs(response_data)
+        
+        # Extract errors and metadata
+        errors, enriched_metadata = self._extract_errors_and_metadata(response_data, raw_response)
+                
+        return LiteLLMBatch(
+            id=job_arn,
+            object="batch",
+            endpoint="/v1/chat/completions",
+            errors=errors,
+            input_file_id=input_file_id,
+            completion_window="24h",
+            status=openai_status,
+            output_file_id=output_file_id,
+            error_file_id=None,
+            created_at=created_at or int(time.time()),
+            in_progress_at=in_progress_at,
+            expires_at=expires_at,
+            finalizing_at=None,
+            completed_at=completed_at,
+            failed_at=failed_at,
+            expired_at=None,
+            cancelling_at=None,
+            cancelled_at=cancelled_at,
+            request_counts=None,
+            metadata=enriched_metadata,
+        )
+
     def get_error_class(
         self, error_message: str, status_code: int, headers: Union[Dict, Headers]
     ) -> BaseLLMException:

litellm/llms/bedrock/chat/converse_handler.py

@@ -307,6 +307,7 @@ class BedrockConverseLLM(BaseAWSLLM):
         )  # https://bedrock-runtime.{region_name}.amazonaws.com
         aws_web_identity_token = optional_params.pop("aws_web_identity_token", None)
         aws_sts_endpoint = optional_params.pop("aws_sts_endpoint", None)
+        aws_external_id = optional_params.pop("aws_external_id", None)
         optional_params.pop("aws_region_name", None)
 
         litellm_params[
@@ -323,6 +324,7 @@ class BedrockConverseLLM(BaseAWSLLM):
             aws_role_name=aws_role_name,
             aws_web_identity_token=aws_web_identity_token,
             aws_sts_endpoint=aws_sts_endpoint,
+            aws_external_id=aws_external_id,
         )
 
         ### SET RUNTIME ENDPOINT ###

litellm/llms/bedrock/chat/converse_transformation.py

@@ -14,7 +14,7 @@ from litellm._logging import verbose_logger
 from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
 from litellm.litellm_core_utils.core_helpers import map_finish_reason
 from litellm.litellm_core_utils.litellm_logging import Logging
-from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
+from litellm.litellm_core_utils.prompt_templates.common_utils import (
     _parse_content_for_reasoning,
 )
 from litellm.litellm_core_utils.prompt_templates.factory import (
@@ -102,6 +102,61 @@ class AmazonConverseConfig(BaseConfig):
             "performanceConfig": PerformanceConfigBlock,
         }
 
+    @staticmethod
+    def _convert_consecutive_user_messages_to_guarded_text(
+        messages: List[AllMessageValues], optional_params: dict
+    ) -> List[AllMessageValues]:
+        """
+        Convert consecutive user messages at the end to guarded_text type if guardrailConfig is present
+        and no guarded_text is already present in those messages.
+        """
+        # Check if guardrailConfig is present
+        if "guardrailConfig" not in optional_params:
+            return messages
+
+        # Find all consecutive user messages at the end
+        consecutive_user_message_indices = []
+        for i in range(len(messages) - 1, -1, -1):
+            if messages[i].get("role") == "user":
+                consecutive_user_message_indices.append(i)
+            else:
+                break
+
+        if not consecutive_user_message_indices:
+            return messages
+
+        # Process each consecutive user message
+        messages_copy = copy.deepcopy(messages)
+        for user_message_index in consecutive_user_message_indices:
+            user_message = messages_copy[user_message_index]
+            content = user_message.get("content", [])
+
+            if isinstance(content, list):
+                has_guarded_text = any(
+                    isinstance(item, dict) and item.get("type") == "guarded_text"
+                    for item in content
+                )
+                if has_guarded_text:
+                    continue  # Skip this message if it already has guarded_text
+
+                # Convert text elements to guarded_text
+                new_content = []
+                for item in content:
+                    if isinstance(item, dict) and item.get("type") == "text":
+                        new_item = {"type": "guarded_text", "text": item["text"]}  # type: ignore
+                        new_content.append(new_item)
+                    else:
+                        new_content.append(item)
+
+                messages_copy[user_message_index]["content"] = new_content  # type: ignore
+            elif isinstance(content, str):
+                # If content is a string, convert it to guarded_text
+                messages_copy[user_message_index]["content"] = [  # type: ignore
+                    {"type": "guarded_text", "text": content}  # type: ignore
+                ]
+
+        return messages_copy
+
     @classmethod
     def get_config(cls):
         return {
@@ -397,7 +452,11 @@ class AmazonConverseConfig(BaseConfig):
         for param, value in non_default_params.items():
             if param == "response_format" and isinstance(value, dict):
                 optional_params = self._translate_response_format_param(
-                    value=value, model=model, optional_params=optional_params, non_default_params=non_default_params, is_thinking_enabled=is_thinking_enabled
+                    value=value,
+                    model=model,
+                    optional_params=optional_params,
+                    non_default_params=non_default_params,
+                    is_thinking_enabled=is_thinking_enabled,
                 )
             if param == "max_tokens" or param == "max_completion_tokens":
                 optional_params["maxTokens"] = value
@@ -446,11 +505,11 @@ class AmazonConverseConfig(BaseConfig):
             )
 
         return optional_params
-    
+
     def _translate_response_format_param(
-        self, 
-        value: dict, 
-        model: str, 
+        self,
+        value: dict,
+        model: str,
         optional_params: dict,
         non_default_params: dict,
         is_thinking_enabled: bool,
@@ -497,14 +556,13 @@ class AmazonConverseConfig(BaseConfig):
             )
             and not is_thinking_enabled
         ):
-
             optional_params["tool_choice"] = ToolChoiceValuesBlock(
                 tool=SpecificToolChoiceBlock(name=RESPONSE_FORMAT_TOOL_NAME)
             )
         optional_params["json_mode"] = True
         if non_default_params.get("stream", False) is True:
             optional_params["fake_stream"] = True
-        
+
         return optional_params
 
     def update_optional_params_with_thinking_tokens(
@@ -766,6 +824,11 @@ class AmazonConverseConfig(BaseConfig):
         headers: Optional[dict] = None,
     ) -> RequestObject:
         messages, system_content_blocks = self._transform_system_message(messages)
+
+        # Convert last user message to guarded_text if guardrailConfig is present
+        messages = self._convert_consecutive_user_messages_to_guarded_text(
+            messages, optional_params
+        )
         ## TRANSFORMATION ##
 
         _data: CommonRequestObject = self._transform_request_helper(
@@ -818,6 +881,11 @@ class AmazonConverseConfig(BaseConfig):
     ) -> RequestObject:
         messages, system_content_blocks = self._transform_system_message(messages)
 
+        # Convert last user message to guarded_text if guardrailConfig is present
+        messages = self._convert_consecutive_user_messages_to_guarded_text(
+            messages, optional_params
+        )
+
         _data: CommonRequestObject = self._transform_request_helper(
             model=model,
             system_content_blocks=system_content_blocks,
@@ -991,7 +1059,9 @@ class AmazonConverseConfig(BaseConfig):
 
         return message, returned_finish_reason
 
-    def _translate_message_content(self, content_blocks: List[ContentBlock]) -> Tuple[
+    def _translate_message_content(
+        self, content_blocks: List[ContentBlock]
+    ) -> Tuple[
         str,
         List[ChatCompletionToolCallChunk],
         Optional[List[BedrockConverseReasoningContentBlock]],
@@ -1006,9 +1076,9 @@ class AmazonConverseConfig(BaseConfig):
         """
         content_str = ""
         tools: List[ChatCompletionToolCallChunk] = []
-        reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]] = (
-            None
-        )
+        reasoningContentBlocks: Optional[
+            List[BedrockConverseReasoningContentBlock]
+        ] = None
         for idx, content in enumerate(content_blocks):
             """
             - Content is either a tool response or text
@@ -1129,9 +1199,9 @@ class AmazonConverseConfig(BaseConfig):
         chat_completion_message: ChatCompletionResponseMessage = {"role": "assistant"}
         content_str = ""
         tools: List[ChatCompletionToolCallChunk] = []
-        reasoningContentBlocks: Optional[List[BedrockConverseReasoningContentBlock]] = (
-            None
-        )
+        reasoningContentBlocks: Optional[
+            List[BedrockConverseReasoningContentBlock]
+        ] = None
 
         if message is not None:
             (
@@ -1144,12 +1214,12 @@ class AmazonConverseConfig(BaseConfig):
             chat_completion_message["provider_specific_fields"] = {
                 "reasoningContentBlocks": reasoningContentBlocks,
             }
-            chat_completion_message["reasoning_content"] = (
-                self._transform_reasoning_content(reasoningContentBlocks)
-            )
-            chat_completion_message["thinking_blocks"] = (
-                self._transform_thinking_blocks(reasoningContentBlocks)
-            )
+            chat_completion_message[
+                "reasoning_content"
+            ] = self._transform_reasoning_content(reasoningContentBlocks)
+            chat_completion_message[
+                "thinking_blocks"
+            ] = self._transform_thinking_blocks(reasoningContentBlocks)
         chat_completion_message["content"] = content_str
         if (
             json_mode is True
@@ -1167,7 +1237,6 @@ class AmazonConverseConfig(BaseConfig):
                 # Bedrock returns the response wrapped in a "properties" object
                 # We need to extract the actual content from this wrapper
                 try:
-
                     response_data = json.loads(json_mode_content_str)
 
                     # If Bedrock wrapped the response in "properties", extract the content

litellm/llms/bedrock/chat/invoke_transformations/amazon_deepseek_transformation.py

@@ -3,7 +3,7 @@ from typing import Any, List, Optional, cast
 from httpx import Response
 
 from litellm import verbose_logger
-from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
+from litellm.litellm_core_utils.prompt_templates.common_utils import (
     _parse_content_for_reasoning,
 )
 from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator