From 703159eaabaf6a2c52e334041a9b144a45c3ff31 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C4=81na=28Bass=20Ver=2E=29?= <1759138827@qq.com>
Date: Mon, 27 Jul 2026 12:43:26 +0800
Subject: [PATCH 01/37] fix(proxy): allow unblocking customers via
 /customer/update

update_end_user filtered out non-default values with v not in
([], {}, 0). Since False == 0 in Python, blocked: False was stripped
from the update payload. Treat bools as explicit values while preserving
the existing skips for empty containers and numeric zero

Fixes #34379
---
 .../customer_endpoints.py                     |  6 +-----
 .../test_customer_endpoints.py                | 20 +++++++++++++++++++
 2 files changed, 21 insertions(+), 5 deletions(-)

diff --git a/litellm/proxy/management_endpoints/customer_endpoints.py b/litellm/proxy/management_endpoints/customer_endpoints.py
index a46481d5bb7..388888d960d 100644
--- a/litellm/proxy/management_endpoints/customer_endpoints.py
+++ b/litellm/proxy/management_endpoints/customer_endpoints.py
@@ -553,11 +553,7 @@ async def update_end_user(
         # get non default values for key
         non_default_values = {}
         for k, v in data_json.items():
-            if v is not None and v not in (
-                [],
-                {},
-                0,
-            ):  # models default to [], spend defaults to 0, we should not reset these values
+            if v is not None and (isinstance(v, bool) or v not in ([], {}, 0)):
                 non_default_values[k] = v
 
         ## Get end user table data ##
diff --git a/tests/test_litellm/proxy/management_endpoints/test_customer_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_customer_endpoints.py
index 98e93eea5f9..c1479d4539c 100644
--- a/tests/test_litellm/proxy/management_endpoints/test_customer_endpoints.py
+++ b/tests/test_litellm/proxy/management_endpoints/test_customer_endpoints.py
@@ -85,6 +85,26 @@ def test_update_customer_success(mock_prisma_client, mock_user_api_key_auth):
     assert response.json()["alias"] == "Updated Test User"
 
 
+def test_update_customer_unblock(mock_prisma_client, mock_user_api_key_auth):
+    mock_end_user = LiteLLM_EndUserTable(user_id="test-user-1", blocked=True)
+    updated_mock_end_user = LiteLLM_EndUserTable(user_id="test-user-1", blocked=False)
+
+    mock_prisma_client.db.litellm_endusertable.find_first = AsyncMock(return_value=mock_end_user)
+    mock_prisma_client.db.litellm_endusertable.update = AsyncMock(return_value=updated_mock_end_user)
+
+    response = client.post(
+        "/customer/update",
+        json={"user_id": "test-user-1", "blocked": False},
+        headers={"Authorization": "Bearer test-key"},
+    )
+
+    assert response.status_code == 200
+    assert response.json()["blocked"] is False
+    update_mock = mock_prisma_client.db.litellm_endusertable.update
+    update_mock.assert_called_once()
+    assert update_mock.call_args.kwargs["data"]["blocked"] is False
+
+
 def test_update_customer_not_found(mock_prisma_client, mock_user_api_key_auth):
     """
     Test that update_end_user raises a 404 ProxyException when user_id does not exist.

From 463e9cd7ff3612070aaab7657ce31d98373e5f5b Mon Sep 17 00:00:00 2001
From: cat0825 <cat0825@users.noreply.github.com>
Date: Tue, 4 Aug 2026 11:27:09 +0800
Subject: [PATCH 02/37] fix(proxy): only apply blocked when explicitly supplied

The model default blocked=False was being written on every customer
update that omitted the field, silently unblocking blocked customers
when admins changed unrelated fields like alias or budget.

Only accept bool values for fields the caller explicitly supplied
(data.fields_set()), keeping the isinstance(v, bool) semantics for
explicit updates like blocked=True/False.

Adds a regression test: updating a blocked customer without the
blocked field must not reset the block.
---
 .../customer_endpoints.py                     |  5 +++-
 .../test_customer_endpoints.py                | 24 +++++++++++++++++++
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/litellm/proxy/management_endpoints/customer_endpoints.py b/litellm/proxy/management_endpoints/customer_endpoints.py
index 388888d960d..4127bc973e4 100644
--- a/litellm/proxy/management_endpoints/customer_endpoints.py
+++ b/litellm/proxy/management_endpoints/customer_endpoints.py
@@ -553,7 +553,10 @@ async def update_end_user(
         # get non default values for key
         non_default_values = {}
         for k, v in data_json.items():
-            if v is not None and (isinstance(v, bool) or v not in ([], {}, 0)):
+            if v is not None and (
+                (isinstance(v, bool) and k in data.fields_set())
+                or v not in ([], {}, 0)
+            ):
                 non_default_values[k] = v
 
         ## Get end user table data ##
diff --git a/tests/test_litellm/proxy/management_endpoints/test_customer_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_customer_endpoints.py
index c1479d4539c..6eb03256b67 100644
--- a/tests/test_litellm/proxy/management_endpoints/test_customer_endpoints.py
+++ b/tests/test_litellm/proxy/management_endpoints/test_customer_endpoints.py
@@ -105,6 +105,30 @@ def test_update_customer_unblock(mock_prisma_client, mock_user_api_key_auth):
     assert update_mock.call_args.kwargs["data"]["blocked"] is False
 
 
+def test_update_customer_keeps_blocked_when_omitted(mock_prisma_client, mock_user_api_key_auth):
+    """
+    Regression test: updating a blocked customer without supplying `blocked`
+    must NOT reset it to unblocked. `blocked=False` is the model default and
+    should only be applied when explicitly provided by the caller.
+    """
+    mock_end_user = LiteLLM_EndUserTable(user_id="test-user-1", blocked=True)
+    updated_mock_end_user = LiteLLM_EndUserTable(user_id="test-user-1", blocked=True)
+
+    mock_prisma_client.db.litellm_endusertable.find_first = AsyncMock(return_value=mock_end_user)
+    mock_prisma_client.db.litellm_endusertable.update = AsyncMock(return_value=updated_mock_end_user)
+
+    response = client.post(
+        "/customer/update",
+        json={"user_id": "test-user-1", "alias": "Updated Test User"},
+        headers={"Authorization": "Bearer test-key"},
+    )
+
+    assert response.status_code == 200
+    update_mock = mock_prisma_client.db.litellm_endusertable.update
+    update_mock.assert_called_once()
+    assert "blocked" not in update_mock.call_args.kwargs["data"]
+
+
 def test_update_customer_not_found(mock_prisma_client, mock_user_api_key_auth):
     """
     Test that update_end_user raises a 404 ProxyException when user_id does not exist.

From c12b2e82f74d49f2377e463addbf9248bf796d94 Mon Sep 17 00:00:00 2001
From: cat0825 <cat0825@users.noreply.github.com>
Date: Tue, 4 Aug 2026 12:18:23 +0800
Subject: [PATCH 03/37] style: ruff format customer_endpoints.py

---
 litellm/proxy/management_endpoints/customer_endpoints.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/litellm/proxy/management_endpoints/customer_endpoints.py b/litellm/proxy/management_endpoints/customer_endpoints.py
index 4127bc973e4..6efb3365cd9 100644
--- a/litellm/proxy/management_endpoints/customer_endpoints.py
+++ b/litellm/proxy/management_endpoints/customer_endpoints.py
@@ -553,10 +553,7 @@ async def update_end_user(
         # get non default values for key
         non_default_values = {}
         for k, v in data_json.items():
-            if v is not None and (
-                (isinstance(v, bool) and k in data.fields_set())
-                or v not in ([], {}, 0)
-            ):
+            if v is not None and ((isinstance(v, bool) and k in data.fields_set()) or v not in ([], {}, 0)):
                 non_default_values[k] = v
 
         ## Get end user table data ##

From b7a7754b0529721dc000cd1cadf68272da7e287d Mon Sep 17 00:00:00 2001
From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Fri, 28 Aug 2026 00:07:40 +0000
Subject: [PATCH 04/37] fix(bedrock): surface Nova Sonic user transcripts,
 speech events, and usage in realtime API

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../llms/bedrock/realtime/transformation.py   | 154 ++++++++++++++--
 ...odel_prices_and_context_window_backup.json |  20 +++
 litellm/types/llms/openai.py                  |  39 ++++
 model_prices_and_context_window.json          |  20 +++
 .../test_bedrock_realtime_transformation.py   | 168 ++++++++++++++++++
 5 files changed, 383 insertions(+), 18 deletions(-)

diff --git a/litellm/llms/bedrock/realtime/transformation.py b/litellm/llms/bedrock/realtime/transformation.py
index 951bf636b2f..20996512295 100644
--- a/litellm/llms/bedrock/realtime/transformation.py
+++ b/litellm/llms/bedrock/realtime/transformation.py
@@ -7,7 +7,7 @@ Transforms between OpenAI Realtime API format and Bedrock Nova Sonic format.
 import base64
 import json
 import uuid as uuid_lib
-from typing import Any, Final
+from typing import Any, Final, cast
 
 from pydantic import BaseModel
 
@@ -20,16 +20,21 @@ from litellm.types.llms.openai import (
     OpenAIRealtimeContentPartDone,
     OpenAIRealtimeDoneEvent,
     OpenAIRealtimeEvents,
+    OpenAIRealtimeInputAudioBufferSpeechEvent,
+    OpenAIRealtimeInputAudioTranscriptionCompleted,
+    OpenAIRealtimeInputAudioTranscriptionDelta,
     OpenAIRealtimeOutputItemDone,
     OpenAIRealtimeResponseAudioDone,
     OpenAIRealtimeResponseContentPartAdded,
     OpenAIRealtimeResponseDelta,
     OpenAIRealtimeResponseDoneObject,
     OpenAIRealtimeResponseTextDone,
+    OpenAIRealtimeResponseUsage,
     OpenAIRealtimeStreamResponseBaseObject,
     OpenAIRealtimeStreamResponseOutputItemAdded,
     OpenAIRealtimeStreamSession,
     OpenAIRealtimeStreamSessionEvents,
+    OpenAIRealtimeUsageTokenDetails,
 )
 from litellm.types.realtime import (
     ALL_DELTA_TYPES,
@@ -43,6 +48,27 @@ class BedrockContentEnd(BaseModel):
     stopReason: str | None = None
 
 
+class BedrockUsageTokenDetails(BaseModel):
+    speechTokens: int = 0
+    textTokens: int = 0
+
+
+class BedrockUsageDetailsTotal(BaseModel):
+    input: BedrockUsageTokenDetails = BedrockUsageTokenDetails()
+    output: BedrockUsageTokenDetails = BedrockUsageTokenDetails()
+
+
+class BedrockUsageDetails(BaseModel):
+    total: BedrockUsageDetailsTotal = BedrockUsageDetailsTotal()
+
+
+class BedrockUsageEvent(BaseModel):
+    totalInputTokens: int = 0
+    totalOutputTokens: int = 0
+    totalTokens: int = 0
+    details: BedrockUsageDetails = BedrockUsageDetails()
+
+
 TRIGGER_AUDIO_SAMPLE_RATE_HERTZ: Final = 16000
 TRIGGER_AUDIO_BYTES_PER_SECOND: Final = TRIGGER_AUDIO_SAMPLE_RATE_HERTZ * 2
 TRIGGER_LEADING_SILENCE: Final = bytes(TRIGGER_AUDIO_BYTES_PER_SECOND // 2)
@@ -87,6 +113,12 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
         # Text configuration
         self.text_media_type = "text/plain"
 
+        # Response-stream state (Bedrock events carry no role on textOutput,
+        # so the USER/ASSISTANT split from contentStart is tracked here)
+        self._user_transcript_active = False
+        self._user_transcript_generation_stage: str | None = None
+        self._latest_usage: OpenAIRealtimeResponseUsage | None = None
+
     def validate_environment(self, headers: dict, model: str, api_key: str | None = None) -> dict:
         """Validate environment - no special validation needed for Bedrock."""
         return headers
@@ -691,6 +723,11 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
         role: Final = content_start.get("role")
 
         if role != "ASSISTANT":
+            if role == "USER" and content_start.get("type") == "TEXT":
+                self._user_transcript_active = True
+                self._user_transcript_generation_stage = self._parse_generation_stage(
+                    content_start.get("additionalModelFields")
+                )
             return (
                 [],
                 current_response_id,
@@ -700,6 +737,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
             )
 
         verbose_logger.debug("Handling ASSISTANT contentStart")
+        is_new_response: Final = current_response_id is None
 
         # Initialize IDs if needed
         if not current_response_id:
@@ -715,7 +753,8 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
 
         returned_messages: Final[list[OpenAIRealtimeEvents]] = []
 
-        # Send response.created
+        # Send response.created only once per response (a response can contain
+        # multiple content blocks, e.g. TEXT then AUDIO)
         response_created: Final = OpenAIRealtimeStreamResponseBaseObject(
             type="response.created",
             event_id=f"event_{uuid.uuid4()}",
@@ -727,7 +766,8 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
                 "conversation_id": current_conversation_id,
             },
         )
-        returned_messages.append(response_created)
+        if is_new_response:
+            returned_messages.append(response_created)
 
         # Send response.output_item.added
         output_item_added: Final = OpenAIRealtimeStreamResponseOutputItemAdded(
@@ -767,6 +807,70 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
             current_delta_type,
         )
 
+    @staticmethod
+    def _parse_generation_stage(additional_model_fields: object) -> str | None:
+        if not isinstance(additional_model_fields, str):
+            return None
+        try:
+            parsed: Final = json.loads(additional_model_fields)
+        except json.JSONDecodeError:
+            return None
+        stage: Final = parsed.get("generationStage") if isinstance(parsed, dict) else None
+        return stage if isinstance(stage, str) else None
+
+    def transform_user_speech_event(self, is_speech_start: bool) -> tuple[OpenAIRealtimeEvents, ...]:
+        """Transform Bedrock userSpeechStart/userSpeechEnd to OpenAI speech boundary events."""
+        verbose_logger.debug("Handling userSpeech%s", "Start" if is_speech_start else "End")
+        speech_event: Final[OpenAIRealtimeInputAudioBufferSpeechEvent] = {
+            "type": "input_audio_buffer.speech_started" if is_speech_start else "input_audio_buffer.speech_stopped",
+            "event_id": f"event_{uuid.uuid4()}",
+            "item_id": f"item_{uuid.uuid4()}",
+        }
+        return (speech_event,)
+
+    def transform_usage_event(self, usage_event: BedrockUsageEvent) -> None:
+        """Record Bedrock usageEvent token totals for the next response.done."""
+        verbose_logger.debug("Handling usageEvent")
+        input_details: Final[OpenAIRealtimeUsageTokenDetails] = {
+            "audio_tokens": usage_event.details.total.input.speechTokens,
+            "text_tokens": usage_event.details.total.input.textTokens,
+            "cached_tokens": 0,
+        }
+        output_details: Final[OpenAIRealtimeUsageTokenDetails] = {
+            "audio_tokens": usage_event.details.total.output.speechTokens,
+            "text_tokens": usage_event.details.total.output.textTokens,
+        }
+        latest_usage: Final[OpenAIRealtimeResponseUsage] = {
+            "input_tokens": usage_event.totalInputTokens,
+            "output_tokens": usage_event.totalOutputTokens,
+            "total_tokens": usage_event.totalTokens,
+            "input_token_details": input_details,
+            "output_token_details": output_details,
+        }
+        self._latest_usage = latest_usage
+
+    def transform_user_transcript_event(self, transcript: str) -> tuple[OpenAIRealtimeEvents, ...]:
+        """Transform a USER-role Bedrock textOutput (ASR transcript) to OpenAI transcription events."""
+        verbose_logger.debug("Handling USER textOutput (ASR transcript)")
+        item_id: Final = f"item_{uuid.uuid4()}"
+        delta_event: Final[OpenAIRealtimeInputAudioTranscriptionDelta] = {
+            "type": "conversation.item.input_audio_transcription.delta",
+            "event_id": f"event_{uuid.uuid4()}",
+            "item_id": item_id,
+            "content_index": 0,
+            "delta": transcript,
+        }
+        if self._user_transcript_generation_stage == "SPECULATIVE":
+            return (delta_event,)
+        completed_event: Final[OpenAIRealtimeInputAudioTranscriptionCompleted] = {
+            "type": "conversation.item.input_audio_transcription.completed",
+            "event_id": f"event_{uuid.uuid4()}",
+            "item_id": item_id,
+            "content_index": 0,
+            "transcript": transcript,
+        }
+        return (delta_event, completed_event)
+
     def transform_text_output_event(
         self,
         event: dict,
@@ -985,7 +1089,14 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
         if not current_response_id or not current_conversation_id:
             return [], None, None, None
 
-        usage_obj: Final = get_empty_usage()
+        empty_usage: Final = get_empty_usage()
+        zero_usage: Final[OpenAIRealtimeResponseUsage] = {
+            "input_tokens": empty_usage.prompt_tokens,
+            "output_tokens": empty_usage.completion_tokens,
+            "total_tokens": empty_usage.total_tokens,
+        }
+        usage: Final = self._latest_usage or zero_usage
+        self._latest_usage = None
         response_done: Final = OpenAIRealtimeDoneEvent(
             type="response.done",
             event_id=f"event_{uuid.uuid4()}",
@@ -995,11 +1106,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
                 status="completed",
                 output=[],
                 conversation_id=current_conversation_id,
-                usage={
-                    "prompt_tokens": usage_obj.prompt_tokens,
-                    "completion_tokens": usage_obj.completion_tokens,
-                    "total_tokens": usage_obj.total_tokens,
-                },
+                usage=dict(usage),
             ),
         )
 
@@ -1042,8 +1149,6 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
 
         # Create a function call arguments done event
         # This is a custom event format that matches what clients expect
-        from typing import cast
-
         function_call_event: Final[dict[str, Any]] = {
             "type": "response.function_call_arguments.done",
             "event_id": f"event_{uuid.uuid4()}",
@@ -1194,18 +1299,25 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
             returned_messages.extend(events)
 
         elif "textOutput" in event:
-            events, current_delta_chunks = self.transform_text_output_event(
-                event,
-                current_output_item_id,
-                current_response_id,
-                current_delta_chunks,
-            )
-            returned_messages.extend(events)
+            if self._user_transcript_active:
+                returned_messages.extend(self.transform_user_transcript_event(event["textOutput"].get("content", "")))
+            else:
+                events, current_delta_chunks = self.transform_text_output_event(
+                    event,
+                    current_output_item_id,
+                    current_response_id,
+                    current_delta_chunks,
+                )
+                returned_messages.extend(events)
 
         elif "audioOutput" in event:
             events = self.transform_audio_output_event(event, current_output_item_id, current_response_id)
             returned_messages.extend(events)
 
+        elif "contentEnd" in event and self._user_transcript_active:
+            self._user_transcript_active = False
+            self._user_transcript_generation_stage = None
+
         elif "contentEnd" in event:
             events, current_delta_chunks = self.transform_content_end_event(
                 event,
@@ -1224,6 +1336,12 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
                 ) = self._response_done_events(current_response_id, current_conversation_id)
                 returned_messages.extend(done_events)
 
+        elif "userSpeechStart" in event or "userSpeechEnd" in event:
+            returned_messages.extend(self.transform_user_speech_event("userSpeechStart" in event))
+
+        elif "usageEvent" in event:
+            self.transform_usage_event(BedrockUsageEvent.model_validate(event["usageEvent"]))
+
         elif "toolUse" in event:
             events, tool_call_id, tool_name = self.transform_tool_use_event(
                 event, current_output_item_id, current_response_id
diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json
index 77572f69b8b..8fb6108bdda 100644
--- a/litellm/model_prices_and_context_window_backup.json
+++ b/litellm/model_prices_and_context_window_backup.json
@@ -553,6 +553,26 @@
         "supports_response_schema": true,
         "supports_vision": true
     },
+    "amazon.nova-sonic-v1:0": {
+        "input_cost_per_audio_token": 3.4e-06,
+        "input_cost_per_token": 6e-08,
+        "litellm_provider": "bedrock",
+        "mode": "realtime",
+        "output_cost_per_audio_token": 1.36e-05,
+        "output_cost_per_token": 2.4e-07,
+        "supports_audio_input": true,
+        "supports_audio_output": true
+    },
+    "amazon.nova-2-sonic-v1:0": {
+        "input_cost_per_audio_token": 3e-06,
+        "input_cost_per_token": 3.3e-07,
+        "litellm_provider": "bedrock",
+        "mode": "realtime",
+        "output_cost_per_audio_token": 1.2e-05,
+        "output_cost_per_token": 2.75e-06,
+        "supports_audio_input": true,
+        "supports_audio_output": true
+    },
     "amazon.rerank-v1:0": {
         "input_cost_per_query": 0.001,
         "input_cost_per_token": 0.0,
diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py
index a6115640d78..fcade835cce 100644
--- a/litellm/types/llms/openai.py
+++ b/litellm/types/llms/openai.py
@@ -2162,6 +2162,42 @@ class OpenAIRealtimeDoneEvent(TypedDict):
     type: Literal["response.done"]
 
 
+class OpenAIRealtimeInputAudioBufferSpeechEvent(TypedDict):
+    type: ReadOnly[Literal["input_audio_buffer.speech_started", "input_audio_buffer.speech_stopped"]]
+    event_id: ReadOnly[str]
+    item_id: ReadOnly[str]
+
+
+class OpenAIRealtimeInputAudioTranscriptionDelta(TypedDict):
+    type: ReadOnly[Literal["conversation.item.input_audio_transcription.delta"]]
+    event_id: ReadOnly[str]
+    item_id: ReadOnly[str]
+    content_index: ReadOnly[int]
+    delta: ReadOnly[str]
+
+
+class OpenAIRealtimeInputAudioTranscriptionCompleted(TypedDict):
+    type: ReadOnly[Literal["conversation.item.input_audio_transcription.completed"]]
+    event_id: ReadOnly[str]
+    item_id: ReadOnly[str]
+    content_index: ReadOnly[int]
+    transcript: ReadOnly[str]
+
+
+class OpenAIRealtimeUsageTokenDetails(TypedDict):
+    audio_tokens: ReadOnly[int]
+    text_tokens: ReadOnly[int]
+    cached_tokens: NotRequired[ReadOnly[int]]
+
+
+class OpenAIRealtimeResponseUsage(TypedDict):
+    input_tokens: ReadOnly[int]
+    output_tokens: ReadOnly[int]
+    total_tokens: ReadOnly[int]
+    input_token_details: NotRequired[ReadOnly[OpenAIRealtimeUsageTokenDetails]]
+    output_token_details: NotRequired[ReadOnly[OpenAIRealtimeUsageTokenDetails]]
+
+
 class OpenAIRealtimeEventTypes(Enum):
     SESSION_CREATED = "session.created"
     # Beta delta event names
@@ -2199,6 +2235,9 @@ OpenAIRealtimeEvents = (
     | OpenAIRealtimeOutputItemDone
     | OpenAIRealtimeFunctionCallArgumentsDone
     | OpenAIRealtimeDoneEvent
+    | OpenAIRealtimeInputAudioBufferSpeechEvent
+    | OpenAIRealtimeInputAudioTranscriptionDelta
+    | OpenAIRealtimeInputAudioTranscriptionCompleted
 )
 
 OpenAIRealtimeStreamList = list[OpenAIRealtimeEvents]
diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json
index 77572f69b8b..8fb6108bdda 100644
--- a/model_prices_and_context_window.json
+++ b/model_prices_and_context_window.json
@@ -553,6 +553,26 @@
         "supports_response_schema": true,
         "supports_vision": true
     },
+    "amazon.nova-sonic-v1:0": {
+        "input_cost_per_audio_token": 3.4e-06,
+        "input_cost_per_token": 6e-08,
+        "litellm_provider": "bedrock",
+        "mode": "realtime",
+        "output_cost_per_audio_token": 1.36e-05,
+        "output_cost_per_token": 2.4e-07,
+        "supports_audio_input": true,
+        "supports_audio_output": true
+    },
+    "amazon.nova-2-sonic-v1:0": {
+        "input_cost_per_audio_token": 3e-06,
+        "input_cost_per_token": 3.3e-07,
+        "litellm_provider": "bedrock",
+        "mode": "realtime",
+        "output_cost_per_audio_token": 1.2e-05,
+        "output_cost_per_token": 2.75e-06,
+        "supports_audio_input": true,
+        "supports_audio_output": true
+    },
     "amazon.rerank-v1:0": {
         "input_cost_per_query": 0.001,
         "input_cost_per_token": 0.0,
diff --git a/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_transformation.py b/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_transformation.py
index ae6b1febd6b..b910018e6c4 100644
--- a/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_transformation.py
+++ b/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_transformation.py
@@ -827,5 +827,173 @@ class TestBedrockRealtimeSessionEvents:
         assert event["session"]["modalities"] == ["text", "audio"]
 
 
+class TestBedrockRealtimeUserEventsAndUsage:
+    """Regression tests for #38346: USER ASR transcripts, speech boundary events,
+    usage propagation, and duplicate response.created"""
+
+    @staticmethod
+    def _run(config, messages):
+        logging_obj = MagicMock()
+        logging_obj.litellm_trace_id = "trace_123"
+        state = {
+            "session_configuration_request": json.dumps({"configured": True}),
+            "current_output_item_id": None,
+            "current_response_id": None,
+            "current_conversation_id": None,
+            "current_delta_chunks": [],
+            "current_item_chunks": [],
+            "current_delta_type": None,
+        }
+        all_events = []
+        for msg in messages:
+            result = config.transform_realtime_response(
+                json.dumps(msg),
+                "amazon.nova-2-sonic-v1:0",
+                logging_obj,
+                realtime_response_transform_input=dict(state),
+            )
+            all_events.extend(result["response"])
+            state.update(
+                {
+                    "current_output_item_id": result["current_output_item_id"],
+                    "current_response_id": result["current_response_id"],
+                    "current_conversation_id": result["current_conversation_id"],
+                    "current_delta_chunks": result["current_delta_chunks"],
+                    "current_delta_type": result["current_delta_type"],
+                }
+            )
+        return all_events
+
+    def test_user_speech_start_and_stop_events(self):
+        events = self._run(
+            BedrockRealtimeConfig(),
+            [{"event": {"userSpeechStart": {}}}, {"event": {"userSpeechEnd": {}}}],
+        )
+        assert [e["type"] for e in events] == [
+            "input_audio_buffer.speech_started",
+            "input_audio_buffer.speech_stopped",
+        ]
+        assert all(e["event_id"] and e["item_id"] for e in events)
+
+    def test_user_transcript_emits_input_audio_transcription_events(self):
+        events = self._run(
+            BedrockRealtimeConfig(),
+            [
+                {
+                    "event": {
+                        "contentStart": {
+                            "role": "USER",
+                            "type": "TEXT",
+                            "additionalModelFields": json.dumps({"generationStage": "FINAL"}),
+                        }
+                    }
+                },
+                {"event": {"textOutput": {"content": "ready"}}},
+                {"event": {"contentEnd": {"stopReason": "PARTIAL_TURN"}}},
+            ],
+        )
+        deltas = [e for e in events if e["type"] == "conversation.item.input_audio_transcription.delta"]
+        completed = [e for e in events if e["type"] == "conversation.item.input_audio_transcription.completed"]
+        assert len(deltas) == 1 and deltas[0]["delta"] == "ready"
+        assert len(completed) == 1 and completed[0]["transcript"] == "ready"
+        assert deltas[0]["item_id"] == completed[0]["item_id"]
+        assert not any(e["type"] == "response.text.delta" for e in events)
+
+    def test_speculative_user_transcript_emits_delta_only(self):
+        events = self._run(
+            BedrockRealtimeConfig(),
+            [
+                {
+                    "event": {
+                        "contentStart": {
+                            "role": "USER",
+                            "type": "TEXT",
+                            "additionalModelFields": json.dumps({"generationStage": "SPECULATIVE"}),
+                        }
+                    }
+                },
+                {"event": {"textOutput": {"content": "rea"}}},
+            ],
+        )
+        assert [e["type"] for e in events] == ["conversation.item.input_audio_transcription.delta"]
+
+    def test_user_transcript_state_resets_on_content_end(self):
+        events = self._run(
+            BedrockRealtimeConfig(),
+            [
+                {"event": {"contentStart": {"role": "USER", "type": "TEXT"}}},
+                {"event": {"contentEnd": {"stopReason": "PARTIAL_TURN"}}},
+                {"event": {"contentStart": {"role": "ASSISTANT", "type": "TEXT"}}},
+                {"event": {"textOutput": {"content": "Hi there"}}},
+            ],
+        )
+        text_deltas = [e for e in events if e["type"] == "response.text.delta"]
+        assert len(text_deltas) == 1 and text_deltas[0]["delta"] == "Hi there"
+        assert not any(e["type"].startswith("conversation.item.input_audio_transcription") for e in events)
+
+    def test_response_created_emitted_once_per_response(self):
+        events = self._run(
+            BedrockRealtimeConfig(),
+            [
+                {"event": {"contentStart": {"role": "ASSISTANT", "type": "TEXT"}}},
+                {"event": {"textOutput": {"content": "Hi"}}},
+                {"event": {"contentEnd": {"stopReason": "PARTIAL_TURN"}}},
+                {"event": {"contentStart": {"role": "ASSISTANT", "type": "AUDIO"}}},
+            ],
+        )
+        assert sum(1 for e in events if e["type"] == "response.created") == 1
+
+    def test_usage_event_propagates_to_response_done(self):
+        events = self._run(
+            BedrockRealtimeConfig(),
+            [
+                {
+                    "event": {
+                        "usageEvent": {
+                            "totalInputTokens": 25,
+                            "totalOutputTokens": 40,
+                            "totalTokens": 65,
+                            "details": {
+                                "total": {
+                                    "input": {"speechTokens": 20, "textTokens": 5},
+                                    "output": {"speechTokens": 30, "textTokens": 10},
+                                }
+                            },
+                        }
+                    }
+                },
+                {"event": {"contentStart": {"role": "ASSISTANT", "type": "TEXT"}}},
+                {"event": {"textOutput": {"content": "Hi"}}},
+                {"event": {"contentEnd": {"stopReason": "END_TURN"}}},
+            ],
+        )
+        done_events = [e for e in events if e["type"] == "response.done"]
+        assert len(done_events) == 1
+        usage = done_events[0]["response"]["usage"]
+        assert usage["input_tokens"] == 25
+        assert usage["output_tokens"] == 40
+        assert usage["total_tokens"] == 65
+        assert usage["input_token_details"]["audio_tokens"] == 20
+        assert usage["input_token_details"]["text_tokens"] == 5
+        assert usage["output_token_details"]["audio_tokens"] == 30
+        assert usage["output_token_details"]["text_tokens"] == 10
+
+    def test_response_done_without_usage_event_reports_zero_usage(self):
+        events = self._run(
+            BedrockRealtimeConfig(),
+            [
+                {"event": {"contentStart": {"role": "ASSISTANT", "type": "TEXT"}}},
+                {"event": {"textOutput": {"content": "Hi"}}},
+                {"event": {"contentEnd": {"stopReason": "END_TURN"}}},
+            ],
+        )
+        done_events = [e for e in events if e["type"] == "response.done"]
+        assert len(done_events) == 1
+        usage = done_events[0]["response"]["usage"]
+        assert usage["input_tokens"] == 0
+        assert usage["output_tokens"] == 0
+        assert usage["total_tokens"] == 0
+
+
 if __name__ == "__main__":
     pytest.main([__file__, "-v"])

From 5eee3bd9f952caa50edfd42927945da86848ed8d Mon Sep 17 00:00:00 2001
From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Fri, 28 Aug 2026 00:28:22 +0000
Subject: [PATCH 05/37] fix(bedrock): dispatch success handlers for realtime
 sessions so spend is logged

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/llms/bedrock/realtime/handler.py      | 12 +++++
 .../realtime/test_bedrock_realtime_handler.py | 48 +++++++++++++++++++
 2 files changed, 60 insertions(+)

diff --git a/litellm/llms/bedrock/realtime/handler.py b/litellm/llms/bedrock/realtime/handler.py
index 3eeb3cb9fc6..b1b598039f8 100644
--- a/litellm/llms/bedrock/realtime/handler.py
+++ b/litellm/llms/bedrock/realtime/handler.py
@@ -13,6 +13,8 @@ from pydantic import TypeAdapter
 
 from litellm._logging import _redact_string, verbose_proxy_logger
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
+from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
+from litellm.types.llms.openai import OpenAIRealtimeEvents
 
 from ..base_aws_llm import BaseAWSLLM
 from ..common_utils import BedrockError
@@ -154,6 +156,7 @@ class BedrockRealtime(BaseAWSLLM):
                 )
             )
 
+            logged_events: Final[list[OpenAIRealtimeEvents]] = []  # mutable-ok: events accumulate across stream loop iterations for spend logging
             bedrock_to_client_task: Final = asyncio.create_task(
                 self._forward_bedrock_to_client(
                     bedrock_stream,
@@ -162,6 +165,7 @@ class BedrockRealtime(BaseAWSLLM):
                     model,
                     logging_obj,
                     session_state,
+                    logged_events,
                 )
             )
 
@@ -172,6 +176,11 @@ class BedrockRealtime(BaseAWSLLM):
                 return_exceptions=True,
             )
 
+            if logged_events:
+                GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(
+                    logging_obj.dispatch_success_handlers(logged_events, prefer_async_handlers=True)
+                )
+
         except Exception as e:
             verbose_proxy_logger.exception("Error in BedrockRealtime.async_realtime: %s", e)
             try:
@@ -252,6 +261,7 @@ class BedrockRealtime(BaseAWSLLM):
         model: str,
         logging_obj: LiteLLMLogging,
         session_state: dict,
+        logged_events: "list[OpenAIRealtimeEvents] | None" = None,
     ):
         """Forward messages from Bedrock stream to client WebSocket."""
         try:
@@ -304,6 +314,8 @@ class BedrockRealtime(BaseAWSLLM):
                     # Send transformed messages to client
                     openai_messages = transformed_response.get("response", [])
                     for openai_message in openai_messages:
+                        if logged_events is not None and isinstance(openai_message, dict):
+                            logged_events.append(openai_message)
                         message_json = json.dumps(openai_message)
                         await client_ws.send_text(message_json)
                         verbose_proxy_logger.debug("Bedrock Realtime: Sent to client: %s", message_json[:200])
diff --git a/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_handler.py b/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_handler.py
index 9efcee192b1..76a564845dd 100644
--- a/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_handler.py
+++ b/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_handler.py
@@ -104,6 +104,26 @@ class RealtimeClientWS:
         self.closed = True
 
 
+class ScriptedBedrockReceiver:
+    def __init__(self, payloads):
+        self._payloads = list(payloads)
+
+    async def receive(self):
+        if not self._payloads:
+            return None
+        payload = self._payloads.pop(0)
+        return SimpleNamespace(value=SimpleNamespace(bytes_=payload.encode("utf-8")))
+
+
+class ScriptedBedrockStream:
+    def __init__(self, payloads):
+        self.input_stream = FakeInputStream()
+        self._receiver = ScriptedBedrockReceiver(payloads)
+
+    async def await_output(self):
+        return (None, self._receiver)
+
+
 class ImmediatelyEndingBedrockStream:
     def __init__(self):
         self.input_stream = FakeInputStream()
@@ -271,6 +291,34 @@ class TestBedrockRealtimeHandler:
         assert "sessionEnd" in event_names
         assert stream.input_stream.closed
 
+    @pytest.mark.asyncio
+    async def test_forwarded_events_are_collected_for_spend_logging(self):
+        handler = BedrockRealtime()
+        stream = ScriptedBedrockStream(
+            [
+                json.dumps({"event": {"userSpeechStart": {}}}),
+                json.dumps({"event": {"userSpeechEnd": {}}}),
+            ]
+        )
+        client_ws = RealtimeClientWS()
+        logged_events = []
+
+        await handler._forward_bedrock_to_client(
+            stream,
+            client_ws,
+            BedrockRealtimeConfig(),
+            "amazon.nova-sonic-v1:0",
+            FakeLogging(),
+            {},
+            logged_events,
+        )
+
+        assert [event["type"] for event in logged_events] == [
+            "input_audio_buffer.speech_started",
+            "input_audio_buffer.speech_stopped",
+        ]
+        assert client_ws.closed
+
     @pytest.mark.asyncio
     async def test_bedrock_stream_end_closes_client_websocket(self):
         handler = BedrockRealtime()

From eee47dcdaa9a3531f5f78e3bb2a70315626ea175 Mon Sep 17 00:00:00 2001
From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Fri, 28 Aug 2026 00:39:10 +0000
Subject: [PATCH 06/37] fix(bedrock): share one item_id across a user
 utterance's realtime events

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/llms/bedrock/realtime/handler.py      |  4 ++-
 .../llms/bedrock/realtime/transformation.py   | 11 +++++--
 .../realtime/test_bedrock_realtime_handler.py |  8 ++---
 .../test_bedrock_realtime_transformation.py   | 31 +++++++++++++++++++
 4 files changed, 45 insertions(+), 9 deletions(-)

diff --git a/litellm/llms/bedrock/realtime/handler.py b/litellm/llms/bedrock/realtime/handler.py
index b1b598039f8..0891379ef8c 100644
--- a/litellm/llms/bedrock/realtime/handler.py
+++ b/litellm/llms/bedrock/realtime/handler.py
@@ -156,7 +156,9 @@ class BedrockRealtime(BaseAWSLLM):
                 )
             )
 
-            logged_events: Final[list[OpenAIRealtimeEvents]] = []  # mutable-ok: events accumulate across stream loop iterations for spend logging
+            logged_events: Final[
+                list[OpenAIRealtimeEvents]
+            ] = []  # mutable-ok: events accumulate across stream loop iterations for spend logging
             bedrock_to_client_task: Final = asyncio.create_task(
                 self._forward_bedrock_to_client(
                     bedrock_stream,
diff --git a/litellm/llms/bedrock/realtime/transformation.py b/litellm/llms/bedrock/realtime/transformation.py
index 20996512295..ec5eff0a6fc 100644
--- a/litellm/llms/bedrock/realtime/transformation.py
+++ b/litellm/llms/bedrock/realtime/transformation.py
@@ -117,6 +117,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
         # so the USER/ASSISTANT split from contentStart is tracked here)
         self._user_transcript_active = False
         self._user_transcript_generation_stage: str | None = None
+        self._user_item_id: str | None = None
         self._latest_usage: OpenAIRealtimeResponseUsage | None = None
 
     def validate_environment(self, headers: dict, model: str, api_key: str | None = None) -> dict:
@@ -818,13 +819,19 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
         stage: Final = parsed.get("generationStage") if isinstance(parsed, dict) else None
         return stage if isinstance(stage, str) else None
 
+    def _current_user_item_id(self, new_utterance: bool = False) -> str:
+        """Item id shared by all events of one user utterance (speech boundaries and transcript)."""
+        if new_utterance or self._user_item_id is None:
+            self._user_item_id = f"item_{uuid.uuid4()}"
+        return self._user_item_id
+
     def transform_user_speech_event(self, is_speech_start: bool) -> tuple[OpenAIRealtimeEvents, ...]:
         """Transform Bedrock userSpeechStart/userSpeechEnd to OpenAI speech boundary events."""
         verbose_logger.debug("Handling userSpeech%s", "Start" if is_speech_start else "End")
         speech_event: Final[OpenAIRealtimeInputAudioBufferSpeechEvent] = {
             "type": "input_audio_buffer.speech_started" if is_speech_start else "input_audio_buffer.speech_stopped",
             "event_id": f"event_{uuid.uuid4()}",
-            "item_id": f"item_{uuid.uuid4()}",
+            "item_id": self._current_user_item_id(new_utterance=is_speech_start),
         }
         return (speech_event,)
 
@@ -852,7 +859,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
     def transform_user_transcript_event(self, transcript: str) -> tuple[OpenAIRealtimeEvents, ...]:
         """Transform a USER-role Bedrock textOutput (ASR transcript) to OpenAI transcription events."""
         verbose_logger.debug("Handling USER textOutput (ASR transcript)")
-        item_id: Final = f"item_{uuid.uuid4()}"
+        item_id: Final = self._current_user_item_id()
         delta_event: Final[OpenAIRealtimeInputAudioTranscriptionDelta] = {
             "type": "conversation.item.input_audio_transcription.delta",
             "event_id": f"event_{uuid.uuid4()}",
diff --git a/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_handler.py b/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_handler.py
index 76a564845dd..aa6573884e2 100644
--- a/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_handler.py
+++ b/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_handler.py
@@ -368,9 +368,7 @@ class TestBedrockRealtimeSessionLifecycle:
             [json.dumps({"type": "session.update", "session": {"instructions": "hi", "modalities": ["text"]}})]
         )
 
-        await handler._forward_client_to_bedrock(
-            client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, FakeLogging()
-        )
+        await handler._forward_client_to_bedrock(client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, FakeLogging())
 
         acked = [json.loads(message) for message in client_ws.sent_to_client]
         updated = [event for event in acked if event["type"] == "session.updated"]
@@ -382,9 +380,7 @@ class TestBedrockRealtimeSessionLifecycle:
         handler = BedrockRealtime()
         config = BedrockRealtimeConfig()
         stream = FakeBedrockStream()
-        client_ws = DisconnectingClientWS(
-            [json.dumps({"type": "session.update", "session": {"instructions": "hi"}})]
-        )
+        client_ws = DisconnectingClientWS([json.dumps({"type": "session.update", "session": {"instructions": "hi"}})])
 
         await handler._forward_client_to_bedrock(client_ws, stream, config, "amazon.nova-sonic-v1:0", {})
 
diff --git a/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_transformation.py b/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_transformation.py
index b910018e6c4..dbe84cacb29 100644
--- a/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_transformation.py
+++ b/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_transformation.py
@@ -874,6 +874,37 @@ class TestBedrockRealtimeUserEventsAndUsage:
             "input_audio_buffer.speech_stopped",
         ]
         assert all(e["event_id"] and e["item_id"] for e in events)
+        assert events[0]["item_id"] == events[1]["item_id"]
+
+    def test_utterance_lifecycle_shares_one_item_id(self):
+        events = self._run(
+            BedrockRealtimeConfig(),
+            [
+                {"event": {"userSpeechStart": {}}},
+                {"event": {"userSpeechEnd": {}}},
+                {
+                    "event": {
+                        "contentStart": {
+                            "role": "USER",
+                            "type": "TEXT",
+                            "additionalModelFields": json.dumps({"generationStage": "FINAL"}),
+                        }
+                    }
+                },
+                {"event": {"textOutput": {"content": "ready"}}},
+                {"event": {"contentEnd": {"stopReason": "PARTIAL_TURN"}}},
+            ],
+        )
+        item_ids = {e["item_id"] for e in events if "item_id" in e}
+        assert len(item_ids) == 1
+
+    def test_new_utterance_gets_new_item_id(self):
+        config = BedrockRealtimeConfig()
+        first = self._run(config, [{"event": {"userSpeechStart": {}}}, {"event": {"userSpeechEnd": {}}}])
+        second = self._run(config, [{"event": {"userSpeechStart": {}}}, {"event": {"userSpeechEnd": {}}}])
+        assert first[0]["item_id"] == first[1]["item_id"]
+        assert second[0]["item_id"] == second[1]["item_id"]
+        assert first[0]["item_id"] != second[0]["item_id"]
 
     def test_user_transcript_emits_input_audio_transcription_events(self):
         events = self._run(

From 3ea4b715ba8d6c1666205443a12475d74a1af040 Mon Sep 17 00:00:00 2001
From: siyoon <siyoon@friendli.ai>
Date: Sun, 30 Aug 2026 14:22:41 +0900
Subject: [PATCH 07/37] feat(friendli): add zai-org/GLM-5.3-Flash model pricing

Per https://api.friendli.ai/serverless/v1/models:
- $0.15 input / $0.50 output / $0.03 cached input per MTok
- 1M context, 1M max output, reasoning with effort low/high/max
  (per HF chat_template.jinja: low/high honored, anything else -> max)
- tool calling, parallel tool calls, structured output, prompt caching
- image + video input (native multimodal)
---
 model_prices_and_context_window.json          | 25 +++++++++++++
 ...t_friendli_glm_5_3_flash_model_metadata.py | 37 +++++++++++++++++++
 2 files changed, 62 insertions(+)
 create mode 100644 tests/test_litellm/test_friendli_glm_5_3_flash_model_metadata.py

diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json
index 05c1cfd3179..fd6b9e11adf 100644
--- a/model_prices_and_context_window.json
+++ b/model_prices_and_context_window.json
@@ -19564,6 +19564,31 @@
         "supports_system_messages": true,
         "supports_tool_choice": true
     },
+    "friendliai/zai-org/GLM-5.3-Flash": {
+        "litellm_provider": "friendliai",
+        "supports_reasoning": true,
+        "supports_function_calling": true,
+        "max_input_tokens": 1048576,
+        "max_tokens": 1048576,
+        "max_output_tokens": 1048576,
+        "input_cost_per_token": 1.5e-07,
+        "output_cost_per_token": 5e-07,
+        "cache_read_input_token_cost": 3e-08,
+        "supports_prompt_caching": true,
+        "supports_max_reasoning_effort": true,
+        "supports_low_reasoning_effort": true,
+        "supports_parallel_function_calling": true,
+        "supports_response_schema": true,
+        "supports_native_structured_output": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "mode": "chat",
+        "comment": "Native multimodal GLM model for efficient coding and long-horizon agent tasks",
+        "source": "https://api.friendli.ai/serverless/v1/models",
+        "supports_vision": true,
+        "supports_image_input": true,
+        "supports_video_input": true
+    },
     "ft:babbage-002": {
         "deprecation_date": "2026-10-23",
         "input_cost_per_token": 1.6e-06,
diff --git a/tests/test_litellm/test_friendli_glm_5_3_flash_model_metadata.py b/tests/test_litellm/test_friendli_glm_5_3_flash_model_metadata.py
new file mode 100644
index 00000000000..5110307e5bb
--- /dev/null
+++ b/tests/test_litellm/test_friendli_glm_5_3_flash_model_metadata.py
@@ -0,0 +1,37 @@
+import json
+from pathlib import Path
+
+from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
+
+
+def test_friendli_glm_5_3_flash_model_info():
+    model = "friendliai/zai-org/GLM-5.3-Flash"
+    json_path = Path(__file__).parents[2] / "model_prices_and_context_window.json"
+    with open(json_path) as f:
+        model_cost = json.load(f)
+
+    info = model_cost.get(model)
+    assert (
+        info is not None
+    ), f"{model} not found in model_prices_and_context_window.json"
+    assert info["litellm_provider"] == "friendliai"
+    assert info["mode"] == "chat"
+    assert info["input_cost_per_token"] == 1.5e-07
+    assert info["output_cost_per_token"] == 5e-07
+    assert info["cache_read_input_token_cost"] == 3e-08
+    assert info["max_input_tokens"] == 1048576
+    assert info["max_output_tokens"] == 1048576
+    assert info["supports_function_calling"] is True
+    assert info["supports_reasoning"] is True
+    assert info["supports_low_reasoning_effort"] is True
+    assert info["supports_max_reasoning_effort"] is True
+    assert info["supports_tool_choice"] is True
+    assert info["supports_prompt_caching"] is True
+    # Friendli serves image and video input
+    assert info["supports_vision"] is True
+    assert info["supports_image_input"] is True
+    assert info["supports_video_input"] is True
+
+    routed_model, provider, _, _ = get_llm_provider(model=model)
+    assert routed_model == "zai-org/GLM-5.3-Flash"
+    assert provider == "friendliai"

From e9f1af84730ffd1c11b9f77a10c38d0bf20351be Mon Sep 17 00:00:00 2001
From: siyoon <siyoon@friendli.ai>
Date: Sun, 30 Aug 2026 14:29:28 +0900
Subject: [PATCH 08/37] chore(tests): drop redundant capability comment

Per greptile review + CLAUDE.md comment policy: the comment restated
the immediately following assertions without adding value.
---
 tests/test_litellm/test_friendli_glm_5_3_flash_model_metadata.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/test_litellm/test_friendli_glm_5_3_flash_model_metadata.py b/tests/test_litellm/test_friendli_glm_5_3_flash_model_metadata.py
index 5110307e5bb..0acd750e49a 100644
--- a/tests/test_litellm/test_friendli_glm_5_3_flash_model_metadata.py
+++ b/tests/test_litellm/test_friendli_glm_5_3_flash_model_metadata.py
@@ -27,7 +27,6 @@ def test_friendli_glm_5_3_flash_model_info():
     assert info["supports_max_reasoning_effort"] is True
     assert info["supports_tool_choice"] is True
     assert info["supports_prompt_caching"] is True
-    # Friendli serves image and video input
     assert info["supports_vision"] is True
     assert info["supports_image_input"] is True
     assert info["supports_video_input"] is True

From 693279afb443d8a1775fb82b83ae8c2ad0c7a295 Mon Sep 17 00:00:00 2001
From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Sun, 30 Aug 2026 07:57:16 +0000
Subject: [PATCH 09/37] chore(techdebt): clear fresh debt from the 2026-08-29
 window

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 basedpyright-code-budget.json                                 | 4 ++--
 litellm/litellm_core_utils/model_response_utils.py            | 2 --
 litellm/llms/oci/chat/cohere.py                               | 4 ++--
 .../llms/vertex_ai/image_edit/vertex_gemini_transformation.py | 1 -
 .../llms/vertex_ai/vector_stores/rag_api/transformation.py    | 2 --
 litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py | 4 ++--
 .../proxy/guardrails/guardrail_hooks/repelloai/repelloai.py   | 2 +-
 litellm/rag/ingestion/vertex_ai_ingestion.py                  | 2 --
 ruff-strict-budget.json                                       | 4 ++--
 type-discipline-budget.json                                   | 2 +-
 10 files changed, 10 insertions(+), 17 deletions(-)

diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json
index f8ef0ce1732..529a36d2614 100644
--- a/basedpyright-code-budget.json
+++ b/basedpyright-code-budget.json
@@ -1,6 +1,6 @@
 {
   "reportAny": {
-    "limit": 16389
+    "limit": 16387
   },
   "reportArgumentType": {
     "limit": 2229
@@ -24,7 +24,7 @@
     "limit": 19
   },
   "reportExplicitAny": {
-    "limit": 5242
+    "limit": 5241
   },
   "reportFunctionMemberAccess": {
     "limit": 7
diff --git a/litellm/litellm_core_utils/model_response_utils.py b/litellm/litellm_core_utils/model_response_utils.py
index 7bf667164ae..dc4f375daa7 100644
--- a/litellm/litellm_core_utils/model_response_utils.py
+++ b/litellm/litellm_core_utils/model_response_utils.py
@@ -144,7 +144,6 @@ def _is_choice_non_empty(choice: StreamingChoices) -> bool:
     # Check model_extra for dynamically added fields on the choice
     choice_extra_fields: Final[Mapping[str, object]] = choice.model_extra or {}
     for extra_field_name, extra_field_value in choice_extra_fields.items():
-        # Skip certain structural fields that are just default/None placeholders
         if extra_field_name == "index" and extra_field_value == 0:
             continue
         if extra_field_name in {"finish_reason", "logprobs"} and extra_field_value is None:
@@ -192,7 +191,6 @@ def _is_delta_non_empty(delta: Delta) -> bool:
     # Check model_extra for dynamically added fields (this is where Pydantic stores them)
     delta_extra_fields: Final[Mapping[str, object]] = delta.model_extra or {}
     for extra_field_value in delta_extra_fields.values():
-        # Even structural fields are meaningful if they have actual content
         if _has_meaningful_content(extra_field_value):
             return True
 
diff --git a/litellm/llms/oci/chat/cohere.py b/litellm/llms/oci/chat/cohere.py
index 384e7ec4cf8..6e9bb83b0a0 100644
--- a/litellm/llms/oci/chat/cohere.py
+++ b/litellm/llms/oci/chat/cohere.py
@@ -9,7 +9,7 @@ response parsing, and streaming chunk parsing for models served with
 import datetime
 import json
 from collections.abc import Iterable, Mapping, Sequence
-from typing import Any, Final
+from typing import Final
 
 import httpx
 from pydantic import JsonValue, TypeAdapter, ValidationError
@@ -76,7 +76,7 @@ def _content_text(content: str | Iterable[Mapping[str, object]] | None) -> str:
     return str(content)
 
 
-def _extract_text_content(content: Any) -> str:
+def _extract_text_content(content: str | Iterable[Mapping[str, object]] | None) -> str:
     """Return the plain-text representation of a message content value."""
     return _content_text(content)
 
diff --git a/litellm/llms/vertex_ai/image_edit/vertex_gemini_transformation.py b/litellm/llms/vertex_ai/image_edit/vertex_gemini_transformation.py
index 5889a8eba06..725a7f39917 100644
--- a/litellm/llms/vertex_ai/image_edit/vertex_gemini_transformation.py
+++ b/litellm/llms/vertex_ai/image_edit/vertex_gemini_transformation.py
@@ -182,7 +182,6 @@ class VertexAIGeminiImageEditConfig(BaseImageEditConfig, VertexLLM):
             else None
         )
 
-        # Generation config with proper structure for image editing
         generation_config: Final[dict[str, object]] = {
             key: value for key, value in (("response_modalities", ["IMAGE"]), ("image_config", image_config)) if value
         }
diff --git a/litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py b/litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py
index eedd488ecdb..5c250fc1a7e 100644
--- a/litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py
+++ b/litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py
@@ -203,7 +203,6 @@ class VertexVectorStoreConfig(BaseVectorStoreConfig, VertexBase):
             if value is not None
         }
 
-        # Build the request body for Vertex AI RAG API
         query_body: Final[Mapping[str, object]] = {
             key: value
             for key, value in (("text", query), ("rag_retrieval_config", rag_retrieval_config or None))
@@ -294,7 +293,6 @@ class VertexVectorStoreConfig(BaseVectorStoreConfig, VertexBase):
         # Add metadata if provided
         metadata: Final = vector_store_create_optional_params.get("metadata")
 
-        # Build the request body for Vertex AI RAG Corpus creation
         request_body: Final[dict[str, object]] = {
             key: value
             for key, value in (
diff --git a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py
index daa15fb5e5f..d8c8c2f4974 100644
--- a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py
+++ b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py
@@ -433,7 +433,7 @@ class HeadroomGuardrail(CustomGuardrail):
             payload["model"] = model
 
         try:
-            raw_response: HttpxResponse = await self.async_handler.post(  # pyright: ignore[reportUnknownMemberType]
+            raw_response: HttpxResponse = await self.async_handler.post(  # pyright: ignore[reportUnknownMemberType]  # AsyncHTTPHandler.post is untyped
                 url=f"{self.headroom_api_base}/v1/compress",
                 json=payload,
                 headers=self._request_headers(),
@@ -570,7 +570,7 @@ class HeadroomGuardrail(CustomGuardrail):
             params["query"] = query
 
         try:
-            raw_response: HttpxResponse = await self.async_handler.get(  # pyright: ignore[reportUnknownMemberType]
+            raw_response: HttpxResponse = await self.async_handler.get(  # pyright: ignore[reportUnknownMemberType]  # AsyncHTTPHandler.get is untyped
                 url=f"{self.headroom_api_base}/v1/retrieve/{hash_value}",
                 params=params,
                 headers=self._request_headers(),
diff --git a/litellm/proxy/guardrails/guardrail_hooks/repelloai/repelloai.py b/litellm/proxy/guardrails/guardrail_hooks/repelloai/repelloai.py
index d842a9b9b6a..8925cc5b3a6 100644
--- a/litellm/proxy/guardrails/guardrail_hooks/repelloai/repelloai.py
+++ b/litellm/proxy/guardrails/guardrail_hooks/repelloai/repelloai.py
@@ -197,7 +197,7 @@ class RepelloAIGuardrail(CustomGuardrail):
         repelloai_response: RepelloAIAnalyzeResponse | None = None
         try:
             verbose_proxy_logger.debug("RepelloAI Argus request: %s", request)
-            response: Final[HttpxResponse] = await self.async_handler.post(  # pyright: ignore[reportUnknownMemberType]
+            response: Final[HttpxResponse] = await self.async_handler.post(  # pyright: ignore[reportUnknownMemberType]  # AsyncHTTPHandler.post is untyped
                 url=endpoint,
                 headers={"X-API-Key": self.repelloai_api_key},
                 json=request,
diff --git a/litellm/rag/ingestion/vertex_ai_ingestion.py b/litellm/rag/ingestion/vertex_ai_ingestion.py
index 07f9f346d08..eff8ad1b8cb 100644
--- a/litellm/rag/ingestion/vertex_ai_ingestion.py
+++ b/litellm/rag/ingestion/vertex_ai_ingestion.py
@@ -186,7 +186,6 @@ class VertexAIRAGIngestion(BaseRAGIngestion, VertexBase):
         base_url: Final = get_vertex_base_url(self.location)
         url: Final = f"{base_url}/v1beta1/projects/{self.project_id}/locations/{self.location}/ragCorpora"
 
-        # Build request body with camelCase keys (Vertex AI API format)
         vector_db_config: Final = self.vector_store_config.get("vector_db_config")
         embedding_model: Final = self.vector_store_config.get("embedding_model")
         embedding_model_config: Final = (
@@ -447,7 +446,6 @@ class VertexAIRAGIngestion(BaseRAGIngestion, VertexBase):
         # Add max embedding requests per minute if specified
         max_embedding_qpm: Final = self.vector_store_config.get("max_embedding_requests_per_min")
 
-        # Build request body with camelCase keys (Vertex AI API format)
         chunking_config: Final = (
             {"chunkSize": chunk_size or 1024, "chunkOverlap": chunk_overlap or 200}
             if chunk_size or chunk_overlap
diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json
index fce91d57d88..5d0e7b617cb 100644
--- a/ruff-strict-budget.json
+++ b/ruff-strict-budget.json
@@ -24,7 +24,7 @@
     "limit": 133
   },
   "ANN401": {
-    "limit": 597
+    "limit": 596
   },
   "ASYNC230": {
     "limit": 11
@@ -231,7 +231,7 @@
     "limit": 5
   },
   "TID251": {
-    "limit": 1111
+    "limit": 1110
   },
   "TRY002": {
     "limit": 524
diff --git a/type-discipline-budget.json b/type-discipline-budget.json
index bb02618b11e..9c805bd7b61 100644
--- a/type-discipline-budget.json
+++ b/type-discipline-budget.json
@@ -9,7 +9,7 @@
     "limit": 269
   },
   "LIT004": {
-    "limit": 43
+    "limit": 40
   },
   "LIT005": {
     "limit": 0

From 14f392bb9b10bdcee7e134e8132f1b070e913e0a Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Sun, 30 Aug 2026 13:11:32 -0700
Subject: [PATCH 10/37] fix(docker): bump wolfi-base for glibc 2.44 and pin apk
 python to 3.13

---
 Dockerfile                 | 15 ++++++++-------
 backend/Dockerfile         | 12 ++++++------
 docker/Dockerfile.database | 15 ++++++++-------
 docker/Dockerfile.non_root | 17 +++++++++--------
 gateway/Dockerfile         | 12 ++++++------
 5 files changed, 37 insertions(+), 34 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 700b0d6525e..b3ee85e9ed1 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,10 +1,10 @@
 # syntax=docker/dockerfile:1.7
 
 # Base image for building
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
 
 # Runtime image
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 # Pinned by digest like the other base images; bump explicitly on Node upgrades.
 ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
@@ -40,8 +40,8 @@ COPY --from=uvbin /uvx /usr/local/bin/uvx
 RUN apk add --no-cache \
     bash \
     gcc \
-    python3 \
-    python3-dev \
+    python-3.13 \
+    python-3.13-dev \
     rust \
     openssl \
     openssl-dev \
@@ -51,6 +51,7 @@ RUN apk add --no-cache \
 
 ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
     UV_LINK_MODE=copy \
+    UV_PYTHON_DOWNLOADS=0 \
     PATH="/app/.venv/bin:${PATH}"
 
 # Copy dependency metadata first for layer caching
@@ -65,7 +66,7 @@ RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-gr
     --extra extra_proxy \
     --extra semantic-router \
     --extra saml \
-    --python python3
+    --python python3.13
 
 # Copy full source tree
 COPY . .
@@ -86,7 +87,7 @@ RUN uv sync --frozen --no-default-groups --no-editable \
     --extra extra_proxy \
     --extra semantic-router \
     --extra saml \
-    --python python3
+    --python python3.13
 
 RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
     npm_config_cache=/root/.npm \
@@ -101,7 +102,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
 USER root
 
 # node (without npm) is required by the prisma CLI at runtime
-RUN apk add --no-cache bash openssl tzdata nodejs python3 libsndfile
+RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile
 
 WORKDIR /app
 ENV PATH="/app/.venv/bin:${PATH}" \
diff --git a/backend/Dockerfile b/backend/Dockerfile
index 4ca40944606..aa01b9fba8b 100644
--- a/backend/Dockerfile
+++ b/backend/Dockerfile
@@ -1,5 +1,5 @@
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 
 FROM $UV_IMAGE AS uvbin
@@ -16,7 +16,7 @@ COPY --from=uvbin /uv /uvx /usr/local/bin/
 # instead of nodeenv downloading one whose dynamic deps may not be in Wolfi
 # (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes.
 RUN for i in 1 2 3; do \
-      apk add --no-cache bash gcc python3 python3-dev openssl openssl-dev libsndfile nodejs npm && break; \
+      apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev libsndfile nodejs npm && break; \
       [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
       sleep 5; \
     done
@@ -46,7 +46,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
         --extra proxy-runtime \
         --extra extra_proxy \
         --extra semantic-router \
-        --python python3
+        --python python3.13
 
 # Stage 2 — copy source and install the project + workspace members.
 COPY . .
@@ -57,7 +57,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
         --extra proxy-runtime \
         --extra extra_proxy \
         --extra semantic-router \
-        --python python3
+        --python python3.13
 
 RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
     npm_config_cache=/root/.npm \
@@ -71,7 +71,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
 USER root
 
 RUN for i in 1 2 3; do \
-      apk add --no-cache bash openssl tzdata python3 libsndfile libatomic && break; \
+      apk add --no-cache bash openssl tzdata python-3.13 libsndfile libatomic && break; \
       [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
       sleep 5; \
     done
diff --git a/docker/Dockerfile.database b/docker/Dockerfile.database
index f0d6d02fccf..c1348f68231 100644
--- a/docker/Dockerfile.database
+++ b/docker/Dockerfile.database
@@ -1,10 +1,10 @@
 # syntax=docker/dockerfile:1.7
 
 # Base image for building
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
 
 # Runtime image
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 # Pinned by digest like the other base images; bump explicitly on Node upgrades.
 ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
@@ -39,8 +39,8 @@ COPY --from=uvbin /uvx /usr/local/bin/uvx
 RUN apk add --no-cache \
     bash \
     gcc \
-    python3 \
-    python3-dev \
+    python-3.13 \
+    python-3.13-dev \
     openssl \
     openssl-dev \
     nodejs \
@@ -49,6 +49,7 @@ RUN apk add --no-cache \
 
 ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
     UV_LINK_MODE=copy \
+    UV_PYTHON_DOWNLOADS=0 \
     PATH="/app/.venv/bin:${PATH}"
 
 # Copy dependency metadata first for layer caching
@@ -63,7 +64,7 @@ RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-gr
     --extra extra_proxy \
     --extra semantic-router \
     --extra saml \
-    --python python3
+    --python python3.13
 
 # Copy full source tree
 COPY . .
@@ -84,7 +85,7 @@ RUN uv sync --frozen --no-default-groups --no-editable \
     --extra extra_proxy \
     --extra semantic-router \
     --extra saml \
-    --python python3
+    --python python3.13
 
 RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
     npm_config_cache=/root/.npm \
@@ -98,7 +99,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
 USER root
 
 # node (without npm) is required by the prisma CLI at runtime
-RUN apk add --no-cache bash openssl tzdata nodejs python3 libsndfile
+RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile
 
 WORKDIR /app
 ENV PATH="/app/.venv/bin:${PATH}" \
diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root
index 4a5df6ecd69..2221435a83a 100644
--- a/docker/Dockerfile.non_root
+++ b/docker/Dockerfile.non_root
@@ -1,8 +1,8 @@
 # syntax=docker/dockerfile:1.7
 
 # Base images
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
 ARG PROXY_EXTRAS_SOURCE=published
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 # Pinned by digest like the other base images; bump explicitly on Node upgrades.
@@ -37,8 +37,8 @@ COPY --from=uvbin /uvx /usr/local/bin/uvx
 
 RUN for i in 1 2 3; do \
       apk add --no-cache \
-        python3 \
-        python3-dev \
+        python-3.13 \
+        python-3.13-dev \
         gcc \
         rust \
         bash \
@@ -52,6 +52,7 @@ RUN for i in 1 2 3; do \
 
 ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
     UV_LINK_MODE=copy \
+    UV_PYTHON_DOWNLOADS=0 \
     PATH="/app/.venv/bin:${PATH}" \
     LITELLM_NON_ROOT=true \
     XDG_CACHE_HOME=/app/.cache
@@ -69,7 +70,7 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
     --extra extra_proxy \
     --extra semantic-router \
     --extra saml \
-    --python python3
+    --python python3.13
 
 # Copy full source tree
 COPY . .
@@ -96,7 +97,7 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
         --extra extra_proxy \
         --extra semantic-router \
         --extra saml \
-        --python python3 \
+        --python python3.13 \
         --no-sources-package litellm-proxy-extras; \
     else \
       uv sync --frozen --no-default-groups --no-editable \
@@ -105,7 +106,7 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
         --extra extra_proxy \
         --extra semantic-router \
         --extra saml \
-        --python python3; \
+        --python python3.13; \
     fi
 
 RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
@@ -124,7 +125,7 @@ RUN for i in 1 2 3; do \
       apk upgrade --no-cache && break || sleep 5; \
     done && \
     for i in 1 2 3; do \
-      apk add --no-cache python3 bash openssl tzdata libsndfile nodejs && break || sleep 5; \
+      apk add --no-cache python-3.13 bash openssl tzdata libsndfile nodejs && break || sleep 5; \
     done
 
 # Copy only what runtime needs. The application is installed inside the venv;
diff --git a/gateway/Dockerfile b/gateway/Dockerfile
index 4a2e32e186e..308d70a6b26 100644
--- a/gateway/Dockerfile
+++ b/gateway/Dockerfile
@@ -1,5 +1,5 @@
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 
 FROM $UV_IMAGE AS uvbin
@@ -16,7 +16,7 @@ COPY --from=uvbin /uv /uvx /usr/local/bin/
 # instead of nodeenv downloading one whose dynamic deps may not be in Wolfi
 # (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes.
 RUN for i in 1 2 3; do \
-      apk add --no-cache bash gcc python3 python3-dev openssl openssl-dev libsndfile nodejs npm && break; \
+      apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev libsndfile nodejs npm && break; \
       [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
       sleep 5; \
     done
@@ -47,7 +47,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
         --extra extra_proxy \
         --extra semantic-router \
         --extra bedrock-realtime \
-        --python python3
+        --python python3.13
 
 # Stage 2 — copy source and install the project + workspace members.
 COPY . .
@@ -59,7 +59,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
         --extra extra_proxy \
         --extra semantic-router \
         --extra bedrock-realtime \
-        --python python3
+        --python python3.13
 
 RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
     npm_config_cache=/root/.npm \
@@ -73,7 +73,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
 USER root
 
 RUN for i in 1 2 3; do \
-      apk add --no-cache bash openssl tzdata python3 libsndfile libatomic && break; \
+      apk add --no-cache bash openssl tzdata python-3.13 libsndfile libatomic && break; \
       [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
       sleep 5; \
     done

From 1f80f93750a4c0c4c8717d9d34a69b6d93208caa Mon Sep 17 00:00:00 2001
From: samtsai15 <6171228+samtsai15@users.noreply.github.com>
Date: Thu, 27 Aug 2026 09:57:27 +0800
Subject: [PATCH 11/37] fix(guardrails): carry Anthropic url and file image
 sources through to guardrails

_image_sources returned source["data"] only. An Anthropic image block has three
shapes (types/llms/anthropic.py:259) and only the base64 one carries "data", so
{"type": "url", "url": ...} yielded nothing and the image never reached any
guardrail at all.

This is not Bedrock-specific. Five guardrails consume
GenericGuardrailAPIInputs["images"] (vigil_guard, custom_code, deepkeep, straiker,
generic_guardrail_api) and every one of them was blind to url sources on
/v1/messages.

base64 now returns a data URI rather than the bare payload. A consumer otherwise
has no way to recover media_type, and an API like Bedrock's ApplyGuardrail needs
the format to build its request.

The file shape stays unresolvable here: the bytes live behind the Files API and
this extractor has no client to fetch them. Documented rather than silently
dropped, so a consumer treating a missing entry as "no image to scan" is a known
gap and not a surprise.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 .../chat/guardrail_translation/handler.py     | 32 +++++++++++++++++--
 1 file changed, 30 insertions(+), 2 deletions(-)

diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py
index eb6dbd3e5b3..4b763d11652 100644
--- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py
+++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py
@@ -859,12 +859,40 @@ class AnthropicMessagesHandler(BaseTranslation):
 
     @staticmethod
     def _image_sources(block: Mapping[str, object]) -> tuple[str, ...]:
+        """Normalize an Anthropic image block into strings a guardrail can read.
+
+        `source` is one of three shapes (types/llms/anthropic.py:259):
+
+            {"type": "base64", "media_type": "image/png", "data": "<b64>"}
+            {"type": "url",    "url": "https://..."}
+            {"type": "file",   "file_id": "..."}
+
+        base64 is returned as a data URI rather than the bare payload: consumers of
+        ``GenericGuardrailAPIInputs["images"]`` otherwise have no way to know the
+        format, and an API like Bedrock's ApplyGuardrail requires it. url is passed
+        through so the consumer can fetch it under its own SSRF policy.
+
+        file is not resolvable here (the bytes live behind the Files API), so it
+        yields nothing. That is a silent gap for any consumer that treats a missing
+        entry as "no image to scan"; scanning a file_id needs a fetch this extractor
+        has no client for.
+        """
         source: Final = block.get("source")
         if not isinstance(source, Mapping):
             return ()
-        # Could be base64 or url
+
+        source_type: Final = source.get("type")
+        if source_type == "url":
+            url: Final = source.get("url")
+            return (url,) if isinstance(url, str) and url else ()
+
         data: Final = source.get("data")
-        return (data,) if data else ()
+        if not isinstance(data, str) or not data:
+            return ()
+        media_type: Final = source.get("media_type")
+        if isinstance(media_type, str) and media_type:
+            return (f"data:{media_type};base64,{data}",)
+        return (data,)
 
     async def _apply_guardrail_responses_to_input(
         self,

From 0e7562dbc692812dfb5cb02ad72434d54a25b715 Mon Sep 17 00:00:00 2001
From: samtsai15 <6171228+samtsai15@users.noreply.github.com>
Date: Thu, 27 Aug 2026 11:22:51 +0800
Subject: [PATCH 12/37] test(guardrails): cover every Anthropic image source
 shape in the extractor's own suite

_image_sources had no test asserting what it extracts. The existing image tests
live on the Bedrock side and all use base64 without a media_type, which is the one
path the fix left unchanged, so both behaviors it does change went unverified: the
url shape reaching the guardrail at all, and base64 arriving as a data URI.

Against the pre-fix extractor the url case sees [] and the media_type case sees
['AAAA'] instead of ['data:image/png;base64,AAAA'].

The remaining three assert behavior the fix deliberately preserves -- bare base64
passed through, a file source yielding nothing, a malformed source dropped rather
than handed on for a consumer to choke on.

Each message carries a text block because a message with no text never reaches the
guardrail, which would make every source shape look equally dropped.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 .../test_anthropic_guardrail_handler.py       | 86 +++++++++++++++++++
 1 file changed, 86 insertions(+)

diff --git a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py
index af3ccd65b11..2b606b9639a 100644
--- a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py
+++ b/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py
@@ -1490,6 +1490,92 @@ class MockCanaryMaskingGuardrail(CustomGuardrail):
         return inputs
 
 
+class TestAnthropicMessagesImageSources:
+    """An Anthropic image block has three source shapes (types/llms/anthropic.py:259).
+
+    Only the base64 one carries "data", so reading that key alone drops url images
+    entirely -- for every guardrail consuming GenericGuardrailAPIInputs["images"],
+    not just Bedrock.
+    """
+
+    def _data(self, messages):
+        return {"model": "claude-sonnet-4-5", "messages": messages}
+
+    async def _images_seen(self, content) -> list[str]:
+        handler = AnthropicMessagesHandler()
+
+        class ImageRecordingGuardrail(MockCanaryMaskingGuardrail):
+            def __init__(self):
+                super().__init__()
+                self.seen_images: list[str] = []  # mutable-ok: accumulator for the assertion
+
+            async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None):
+                self.seen_images.extend(inputs.get("images") or [])
+                return await super().apply_guardrail(inputs, request_data, input_type, logging_obj)
+
+        guardrail = ImageRecordingGuardrail()
+        # The text block is what gets the guardrail invoked at all: a message with
+        # no text gives the handler nothing to scan, so it never reaches the
+        # guardrail and every source shape would look equally "dropped".
+        await handler.process_input_messages(
+            data=self._data([{"role": "user", "content": [{"type": "text", "text": "describe it"}, *content]}]),
+            guardrail_to_apply=guardrail,
+        )
+        return guardrail.seen_images
+
+    @pytest.mark.asyncio
+    async def test_url_source_reaches_the_guardrail(self):
+        """A url source has no "data" key, so it used to yield nothing at all."""
+        seen = await self._images_seen(
+            [{"type": "image", "source": {"type": "url", "url": "https://example.com/a.png"}}]
+        )
+
+        assert seen == ["https://example.com/a.png"]
+
+    @pytest.mark.asyncio
+    async def test_base64_source_carries_its_media_type(self):
+        """Bare base64 leaves the consumer no way to recover the format.
+
+        An API like Bedrock's ApplyGuardrail needs it to build the request, so the
+        media_type travels with the payload as a data URI.
+        """
+        seen = await self._images_seen(
+            [{"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "AAAA"}}]
+        )
+
+        assert seen == ["data:image/png;base64,AAAA"]
+
+    @pytest.mark.asyncio
+    async def test_base64_source_without_a_media_type_is_passed_through(self):
+        """There is no format to attach, so the payload goes through unchanged."""
+        seen = await self._images_seen([{"type": "image", "source": {"type": "base64", "data": "AAAA"}}])
+
+        assert seen == ["AAAA"]
+
+    @pytest.mark.asyncio
+    async def test_file_source_yields_nothing(self):
+        """The bytes live behind the Files API and this extractor has no client.
+
+        Documented as a known gap rather than silently handed on as a file_id string,
+        which a consumer would try to decode as an image.
+        """
+        seen = await self._images_seen([{"type": "image", "source": {"type": "file", "file_id": "file_abc"}}])
+
+        assert seen == []
+
+    @pytest.mark.asyncio
+    async def test_a_malformed_source_is_dropped_rather_than_passed_on(self):
+        seen = await self._images_seen(
+            [
+                {"type": "image", "source": {"type": "base64"}},
+                {"type": "image", "source": {"type": "url"}},
+                {"type": "image", "source": {"type": "base64", "data": ""}},
+            ]
+        )
+
+        assert seen == []
+
+
 class TestAnthropicMessagesToolResultScanning:
     """LIT-5251: tool_result blocks carry whatever a client's local tool fetched, so
     they are the request-path payload an indirect prompt injection actually arrives in.

From bb51c121cf943b384fd00b0a2b106deca4dd5d53 Mon Sep 17 00:00:00 2001
From: "feng.tsai" <samtsai15@gmail.com>
Date: Mon, 31 Aug 2026 12:21:24 +0800
Subject: [PATCH 13/37] docs: reference the source union by type instead of a
 line number

The line number went stale when the base moved.
---
 litellm/llms/anthropic/chat/guardrail_translation/handler.py    | 2 +-
 .../guardrail_translation/test_anthropic_guardrail_handler.py   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py
index 4b763d11652..5aa0dc94b89 100644
--- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py
+++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py
@@ -861,7 +861,7 @@ class AnthropicMessagesHandler(BaseTranslation):
     def _image_sources(block: Mapping[str, object]) -> tuple[str, ...]:
         """Normalize an Anthropic image block into strings a guardrail can read.
 
-        `source` is one of three shapes (types/llms/anthropic.py:259):
+        `source` is one of three shapes (`AnthropicMessagesImageParam.source`):
 
             {"type": "base64", "media_type": "image/png", "data": "<b64>"}
             {"type": "url",    "url": "https://..."}
diff --git a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py
index 2b606b9639a..0fe7730e91e 100644
--- a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py
+++ b/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py
@@ -1491,7 +1491,7 @@ class MockCanaryMaskingGuardrail(CustomGuardrail):
 
 
 class TestAnthropicMessagesImageSources:
-    """An Anthropic image block has three source shapes (types/llms/anthropic.py:259).
+    """An Anthropic image block has three source shapes (`AnthropicMessagesImageParam.source`).
 
     Only the base64 one carries "data", so reading that key alone drops url images
     entirely -- for every guardrail consuming GenericGuardrailAPIInputs["images"],

From dc034a086a17d349e005fc2b547ed8a6b9c2654e Mon Sep 17 00:00:00 2001
From: "feng.tsai" <samtsai15@gmail.com>
Date: Mon, 31 Aug 2026 13:26:02 +0800
Subject: [PATCH 14/37] docs: trim the _image_sources docstring

It restated the source union that the type definition already carries.
---
 .../chat/guardrail_translation/handler.py      | 18 +++---------------
 1 file changed, 3 insertions(+), 15 deletions(-)

diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py
index 5aa0dc94b89..9395cc3d33e 100644
--- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py
+++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py
@@ -861,21 +861,9 @@ class AnthropicMessagesHandler(BaseTranslation):
     def _image_sources(block: Mapping[str, object]) -> tuple[str, ...]:
         """Normalize an Anthropic image block into strings a guardrail can read.
 
-        `source` is one of three shapes (`AnthropicMessagesImageParam.source`):
-
-            {"type": "base64", "media_type": "image/png", "data": "<b64>"}
-            {"type": "url",    "url": "https://..."}
-            {"type": "file",   "file_id": "..."}
-
-        base64 is returned as a data URI rather than the bare payload: consumers of
-        ``GenericGuardrailAPIInputs["images"]`` otherwise have no way to know the
-        format, and an API like Bedrock's ApplyGuardrail requires it. url is passed
-        through so the consumer can fetch it under its own SSRF policy.
-
-        file is not resolvable here (the bytes live behind the Files API), so it
-        yields nothing. That is a silent gap for any consumer that treats a missing
-        entry as "no image to scan"; scanning a file_id needs a fetch this extractor
-        has no client for.
+        base64 becomes a data URI so the format travels with the payload, which is what
+        the OpenAI path already puts in this field. A file source yields nothing: those
+        bytes live behind the Files API and this extractor has no client to fetch them.
         """
         source: Final = block.get("source")
         if not isinstance(source, Mapping):

From 69fc4496664f3b468bb4384cbcae91d0b0990c23 Mon Sep 17 00:00:00 2001
From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Mon, 31 Aug 2026 07:54:31 +0000
Subject: [PATCH 15/37] chore(techdebt): drop restating comments from the
 2026-08-30 window

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/google_genai/adapters/transformation.py        | 1 -
 litellm/llms/runwayml/videos/transformation.py         | 2 --
 litellm/proxy/response_polling/background_streaming.py | 7 ++-----
 3 files changed, 2 insertions(+), 8 deletions(-)

diff --git a/litellm/google_genai/adapters/transformation.py b/litellm/google_genai/adapters/transformation.py
index b73a1443330..8ea19deef4c 100644
--- a/litellm/google_genai/adapters/transformation.py
+++ b/litellm/google_genai/adapters/transformation.py
@@ -108,7 +108,6 @@ class GoogleGenAIStreamWrapper(AdapterCompletionStreamWrapper):
 
     def __init__(self, completion_stream: object):
         self.sent_first_chunk = False
-        # State tracking for accumulating partial tool calls
         self.accumulated_tool_calls = dict[int, _ToolCallAccumulator]()
         self._returned_response = False
         super().__init__(completion_stream)
diff --git a/litellm/llms/runwayml/videos/transformation.py b/litellm/llms/runwayml/videos/transformation.py
index c7289eb47f5..c7696a1cb29 100644
--- a/litellm/llms/runwayml/videos/transformation.py
+++ b/litellm/llms/runwayml/videos/transformation.py
@@ -160,14 +160,12 @@ class RunwayMLVideoConfig(BaseVideoConfig):
             **self._prompt_image_param(video_create_optional_params),
             **self._ratio_param(video_create_optional_params),
             **self._duration_param(video_create_optional_params),
-            # Pass through other parameters that aren't OpenAI-specific
             **{key: value for key, value in video_create_optional_params.items() if key not in supported_openai_params},
         }
 
     @staticmethod
     def _prompt_image_param(video_create_optional_params: VideoCreateOptionalRequestParams) -> Mapping[str, object]:
         # Handle input_reference parameter - map to promptImage
-        # RunwayML supports URLs and data URIs directly
         if "input_reference" in video_create_optional_params:
             return {"promptImage": video_create_optional_params["input_reference"]}
         return {}
diff --git a/litellm/proxy/response_polling/background_streaming.py b/litellm/proxy/response_polling/background_streaming.py
index f755c6d478b..b03122966c0 100644
--- a/litellm/proxy/response_polling/background_streaming.py
+++ b/litellm/proxy/response_polling/background_streaming.py
@@ -144,10 +144,8 @@ async def background_streaming_task(
 
         # Process streaming response following OpenAI events format
         # https://platform.openai.com/docs/api-reference/responses-streaming
-        output_items: Final = dict[str, _OutputItem]()  # Track output items by ID
-        accumulated_text: Final = dict[
-            tuple[str, int], str
-        ]()  # Track accumulated text deltas by (item_id, content_index)
+        output_items: Final = dict[str, _OutputItem]()
+        accumulated_text: Final = dict[tuple[str, int], str]()
 
         # ResponsesAPIResponse fields to extract from response.completed
         usage_data = None
@@ -262,7 +260,6 @@ async def background_streaming_task(
                                 if "content" in delta_item:
                                     content_list = delta_item["content"]
                                     if content_index < len(content_list):
-                                        # Update existing content part with accumulated text
                                         content_entry = content_list[content_index]
                                         if isinstance(content_entry, dict):
                                             content_entry["text"] = accumulated_text[key]

From 39473745ddb6759b14397df3e9f9499e7b49ce0b Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Mon, 31 Aug 2026 09:53:01 -0700
Subject: [PATCH 16/37] fix(docker): bump wolfi-base for glibc 2.44 and pin apk
 python to 3.13 in migrations image

---
 migrations/Dockerfile | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/migrations/Dockerfile b/migrations/Dockerfile
index 6335e6f6bd8..c6d1b0cc46e 100644
--- a/migrations/Dockerfile
+++ b/migrations/Dockerfile
@@ -1,5 +1,5 @@
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 
 FROM $UV_IMAGE AS uvbin
@@ -35,7 +35,7 @@ COPY --from=uvbin /uv /uvx /usr/local/bin/
 # instead of nodeenv downloading one whose dynamic deps may not be in Wolfi
 # (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes.
 RUN for i in 1 2 3; do \
-      apk add --no-cache bash gcc python3 python3-dev openssl openssl-dev libsndfile nodejs npm && break; \
+      apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev libsndfile nodejs npm && break; \
       [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
       sleep 5; \
     done
@@ -56,7 +56,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
     uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
         --extra proxy \
         --extra extra_proxy \
-        --python python3
+        --python python3.13
 
 # Stage 2 — copy source and install the project + workspace members.
 COPY . .
@@ -65,7 +65,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
     uv sync --frozen --no-default-groups --no-editable \
         --extra proxy \
         --extra extra_proxy \
-        --python python3
+        --python python3.13
 
 COPY migrations/run.py /app/run.py
 
@@ -87,7 +87,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
 USER root
 
 RUN for i in 1 2 3; do \
-      apk add --no-cache bash openssl tzdata python3 nodejs libsndfile libatomic && break; \
+      apk add --no-cache bash openssl tzdata python-3.13 nodejs libsndfile libatomic && break; \
       [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
       sleep 5; \
     done

From 6b2ada2a780e1ebd4f4899833fbc7be9a3214123 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Mon, 31 Aug 2026 11:49:09 -0700
Subject: [PATCH 17/37] fix(bedrock): per-response realtime usage deltas,
 spend-log event filter, single transcript completed

---
 litellm/llms/bedrock/realtime/handler.py      |  70 +++++++---
 .../llms/bedrock/realtime/transformation.py   |  81 ++++++++----
 .../realtime/test_bedrock_realtime_handler.py | 125 ++++++++++++++----
 .../test_bedrock_realtime_transformation.py   | 106 +++++++++++++++
 4 files changed, 313 insertions(+), 69 deletions(-)

diff --git a/litellm/llms/bedrock/realtime/handler.py b/litellm/llms/bedrock/realtime/handler.py
index ecd143dd087..42fe8941443 100644
--- a/litellm/llms/bedrock/realtime/handler.py
+++ b/litellm/llms/bedrock/realtime/handler.py
@@ -7,14 +7,17 @@ This uses aws_sdk_bedrock_runtime for bidirectional streaming with Nova Sonic.
 import asyncio
 import contextlib
 import json
+from collections.abc import AsyncIterator, Mapping
 from typing import Final, Protocol
 
 from pydantic import JsonValue, TypeAdapter
 
+import litellm
 from litellm._logging import _redact_string, verbose_proxy_logger
 from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
 from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
+from litellm.litellm_core_utils.realtime_streaming import DefaultLoggedRealTimeEventTypes
 from litellm.types.llms.openai import OpenAIRealtimeEvents
 from litellm.types.realtime import RealtimeResponseTransformInput
 
@@ -34,6 +37,17 @@ def _json_str(value: JsonValue) -> str | None:
     return value if isinstance(value, str) else None
 
 
+def _should_log_event(openai_message: Mapping[str, object]) -> bool:
+    logged_types: Final = (
+        litellm.logged_real_time_event_types
+        if litellm.logged_real_time_event_types is not None
+        else DefaultLoggedRealTimeEventTypes
+    )
+    if logged_types == "*":
+        return True
+    return openai_message.get("type") in logged_types
+
+
 class RealtimeClientWebSocket(Protocol):
     """The client-facing websocket surface the realtime bridge talks to."""
 
@@ -207,18 +221,22 @@ class BedrockRealtime(BaseAWSLLM):
                 )
             )
 
-            logged_events: Final[list[OpenAIRealtimeEvents]] = []  # mutable-ok: filled across the stream loop
-            bedrock_to_client_task: Final = asyncio.create_task(
-                self._forward_bedrock_to_client(
-                    bedrock_stream,
-                    websocket,
-                    transformation_config,
-                    model,
-                    logging_obj,
-                    session_state,
-                    logged_events,
+            async def forward_bedrock_and_collect_logged_events() -> tuple[OpenAIRealtimeEvents, ...]:
+                return tuple(
+                    [
+                        event
+                        async for event in self._forward_bedrock_to_client(
+                            bedrock_stream,
+                            websocket,
+                            transformation_config,
+                            model,
+                            logging_obj,
+                            session_state,
+                        )
+                    ]
                 )
-            )
+
+            bedrock_to_client_task: Final = asyncio.create_task(forward_bedrock_and_collect_logged_events())
 
             # Wait for both tasks to complete
             await asyncio.gather(
@@ -227,9 +245,25 @@ class BedrockRealtime(BaseAWSLLM):
                 return_exceptions=True,
             )
 
+            forwarded_logged_events: Final = (
+                bedrock_to_client_task.result()
+                if not bedrock_to_client_task.cancelled() and bedrock_to_client_task.exception() is None
+                else ()
+            )
+            logged_events: Final = (
+                *forwarded_logged_events,
+                *(
+                    leftover_event
+                    for leftover_event in transformation_config.leftover_usage_done_events()
+                    if _should_log_event(leftover_event)
+                ),
+            )
             if logged_events:
                 GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(
-                    logging_obj.dispatch_success_handlers(logged_events, prefer_async_handlers=True)
+                    logging_obj.dispatch_success_handlers(
+                        list(logged_events),  # mutable-ok: realtime spend logging requires a list result
+                        prefer_async_handlers=True,
+                    )
                 )
 
         except Exception as e:
@@ -313,9 +347,8 @@ class BedrockRealtime(BaseAWSLLM):
         model: str,
         logging_obj: LiteLLMLogging,
         session_state: RealtimeResponseTransformInput,
-        logged_events: "list[OpenAIRealtimeEvents] | None" = None,  # mutable-ok: caller-owned spend log accumulator
-    ):
-        """Forward messages from Bedrock stream to client WebSocket."""
+    ) -> AsyncIterator[OpenAIRealtimeEvents]:
+        """Forward messages from Bedrock to the client, yielding the ones to record for spend logging."""
         try:
             while True:
                 # Receive from Bedrock
@@ -363,13 +396,14 @@ class BedrockRealtime(BaseAWSLLM):
                     )
 
                     # Send transformed messages to client
-                    openai_messages = transformed_response.get("response", [])
+                    response_value = transformed_response["response"]
+                    openai_messages = response_value if isinstance(response_value, list) else (response_value,)
                     for openai_message in openai_messages:
-                        if logged_events is not None and isinstance(openai_message, dict):
-                            logged_events.append(openai_message)
                         message_json = json.dumps(openai_message)
                         await client_ws.send_text(message_json)
                         verbose_proxy_logger.debug("Bedrock Realtime: Sent to client: %s", message_json[:200])
+                        if _should_log_event(openai_message):
+                            yield openai_message
 
         except Exception as e:
             verbose_proxy_logger.debug("Bedrock to client forwarding ended: %s", e, exc_info=True)
diff --git a/litellm/llms/bedrock/realtime/transformation.py b/litellm/llms/bedrock/realtime/transformation.py
index ec5eff0a6fc..28c2e446d10 100644
--- a/litellm/llms/bedrock/realtime/transformation.py
+++ b/litellm/llms/bedrock/realtime/transformation.py
@@ -41,7 +41,6 @@ from litellm.types.realtime import (
     RealtimeResponseTransformInput,
     RealtimeResponseTypedDict,
 )
-from litellm.utils import get_empty_usage
 
 
 class BedrockContentEnd(BaseModel):
@@ -118,7 +117,9 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
         self._user_transcript_active = False
         self._user_transcript_generation_stage: str | None = None
         self._user_item_id: str | None = None
-        self._latest_usage: OpenAIRealtimeResponseUsage | None = None
+        self._user_transcript_buffer = ""
+        self._cumulative_usage = BedrockUsageEvent()
+        self._reported_usage = BedrockUsageEvent()
 
     def validate_environment(self, headers: dict, model: str, api_key: str | None = None) -> dict:
         """Validate environment - no special validation needed for Bedrock."""
@@ -836,47 +837,79 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
         return (speech_event,)
 
     def transform_usage_event(self, usage_event: BedrockUsageEvent) -> None:
-        """Record Bedrock usageEvent token totals for the next response.done."""
+        """Record Bedrock's session-cumulative usage totals for the next response.done."""
         verbose_logger.debug("Handling usageEvent")
+        self._cumulative_usage = usage_event
+
+    def _take_usage_delta(self) -> OpenAIRealtimeResponseUsage:
+        """Usage for the response now completing: cumulative totals minus what prior response.done events reported."""
+        prior: Final = self._reported_usage
+        latest: Final = self._cumulative_usage
+        self._reported_usage = latest
         input_details: Final[OpenAIRealtimeUsageTokenDetails] = {
-            "audio_tokens": usage_event.details.total.input.speechTokens,
-            "text_tokens": usage_event.details.total.input.textTokens,
+            "audio_tokens": latest.details.total.input.speechTokens - prior.details.total.input.speechTokens,
+            "text_tokens": latest.details.total.input.textTokens - prior.details.total.input.textTokens,
             "cached_tokens": 0,
         }
         output_details: Final[OpenAIRealtimeUsageTokenDetails] = {
-            "audio_tokens": usage_event.details.total.output.speechTokens,
-            "text_tokens": usage_event.details.total.output.textTokens,
+            "audio_tokens": latest.details.total.output.speechTokens - prior.details.total.output.speechTokens,
+            "text_tokens": latest.details.total.output.textTokens - prior.details.total.output.textTokens,
         }
-        latest_usage: Final[OpenAIRealtimeResponseUsage] = {
-            "input_tokens": usage_event.totalInputTokens,
-            "output_tokens": usage_event.totalOutputTokens,
-            "total_tokens": usage_event.totalTokens,
+        usage_delta: Final[OpenAIRealtimeResponseUsage] = {
+            "input_tokens": latest.totalInputTokens - prior.totalInputTokens,
+            "output_tokens": latest.totalOutputTokens - prior.totalOutputTokens,
+            "total_tokens": latest.totalTokens - prior.totalTokens,
             "input_token_details": input_details,
             "output_token_details": output_details,
         }
-        self._latest_usage = latest_usage
+        return usage_delta
+
+    def leftover_usage_done_events(self) -> tuple[OpenAIRealtimeEvents, ...]:
+        """Logged-only response.done for usage Bedrock reports after the final turn's contentEnd."""
+        if self._cumulative_usage == self._reported_usage:
+            return ()
+        usage: Final = self._take_usage_delta()
+        leftover_done: Final = OpenAIRealtimeDoneEvent(
+            type="response.done",
+            event_id=f"event_{uuid.uuid4()}",
+            response=OpenAIRealtimeResponseDoneObject(
+                object="realtime.response",
+                id=f"resp_{uuid.uuid4()}",
+                status="completed",
+                conversation_id=f"conv_{uuid.uuid4()}",
+                usage=dict(usage),  # mutable-ok: OpenAIRealtimeResponseDoneObject types usage as plain dict
+            ),
+        )
+        return (leftover_done,)
 
     def transform_user_transcript_event(self, transcript: str) -> tuple[OpenAIRealtimeEvents, ...]:
-        """Transform a USER-role Bedrock textOutput (ASR transcript) to OpenAI transcription events."""
+        """Transform a USER-role Bedrock textOutput (ASR transcript) to an OpenAI transcription delta."""
         verbose_logger.debug("Handling USER textOutput (ASR transcript)")
-        item_id: Final = self._current_user_item_id()
         delta_event: Final[OpenAIRealtimeInputAudioTranscriptionDelta] = {
             "type": "conversation.item.input_audio_transcription.delta",
             "event_id": f"event_{uuid.uuid4()}",
-            "item_id": item_id,
+            "item_id": self._current_user_item_id(),
             "content_index": 0,
             "delta": transcript,
         }
-        if self._user_transcript_generation_stage == "SPECULATIVE":
-            return (delta_event,)
+        if self._user_transcript_generation_stage != "SPECULATIVE":
+            self._user_transcript_buffer += transcript
+        return (delta_event,)
+
+    def user_transcript_completed_events(self) -> tuple[OpenAIRealtimeEvents, ...]:
+        """One completed event with the full transcript once the FINAL user content block ends."""
+        transcript: Final = self._user_transcript_buffer
+        if not transcript:
+            return ()
+        self._user_transcript_buffer = ""
         completed_event: Final[OpenAIRealtimeInputAudioTranscriptionCompleted] = {
             "type": "conversation.item.input_audio_transcription.completed",
             "event_id": f"event_{uuid.uuid4()}",
-            "item_id": item_id,
+            "item_id": self._current_user_item_id(),
             "content_index": 0,
             "transcript": transcript,
         }
-        return (delta_event, completed_event)
+        return (completed_event,)
 
     def transform_text_output_event(
         self,
@@ -1096,14 +1129,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
         if not current_response_id or not current_conversation_id:
             return [], None, None, None
 
-        empty_usage: Final = get_empty_usage()
-        zero_usage: Final[OpenAIRealtimeResponseUsage] = {
-            "input_tokens": empty_usage.prompt_tokens,
-            "output_tokens": empty_usage.completion_tokens,
-            "total_tokens": empty_usage.total_tokens,
-        }
-        usage: Final = self._latest_usage or zero_usage
-        self._latest_usage = None
+        usage: Final = self._take_usage_delta()
         response_done: Final = OpenAIRealtimeDoneEvent(
             type="response.done",
             event_id=f"event_{uuid.uuid4()}",
@@ -1324,6 +1350,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
         elif "contentEnd" in event and self._user_transcript_active:
             self._user_transcript_active = False
             self._user_transcript_generation_stage = None
+            returned_messages.extend(self.user_transcript_completed_events())
 
         elif "contentEnd" in event:
             events, current_delta_chunks = self.transform_content_end_event(
diff --git a/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_handler.py b/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_handler.py
index aa6573884e2..0ea5b7ad4a1 100644
--- a/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_handler.py
+++ b/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_handler.py
@@ -6,7 +6,7 @@ from unittest.mock import MagicMock
 
 import pytest
 
-
+import litellm
 from litellm.llms.bedrock.common_utils import BedrockError
 from litellm.llms.bedrock.realtime.handler import BedrockRealtime
 from litellm.llms.bedrock.realtime.transformation import BedrockRealtimeConfig
@@ -124,14 +124,6 @@ class ScriptedBedrockStream:
         return (None, self._receiver)
 
 
-class ImmediatelyEndingBedrockStream:
-    def __init__(self):
-        self.input_stream = FakeInputStream()
-
-    async def await_output(self):
-        return (None, EndedBedrockReceiver())
-
-
 class FakeStaticCredentialsResolver:
     pass
 
@@ -171,7 +163,7 @@ def stub_aws_sdk_client(monkeypatch):
 
         async def invoke_model_with_bidirectional_stream(self, operation_input):
             captured["operation_input"] = operation_input
-            return ImmediatelyEndingBedrockStream()
+            return ScriptedBedrockStream(captured.get("scripted_payloads", []))
 
     package = types.ModuleType("aws_sdk_bedrock_runtime")
     client_module = types.ModuleType("aws_sdk_bedrock_runtime.client")
@@ -292,7 +284,40 @@ class TestBedrockRealtimeHandler:
         assert stream.input_stream.closed
 
     @pytest.mark.asyncio
-    async def test_forwarded_events_are_collected_for_spend_logging(self):
+    async def test_forwarded_events_are_filtered_to_logged_types_for_spend_logging(self):
+        handler = BedrockRealtime()
+        stream = ScriptedBedrockStream(
+            [
+                json.dumps({"event": {"userSpeechStart": {}}}),
+                json.dumps({"event": {"contentStart": {"role": "ASSISTANT", "type": "TEXT"}}}),
+                json.dumps({"event": {"textOutput": {"content": "Hi"}}}),
+                json.dumps({"event": {"contentEnd": {"stopReason": "END_TURN"}}}),
+            ]
+        )
+        client_ws = RealtimeClientWS()
+
+        logged_events = [
+            event
+            async for event in handler._forward_bedrock_to_client(
+                stream,
+                client_ws,
+                BedrockRealtimeConfig(),
+                "amazon.nova-sonic-v1:0",
+                FakeLogging(),
+                {},
+            )
+        ]
+
+        assert [event["type"] for event in logged_events] == ["response.done"]
+        sent_types = [json.loads(message)["type"] for message in client_ws.sent_to_client]
+        assert "input_audio_buffer.speech_started" in sent_types
+        assert "response.text.delta" in sent_types
+        assert "response.done" in sent_types
+        assert client_ws.closed
+
+    @pytest.mark.asyncio
+    async def test_logged_event_types_star_collects_every_forwarded_event(self, monkeypatch):
+        monkeypatch.setattr(litellm, "logged_real_time_event_types", "*")
         handler = BedrockRealtime()
         stream = ScriptedBedrockStream(
             [
@@ -301,37 +326,89 @@ class TestBedrockRealtimeHandler:
             ]
         )
         client_ws = RealtimeClientWS()
-        logged_events = []
 
-        await handler._forward_bedrock_to_client(
-            stream,
-            client_ws,
-            BedrockRealtimeConfig(),
-            "amazon.nova-sonic-v1:0",
-            FakeLogging(),
-            {},
-            logged_events,
-        )
+        logged_events = [
+            event
+            async for event in handler._forward_bedrock_to_client(
+                stream,
+                client_ws,
+                BedrockRealtimeConfig(),
+                "amazon.nova-sonic-v1:0",
+                FakeLogging(),
+                {},
+            )
+        ]
 
         assert [event["type"] for event in logged_events] == [
             "input_audio_buffer.speech_started",
             "input_audio_buffer.speech_stopped",
         ]
-        assert client_ws.closed
+
+    @pytest.mark.asyncio
+    async def test_trailing_usage_after_last_done_is_dispatched_for_spend(self, stub_aws_sdk_client, monkeypatch):
+        import litellm.llms.bedrock.realtime.handler as handler_module
+
+        dispatched = {}
+
+        class RecordingLogging(FakeLogging):
+            async def dispatch_success_handlers(self, result=None, prefer_async_handlers=False, **kwargs):
+                dispatched["events"] = result
+
+        class RecordingLoggingWorker:
+            def ensure_initialized_and_enqueue(self, coro):
+                dispatched["coro"] = coro
+
+        monkeypatch.setattr(handler_module, "GLOBAL_LOGGING_WORKER", RecordingLoggingWorker())
+        stub_aws_sdk_client["scripted_payloads"] = [
+            json.dumps(
+                {
+                    "event": {
+                        "usageEvent": {
+                            "totalInputTokens": 3,
+                            "totalOutputTokens": 6,
+                            "totalTokens": 9,
+                            "details": {
+                                "total": {
+                                    "input": {"speechTokens": 3, "textTokens": 0},
+                                    "output": {"speechTokens": 0, "textTokens": 6},
+                                }
+                            },
+                        }
+                    }
+                }
+            )
+        ]
+
+        await BedrockRealtime().async_realtime(
+            model="amazon.nova-sonic-v1:0",
+            websocket=RealtimeClientWS(),
+            logging_obj=RecordingLogging(),
+            aws_region_name="us-east-1",
+            aws_access_key_id="k",
+            aws_secret_access_key="s",
+        )
+        await dispatched["coro"]
+
+        assert [event["type"] for event in dispatched["events"]] == ["response.done"]
+        usage = dispatched["events"][0]["response"]["usage"]
+        assert (usage["input_tokens"], usage["output_tokens"], usage["total_tokens"]) == (3, 6, 9)
+        assert usage["input_token_details"] == {"audio_tokens": 3, "text_tokens": 0, "cached_tokens": 0}
+        assert usage["output_token_details"] == {"audio_tokens": 0, "text_tokens": 6}
 
     @pytest.mark.asyncio
     async def test_bedrock_stream_end_closes_client_websocket(self):
         handler = BedrockRealtime()
         client_ws = ClosableClientWS()
 
-        await handler._forward_bedrock_to_client(
+        async for _ in handler._forward_bedrock_to_client(
             EndedBedrockStream(),
             client_ws,
             BedrockRealtimeConfig(),
             "amazon.nova-sonic-v1:0",
             MagicMock(),
             {},
-        )
+        ):
+            pass
 
         assert client_ws.closed
 
diff --git a/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_transformation.py b/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_transformation.py
index dbe84cacb29..a74f03449a1 100644
--- a/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_transformation.py
+++ b/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_transformation.py
@@ -1025,6 +1025,112 @@ class TestBedrockRealtimeUserEventsAndUsage:
         assert usage["output_tokens"] == 0
         assert usage["total_tokens"] == 0
 
+    @staticmethod
+    def _usage_event(total_input, total_output, in_speech, in_text, out_speech, out_text):
+        return {
+            "event": {
+                "usageEvent": {
+                    "totalInputTokens": total_input,
+                    "totalOutputTokens": total_output,
+                    "totalTokens": total_input + total_output,
+                    "details": {
+                        "total": {
+                            "input": {"speechTokens": in_speech, "textTokens": in_text},
+                            "output": {"speechTokens": out_speech, "textTokens": out_text},
+                        }
+                    },
+                }
+            }
+        }
+
+    _ASSISTANT_TURN = (
+        {"event": {"contentStart": {"role": "ASSISTANT", "type": "TEXT"}}},
+        {"event": {"textOutput": {"content": "Hi"}}},
+        {"event": {"contentEnd": {"stopReason": "END_TURN"}}},
+    )
+
+    def test_multi_turn_usage_reports_per_response_deltas_not_cumulative_totals(self):
+        events = self._run(
+            BedrockRealtimeConfig(),
+            [
+                self._usage_event(25, 40, in_speech=20, in_text=5, out_speech=30, out_text=10),
+                *self._ASSISTANT_TURN,
+                self._usage_event(40, 100, in_speech=30, in_text=10, out_speech=75, out_text=25),
+                *self._ASSISTANT_TURN,
+            ],
+        )
+        usages = [e["response"]["usage"] for e in events if e["type"] == "response.done"]
+        assert len(usages) == 2
+        assert (usages[0]["input_tokens"], usages[0]["output_tokens"], usages[0]["total_tokens"]) == (25, 40, 65)
+        assert (usages[1]["input_tokens"], usages[1]["output_tokens"], usages[1]["total_tokens"]) == (15, 60, 75)
+        assert usages[1]["input_token_details"] == {"audio_tokens": 10, "text_tokens": 5, "cached_tokens": 0}
+        assert usages[1]["output_token_details"] == {"audio_tokens": 45, "text_tokens": 15}
+        assert sum(u["total_tokens"] for u in usages) == 140
+
+    def test_usage_reported_after_last_response_done_flushes_as_logged_only_done(self):
+        config = BedrockRealtimeConfig()
+        self._run(
+            config,
+            [
+                self._usage_event(25, 40, in_speech=20, in_text=5, out_speech=30, out_text=10),
+                *self._ASSISTANT_TURN,
+            ],
+        )
+        assert config.leftover_usage_done_events() == ()
+
+        self._run(config, [self._usage_event(25, 46, in_speech=20, in_text=5, out_speech=30, out_text=16)])
+        leftover = config.leftover_usage_done_events()
+        assert len(leftover) == 1
+        assert leftover[0]["type"] == "response.done"
+        usage = leftover[0]["response"]["usage"]
+        assert (usage["input_tokens"], usage["output_tokens"], usage["total_tokens"]) == (0, 6, 6)
+        assert usage["output_token_details"] == {"audio_tokens": 0, "text_tokens": 6}
+        assert config.leftover_usage_done_events() == ()
+
+    def test_final_transcript_fragments_emit_one_completed_with_full_transcript(self):
+        events = self._run(
+            BedrockRealtimeConfig(),
+            [
+                {
+                    "event": {
+                        "contentStart": {
+                            "role": "USER",
+                            "type": "TEXT",
+                            "additionalModelFields": json.dumps({"generationStage": "FINAL"}),
+                        }
+                    }
+                },
+                {"event": {"textOutput": {"content": "What is the "}}},
+                {"event": {"textOutput": {"content": "capital of France?"}}},
+                {"event": {"contentEnd": {"stopReason": "PARTIAL_TURN"}}},
+            ],
+        )
+        deltas = [e for e in events if e["type"] == "conversation.item.input_audio_transcription.delta"]
+        completed = [e for e in events if e["type"] == "conversation.item.input_audio_transcription.completed"]
+        assert [d["delta"] for d in deltas] == ["What is the ", "capital of France?"]
+        assert len(completed) == 1
+        assert completed[0]["transcript"] == "What is the capital of France?"
+        assert {e["item_id"] for e in deltas + completed} == {completed[0]["item_id"]}
+
+    def test_speculative_transcript_block_end_emits_no_completed(self):
+        events = self._run(
+            BedrockRealtimeConfig(),
+            [
+                {
+                    "event": {
+                        "contentStart": {
+                            "role": "USER",
+                            "type": "TEXT",
+                            "additionalModelFields": json.dumps({"generationStage": "SPECULATIVE"}),
+                        }
+                    }
+                },
+                {"event": {"textOutput": {"content": "rea"}}},
+                {"event": {"contentEnd": {"stopReason": "PARTIAL_TURN"}}},
+            ],
+        )
+        assert [e["type"] for e in events] == ["conversation.item.input_audio_transcription.delta"]
+
 
 if __name__ == "__main__":
     pytest.main([__file__, "-v"])

From ae945f4fa31d415c5ef7e56be90911ca0120f6d1 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Mon, 31 Aug 2026 11:54:04 -0700
Subject: [PATCH 18/37] feat(openai): support workload identity federation
 (OIDC token exchange)

---
 basedpyright-code-budget.json                 |   2 +-
 litellm/llms/openai/common_utils.py           |   1 +
 litellm/llms/openai/openai.py                 |  61 ++++--
 .../llms/openai/responses/transformation.py   |  11 +
 litellm/llms/openai/workload_identity.py      |  92 +++++++++
 .../openai/test_openai_workload_identity.py   | 188 ++++++++++++++++++
 6 files changed, 335 insertions(+), 20 deletions(-)
 create mode 100644 litellm/llms/openai/workload_identity.py
 create mode 100644 tests/test_litellm/llms/openai/test_openai_workload_identity.py

diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json
index d60c3e9c0af..229d1eca3e8 100644
--- a/basedpyright-code-budget.json
+++ b/basedpyright-code-budget.json
@@ -3,7 +3,7 @@
     "limit": 16171
   },
   "reportArgumentType": {
-    "limit": 2226
+    "limit": 2224
   },
   "reportAssignmentType": {
     "limit": 319
diff --git a/litellm/llms/openai/common_utils.py b/litellm/llms/openai/common_utils.py
index 1b1ab80e85d..4d774f6f165 100644
--- a/litellm/llms/openai/common_utils.py
+++ b/litellm/llms/openai/common_utils.py
@@ -268,6 +268,7 @@ class BaseOpenAILLM:
             "max_retries",
             "organization",
             "api_base",
+            "workload_identity_config",
         )
         openai_client_fields: Final = (
             BaseOpenAILLM.get_openai_client_initialization_param_fields(client_type=client_type)
diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py
index 6e66c998acf..16fa0017b23 100644
--- a/litellm/llms/openai/openai.py
+++ b/litellm/llms/openai/openai.py
@@ -51,6 +51,7 @@ from .common_utils import (
     drop_params_from_unprocessable_entity_error,
     is_output_token_limit_error,
 )
+from .workload_identity import resolve_openai_workload_identity_config
 
 openaiOSeriesConfig: Final = OpenAIOSeriesConfig()
 openAIGPT5Config: Final = OpenAIGPT5Config()
@@ -349,6 +350,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
         client: OpenAI | AsyncOpenAI | None = None,
         shared_session: Optional["ClientSession"] = None,
     ) -> OpenAI | AsyncOpenAI | None:
+        workload_identity_config: Final = resolve_openai_workload_identity_config(api_key=api_key, api_base=api_base)
         client_initialization_params: Final[dict] = locals()
         if client is None:
             if not isinstance(max_retries, int):
@@ -364,28 +366,49 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
             if cached_client:
                 if isinstance(cached_client, OpenAI) or isinstance(cached_client, AsyncOpenAI):
                     return cached_client
-            http_client: Final[httpx.Client | httpx.AsyncClient | None] = (
-                OpenAIChatCompletion._get_async_http_client(shared_session=shared_session)
-                if is_async
-                else OpenAIChatCompletion._get_sync_http_client()
-            )
             if is_async:
-                _new_client: OpenAI | AsyncOpenAI = AsyncOpenAI(
-                    api_key=api_key,
-                    base_url=api_base,
-                    http_client=http_client,
-                    timeout=timeout,
-                    max_retries=max_retries,
-                    organization=organization,
+                async_http_client: Final = OpenAIChatCompletion._get_async_http_client(shared_session=shared_session)
+                http_client: httpx.Client | httpx.AsyncClient | None = async_http_client
+                _new_client: OpenAI | AsyncOpenAI = (
+                    AsyncOpenAI(
+                        workload_identity=workload_identity_config.to_sdk_workload_identity(),
+                        base_url=api_base,
+                        http_client=async_http_client,
+                        timeout=timeout,
+                        max_retries=max_retries,
+                        organization=organization,
+                    )
+                    if workload_identity_config is not None
+                    else AsyncOpenAI(
+                        api_key=api_key,
+                        base_url=api_base,
+                        http_client=async_http_client,
+                        timeout=timeout,
+                        max_retries=max_retries,
+                        organization=organization,
+                    )
                 )
             else:
-                _new_client = OpenAI(
-                    api_key=api_key,
-                    base_url=api_base,
-                    http_client=http_client,
-                    timeout=timeout,
-                    max_retries=max_retries,
-                    organization=organization,
+                sync_http_client: Final = OpenAIChatCompletion._get_sync_http_client()
+                http_client = sync_http_client
+                _new_client = (
+                    OpenAI(
+                        workload_identity=workload_identity_config.to_sdk_workload_identity(),
+                        base_url=api_base,
+                        http_client=sync_http_client,
+                        timeout=timeout,
+                        max_retries=max_retries,
+                        organization=organization,
+                    )
+                    if workload_identity_config is not None
+                    else OpenAI(
+                        api_key=api_key,
+                        base_url=api_base,
+                        http_client=sync_http_client,
+                        timeout=timeout,
+                        max_retries=max_retries,
+                        organization=organization,
+                    )
                 )
 
             ## SAVE CACHE KEY
diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py
index eac844a790d..1479c378014 100644
--- a/litellm/llms/openai/responses/transformation.py
+++ b/litellm/llms/openai/responses/transformation.py
@@ -21,6 +21,7 @@ from litellm.types.router import GenericLiteLLMParams
 from litellm.types.utils import LlmProviders
 
 from ..common_utils import OpenAIError
+from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config
 
 OPENAI_RESPONSES_API_MIN_MAX_OUTPUT_TOKENS: Final = 16
 
@@ -392,6 +393,16 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
         litellm_params = litellm_params or GenericLiteLLMParams()
         api_key = litellm_params.api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
         headers.setdefault("Content-Type", "application/json")
+        workload_identity_config: Final = resolve_openai_workload_identity_config(
+            api_key=api_key,
+            api_base=litellm_params.api_base
+            or litellm.api_base
+            or get_secret_str("OPENAI_BASE_URL")
+            or get_secret_str("OPENAI_API_BASE"),
+        )
+        if workload_identity_config is not None:
+            headers["Authorization"] = f"Bearer {get_workload_identity_bearer_token(workload_identity_config)}"
+            return headers
         headers["Authorization"] = f"Bearer {api_key}"
         return headers
 
diff --git a/litellm/llms/openai/workload_identity.py b/litellm/llms/openai/workload_identity.py
new file mode 100644
index 00000000000..15105d67957
--- /dev/null
+++ b/litellm/llms/openai/workload_identity.py
@@ -0,0 +1,92 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from functools import lru_cache
+from typing import TYPE_CHECKING, Final
+from urllib.parse import urlparse
+
+from litellm.secret_managers.main import get_secret_str
+
+from .common_utils import OpenAIError
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from openai.auth import SubjectTokenProvider, WorkloadIdentity, WorkloadIdentityAuth
+
+OPENAI_WIF_CLIENT_ID: Final = "litellm"
+_OPENAI_API_HOST: Final = "api.openai.com"
+_SDK_UPGRADE_MESSAGE: Final = (
+    "OpenAI workload identity federation requires openai>=2.32.0. "
+    "Upgrade the installed openai package to use OPENAI_IDENTITY_PROVIDER_ID / "
+    "OPENAI_SERVICE_ACCOUNT_ID / OPENAI_IDENTITY_TOKEN_FILE."
+)
+
+
+@dataclass(frozen=True, slots=True)
+class OpenAIWorkloadIdentityConfig:
+    identity_provider_id: str
+    service_account_id: str
+    token_file: str
+
+    def to_sdk_workload_identity(self) -> WorkloadIdentity:
+        k8s_token_provider: Final = _load_sdk_k8s_token_provider()
+        workload_identity: Final[WorkloadIdentity] = {
+            "client_id": OPENAI_WIF_CLIENT_ID,
+            "identity_provider_id": self.identity_provider_id,
+            "service_account_id": self.service_account_id,
+            "provider": k8s_token_provider(self.token_file),
+        }
+        return workload_identity
+
+
+def resolve_openai_workload_identity_config(
+    api_key: str | None,
+    api_base: str | None,
+) -> OpenAIWorkloadIdentityConfig | None:
+    if api_key is not None:
+        return None
+    if not _targets_openai_api(api_base):
+        return None
+    identity_provider_id: Final = get_secret_str("OPENAI_IDENTITY_PROVIDER_ID")
+    service_account_id: Final = get_secret_str("OPENAI_SERVICE_ACCOUNT_ID")
+    token_file: Final = get_secret_str("OPENAI_IDENTITY_TOKEN_FILE")
+    if not identity_provider_id or not service_account_id or not token_file:
+        return None
+    return OpenAIWorkloadIdentityConfig(
+        identity_provider_id=identity_provider_id,
+        service_account_id=service_account_id,
+        token_file=token_file,
+    )
+
+
+def get_workload_identity_bearer_token(config: OpenAIWorkloadIdentityConfig) -> str:
+    return _workload_identity_auth(config).get_token()
+
+
+def _targets_openai_api(api_base: str | None) -> bool:
+    if api_base is None:
+        return True
+    return urlparse(api_base).hostname == _OPENAI_API_HOST
+
+
+@lru_cache(maxsize=16)
+def _workload_identity_auth(config: OpenAIWorkloadIdentityConfig) -> WorkloadIdentityAuth:
+    sdk_workload_identity_auth: Final = _load_sdk_workload_identity_auth()
+    return sdk_workload_identity_auth(workload_identity=config.to_sdk_workload_identity())
+
+
+def _load_sdk_workload_identity_auth() -> type[WorkloadIdentityAuth]:
+    try:
+        from openai.auth import WorkloadIdentityAuth as sdk_workload_identity_auth
+    except ImportError as e:
+        raise OpenAIError(status_code=500, message=_SDK_UPGRADE_MESSAGE) from e
+    return sdk_workload_identity_auth
+
+
+def _load_sdk_k8s_token_provider() -> Callable[[str], SubjectTokenProvider]:
+    try:
+        from openai.auth import k8s_service_account_token_provider
+    except ImportError as e:
+        raise OpenAIError(status_code=500, message=_SDK_UPGRADE_MESSAGE) from e
+    return k8s_service_account_token_provider
diff --git a/tests/test_litellm/llms/openai/test_openai_workload_identity.py b/tests/test_litellm/llms/openai/test_openai_workload_identity.py
new file mode 100644
index 00000000000..b8cc8c9c80f
--- /dev/null
+++ b/tests/test_litellm/llms/openai/test_openai_workload_identity.py
@@ -0,0 +1,188 @@
+import json
+import sys
+from pathlib import Path
+from typing import Final
+
+import httpx
+import pytest
+import respx
+from openai import AsyncOpenAI, OpenAI
+
+import litellm
+from litellm.llms.openai.common_utils import BaseOpenAILLM, OpenAIError
+from litellm.llms.openai.openai import OpenAIChatCompletion
+from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
+from litellm.llms.openai.workload_identity import (
+    OpenAIWorkloadIdentityConfig,
+    _workload_identity_auth,
+    get_workload_identity_bearer_token,
+    resolve_openai_workload_identity_config,
+)
+from litellm.types.router import GenericLiteLLMParams
+
+TOKEN_EXCHANGE_URL: Final = "https://auth.openai.com/oauth/token"
+
+
+@pytest.fixture
+def wif_env(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> OpenAIWorkloadIdentityConfig:
+    token_file: Final = tmp_path / "subject_token.jwt"
+    token_file.write_text("subject-token-from-file")
+    monkeypatch.delenv("OPENAI_API_KEY", raising=False)
+    monkeypatch.setenv("OPENAI_IDENTITY_PROVIDER_ID", "idp_test123")
+    monkeypatch.setenv("OPENAI_SERVICE_ACCOUNT_ID", "user-test456")
+    monkeypatch.setenv("OPENAI_IDENTITY_TOKEN_FILE", str(token_file))
+    _workload_identity_auth.cache_clear()
+    litellm.in_memory_llm_clients_cache.flush_cache()
+    return OpenAIWorkloadIdentityConfig(
+        identity_provider_id="idp_test123",
+        service_account_id="user-test456",
+        token_file=str(token_file),
+    )
+
+
+def mock_token_exchange(access_token: str = "exchanged-bearer-token") -> respx.Route:
+    return respx.post(TOKEN_EXCHANGE_URL).mock(
+        return_value=httpx.Response(200, json={"access_token": access_token, "expires_in": 3600})
+    )
+
+
+class TestResolveConfig:
+    def test_resolves_from_env(self, wif_env: OpenAIWorkloadIdentityConfig) -> None:
+        assert resolve_openai_workload_identity_config(api_key=None, api_base=None) == wif_env
+
+    def test_static_api_key_wins(self, wif_env: OpenAIWorkloadIdentityConfig) -> None:
+        assert resolve_openai_workload_identity_config(api_key="sk-static", api_base=None) is None
+
+    def test_foreign_api_base_disables(self, wif_env: OpenAIWorkloadIdentityConfig) -> None:
+        assert resolve_openai_workload_identity_config(api_key=None, api_base="https://my-vllm.internal/v1") is None
+
+    def test_openai_api_base_allows(self, wif_env: OpenAIWorkloadIdentityConfig) -> None:
+        assert resolve_openai_workload_identity_config(api_key=None, api_base="https://api.openai.com/v1") == wif_env
+
+    @pytest.mark.parametrize(
+        "missing_var",
+        ["OPENAI_IDENTITY_PROVIDER_ID", "OPENAI_SERVICE_ACCOUNT_ID", "OPENAI_IDENTITY_TOKEN_FILE"],
+    )
+    def test_partial_env_disables(
+        self, wif_env: OpenAIWorkloadIdentityConfig, monkeypatch: pytest.MonkeyPatch, missing_var: str
+    ) -> None:
+        monkeypatch.delenv(missing_var)
+        assert resolve_openai_workload_identity_config(api_key=None, api_base=None) is None
+
+
+class TestTokenExchange:
+    @respx.mock
+    def test_exchanges_subject_token_for_bearer(self, wif_env: OpenAIWorkloadIdentityConfig) -> None:
+        route: Final = mock_token_exchange()
+        assert get_workload_identity_bearer_token(wif_env) == "exchanged-bearer-token"
+        request_body: Final = json.loads(route.calls.last.request.content)
+        assert request_body["grant_type"] == "urn:ietf:params:oauth:grant-type:token-exchange"
+        assert request_body["subject_token"] == "subject-token-from-file"
+        assert request_body["subject_token_type"] == "urn:ietf:params:oauth:token-type:jwt"
+        assert request_body["identity_provider_id"] == "idp_test123"
+        assert request_body["service_account_id"] == "user-test456"
+
+    @respx.mock
+    def test_token_cached_across_mints(self, wif_env: OpenAIWorkloadIdentityConfig) -> None:
+        route: Final = mock_token_exchange()
+        first: Final = get_workload_identity_bearer_token(wif_env)
+        second: Final = get_workload_identity_bearer_token(wif_env)
+        assert first == second == "exchanged-bearer-token"
+        assert route.call_count == 1
+
+    def test_old_sdk_raises_upgrade_error(
+        self, wif_env: OpenAIWorkloadIdentityConfig, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        import openai as openai_module
+
+        monkeypatch.delattr(openai_module, "auth", raising=False)
+        monkeypatch.setitem(sys.modules, "openai.auth", None)
+        with pytest.raises(OpenAIError, match=r"openai>=2\.32\.0"):
+            wif_env.to_sdk_workload_identity()
+
+
+class TestClientConstruction:
+    def test_sync_client_uses_workload_identity(self, wif_env: OpenAIWorkloadIdentityConfig) -> None:
+        client: Final = OpenAIChatCompletion()._get_openai_client(is_async=False, api_key=None, api_base=None)
+        assert isinstance(client, OpenAI)
+        assert client.api_key == "workload-identity-auth"
+        assert client._workload_identity_auth is not None
+
+    def test_async_client_uses_workload_identity(self, wif_env: OpenAIWorkloadIdentityConfig) -> None:
+        client: Final = OpenAIChatCompletion()._get_openai_client(is_async=True, api_key=None, api_base=None)
+        assert isinstance(client, AsyncOpenAI)
+        assert client.api_key == "workload-identity-auth"
+        assert client._workload_identity_auth is not None
+
+    def test_static_key_client_unaffected(self, wif_env: OpenAIWorkloadIdentityConfig) -> None:
+        client: Final = OpenAIChatCompletion()._get_openai_client(is_async=False, api_key="sk-static", api_base=None)
+        assert isinstance(client, OpenAI)
+        assert client.api_key == "sk-static"
+        assert client._workload_identity_auth is None
+
+    def test_cache_key_separates_wif_identities(self, wif_env: OpenAIWorkloadIdentityConfig) -> None:
+        other_config: Final = OpenAIWorkloadIdentityConfig(
+            identity_provider_id="idp_other",
+            service_account_id="user-other",
+            token_file=wif_env.token_file,
+        )
+        keys: Final = tuple(
+            BaseOpenAILLM.get_openai_client_cache_key(
+                client_initialization_params={"api_key": None, "is_async": False, "workload_identity_config": config},
+                client_type="openai",
+            )
+            for config in (wif_env, other_config, None)
+        )
+        assert len(set(keys)) == 3
+
+    @respx.mock
+    def test_request_carries_exchanged_bearer(self, wif_env: OpenAIWorkloadIdentityConfig) -> None:
+        mock_token_exchange()
+        completion_route: Final = respx.post("https://api.openai.com/v1/chat/completions").mock(
+            return_value=httpx.Response(
+                200,
+                json={
+                    "id": "chatcmpl-wif",
+                    "object": "chat.completion",
+                    "created": 1,
+                    "model": "gpt-4o-mini",
+                    "choices": [
+                        {
+                            "index": 0,
+                            "message": {"role": "assistant", "content": "ok"},
+                            "finish_reason": "stop",
+                        }
+                    ],
+                    "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
+                },
+            )
+        )
+        client = OpenAIChatCompletion()._get_openai_client(is_async=False, api_key=None, api_base=None)
+        assert isinstance(client, OpenAI)
+        client.chat.completions.create(model="gpt-4o-mini", messages=[{"role": "user", "content": "hi"}])
+        auth_header: Final = completion_route.calls.last.request.headers["Authorization"]
+        assert auth_header == "Bearer exchanged-bearer-token"
+
+
+class TestResponsesValidateEnvironment:
+    @respx.mock
+    def test_mints_bearer_when_wif_configured(self, wif_env: OpenAIWorkloadIdentityConfig) -> None:
+        mock_token_exchange()
+        headers: Final = OpenAIResponsesAPIConfig().validate_environment(
+            headers={}, model="gpt-4o-mini", litellm_params=GenericLiteLLMParams()
+        )
+        assert headers["Authorization"] == "Bearer exchanged-bearer-token"
+
+    def test_static_key_wins(self, wif_env: OpenAIWorkloadIdentityConfig) -> None:
+        headers: Final = OpenAIResponsesAPIConfig().validate_environment(
+            headers={}, model="gpt-4o-mini", litellm_params=GenericLiteLLMParams(api_key="sk-responses")
+        )
+        assert headers["Authorization"] == "Bearer sk-responses"
+
+    def test_foreign_api_base_skips_wif(self, wif_env: OpenAIWorkloadIdentityConfig) -> None:
+        headers: Final = OpenAIResponsesAPIConfig().validate_environment(
+            headers={},
+            model="gpt-4o-mini",
+            litellm_params=GenericLiteLLMParams(api_base="https://my-vllm.internal/v1"),
+        )
+        assert headers["Authorization"] == "Bearer None"

From c7c382402a6ec0e26d32b45b30a2ee1c10ad64d5 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Mon, 31 Aug 2026 12:04:44 -0700
Subject: [PATCH 19/37] feat(proxy): add /v1/responses/input_tokens token
 counting endpoint

---
 litellm/proxy/_types.py                       |   3 +
 .../proxy/response_api_endpoints/endpoints.py | 157 ++++++++++++-
 .../response_api_endpoints/test_endpoints.py  | 215 +++++++++++++-----
 ui/litellm-dashboard/src/lib/http/schema.d.ts | 153 +++++++++++++
 4 files changed, 467 insertions(+), 61 deletions(-)

diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py
index 0f2d97b8b1c..9574eb36ee5 100644
--- a/litellm/proxy/_types.py
+++ b/litellm/proxy/_types.py
@@ -422,6 +422,9 @@ class LiteLLMRoutes(enum.Enum):
         "/responses/{response_id}/cancel",
         "/v1/responses/{response_id}/cancel",
         "/openai/v1/responses/{response_id}/cancel",
+        "/responses/input_tokens",
+        "/v1/responses/input_tokens",
+        "/openai/v1/responses/input_tokens",
         # vector stores
         "/vector_stores",
         "/v1/vector_stores",
diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py
index 5e56e822484..100b42a9e2a 100644
--- a/litellm/proxy/response_api_endpoints/endpoints.py
+++ b/litellm/proxy/response_api_endpoints/endpoints.py
@@ -1,14 +1,18 @@
 import asyncio
 import json
 import time
-from collections.abc import AsyncIterator, Mapping
+from collections.abc import AsyncIterator, Awaitable, Mapping
+from enum import Enum
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Any, Final, NamedTuple, cast, get_args
+from typing import TYPE_CHECKING, Any, Final, NamedTuple, Protocol, cast, get_args
 from uuid import uuid4
 
 import fastapi
 from fastapi import APIRouter, Depends, HTTPException, Request, Response
+from fastapi.responses import JSONResponse
+from openai.types.responses.response_create_params import ResponseInputParam
 from starlette.websockets import WebSocket, WebSocketDisconnect
+from typing_extensions import ReadOnly, TypedDict
 
 from litellm._logging import verbose_proxy_logger
 from litellm.integrations.custom_guardrail import ModifyResponseException
@@ -26,8 +30,13 @@ from litellm.proxy.common_utils.http_parsing_utils import (
     _read_request_body,
     _safe_set_request_parsed_body,
 )
-from litellm.types.llms.openai import REASONING_EFFORT, ResponsesAPIResponse
+from litellm.types.llms.openai import (
+    REASONING_EFFORT,
+    ResponsesAPIOptionalRequestParams,
+    ResponsesAPIResponse,
+)
 from litellm.types.responses.main import DeleteResponseResult
+from litellm.types.utils import TokenCountResponse
 
 if TYPE_CHECKING:
     from litellm.router import Router
@@ -35,7 +44,7 @@ if TYPE_CHECKING:
 router: Final = APIRouter()
 
 _user_api_key_auth_dep: Final = Depends(user_api_key_auth)
-_RESPONSES_TAGS: Final = ["responses"]  # mutable-ok: fastapi's route signature requires List[str] tags
+_RESPONSES_TAGS: Final[list[str | Enum]] = ["responses"]  # mutable-ok: fastapi's route signature requires list tags
 
 _TOOL_PAYLOAD_KEYS: Final[Mapping[str, tuple[str, ...]]] = MappingProxyType(
     {
@@ -1017,6 +1026,146 @@ async def compact_response(
         )
 
 
+class _ResponsesApiErrorDetail(TypedDict):
+    message: ReadOnly[str]
+    type: ReadOnly[str]
+    param: ReadOnly[str | None]
+    code: ReadOnly[str | None]
+
+
+class _ResponsesApiErrorBody(TypedDict):
+    error: ReadOnly[_ResponsesApiErrorDetail]
+
+
+class _ResponsesInputTokensResult(TypedDict):
+    object: ReadOnly[str]
+    input_tokens: ReadOnly[int]
+
+
+class _TokenCountPayload(TypedDict):
+    model: ReadOnly[str]
+    messages: ReadOnly[tuple[Mapping[str, object], ...]]
+    tools: ReadOnly[object]
+
+
+class _TokenCounter(Protocol):
+    def __call__(self, request: TokenCountRequest, call_endpoint: bool) -> Awaitable[TokenCountResponse]: ...
+
+
+def _proxy_token_counter() -> _TokenCounter:
+    from litellm.proxy.proxy_server import token_counter
+
+    return token_counter
+
+
+_token_counter_dep: Final = Depends(_proxy_token_counter)
+
+
+def _responses_invalid_request_response(message: str, param: str | None, code: str | None) -> JSONResponse:
+    body: Final[_ResponsesApiErrorBody] = {
+        "error": {
+            "message": message,
+            "type": "invalid_request_error",
+            "param": param,
+            "code": code,
+        }
+    }
+    return JSONResponse(status_code=400, content=body)
+
+
+def _missing_responses_param_response(param: str) -> JSONResponse:
+    return _responses_invalid_request_response(
+        message=f"Missing required parameter: '{param}'.",
+        param=param,
+        code="missing_required_parameter",
+    )
+
+
+def _responses_input_as_token_count_messages(
+    input_value: str | ResponseInputParam,
+    instructions: str | None,
+) -> tuple[Mapping[str, object], ...]:
+    from litellm.responses.litellm_completion_transformation.transformation import (
+        LiteLLMCompletionResponsesConfig,
+    )
+
+    request_params: Final[ResponsesAPIOptionalRequestParams] = {"instructions": instructions}
+    transformed: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
+        input=input_value,
+        responses_api_request=request_params,
+    )
+    return tuple(
+        message if isinstance(message, dict) else message.model_dump(exclude_none=True) for message in transformed
+    )
+
+
+@router.post(
+    "/v1/responses/input_tokens",
+    dependencies=(_user_api_key_auth_dep,),
+    tags=_RESPONSES_TAGS,
+)
+@router.post(
+    "/responses/input_tokens",
+    dependencies=(_user_api_key_auth_dep,),
+    tags=_RESPONSES_TAGS,
+)
+@router.post(
+    "/openai/v1/responses/input_tokens",
+    dependencies=(_user_api_key_auth_dep,),
+    tags=_RESPONSES_TAGS,
+)
+async def responses_input_tokens(
+    request: Request,
+    token_counter: _TokenCounter = _token_counter_dep,
+):
+    """
+    Count the input tokens of a Responses API request without calling the model.
+
+    Follows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses/input-tokens
+
+    ```bash
+    curl -X POST http://localhost:4000/v1/responses/input_tokens \
+    -H "Content-Type: application/json" \
+    -H "Authorization: Bearer sk-1234" \
+    -d '{
+        "model": "gpt-4o",
+        "input": "Hello, how are you?"
+    }'
+    ```
+
+    Returns: `{"object": "response.input_tokens", "input_tokens": <count>}`
+    """
+    data: Final = await _read_request_body(request=request)
+    model_name: Final = data.get("model")
+    input_value: Final = data.get("input")
+    if not isinstance(model_name, str) or not model_name:
+        return _missing_responses_param_response("model")
+    if input_value is None:
+        return _missing_responses_param_response("input")
+
+    try:
+        payload: Final[_TokenCountPayload] = {
+            "model": model_name,
+            "messages": _responses_input_as_token_count_messages(
+                input_value=input_value,
+                instructions=data.get("instructions"),
+            ),
+            "tools": data.get("tools"),
+        }
+        token_request: Final = TokenCountRequest.model_validate(payload)
+    except Exception as e:
+        return _responses_invalid_request_response(
+            message=f"Invalid request for token counting: {e}", param=None, code=None
+        )
+
+    token_response: Final = await token_counter(request=token_request, call_endpoint=True)
+    result: Final[_ResponsesInputTokensResult] = {
+        "object": "response.input_tokens",
+        "input_tokens": token_response.total_tokens,
+    }
+    return result
+
+
 @router.post(
     "/v1/responses/{response_id}/cancel",
     dependencies=[Depends(user_api_key_auth)],
diff --git a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py
index 791d64c6428..dc43e7f5c06 100644
--- a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py
+++ b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py
@@ -82,11 +82,7 @@ class TestResponsesAPIEndpoints(unittest.TestCase):
                 ResponseOutputMessage(
                     type="message",
                     role="assistant",
-                    content=[
-                        ResponseOutputText(
-                            type="output_text", text="Hello from Cursor!"
-                        )
-                    ],
+                    content=[ResponseOutputText(type="output_text", text="Hello from Cursor!")],
                 )
             ],
         )
@@ -121,9 +117,7 @@ class TestResponsesAPIEndpoints(unittest.TestCase):
     @pytest.mark.asyncio
     @patch("litellm.proxy.proxy_server.llm_router")
     @patch("litellm.proxy.proxy_server.user_api_key_auth")
-    async def test_responses_api_key_spend_header_includes_response_cost(
-        self, mock_auth, mock_router
-    ):
+    async def test_responses_api_key_spend_header_includes_response_cost(self, mock_auth, mock_router):
         """
         Test that x-litellm-key-spend header includes the current request's response_cost
         for /v1/responses endpoint.
@@ -159,9 +153,7 @@ class TestResponsesAPIEndpoints(unittest.TestCase):
                 ResponseOutputMessage(
                     type="message",
                     role="assistant",
-                    content=[
-                        ResponseOutputText(type="output_text", text="Test response")
-                    ],
+                    content=[ResponseOutputText(type="output_text", text="Test response")],
                 )
             ],
         )
@@ -356,6 +348,7 @@ class TestWSModelExtraction:
         from litellm.proxy.response_api_endpoints.endpoints import (
             _extract_model_from_first_ws_event,
         )
+
         event = {"type": "response.create", "model": "gpt-4o", "input": "hello"}
         assert _extract_model_from_first_ws_event(event) == "gpt-4o"
 
@@ -363,6 +356,7 @@ class TestWSModelExtraction:
         from litellm.proxy.response_api_endpoints.endpoints import (
             _extract_model_from_first_ws_event,
         )
+
         event = {"type": "response.create", "response": {"model": "gpt-4o", "input": "hello"}}
         assert _extract_model_from_first_ws_event(event) == "gpt-4o"
 
@@ -370,6 +364,7 @@ class TestWSModelExtraction:
         from litellm.proxy.response_api_endpoints.endpoints import (
             _extract_model_from_first_ws_event,
         )
+
         event = {
             "type": "response.create",
             "model": "flat-model",
@@ -381,6 +376,7 @@ class TestWSModelExtraction:
         from litellm.proxy.response_api_endpoints.endpoints import (
             _extract_model_from_first_ws_event,
         )
+
         event = {"type": "response.create", "input": "hello"}
         assert _extract_model_from_first_ws_event(event) is None
 
@@ -400,9 +396,7 @@ class TestResponsesWSFirstFrameValidation:
         )
 
         ws = MagicMock()
-        ws.receive_text = AsyncMock(
-            return_value=json.dumps({"type": "session.update", "model": "gpt-4o"})
-        )
+        ws.receive_text = AsyncMock(return_value=json.dumps({"type": "session.update", "model": "gpt-4o"}))
         ws.send_text = AsyncMock()
         ws.close = AsyncMock()
 
@@ -412,10 +406,7 @@ class TestResponsesWSFirstFrameValidation:
         ws.send_text.assert_awaited_once()
         ws.close.assert_awaited_once_with(code=1008, reason="Invalid first message")
         error_payload = json.loads(ws.send_text.await_args.args[0])
-        assert (
-            error_payload["error"]["message"]
-            == "First message must be a response.create JSON object."
-        )
+        assert error_payload["error"]["message"] == "First message must be a response.create JSON object."
 
     @pytest.mark.asyncio
     async def test_rejects_non_object_json_first_frame(self):
@@ -484,16 +475,12 @@ class TestResponsesWSFirstFrameModelAuth:
         ws.url = "ws://testserver/v1/responses"
         ws.accept = AsyncMock()
         ws.receive_text = AsyncMock(
-            return_value=json.dumps(
-                {"type": "response.create", "model": "gpt-4o-mini", "input": []}
-            )
+            return_value=json.dumps({"type": "response.create", "model": "gpt-4o-mini", "input": []})
         )
         ws.close = AsyncMock()
 
         processor = MagicMock()
-        processor.common_processing_pre_call_logic = AsyncMock(
-            return_value=({"model": "gpt-4o-mini"}, MagicMock())
-        )
+        processor.common_processing_pre_call_logic = AsyncMock(return_value=({"model": "gpt-4o-mini"}, MagicMock()))
 
         async def fake_llm_call():
             return None
@@ -529,9 +516,7 @@ class TestResponsesWSFirstFrameModelAuth:
             _enforce_responses_ws_first_frame_model_auth,
         )
 
-        request = Request(
-            {"type": "http", "method": "POST", "path": "/v1/responses", "headers": []}
-        )
+        request = Request({"type": "http", "method": "POST", "path": "/v1/responses", "headers": []})
         user_api_key_dict = MagicMock()
         llm_router = MagicMock()
 
@@ -593,9 +578,7 @@ class TestReadWSModelFromFirstFrameErrors:
 
         assert result is None
         ws.send_text.assert_not_awaited()
-        ws.close.assert_awaited_once_with(
-            code=1008, reason="Timed out waiting for first message"
-        )
+        ws.close.assert_awaited_once_with(code=1008, reason="Timed out waiting for first message")
 
     @pytest.mark.asyncio
     async def test_invalid_json_sends_error_and_closes(self):
@@ -613,9 +596,7 @@ class TestReadWSModelFromFirstFrameErrors:
         assert result is None
         payload = json.loads(ws.send_text.await_args.args[0])
         assert payload["error"]["message"] == "First message is not valid JSON."
-        ws.close.assert_awaited_once_with(
-            code=1008, reason="Invalid JSON in first message"
-        )
+        ws.close.assert_awaited_once_with(code=1008, reason="Invalid JSON in first message")
 
     @pytest.mark.asyncio
     async def test_missing_model_sends_error_and_closes(self):
@@ -624,9 +605,7 @@ class TestReadWSModelFromFirstFrameErrors:
         )
 
         ws = MagicMock()
-        ws.receive_text = AsyncMock(
-            return_value=json.dumps({"type": "response.create", "input": []})
-        )
+        ws.receive_text = AsyncMock(return_value=json.dumps({"type": "response.create", "input": []}))
         ws.send_text = AsyncMock()
         ws.close = AsyncMock()
 
@@ -679,10 +658,7 @@ class TestManagedResponsesSameProvider:
         assert self._handler("gpt-4o")._same_provider("gpt-4o-mini") is True
 
     def test_different_provider_is_not_same(self):
-        assert (
-            self._handler("gpt-4o")._same_provider("vertex_ai/gemini-2.0-flash")
-            is False
-        )
+        assert self._handler("gpt-4o")._same_provider("vertex_ai/gemini-2.0-flash") is False
 
     def test_inject_credentials_keeps_provider_for_same_provider_model(self):
         handler = self._handler("gpt-4o", custom_llm_provider="openai")
@@ -697,18 +673,14 @@ class TestManagedResponsesSameProvider:
         assert "custom_llm_provider" not in call_kwargs
 
     def test_unresolvable_connection_model_falls_back_to_custom_provider(self):
-        handler = self._handler(
-            "my-custom-deployment", custom_llm_provider="openai"
-        )
+        handler = self._handler("my-custom-deployment", custom_llm_provider="openai")
         assert handler._same_provider("gpt-4o-mini") is True
         call_kwargs: dict = {}
         handler._inject_credentials(call_kwargs, model="gpt-4o-mini")
         assert call_kwargs["custom_llm_provider"] == "openai"
 
     def test_unresolvable_connection_model_still_drops_cross_provider(self):
-        handler = self._handler(
-            "my-custom-deployment", custom_llm_provider="openai"
-        )
+        handler = self._handler("my-custom-deployment", custom_llm_provider="openai")
         call_kwargs: dict = {}
         handler._inject_credentials(call_kwargs, model="vertex_ai/gemini-2.0-flash")
         assert "custom_llm_provider" not in call_kwargs
@@ -840,9 +812,7 @@ def test_cursor_chat_completions_input_body_uses_responses_pipeline_and_strips_s
                     type="message",
                     role="assistant",
                     status="completed",
-                    content=[
-                        ResponseOutputText(type="output_text", text="agent reply", annotations=[])
-                    ],
+                    content=[ResponseOutputText(type="output_text", text="agent reply", annotations=[])],
                 )
             ],
         )
@@ -851,9 +821,12 @@ def test_cursor_chat_completions_input_body_uses_responses_pipeline_and_strips_s
 
     app.dependency_overrides[user_api_key_auth] = _auth_override
     try:
-        with patch.object(ps, "llm_router", mock_router), patch(
-            "litellm.proxy.response_api_endpoints.endpoints._read_request_body",
-            side_effect=capturing_read_request_body,
+        with (
+            patch.object(ps, "llm_router", mock_router),
+            patch(
+                "litellm.proxy.response_api_endpoints.endpoints._read_request_body",
+                side_effect=capturing_read_request_body,
+            ),
         ):
             client = TestClient(app)
             response = client.post(
@@ -1488,8 +1461,8 @@ def _router_serving_only(base_model: str) -> MagicMock:
     mock_router.router_general_settings.pass_through_all_models = False
     mock_router.default_deployment = None
     mock_router.pattern_router.patterns = {base_model: ["anthropic/*"]}
-    mock_router.pattern_router.get_pattern.side_effect = (
-        lambda model: [{"model_name": "anthropic/*"}] if model == base_model else None
+    mock_router.pattern_router.get_pattern.side_effect = lambda model: (
+        [{"model_name": "anthropic/*"}] if model == base_model else None
     )
     return mock_router
 
@@ -1739,9 +1712,7 @@ class TestCursorGateRecognizesRoutingGroups:
         from litellm.proxy.response_api_endpoints.endpoints import _resolve_cursor_model_variant
 
         router = Router(
-            model_list=[
-                {"model_name": "member-fast", "litellm_params": {"model": "openai/gpt-4o", "api_key": "fake"}}
-            ],
+            model_list=[{"model_name": "member-fast", "litellm_params": {"model": "openai/gpt-4o", "api_key": "fake"}}],
             routing_groups=[
                 {"group_name": "grouped-thinking-high", "models": ["member-fast"], "routing_strategy": "simple-shuffle"}
             ],
@@ -1836,3 +1807,133 @@ class TestGuardrailBlockedResponsesUsage:
         assert usage["input_tokens"] == 0
         assert usage["output_tokens"] == 0
         assert usage["total_tokens"] == 0
+
+
+class TestResponsesInputTokens:
+    """Regression tests for POST /v1/responses/input_tokens.
+
+    The docs promise OpenAI-format token counting on the proxy, but the route was
+    never registered, so the POST fell through to the GET/DELETE-only
+    /v1/responses/{response_id} route and returned 405."""
+
+    def _post_input_tokens(self, body, path="/v1/responses/input_tokens", counter=None):
+        from litellm.proxy._types import UserAPIKeyAuth
+        from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+        from litellm.proxy.response_api_endpoints.endpoints import _proxy_token_counter
+        from litellm.types.utils import TokenCountResponse
+
+        token_counter_mock = (
+            counter
+            if counter is not None
+            else AsyncMock(
+                return_value=TokenCountResponse(
+                    total_tokens=13,
+                    request_model=body.get("model", ""),
+                    model_used=body.get("model", ""),
+                    tokenizer_type="openai_api",
+                )
+            )
+        )
+        app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth(api_key="sk-test", request_route=path)
+        app.dependency_overrides[_proxy_token_counter] = lambda: token_counter_mock
+        try:
+            client = TestClient(app)
+            response = client.post(path, json=body, headers={"Authorization": "Bearer sk-1234"})
+            return response, token_counter_mock
+        finally:
+            app.dependency_overrides.pop(user_api_key_auth, None)
+            app.dependency_overrides.pop(_proxy_token_counter, None)
+
+    def test_string_input_returns_openai_input_tokens_shape(self):
+        response, counter = self._post_input_tokens({"model": "gpt-4o", "input": "Hello, how are you?"})
+
+        assert response.status_code == 200, response.text
+        assert response.json() == {"object": "response.input_tokens", "input_tokens": 13}
+        counter.assert_awaited_once()
+        assert counter.call_args.kwargs["call_endpoint"] is True
+        token_request = counter.call_args.kwargs["request"]
+        assert token_request.model == "gpt-4o"
+        assert token_request.messages == [{"role": "user", "content": "Hello, how are you?"}]
+
+    def test_every_route_alias_is_registered(self):
+        for path in ("/v1/responses/input_tokens", "/responses/input_tokens", "/openai/v1/responses/input_tokens"):
+            response, _ = self._post_input_tokens({"model": "gpt-4o", "input": "hi"}, path=path)
+            assert response.status_code == 200, f"{path}: {response.status_code} {response.text}"
+
+    def test_input_items_instructions_and_tools_are_forwarded(self):
+        tools = [
+            {
+                "type": "function",
+                "name": "get_weather",
+                "description": "Get weather for a city",
+                "parameters": {"type": "object", "properties": {"city": {"type": "string"}}},
+            }
+        ]
+        response, counter = self._post_input_tokens(
+            {
+                "model": "gpt-4o",
+                "input": [{"role": "user", "content": "What is the weather in Paris?"}],
+                "instructions": "You are terse.",
+                "tools": tools,
+            }
+        )
+
+        assert response.status_code == 200, response.text
+        token_request = counter.call_args.kwargs["request"]
+        assert token_request.messages == [
+            {"role": "system", "content": "You are terse."},
+            {"role": "user", "content": "What is the weather in Paris?"},
+        ]
+        assert token_request.tools == tools
+
+    def test_missing_model_returns_openai_400(self):
+        response, counter = self._post_input_tokens({"input": "Hello"})
+
+        assert response.status_code == 400, response.text
+        assert response.json() == {
+            "error": {
+                "message": "Missing required parameter: 'model'.",
+                "type": "invalid_request_error",
+                "param": "model",
+                "code": "missing_required_parameter",
+            }
+        }
+        counter.assert_not_awaited()
+
+    def test_missing_input_returns_openai_400(self):
+        response, counter = self._post_input_tokens({"model": "gpt-4o"})
+
+        assert response.status_code == 400, response.text
+        assert response.json() == {
+            "error": {
+                "message": "Missing required parameter: 'input'.",
+                "type": "invalid_request_error",
+                "param": "input",
+                "code": "missing_required_parameter",
+            }
+        }
+        counter.assert_not_awaited()
+
+    def test_invalid_tools_returns_openai_400(self):
+        response, counter = self._post_input_tokens({"model": "gpt-4o", "input": "hi", "tools": "not-a-list"})
+
+        assert response.status_code == 400, response.text
+        error = response.json()["error"]
+        assert error["type"] == "invalid_request_error"
+        counter.assert_not_awaited()
+
+    def test_provider_error_maps_status_code(self):
+        from litellm.proxy._types import ProxyException
+
+        failing_counter = AsyncMock(
+            side_effect=ProxyException(
+                message="rate limited",
+                type="token_counting_error",
+                param="model",
+                code="429",
+            )
+        )
+        response, _ = self._post_input_tokens({"model": "gpt-4o", "input": "hi"}, counter=failing_counter)
+
+        assert response.status_code == 429, response.text
+        assert response.json()["error"]["message"] == "rate limited"
diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts
index 81a2fefe32e..01a1b516d11 100644
--- a/ui/litellm-dashboard/src/lib/http/schema.d.ts
+++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts
@@ -9700,6 +9700,37 @@ export interface paths {
         patch?: never;
         trace?: never;
     };
+    "/openai/v1/responses/input_tokens": {
+        parameters: {
+            query?: never;
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        get?: never;
+        put?: never;
+        /**
+         * Responses Input Tokens
+         * @description Count the input tokens of a Responses API request without calling the model.
+         *
+         *     Follows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses/input-tokens
+         *
+         *     ```bash
+         *     curl -X POST http://localhost:4000/v1/responses/input_tokens     -H "Content-Type: application/json"     -H "Authorization: Bearer sk-1234"     -d '{
+         *         "model": "gpt-4o",
+         *         "input": "Hello, how are you?"
+         *     }'
+         *     ```
+         *
+         *     Returns: `{"object": "response.input_tokens", "input_tokens": <count>}`
+         */
+        post: operations["responses_input_tokens_openai_v1_responses_input_tokens_post"];
+        delete?: never;
+        options?: never;
+        head?: never;
+        patch?: never;
+        trace?: never;
+    };
     "/openai/v1/responses/{response_id}": {
         parameters: {
             query?: never;
@@ -12619,6 +12650,37 @@ export interface paths {
         patch?: never;
         trace?: never;
     };
+    "/responses/input_tokens": {
+        parameters: {
+            query?: never;
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        get?: never;
+        put?: never;
+        /**
+         * Responses Input Tokens
+         * @description Count the input tokens of a Responses API request without calling the model.
+         *
+         *     Follows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses/input-tokens
+         *
+         *     ```bash
+         *     curl -X POST http://localhost:4000/v1/responses/input_tokens     -H "Content-Type: application/json"     -H "Authorization: Bearer sk-1234"     -d '{
+         *         "model": "gpt-4o",
+         *         "input": "Hello, how are you?"
+         *     }'
+         *     ```
+         *
+         *     Returns: `{"object": "response.input_tokens", "input_tokens": <count>}`
+         */
+        post: operations["responses_input_tokens_responses_input_tokens_post"];
+        delete?: never;
+        options?: never;
+        head?: never;
+        patch?: never;
+        trace?: never;
+    };
     "/responses/{response_id}": {
         parameters: {
             query?: never;
@@ -19184,6 +19246,37 @@ export interface paths {
         patch?: never;
         trace?: never;
     };
+    "/v1/responses/input_tokens": {
+        parameters: {
+            query?: never;
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        get?: never;
+        put?: never;
+        /**
+         * Responses Input Tokens
+         * @description Count the input tokens of a Responses API request without calling the model.
+         *
+         *     Follows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses/input-tokens
+         *
+         *     ```bash
+         *     curl -X POST http://localhost:4000/v1/responses/input_tokens     -H "Content-Type: application/json"     -H "Authorization: Bearer sk-1234"     -d '{
+         *         "model": "gpt-4o",
+         *         "input": "Hello, how are you?"
+         *     }'
+         *     ```
+         *
+         *     Returns: `{"object": "response.input_tokens", "input_tokens": <count>}`
+         */
+        post: operations["responses_input_tokens_v1_responses_input_tokens_post"];
+        delete?: never;
+        options?: never;
+        head?: never;
+        patch?: never;
+        trace?: never;
+    };
     "/v1/responses/{response_id}": {
         parameters: {
             query?: never;
@@ -51476,6 +51569,26 @@ export interface operations {
             };
         };
     };
+    responses_input_tokens_openai_v1_responses_input_tokens_post: {
+        parameters: {
+            query?: never;
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        requestBody?: never;
+        responses: {
+            /** @description Successful Response */
+            200: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": unknown;
+                };
+            };
+        };
+    };
     get_response_openai_v1_responses__response_id__get: {
         parameters: {
             query?: never;
@@ -54440,6 +54553,26 @@ export interface operations {
             };
         };
     };
+    responses_input_tokens_responses_input_tokens_post: {
+        parameters: {
+            query?: never;
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        requestBody?: never;
+        responses: {
+            /** @description Successful Response */
+            200: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": unknown;
+                };
+            };
+        };
+    };
     get_response_responses__response_id__get: {
         parameters: {
             query?: never;
@@ -62862,6 +62995,26 @@ export interface operations {
             };
         };
     };
+    responses_input_tokens_v1_responses_input_tokens_post: {
+        parameters: {
+            query?: never;
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        requestBody?: never;
+        responses: {
+            /** @description Successful Response */
+            200: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": unknown;
+                };
+            };
+        };
+    };
     get_response_v1_responses__response_id__get: {
         parameters: {
             query?: never;

From 72adeda9ce3bb5cf61ca6816f01fc5693b85a1e2 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Mon, 31 Aug 2026 12:15:52 -0700
Subject: [PATCH 20/37] fix(openai): scope workload identity to the openai
 provider and env-resolved base/key

---
 .../llms/openai/responses/transformation.py   | 10 +++---
 litellm/llms/openai/workload_identity.py      |  8 +++--
 .../openai/test_openai_workload_identity.py   | 34 +++++++++++++++++++
 3 files changed, 44 insertions(+), 8 deletions(-)

diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py
index 1479c378014..eadc087383a 100644
--- a/litellm/llms/openai/responses/transformation.py
+++ b/litellm/llms/openai/responses/transformation.py
@@ -393,12 +393,10 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
         litellm_params = litellm_params or GenericLiteLLMParams()
         api_key = litellm_params.api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
         headers.setdefault("Content-Type", "application/json")
-        workload_identity_config: Final = resolve_openai_workload_identity_config(
-            api_key=api_key,
-            api_base=litellm_params.api_base
-            or litellm.api_base
-            or get_secret_str("OPENAI_BASE_URL")
-            or get_secret_str("OPENAI_API_BASE"),
+        workload_identity_config: Final = (
+            resolve_openai_workload_identity_config(api_key=api_key, api_base=litellm_params.api_base)
+            if self.custom_llm_provider is LlmProviders.OPENAI
+            else None
         )
         if workload_identity_config is not None:
             headers["Authorization"] = f"Bearer {get_workload_identity_bearer_token(workload_identity_config)}"
diff --git a/litellm/llms/openai/workload_identity.py b/litellm/llms/openai/workload_identity.py
index 15105d67957..be4d015af8a 100644
--- a/litellm/llms/openai/workload_identity.py
+++ b/litellm/llms/openai/workload_identity.py
@@ -5,6 +5,7 @@ from functools import lru_cache
 from typing import TYPE_CHECKING, Final
 from urllib.parse import urlparse
 
+import litellm
 from litellm.secret_managers.main import get_secret_str
 
 from .common_utils import OpenAIError
@@ -44,9 +45,12 @@ def resolve_openai_workload_identity_config(
     api_key: str | None,
     api_base: str | None,
 ) -> OpenAIWorkloadIdentityConfig | None:
-    if api_key is not None:
+    if api_key is not None or get_secret_str("OPENAI_API_KEY") is not None:
         return None
-    if not _targets_openai_api(api_base):
+    effective_api_base: Final = (
+        api_base or litellm.api_base or get_secret_str("OPENAI_BASE_URL") or get_secret_str("OPENAI_API_BASE")
+    )
+    if not _targets_openai_api(effective_api_base):
         return None
     identity_provider_id: Final = get_secret_str("OPENAI_IDENTITY_PROVIDER_ID")
     service_account_id: Final = get_secret_str("OPENAI_SERVICE_ACCOUNT_ID")
diff --git a/tests/test_litellm/llms/openai/test_openai_workload_identity.py b/tests/test_litellm/llms/openai/test_openai_workload_identity.py
index b8cc8c9c80f..d7257d6af89 100644
--- a/tests/test_litellm/llms/openai/test_openai_workload_identity.py
+++ b/tests/test_litellm/llms/openai/test_openai_workload_identity.py
@@ -9,6 +9,7 @@ import respx
 from openai import AsyncOpenAI, OpenAI
 
 import litellm
+from litellm.llms.litellm_proxy.responses.transformation import LiteLLMProxyResponsesAPIConfig
 from litellm.llms.openai.common_utils import BaseOpenAILLM, OpenAIError
 from litellm.llms.openai.openai import OpenAIChatCompletion
 from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
@@ -28,6 +29,9 @@ def wif_env(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> OpenAIWorkloadId
     token_file: Final = tmp_path / "subject_token.jwt"
     token_file.write_text("subject-token-from-file")
     monkeypatch.delenv("OPENAI_API_KEY", raising=False)
+    monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
+    monkeypatch.delenv("OPENAI_API_BASE", raising=False)
+    monkeypatch.setattr(litellm, "api_base", None)
     monkeypatch.setenv("OPENAI_IDENTITY_PROVIDER_ID", "idp_test123")
     monkeypatch.setenv("OPENAI_SERVICE_ACCOUNT_ID", "user-test456")
     monkeypatch.setenv("OPENAI_IDENTITY_TOKEN_FILE", str(token_file))
@@ -53,12 +57,36 @@ class TestResolveConfig:
     def test_static_api_key_wins(self, wif_env: OpenAIWorkloadIdentityConfig) -> None:
         assert resolve_openai_workload_identity_config(api_key="sk-static", api_base=None) is None
 
+    def test_env_openai_api_key_wins(
+        self, wif_env: OpenAIWorkloadIdentityConfig, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        monkeypatch.setenv("OPENAI_API_KEY", "sk-from-env")
+        assert resolve_openai_workload_identity_config(api_key=None, api_base=None) is None
+
     def test_foreign_api_base_disables(self, wif_env: OpenAIWorkloadIdentityConfig) -> None:
         assert resolve_openai_workload_identity_config(api_key=None, api_base="https://my-vllm.internal/v1") is None
 
     def test_openai_api_base_allows(self, wif_env: OpenAIWorkloadIdentityConfig) -> None:
         assert resolve_openai_workload_identity_config(api_key=None, api_base="https://api.openai.com/v1") == wif_env
 
+    def test_foreign_env_base_url_disables(
+        self, wif_env: OpenAIWorkloadIdentityConfig, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        monkeypatch.setenv("OPENAI_BASE_URL", "https://my-vllm.internal/v1")
+        assert resolve_openai_workload_identity_config(api_key=None, api_base=None) is None
+
+    def test_openai_env_base_url_allows(
+        self, wif_env: OpenAIWorkloadIdentityConfig, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        monkeypatch.setenv("OPENAI_BASE_URL", "https://api.openai.com/v1")
+        assert resolve_openai_workload_identity_config(api_key=None, api_base=None) == wif_env
+
+    def test_foreign_litellm_api_base_disables(
+        self, wif_env: OpenAIWorkloadIdentityConfig, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        monkeypatch.setattr(litellm, "api_base", "https://my-vllm.internal/v1")
+        assert resolve_openai_workload_identity_config(api_key=None, api_base=None) is None
+
     @pytest.mark.parametrize(
         "missing_var",
         ["OPENAI_IDENTITY_PROVIDER_ID", "OPENAI_SERVICE_ACCOUNT_ID", "OPENAI_IDENTITY_TOKEN_FILE"],
@@ -186,3 +214,9 @@ class TestResponsesValidateEnvironment:
             litellm_params=GenericLiteLLMParams(api_base="https://my-vllm.internal/v1"),
         )
         assert headers["Authorization"] == "Bearer None"
+
+    def test_litellm_proxy_subclass_never_mints_wif(self, wif_env: OpenAIWorkloadIdentityConfig) -> None:
+        headers: Final = LiteLLMProxyResponsesAPIConfig().validate_environment(
+            headers={}, model="gpt-4o-mini", litellm_params=GenericLiteLLMParams()
+        )
+        assert headers["Authorization"] == "Bearer None"

From 666d8737aa8766ac293995e565d91c1faf82069e Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Mon, 31 Aug 2026 12:19:41 -0700
Subject: [PATCH 21/37] fix(init): ignore pydantic ReadOnly TypedDict warning
 that floods proxy boot

---
 litellm/__init__.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/litellm/__init__.py b/litellm/__init__.py
index c83e72a78b4..1447e05fdf7 100644
--- a/litellm/__init__.py
+++ b/litellm/__init__.py
@@ -7,6 +7,9 @@ warnings.filterwarnings("ignore", message=".*conflict with protected namespace.*
 # Suppress Pydantic 2.11+ deprecation warning about accessing model_fields on instances
 # This warning can accumulate during streaming and cause memory leaks
 warnings.filterwarnings("ignore", message=".*Accessing the.*attribute on the instance is deprecated.*")
+# ReadOnly on TypedDict fields is repo-wide static discipline (LIT012); pydantic warns it
+# cannot enforce it at runtime, which floods proxy boot once such a type is schema-walked
+warnings.filterwarnings("ignore", message=".*`ReadOnly` qualifier.*")
 ### INIT VARIABLES #########################
 import threading
 import os

From 6b7159323bebc5bb5f2ad6a7b8680942cbfb8ab8 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Mon, 31 Aug 2026 12:25:45 -0700
Subject: [PATCH 22/37] fix(proxy): match OpenAI on empty input and skip budget
 reservation for token counting

/v1/responses/input_tokens returned 200 with a count for an empty
"input" ("" or []), while OpenAI returns a 400 missing_required_parameter.
The route also went through optimistic budget reservation, which is only
released by LLM success/failure callbacks that a token count never
reaches, so every call leaked a reservation until TTL expiry and could
429 real traffic. Both routes plus the /openai alias now join
/utils/token_counter in the reservation exemption set.
---
 .../proxy/response_api_endpoints/endpoints.py |  6 +++
 .../spend_tracking/budget_reservation.py      |  9 +++-
 .../response_api_endpoints/test_endpoints.py  | 15 ++++++
 .../spend_tracking/test_budget_reservation.py | 48 +++++++++++++++++++
 4 files changed, 77 insertions(+), 1 deletion(-)
 create mode 100644 tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py

diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py
index 100b42a9e2a..aa7595ed13d 100644
--- a/litellm/proxy/response_api_endpoints/endpoints.py
+++ b/litellm/proxy/response_api_endpoints/endpoints.py
@@ -1142,6 +1142,12 @@ async def responses_input_tokens(
         return _missing_responses_param_response("model")
     if input_value is None:
         return _missing_responses_param_response("input")
+    if isinstance(input_value, (str, list)) and not input_value:
+        return _responses_invalid_request_response(
+            message="""One of "input" or "previous_response_id" or 'prompt' or 'conversation' must be provided.""",
+            param=None,
+            code="missing_required_parameter",
+        )
 
     try:
         payload: Final[_TokenCountPayload] = {
diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py
index 8b2a5dd9312..2d113cfe355 100644
--- a/litellm/proxy/spend_tracking/budget_reservation.py
+++ b/litellm/proxy/spend_tracking/budget_reservation.py
@@ -172,7 +172,14 @@ async def reserve_budget_for_request(
 ) -> dict | None:
     if valid_token is None or not RouteChecks.is_llm_api_route(route=route):
         return None
-    if route in {"/models", "/v1/models", "/utils/token_counter"}:
+    if route in {
+        "/models",
+        "/v1/models",
+        "/utils/token_counter",
+        "/responses/input_tokens",
+        "/v1/responses/input_tokens",
+        "/openai/v1/responses/input_tokens",
+    }:
         return None
     if get_model_from_request(request_body, route, llm_router=llm_router) is None:
         return None
diff --git a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py
index dc43e7f5c06..b31c53c14a8 100644
--- a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py
+++ b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py
@@ -1914,6 +1914,21 @@ class TestResponsesInputTokens:
         }
         counter.assert_not_awaited()
 
+    @pytest.mark.parametrize("empty_input", ["", []])
+    def test_empty_input_returns_openai_400(self, empty_input):
+        response, counter = self._post_input_tokens({"model": "gpt-4o", "input": empty_input})
+
+        assert response.status_code == 400, response.text
+        assert response.json() == {
+            "error": {
+                "message": """One of "input" or "previous_response_id" or 'prompt' or 'conversation' must be provided.""",
+                "type": "invalid_request_error",
+                "param": None,
+                "code": "missing_required_parameter",
+            }
+        }
+        counter.assert_not_awaited()
+
     def test_invalid_tools_returns_openai_400(self):
         response, counter = self._post_input_tokens({"model": "gpt-4o", "input": "hi", "tools": "not-a-list"})
 
diff --git a/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py b/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py
new file mode 100644
index 00000000000..f65f68812a2
--- /dev/null
+++ b/tests/test_litellm/proxy/spend_tracking/test_budget_reservation.py
@@ -0,0 +1,48 @@
+from typing import Final
+
+import pytest
+
+from litellm.caching import DualCache
+from litellm.proxy._types import UserAPIKeyAuth
+from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
+from litellm.proxy.spend_tracking.budget_reservation import reserve_budget_for_request
+from litellm.proxy.utils import ProxyLogging
+
+TOKEN_COUNTING_ROUTES: Final = (
+    "/responses/input_tokens",
+    "/v1/responses/input_tokens",
+    "/openai/v1/responses/input_tokens",
+    "/utils/token_counter",
+)
+
+
+def _budgeted_token() -> UserAPIKeyAuth:
+    return UserAPIKeyAuth(api_key="sk-test", token="hashed-token", max_budget=100.0, spend=0.0)
+
+
+async def _reserve(route: str) -> dict | None:
+    return await reserve_budget_for_request(
+        request_body={"model": "gpt-4o", "input": "hello"},
+        route=route,
+        llm_router=None,
+        valid_token=_budgeted_token(),
+        team_object=None,
+        user_object=None,
+        prisma_client=None,
+        user_api_key_cache=UserApiKeyCache(),
+        proxy_logging_obj=ProxyLogging(user_api_key_cache=DualCache()),
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("route", TOKEN_COUNTING_ROUTES)
+async def test_token_counting_routes_are_exempt_from_budget_reservation(route):
+    assert await _reserve(route) is None
+
+
+@pytest.mark.asyncio
+async def test_non_exempt_llm_route_still_reserves_budget():
+    reservation: Final = await _reserve("/v1/responses")
+
+    assert reservation is not None
+    assert reservation["reserved_cost"] > 0

From ef72e7b37dc66d0d755af8dd67934d6f2ae1824d Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Mon, 31 Aug 2026 12:32:00 -0700
Subject: [PATCH 23/37] fix(openai): require https for workload identity
 api_base targets

---
 litellm/llms/openai/workload_identity.py                       | 3 ++-
 .../test_litellm/llms/openai/test_openai_workload_identity.py  | 3 +++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/litellm/llms/openai/workload_identity.py b/litellm/llms/openai/workload_identity.py
index be4d015af8a..8a914039bd0 100644
--- a/litellm/llms/openai/workload_identity.py
+++ b/litellm/llms/openai/workload_identity.py
@@ -71,7 +71,8 @@ def get_workload_identity_bearer_token(config: OpenAIWorkloadIdentityConfig) ->
 def _targets_openai_api(api_base: str | None) -> bool:
     if api_base is None:
         return True
-    return urlparse(api_base).hostname == _OPENAI_API_HOST
+    parsed: Final = urlparse(api_base)
+    return parsed.scheme == "https" and parsed.hostname == _OPENAI_API_HOST
 
 
 @lru_cache(maxsize=16)
diff --git a/tests/test_litellm/llms/openai/test_openai_workload_identity.py b/tests/test_litellm/llms/openai/test_openai_workload_identity.py
index d7257d6af89..852e499dc54 100644
--- a/tests/test_litellm/llms/openai/test_openai_workload_identity.py
+++ b/tests/test_litellm/llms/openai/test_openai_workload_identity.py
@@ -69,6 +69,9 @@ class TestResolveConfig:
     def test_openai_api_base_allows(self, wif_env: OpenAIWorkloadIdentityConfig) -> None:
         assert resolve_openai_workload_identity_config(api_key=None, api_base="https://api.openai.com/v1") == wif_env
 
+    def test_plaintext_http_api_base_disables(self, wif_env: OpenAIWorkloadIdentityConfig) -> None:
+        assert resolve_openai_workload_identity_config(api_key=None, api_base="http://api.openai.com/v1") is None
+
     def test_foreign_env_base_url_disables(
         self, wif_env: OpenAIWorkloadIdentityConfig, monkeypatch: pytest.MonkeyPatch
     ) -> None:

From 73ab647b1c26c8b1fcff137c87733e46e8d90326 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Mon, 31 Aug 2026 12:43:17 -0700
Subject: [PATCH 24/37] fix(count_tokens): preserve image inputs when counting
 Responses API tokens

The chat-to-Responses reverse transform kept only text blocks, so an image
input was dropped before the count went to OpenAI. A 256x256 image request
counted 13 tokens instead of 268.
---
 .../responses/count_tokens/transformation.py  | 73 ++++++++++++--
 ...test_openai_count_tokens_transformation.py | 97 +++++++++++++++++++
 .../response_api_endpoints/test_endpoints.py  |  9 +-
 3 files changed, 169 insertions(+), 10 deletions(-)

diff --git a/litellm/llms/openai/responses/count_tokens/transformation.py b/litellm/llms/openai/responses/count_tokens/transformation.py
index 6b2f4535df1..72038ae6f6c 100644
--- a/litellm/llms/openai/responses/count_tokens/transformation.py
+++ b/litellm/llms/openai/responses/count_tokens/transformation.py
@@ -4,7 +4,69 @@ OpenAI Responses API token counting transformation logic.
 This module handles the transformation of requests to OpenAI's /v1/responses/input_tokens endpoint.
 """
 
-from typing import Any, Final
+from collections.abc import Mapping, Sequence
+from typing import Any, Final, Literal
+
+from typing_extensions import ReadOnly, TypedDict
+
+
+class ResponsesInputTextPart(TypedDict):
+    type: ReadOnly[Literal["input_text"]]
+    text: ReadOnly[str]
+
+
+class ResponsesInputImagePart(TypedDict):
+    type: ReadOnly[Literal["input_image"]]
+    image_url: ReadOnly[str]
+    detail: ReadOnly[str]
+
+
+ResponsesInputPart = ResponsesInputTextPart | ResponsesInputImagePart
+
+
+def _chat_image_block_to_responses_part(image_url: object) -> ResponsesInputImagePart | None:
+    url: Final = image_url.get("url") if isinstance(image_url, Mapping) else image_url
+    if not isinstance(url, str) or not url:
+        return None
+    detail: Final = image_url.get("detail") if isinstance(image_url, Mapping) else None
+    part: Final[ResponsesInputImagePart] = {
+        "type": "input_image",
+        "image_url": url,
+        "detail": detail if isinstance(detail, str) and detail else "auto",
+    }
+    return part
+
+
+def _chat_block_to_responses_part(block: object) -> ResponsesInputPart | None:
+    if isinstance(block, str):
+        bare: Final[ResponsesInputTextPart] = {"type": "input_text", "text": block}
+        return bare
+    if not isinstance(block, Mapping):
+        return None
+    match block.get("type"):
+        case "text":
+            text_value: Final = block.get("text")
+            text: Final[ResponsesInputTextPart] = {
+                "type": "input_text",
+                "text": text_value if isinstance(text_value, str) else "",
+            }
+            return text
+        case "image_url":
+            return _chat_image_block_to_responses_part(block.get("image_url"))
+        case _:
+            return None
+
+
+def chat_content_blocks_to_responses_content(
+    content: Sequence[object],
+) -> str | tuple[ResponsesInputPart, ...]:
+    """Text-only content collapses to a joined string, so text-only counts stay unchanged."""
+    parts: Final = tuple(
+        part for part in (_chat_block_to_responses_part(block) for block in content) if part is not None
+    )
+    if any(part["type"] != "input_text" for part in parts):
+        return parts
+    return "\n".join(part["text"] for part in parts if part["type"] == "input_text")
 
 
 class OpenAICountTokensConfig:
@@ -120,14 +182,7 @@ class OpenAICountTokensConfig:
                     instructions_parts.append("\n".join(text_parts))
             elif role == "user":
                 if isinstance(content, list):
-                    # Extract text from content blocks for Responses API
-                    text_parts = []
-                    for block in content:
-                        if isinstance(block, dict) and block.get("type") == "text":
-                            text_parts.append(block.get("text", ""))
-                        elif isinstance(block, str):
-                            text_parts.append(block)
-                    content = "\n".join(text_parts)
+                    content = chat_content_blocks_to_responses_content(content)
                 input_items.append({"role": "user", "content": content})
             elif role == "assistant":
                 # Map tool_calls to Responses API function_call items
diff --git a/tests/test_litellm/llms/openai/responses/test_openai_count_tokens_transformation.py b/tests/test_litellm/llms/openai/responses/test_openai_count_tokens_transformation.py
index e1cc6a92927..ca9e7ab52c3 100644
--- a/tests/test_litellm/llms/openai/responses/test_openai_count_tokens_transformation.py
+++ b/tests/test_litellm/llms/openai/responses/test_openai_count_tokens_transformation.py
@@ -163,6 +163,103 @@ def test_messages_to_responses_input_with_tool():
     }
 
 
+def test_messages_to_responses_input_preserves_images():
+    """An image block must survive the round trip, or OpenAI counts only the text.
+
+    A 256x256 image is worth 255 tokens to OpenAI's counting API; dropping it
+    turned a 268-token request into a 13-token one.
+    """
+    messages = [
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "What is in this image?"},
+                {
+                    "type": "image_url",
+                    "image_url": {"url": "data:image/png;base64,iVBORw0KGgo=", "detail": "high"},
+                },
+            ],
+        }
+    ]
+
+    input_items, instructions = OpenAICountTokensConfig.messages_to_responses_input(messages)
+
+    assert instructions is None
+    assert input_items == [
+        {
+            "role": "user",
+            "content": (
+                {"type": "input_text", "text": "What is in this image?"},
+                {
+                    "type": "input_image",
+                    "image_url": "data:image/png;base64,iVBORw0KGgo=",
+                    "detail": "high",
+                },
+            ),
+        }
+    ]
+
+
+def test_messages_to_responses_input_image_without_detail_defaults_to_auto():
+    messages = [
+        {
+            "role": "user",
+            "content": [{"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}}],
+        }
+    ]
+
+    input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages)
+
+    assert input_items[0]["content"] == (
+        {"type": "input_image", "image_url": "https://example.com/cat.png", "detail": "auto"},
+    )
+
+
+def test_messages_to_responses_input_bare_string_image_url_is_preserved():
+    messages = [{"role": "user", "content": [{"type": "image_url", "image_url": "https://example.com/cat.png"}]}]
+
+    input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages)
+
+    assert input_items[0]["content"] == (
+        {"type": "input_image", "image_url": "https://example.com/cat.png", "detail": "auto"},
+    )
+
+
+def test_messages_to_responses_input_text_only_blocks_stay_a_joined_string():
+    """Text-only content must keep collapsing to a string so existing counts do not shift."""
+    messages = [
+        {
+            "role": "user",
+            "content": [{"type": "text", "text": "first"}, {"type": "text", "text": "second"}],
+        }
+    ]
+
+    input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages)
+
+    assert input_items == [{"role": "user", "content": "first\nsecond"}]
+
+
+def test_messages_to_responses_input_drops_unmappable_blocks():
+    """A block with no Responses API equivalent is skipped, never forwarded verbatim."""
+    messages = [
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "hi"},
+                {"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}},
+                {"type": "input_audio", "input_audio": {"data": "AAAA", "format": "wav"}},
+            ],
+        }
+    ]
+
+    input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages)
+
+    assert input_items[0]["content"] == (
+        {"type": "input_text", "text": "hi"},
+        {"type": "input_image", "image_url": "https://example.com/cat.png", "detail": "auto"},
+    )
+
+
 def test_validate_request_valid():
     """Test that valid requests pass validation."""
     config = OpenAICountTokensConfig()
diff --git a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py
index b31c53c14a8..d7010de6405 100644
--- a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py
+++ b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py
@@ -3,10 +3,12 @@ Test for response_api_endpoints/endpoints.py
 """
 
 import unittest
+from typing import Any
 from unittest.mock import AsyncMock, MagicMock, patch
 
 import pytest
 from fastapi.testclient import TestClient
+from httpx import Response
 
 import litellm
 from litellm.proxy.proxy_server import app
@@ -1816,7 +1818,12 @@ class TestResponsesInputTokens:
     never registered, so the POST fell through to the GET/DELETE-only
     /v1/responses/{response_id} route and returned 405."""
 
-    def _post_input_tokens(self, body, path="/v1/responses/input_tokens", counter=None):
+    def _post_input_tokens(
+        self,
+        body: dict[str, Any],
+        path: str = "/v1/responses/input_tokens",
+        counter: AsyncMock | None = None,
+    ) -> tuple[Response, AsyncMock]:
         from litellm.proxy._types import UserAPIKeyAuth
         from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
         from litellm.proxy.response_api_endpoints.endpoints import _proxy_token_counter

From e7dc0213bdd1d88dccf2596aa349e8d771d74311 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Mon, 31 Aug 2026 12:52:55 -0700
Subject: [PATCH 25/37] fix(openai): treat empty api key values as unset for
 workload identity

---
 litellm/llms/openai/workload_identity.py                 | 2 +-
 .../llms/openai/test_openai_workload_identity.py         | 9 +++++++++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/litellm/llms/openai/workload_identity.py b/litellm/llms/openai/workload_identity.py
index 8a914039bd0..e9ac26e26a2 100644
--- a/litellm/llms/openai/workload_identity.py
+++ b/litellm/llms/openai/workload_identity.py
@@ -45,7 +45,7 @@ def resolve_openai_workload_identity_config(
     api_key: str | None,
     api_base: str | None,
 ) -> OpenAIWorkloadIdentityConfig | None:
-    if api_key is not None or get_secret_str("OPENAI_API_KEY") is not None:
+    if api_key or get_secret_str("OPENAI_API_KEY"):
         return None
     effective_api_base: Final = (
         api_base or litellm.api_base or get_secret_str("OPENAI_BASE_URL") or get_secret_str("OPENAI_API_BASE")
diff --git a/tests/test_litellm/llms/openai/test_openai_workload_identity.py b/tests/test_litellm/llms/openai/test_openai_workload_identity.py
index 852e499dc54..228d591b47a 100644
--- a/tests/test_litellm/llms/openai/test_openai_workload_identity.py
+++ b/tests/test_litellm/llms/openai/test_openai_workload_identity.py
@@ -63,6 +63,15 @@ class TestResolveConfig:
         monkeypatch.setenv("OPENAI_API_KEY", "sk-from-env")
         assert resolve_openai_workload_identity_config(api_key=None, api_base=None) is None
 
+    def test_empty_env_openai_api_key_counts_as_unset(
+        self, wif_env: OpenAIWorkloadIdentityConfig, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        monkeypatch.setenv("OPENAI_API_KEY", "")
+        assert resolve_openai_workload_identity_config(api_key=None, api_base=None) == wif_env
+
+    def test_empty_api_key_param_counts_as_unset(self, wif_env: OpenAIWorkloadIdentityConfig) -> None:
+        assert resolve_openai_workload_identity_config(api_key="", api_base=None) == wif_env
+
     def test_foreign_api_base_disables(self, wif_env: OpenAIWorkloadIdentityConfig) -> None:
         assert resolve_openai_workload_identity_config(api_key=None, api_base="https://my-vllm.internal/v1") is None
 

From ae83444a3e0eca7536f49101557bc8ec044b501a Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Mon, 31 Aug 2026 19:54:45 +0000
Subject: [PATCH 26/37] fix(openai): treat empty api_key as unset for WIF
 resolution

---
 litellm/llms/openai/workload_identity.py         |  7 +++++--
 .../llms/openai/test_openai_workload_identity.py | 16 ++++++++++------
 2 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/litellm/llms/openai/workload_identity.py b/litellm/llms/openai/workload_identity.py
index e9ac26e26a2..ecec161ed46 100644
--- a/litellm/llms/openai/workload_identity.py
+++ b/litellm/llms/openai/workload_identity.py
@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Final
 from urllib.parse import urlparse
 
 import litellm
-from litellm.secret_managers.main import get_secret_str
+from litellm.secret_managers.main import get_secret_str, normalize_nonempty_secret_str
 
 from .common_utils import OpenAIError
 
@@ -45,7 +45,10 @@ def resolve_openai_workload_identity_config(
     api_key: str | None,
     api_base: str | None,
 ) -> OpenAIWorkloadIdentityConfig | None:
-    if api_key or get_secret_str("OPENAI_API_KEY"):
+    static_api_key: Final = normalize_nonempty_secret_str(api_key) or normalize_nonempty_secret_str(
+        get_secret_str("OPENAI_API_KEY")
+    )
+    if static_api_key is not None:
         return None
     effective_api_base: Final = (
         api_base or litellm.api_base or get_secret_str("OPENAI_BASE_URL") or get_secret_str("OPENAI_API_BASE")
diff --git a/tests/test_litellm/llms/openai/test_openai_workload_identity.py b/tests/test_litellm/llms/openai/test_openai_workload_identity.py
index 228d591b47a..d8d9936e9a1 100644
--- a/tests/test_litellm/llms/openai/test_openai_workload_identity.py
+++ b/tests/test_litellm/llms/openai/test_openai_workload_identity.py
@@ -63,14 +63,18 @@ class TestResolveConfig:
         monkeypatch.setenv("OPENAI_API_KEY", "sk-from-env")
         assert resolve_openai_workload_identity_config(api_key=None, api_base=None) is None
 
-    def test_empty_env_openai_api_key_counts_as_unset(
-        self, wif_env: OpenAIWorkloadIdentityConfig, monkeypatch: pytest.MonkeyPatch
+    @pytest.mark.parametrize("empty_key", ["", "   "])
+    def test_empty_api_key_arg_does_not_disable_wif(
+        self, wif_env: OpenAIWorkloadIdentityConfig, empty_key: str
     ) -> None:
-        monkeypatch.setenv("OPENAI_API_KEY", "")
-        assert resolve_openai_workload_identity_config(api_key=None, api_base=None) == wif_env
+        assert resolve_openai_workload_identity_config(api_key=empty_key, api_base=None) == wif_env
 
-    def test_empty_api_key_param_counts_as_unset(self, wif_env: OpenAIWorkloadIdentityConfig) -> None:
-        assert resolve_openai_workload_identity_config(api_key="", api_base=None) == wif_env
+    @pytest.mark.parametrize("empty_key", ["", "   "])
+    def test_empty_env_openai_api_key_does_not_disable_wif(
+        self, wif_env: OpenAIWorkloadIdentityConfig, monkeypatch: pytest.MonkeyPatch, empty_key: str
+    ) -> None:
+        monkeypatch.setenv("OPENAI_API_KEY", empty_key)
+        assert resolve_openai_workload_identity_config(api_key=None, api_base=None) == wif_env
 
     def test_foreign_api_base_disables(self, wif_env: OpenAIWorkloadIdentityConfig) -> None:
         assert resolve_openai_workload_identity_config(api_key=None, api_base="https://my-vllm.internal/v1") is None

From fe90c6f6fca6440f302e21bb31f5816225b96770 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Mon, 31 Aug 2026 12:59:40 -0700
Subject: [PATCH 27/37] fix(count_tokens): keep assistant turns on the provider
 counting API

Assistant list content was forwarded to /v1/responses/input_tokens as chat
`text` blocks, which the Responses API rejects (it accepts only output_text
and refusal inside an assistant turn). The 400 sent the whole request to the
local tokenizer, so any conversation with an assistant turn silently lost
provider-exact counting, including the image counting added in 73ab647b1c.

Assistant content now collapses to the plain string the Responses API counts
identically, and image parts are kept to user turns where they are legal.
---
 .../responses/count_tokens/transformation.py  | 19 ++++--
 ...test_openai_count_tokens_transformation.py | 63 +++++++++++++++++++
 2 files changed, 77 insertions(+), 5 deletions(-)

diff --git a/litellm/llms/openai/responses/count_tokens/transformation.py b/litellm/llms/openai/responses/count_tokens/transformation.py
index 72038ae6f6c..62dbebb4fe6 100644
--- a/litellm/llms/openai/responses/count_tokens/transformation.py
+++ b/litellm/llms/openai/responses/count_tokens/transformation.py
@@ -23,6 +23,8 @@ class ResponsesInputImagePart(TypedDict):
 
 ResponsesInputPart = ResponsesInputTextPart | ResponsesInputImagePart
 
+ResponsesContentRole = Literal["user", "assistant"]
+
 
 def _chat_image_block_to_responses_part(image_url: object) -> ResponsesInputImagePart | None:
     url: Final = image_url.get("url") if isinstance(image_url, Mapping) else image_url
@@ -37,7 +39,7 @@ def _chat_image_block_to_responses_part(image_url: object) -> ResponsesInputImag
     return part
 
 
-def _chat_block_to_responses_part(block: object) -> ResponsesInputPart | None:
+def _chat_block_to_responses_part(block: object, role: ResponsesContentRole) -> ResponsesInputPart | None:
     if isinstance(block, str):
         bare: Final[ResponsesInputTextPart] = {"type": "input_text", "text": block}
         return bare
@@ -51,7 +53,7 @@ def _chat_block_to_responses_part(block: object) -> ResponsesInputPart | None:
                 "text": text_value if isinstance(text_value, str) else "",
             }
             return text
-        case "image_url":
+        case "image_url" if role == "user":
             return _chat_image_block_to_responses_part(block.get("image_url"))
         case _:
             return None
@@ -59,10 +61,15 @@ def _chat_block_to_responses_part(block: object) -> ResponsesInputPart | None:
 
 def chat_content_blocks_to_responses_content(
     content: Sequence[object],
+    role: ResponsesContentRole,
 ) -> str | tuple[ResponsesInputPart, ...]:
-    """Text-only content collapses to a joined string, so text-only counts stay unchanged."""
+    """Text-only content collapses to a joined string, which every role accepts and counts identically.
+
+    Only a user turn may carry an image part: the Responses API rejects any part but
+    output_text and refusal inside an assistant turn.
+    """
     parts: Final = tuple(
-        part for part in (_chat_block_to_responses_part(block) for block in content) if part is not None
+        part for part in (_chat_block_to_responses_part(block, role) for block in content) if part is not None
     )
     if any(part["type"] != "input_text" for part in parts):
         return parts
@@ -182,11 +189,13 @@ class OpenAICountTokensConfig:
                     instructions_parts.append("\n".join(text_parts))
             elif role == "user":
                 if isinstance(content, list):
-                    content = chat_content_blocks_to_responses_content(content)
+                    content = chat_content_blocks_to_responses_content(content, "user")
                 input_items.append({"role": "user", "content": content})
             elif role == "assistant":
                 # Map tool_calls to Responses API function_call items
                 tool_calls = msg.get("tool_calls")
+                if isinstance(content, list):
+                    content = chat_content_blocks_to_responses_content(content, "assistant")
                 if content:
                     input_items.append({"role": "assistant", "content": content})
                 if tool_calls:
diff --git a/tests/test_litellm/llms/openai/responses/test_openai_count_tokens_transformation.py b/tests/test_litellm/llms/openai/responses/test_openai_count_tokens_transformation.py
index ca9e7ab52c3..22761272321 100644
--- a/tests/test_litellm/llms/openai/responses/test_openai_count_tokens_transformation.py
+++ b/tests/test_litellm/llms/openai/responses/test_openai_count_tokens_transformation.py
@@ -260,6 +260,69 @@ def test_messages_to_responses_input_drops_unmappable_blocks():
     )
 
 
+def test_messages_to_responses_input_assistant_blocks_collapse_to_a_string():
+    """An assistant turn must never forward chat `text` blocks.
+
+    The Responses API only accepts output_text and refusal inside an assistant turn, so
+    forwarding them 400s the whole request and silently drops the count back to the local
+    tokenizer, which is exactly what defeats the image fix above.
+    """
+    messages = [
+        {"role": "user", "content": [{"type": "text", "text": "What is the capital of France?"}]},
+        {"role": "assistant", "content": [{"type": "text", "text": "Paris."}]},
+    ]
+
+    input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages)
+
+    assert input_items == [
+        {"role": "user", "content": "What is the capital of France?"},
+        {"role": "assistant", "content": "Paris."},
+    ]
+
+
+def test_messages_to_responses_input_assistant_image_block_is_dropped():
+    """An image part is illegal inside an assistant turn, so it must not reach the provider."""
+    messages = [
+        {
+            "role": "assistant",
+            "content": [
+                {"type": "text", "text": "Here it is"},
+                {"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}},
+            ],
+        }
+    ]
+
+    input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages)
+
+    assert input_items == [{"role": "assistant", "content": "Here it is"}]
+
+
+def test_messages_to_responses_input_keeps_user_image_alongside_an_assistant_turn():
+    messages = [
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "What is in this image?"},
+                {"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}},
+            ],
+        },
+        {"role": "assistant", "content": [{"type": "text", "text": "A cat."}]},
+    ]
+
+    input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages)
+
+    assert input_items == [
+        {
+            "role": "user",
+            "content": (
+                {"type": "input_text", "text": "What is in this image?"},
+                {"type": "input_image", "image_url": "https://example.com/cat.png", "detail": "auto"},
+            ),
+        },
+        {"role": "assistant", "content": "A cat."},
+    ]
+
+
 def test_validate_request_valid():
     """Test that valid requests pass validation."""
     config = OpenAICountTokensConfig()

From 25c8d58400a48d1010e73d98987dd44b035764b5 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Mon, 31 Aug 2026 13:09:39 -0700
Subject: [PATCH 28/37] fix(docker): install file and make in wolfi builders so
 the uvloop sdist can build

---
 Dockerfile                 | 2 ++
 docker/Dockerfile.database | 2 ++
 docker/Dockerfile.non_root | 2 ++
 3 files changed, 6 insertions(+)

diff --git a/Dockerfile b/Dockerfile
index 700b0d6525e..4688a733331 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -39,7 +39,9 @@ COPY --from=uvbin /uvx /usr/local/bin/uvx
 
 RUN apk add --no-cache \
     bash \
+    file \
     gcc \
+    make \
     python3 \
     python3-dev \
     rust \
diff --git a/docker/Dockerfile.database b/docker/Dockerfile.database
index f0d6d02fccf..9a243c68dca 100644
--- a/docker/Dockerfile.database
+++ b/docker/Dockerfile.database
@@ -38,7 +38,9 @@ COPY --from=uvbin /uvx /usr/local/bin/uvx
 
 RUN apk add --no-cache \
     bash \
+    file \
     gcc \
+    make \
     python3 \
     python3-dev \
     openssl \
diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root
index 4a5df6ecd69..2cf70f5b01d 100644
--- a/docker/Dockerfile.non_root
+++ b/docker/Dockerfile.non_root
@@ -39,7 +39,9 @@ RUN for i in 1 2 3; do \
       apk add --no-cache \
         python3 \
         python3-dev \
+        file \
         gcc \
+        make \
         rust \
         bash \
         coreutils \

From c9908ffabb732f902f3409a7d763c5ea6a1541a8 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Mon, 31 Aug 2026 13:17:43 -0700
Subject: [PATCH 29/37] fix(responses): count input_file tokens instead of
 silently dropping the file

The Responses-to-chat transform dropped the filename OpenAI requires next to
file_data, so a request carrying an inline PDF counted 13 tokens instead of 36
and a real completion through the chat bridge got a 400.
---
 .../responses/count_tokens/transformation.py  | 28 ++++++-
 .../transformation.py                         |  2 +
 ...test_openai_count_tokens_transformation.py | 74 +++++++++++++++++++
 .../test_litellm_completion_responses.py      | 19 +++++
 4 files changed, 121 insertions(+), 2 deletions(-)

diff --git a/litellm/llms/openai/responses/count_tokens/transformation.py b/litellm/llms/openai/responses/count_tokens/transformation.py
index 62dbebb4fe6..88f04c59e01 100644
--- a/litellm/llms/openai/responses/count_tokens/transformation.py
+++ b/litellm/llms/openai/responses/count_tokens/transformation.py
@@ -21,7 +21,13 @@ class ResponsesInputImagePart(TypedDict):
     detail: ReadOnly[str]
 
 
-ResponsesInputPart = ResponsesInputTextPart | ResponsesInputImagePart
+class ResponsesInputFilePart(TypedDict):
+    type: ReadOnly[Literal["input_file"]]
+    filename: ReadOnly[str]
+    file_data: ReadOnly[str]
+
+
+ResponsesInputPart = ResponsesInputTextPart | ResponsesInputImagePart | ResponsesInputFilePart
 
 ResponsesContentRole = Literal["user", "assistant"]
 
@@ -39,6 +45,22 @@ def _chat_image_block_to_responses_part(image_url: object) -> ResponsesInputImag
     return part
 
 
+def _chat_file_block_to_responses_part(file_value: object) -> ResponsesInputFilePart | None:
+    """Only an inline file round trips: OpenAI rejects `file_data` without the `filename` beside it."""
+    if not isinstance(file_value, Mapping):
+        return None
+    filename: Final = file_value.get("filename")
+    file_data: Final = file_value.get("file_data")
+    if not isinstance(filename, str) or not filename or not isinstance(file_data, str) or not file_data:
+        return None
+    part: Final[ResponsesInputFilePart] = {
+        "type": "input_file",
+        "filename": filename,
+        "file_data": file_data,
+    }
+    return part
+
+
 def _chat_block_to_responses_part(block: object, role: ResponsesContentRole) -> ResponsesInputPart | None:
     if isinstance(block, str):
         bare: Final[ResponsesInputTextPart] = {"type": "input_text", "text": block}
@@ -55,6 +77,8 @@ def _chat_block_to_responses_part(block: object, role: ResponsesContentRole) ->
             return text
         case "image_url" if role == "user":
             return _chat_image_block_to_responses_part(block.get("image_url"))
+        case "file" if role == "user":
+            return _chat_file_block_to_responses_part(block.get("file"))
         case _:
             return None
 
@@ -65,7 +89,7 @@ def chat_content_blocks_to_responses_content(
 ) -> str | tuple[ResponsesInputPart, ...]:
     """Text-only content collapses to a joined string, which every role accepts and counts identically.
 
-    Only a user turn may carry an image part: the Responses API rejects any part but
+    Only a user turn may carry an image or file part: the Responses API rejects any part but
     output_text and refusal inside an assistant turn.
     """
     parts: Final = tuple(
diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py
index f39df38d069..4a416f61e1b 100644
--- a/litellm/responses/litellm_completion_transformation/transformation.py
+++ b/litellm/responses/litellm_completion_transformation/transformation.py
@@ -1629,6 +1629,8 @@ class LiteLLMCompletionResponsesConfig:
             file_dict["file_id"] = file_id
         if item.get("file_data"):
             file_dict["file_data"] = item["file_data"]
+        if item.get("filename"):
+            file_dict["filename"] = item["filename"]
 
         new_item: Final[dict[str, object]] = {"type": "file", "file": file_dict}
         if "cache_control" in item:
diff --git a/tests/test_litellm/llms/openai/responses/test_openai_count_tokens_transformation.py b/tests/test_litellm/llms/openai/responses/test_openai_count_tokens_transformation.py
index 22761272321..c2efc1acdb9 100644
--- a/tests/test_litellm/llms/openai/responses/test_openai_count_tokens_transformation.py
+++ b/tests/test_litellm/llms/openai/responses/test_openai_count_tokens_transformation.py
@@ -323,6 +323,80 @@ def test_messages_to_responses_input_keeps_user_image_alongside_an_assistant_tur
     ]
 
 
+def test_messages_to_responses_input_preserves_inline_files():
+    """An inline file must survive the round trip, or the count silently drops the file.
+
+    A small PDF is worth 36 tokens to OpenAI's counting API; dropping it left the same
+    request counting 13, the text-only total.
+    """
+    messages = [
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "Summarize this file."},
+                {
+                    "type": "file",
+                    "file": {"filename": "report.pdf", "file_data": "data:application/pdf;base64,JVBERi0="},
+                },
+            ],
+        }
+    ]
+
+    input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages)
+
+    assert input_items == [
+        {
+            "role": "user",
+            "content": (
+                {"type": "input_text", "text": "Summarize this file."},
+                {
+                    "type": "input_file",
+                    "filename": "report.pdf",
+                    "file_data": "data:application/pdf;base64,JVBERi0=",
+                },
+            ),
+        }
+    ]
+
+
+def test_messages_to_responses_input_drops_a_file_with_no_inline_data():
+    """OpenAI rejects `file_data` without a `filename`, and a rejected request loses the whole count."""
+    messages = [
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "Summarize this file."},
+                {"type": "file", "file": {"file_data": "data:application/pdf;base64,JVBERi0="}},
+                {"type": "file", "file": {"file_id": "file-abc123"}},
+            ],
+        }
+    ]
+
+    input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages)
+
+    assert input_items == [{"role": "user", "content": "Summarize this file."}]
+
+
+def test_messages_to_responses_input_assistant_file_block_is_dropped():
+    """A file part is illegal inside an assistant turn, so it must not reach the provider."""
+    messages = [
+        {
+            "role": "assistant",
+            "content": [
+                {"type": "text", "text": "Here it is"},
+                {
+                    "type": "file",
+                    "file": {"filename": "report.pdf", "file_data": "data:application/pdf;base64,JVBERi0="},
+                },
+            ],
+        }
+    ]
+
+    input_items, _ = OpenAICountTokensConfig.messages_to_responses_input(messages)
+
+    assert input_items == [{"role": "assistant", "content": "Here it is"}]
+
+
 def test_validate_request_valid():
     """Test that valid requests pass validation."""
     config = OpenAICountTokensConfig()
diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py
index b96d2eb5322..bb59e576568 100644
--- a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py
+++ b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py
@@ -124,6 +124,25 @@ class TestLiteLLMCompletionResponsesConfig:
         assert "extra_field" not in result["file"]
         assert "another_field" not in result["file"]
 
+    def test_transform_input_file_item_to_file_item_keeps_filename(self):
+        """OpenAI rejects file_data with no filename beside it, so dropping it 400s the request"""
+        result = (
+            LiteLLMCompletionResponsesConfig._transform_input_file_item_to_file_item(
+                {
+                    "type": "input_file",
+                    "filename": "report.pdf",
+                    "file_data": "data:application/pdf;base64,JVBERi0=",
+                }
+            )
+        )
+        assert result == {
+            "type": "file",
+            "file": {
+                "file_data": "data:application/pdf;base64,JVBERi0=",
+                "filename": "report.pdf",
+            },
+        }
+
     def test_transform_input_file_item_to_file_item_with_file_url(self):
         """file_url should be mapped to file_id for downstream URL handling"""
         result = (

From a90fb538bfd0a71e39d82b344c5c666200d01a63 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Mon, 31 Aug 2026 13:25:51 -0700
Subject: [PATCH 30/37] fix(friendli): declare GLM-5.3-Flash reasoning efforts
 as explicit levels

---
 model_prices_and_context_window.json                       | 7 +++++--
 .../test_friendli_glm_5_3_flash_model_metadata.py          | 3 +--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json
index fd6b9e11adf..f0e95d77018 100644
--- a/model_prices_and_context_window.json
+++ b/model_prices_and_context_window.json
@@ -19575,8 +19575,11 @@
         "output_cost_per_token": 5e-07,
         "cache_read_input_token_cost": 3e-08,
         "supports_prompt_caching": true,
-        "supports_max_reasoning_effort": true,
-        "supports_low_reasoning_effort": true,
+        "reasoning_effort_levels": [
+            "low",
+            "high",
+            "max"
+        ],
         "supports_parallel_function_calling": true,
         "supports_response_schema": true,
         "supports_native_structured_output": true,
diff --git a/tests/test_litellm/test_friendli_glm_5_3_flash_model_metadata.py b/tests/test_litellm/test_friendli_glm_5_3_flash_model_metadata.py
index 0acd750e49a..7e94205fb09 100644
--- a/tests/test_litellm/test_friendli_glm_5_3_flash_model_metadata.py
+++ b/tests/test_litellm/test_friendli_glm_5_3_flash_model_metadata.py
@@ -23,8 +23,7 @@ def test_friendli_glm_5_3_flash_model_info():
     assert info["max_output_tokens"] == 1048576
     assert info["supports_function_calling"] is True
     assert info["supports_reasoning"] is True
-    assert info["supports_low_reasoning_effort"] is True
-    assert info["supports_max_reasoning_effort"] is True
+    assert info["reasoning_effort_levels"] == ["low", "high", "max"]
     assert info["supports_tool_choice"] is True
     assert info["supports_prompt_caching"] is True
     assert info["supports_vision"] is True

From ebdb54d4f0c51c7c8df038dd5b189f4e84577203 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Mon, 31 Aug 2026 13:32:19 -0700
Subject: [PATCH 31/37] fix(docker): keep image venvs on the apk python and
 bump the wolfi digest

Since the requires-python cap moved to <3.15, uv resolved the project
python to 3.14, downloaded a managed interpreter under
/root/.local/share/uv that the runtime stage never receives, and every
layer-cache-miss image build broke: first at uvloop's cp314 sdist
configure step, then, with file/make added, at the runtime stage where
the copied venv's python symlink dangles and prisma imports fall through
to the system python. UV_PYTHON_DOWNLOADS=0 (already the convention in
migrations/backend/gateway) roots the venv on the apk python3.

The wolfi-base digest bump is required alongside it: the pinned 08-22
base ships glibc-2.43 while the current apk python-3.13 needs
GLIBC_2.44, and wolfi version-names glibc packages so apk upgrade
cannot cross that boundary.

With the venv on system 3.13 every dependency installs from wheels
again, so the file and make packages added for the sdist build are
reverted.
---
 Dockerfile                 | 11 +++++++----
 backend/Dockerfile         |  4 ++--
 docker/Dockerfile.database | 11 +++++++----
 docker/Dockerfile.non_root | 11 +++++++----
 gateway/Dockerfile         |  4 ++--
 migrations/Dockerfile      |  4 ++--
 6 files changed, 27 insertions(+), 18 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 4688a733331..675a79b0686 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,10 +1,10 @@
 # syntax=docker/dockerfile:1.7
 
 # Base image for building
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:57108e597a8cf3bd376b810f1c3539c21942daefa242cb9dddaae30f8aac735d
 
 # Runtime image
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:57108e597a8cf3bd376b810f1c3539c21942daefa242cb9dddaae30f8aac735d
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 # Pinned by digest like the other base images; bump explicitly on Node upgrades.
 ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
@@ -39,9 +39,7 @@ COPY --from=uvbin /uvx /usr/local/bin/uvx
 
 RUN apk add --no-cache \
     bash \
-    file \
     gcc \
-    make \
     python3 \
     python3-dev \
     rust \
@@ -51,8 +49,13 @@ RUN apk add --no-cache \
     npm \
     libsndfile
 
+# UV_PYTHON_DOWNLOADS=0 keeps the venv on the apk python3 above. Without it,
+# uv resolves requires-python to the newest allowed minor, downloads a managed
+# interpreter under /root/.local/share/uv that the runtime stage never
+# receives, and the copied venv's python symlink dangles at runtime.
 ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
     UV_LINK_MODE=copy \
+    UV_PYTHON_DOWNLOADS=0 \
     PATH="/app/.venv/bin:${PATH}"
 
 # Copy dependency metadata first for layer caching
diff --git a/backend/Dockerfile b/backend/Dockerfile
index 4ca40944606..8aea8312df9 100644
--- a/backend/Dockerfile
+++ b/backend/Dockerfile
@@ -1,5 +1,5 @@
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:57108e597a8cf3bd376b810f1c3539c21942daefa242cb9dddaae30f8aac735d
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:57108e597a8cf3bd376b810f1c3539c21942daefa242cb9dddaae30f8aac735d
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 
 FROM $UV_IMAGE AS uvbin
diff --git a/docker/Dockerfile.database b/docker/Dockerfile.database
index 9a243c68dca..4243eea5796 100644
--- a/docker/Dockerfile.database
+++ b/docker/Dockerfile.database
@@ -1,10 +1,10 @@
 # syntax=docker/dockerfile:1.7
 
 # Base image for building
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:57108e597a8cf3bd376b810f1c3539c21942daefa242cb9dddaae30f8aac735d
 
 # Runtime image
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:57108e597a8cf3bd376b810f1c3539c21942daefa242cb9dddaae30f8aac735d
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 # Pinned by digest like the other base images; bump explicitly on Node upgrades.
 ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
@@ -38,9 +38,7 @@ COPY --from=uvbin /uvx /usr/local/bin/uvx
 
 RUN apk add --no-cache \
     bash \
-    file \
     gcc \
-    make \
     python3 \
     python3-dev \
     openssl \
@@ -49,8 +47,13 @@ RUN apk add --no-cache \
     npm \
     libsndfile
 
+# UV_PYTHON_DOWNLOADS=0 keeps the venv on the apk python3 above. Without it,
+# uv resolves requires-python to the newest allowed minor, downloads a managed
+# interpreter under /root/.local/share/uv that the runtime stage never
+# receives, and the copied venv's python symlink dangles at runtime.
 ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
     UV_LINK_MODE=copy \
+    UV_PYTHON_DOWNLOADS=0 \
     PATH="/app/.venv/bin:${PATH}"
 
 # Copy dependency metadata first for layer caching
diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root
index 2cf70f5b01d..19ef97a4f46 100644
--- a/docker/Dockerfile.non_root
+++ b/docker/Dockerfile.non_root
@@ -1,8 +1,8 @@
 # syntax=docker/dockerfile:1.7
 
 # Base images
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:57108e597a8cf3bd376b810f1c3539c21942daefa242cb9dddaae30f8aac735d
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:57108e597a8cf3bd376b810f1c3539c21942daefa242cb9dddaae30f8aac735d
 ARG PROXY_EXTRAS_SOURCE=published
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 # Pinned by digest like the other base images; bump explicitly on Node upgrades.
@@ -39,9 +39,7 @@ RUN for i in 1 2 3; do \
       apk add --no-cache \
         python3 \
         python3-dev \
-        file \
         gcc \
-        make \
         rust \
         bash \
         coreutils \
@@ -52,8 +50,13 @@ RUN for i in 1 2 3; do \
         npm && break || sleep 5; \
     done
 
+# UV_PYTHON_DOWNLOADS=0 keeps the venv on the apk python3 above. Without it,
+# uv resolves requires-python to the newest allowed minor, downloads a managed
+# interpreter under /root/.local/share/uv that the runtime stage never
+# receives, and the copied venv's python symlink dangles at runtime.
 ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
     UV_LINK_MODE=copy \
+    UV_PYTHON_DOWNLOADS=0 \
     PATH="/app/.venv/bin:${PATH}" \
     LITELLM_NON_ROOT=true \
     XDG_CACHE_HOME=/app/.cache
diff --git a/gateway/Dockerfile b/gateway/Dockerfile
index 4a2e32e186e..235c535f9f1 100644
--- a/gateway/Dockerfile
+++ b/gateway/Dockerfile
@@ -1,5 +1,5 @@
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:57108e597a8cf3bd376b810f1c3539c21942daefa242cb9dddaae30f8aac735d
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:57108e597a8cf3bd376b810f1c3539c21942daefa242cb9dddaae30f8aac735d
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 
 FROM $UV_IMAGE AS uvbin
diff --git a/migrations/Dockerfile b/migrations/Dockerfile
index 6335e6f6bd8..524450024e0 100644
--- a/migrations/Dockerfile
+++ b/migrations/Dockerfile
@@ -1,5 +1,5 @@
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:57108e597a8cf3bd376b810f1c3539c21942daefa242cb9dddaae30f8aac735d
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:57108e597a8cf3bd376b810f1c3539c21942daefa242cb9dddaae30f8aac735d
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 
 FROM $UV_IMAGE AS uvbin

From b7da4717843e61cf6bd1aeb10b1cbaf70e946e1d Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Mon, 31 Aug 2026 13:53:22 -0700
Subject: [PATCH 32/37] fix(count_tokens): price an inline file block instead
 of raising on it

`ChatCompletionFileObject` is in the union `_count_content_list` accepts, but
`file` was missing from its match, so every local count of a Responses
`input_file` raised `Invalid content item type: file`. On
/v1/responses/input_tokens that surfaced as an opaque 500 whenever the model's
provider counting API refused the block and the local tokenizer took over.

Count it the way the module already counts the same thing in Anthropic's
dialect: the filename like a document title, the inline bytes through the
image pricer.
---
 litellm/litellm_core_utils/token_counter.py   | 28 ++++++++++++++-
 .../litellm_core_utils/test_token_counter.py  | 35 +++++++++++++++++++
 2 files changed, 62 insertions(+), 1 deletion(-)

diff --git a/litellm/litellm_core_utils/token_counter.py b/litellm/litellm_core_utils/token_counter.py
index 256bee7b348..c350bc5569e 100644
--- a/litellm/litellm_core_utils/token_counter.py
+++ b/litellm/litellm_core_utils/token_counter.py
@@ -693,6 +693,26 @@ def _count_document_tokens(
     )
 
 
+def _count_file_tokens(
+    file_value: object,
+    count_function: TokenCounterFunction,
+    use_default_image_token_count: bool,
+) -> int:
+    """An OpenAI `file` block is the chat-completions spelling of a document, so it prices like one."""
+    if not isinstance(file_value, Mapping):
+        return 0
+    filename: Final = file_value.get("filename")
+    file_data: Final = file_value.get("file_data")
+    name_tokens: Final = count_function(filename) if isinstance(filename, str) and filename else 0
+    if not isinstance(file_data, str) or not file_data:
+        return name_tokens
+    return name_tokens + calculate_img_tokens(
+        data=file_data,
+        mode="auto",
+        use_default_image_token_count=use_default_image_token_count,
+    )
+
+
 def _count_anthropic_content(
     content: Mapping[str, Any],
     count_function: TokenCounterFunction,
@@ -778,6 +798,12 @@ def _count_content_list(
                     use_default_image_token_count,
                     default_token_count,
                 )
+            elif c["type"] == "file":
+                num_tokens += _count_file_tokens(
+                    c.get("file"),
+                    count_function,
+                    use_default_image_token_count,
+                )
             elif c["type"] in ("tool_use", "tool_result"):
                 num_tokens += _count_anthropic_content(
                     c,
@@ -807,7 +833,7 @@ def _count_content_list(
                 raise ValueError(
                     f"Invalid content item type: {content_type}. "
                     f"Expected str or dict with 'type' field "
-                    f"(text, image_url, image, document, tool_use, tool_result, thinking, tool_reference)."
+                    f"(text, image_url, image, document, file, tool_use, tool_result, thinking, tool_reference)."
                 )
         return num_tokens
     except Exception as e:
diff --git a/tests/test_litellm/litellm_core_utils/test_token_counter.py b/tests/test_litellm/litellm_core_utils/test_token_counter.py
index 572b505e94c..4694fa8fbed 100644
--- a/tests/test_litellm/litellm_core_utils/test_token_counter.py
+++ b/tests/test_litellm/litellm_core_utils/test_token_counter.py
@@ -1377,3 +1377,38 @@ def test_anthropic_document_title_and_context_add_their_tokens():
             {"type": "document", "source": source},
         ]
     )
+
+
+def test_openai_file_block_prices_like_the_equivalent_anthropic_document():
+    """An inline `file` is a `document` in the chat-completions dialect, so it must price identically, not raise.
+
+    Before the fix `file` was missing from the content-block match even though `ChatCompletionFileObject`
+    is in the union this counter accepts, so every local count of a Responses `input_file` raised
+    `Invalid content item type: file` and surfaced as a 500 on /v1/responses/input_tokens.
+    """
+    prompt = {"type": "text", "text": "Summarize this file."}
+    inline_file = {
+        "type": "file",
+        "file": {"filename": "report.pdf", "file_data": "data:application/pdf;base64,JVBERi0xLjQK"},
+    }
+    document = {
+        "type": "document",
+        "title": "report.pdf",
+        "source": {"type": "base64", "media_type": "application/pdf", "data": "JVBERi0xLjQK"},
+    }
+
+    assert _count_user_content([prompt, inline_file]) == _count_user_content([prompt, document])
+    assert _count_user_content([prompt, inline_file]) > _count_user_content([prompt])
+
+
+def test_openai_file_block_without_inline_bytes_counts_what_it_carries():
+    """A `file` block naming an uploaded file has no bytes to price, so it adds only the filename's tokens."""
+    prompt = {"type": "text", "text": "Summarize this file."}
+
+    by_id = {"type": "file", "file": {"file_id": "file-abc123"}}
+    assert _count_user_content([prompt, by_id]) == _count_user_content([prompt])
+
+    named = {"type": "file", "file": {"file_id": "file-abc123", "filename": "report.pdf"}}
+    assert _count_user_content([prompt, named]) == _count_user_content(
+        [prompt, {"type": "text", "text": "report.pdf"}]
+    )

From 2bbf5135a5782a07e62d51d6df1fc2a774ec501e Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Mon, 31 Aug 2026 13:55:39 -0700
Subject: [PATCH 33/37] fix(friendli): ship GLM-5.3-Flash in the bundled backup
 cost map

---
 ...odel_prices_and_context_window_backup.json | 28 +++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json
index 05c1cfd3179..f0e95d77018 100644
--- a/litellm/model_prices_and_context_window_backup.json
+++ b/litellm/model_prices_and_context_window_backup.json
@@ -19564,6 +19564,34 @@
         "supports_system_messages": true,
         "supports_tool_choice": true
     },
+    "friendliai/zai-org/GLM-5.3-Flash": {
+        "litellm_provider": "friendliai",
+        "supports_reasoning": true,
+        "supports_function_calling": true,
+        "max_input_tokens": 1048576,
+        "max_tokens": 1048576,
+        "max_output_tokens": 1048576,
+        "input_cost_per_token": 1.5e-07,
+        "output_cost_per_token": 5e-07,
+        "cache_read_input_token_cost": 3e-08,
+        "supports_prompt_caching": true,
+        "reasoning_effort_levels": [
+            "low",
+            "high",
+            "max"
+        ],
+        "supports_parallel_function_calling": true,
+        "supports_response_schema": true,
+        "supports_native_structured_output": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "mode": "chat",
+        "comment": "Native multimodal GLM model for efficient coding and long-horizon agent tasks",
+        "source": "https://api.friendli.ai/serverless/v1/models",
+        "supports_vision": true,
+        "supports_image_input": true,
+        "supports_video_input": true
+    },
     "ft:babbage-002": {
         "deprecation_date": "2026-10-23",
         "input_cost_per_token": 1.6e-06,

From db7eb641cbc0a81dcd640350501b357df53abf94 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Mon, 31 Aug 2026 13:58:53 -0700
Subject: [PATCH 34/37] ci(osv): ignore GHSA-h7x2-h6g9-p789 until mlflow ships
 a fix

The advisory was modified 2026-08-31 and flags mlflow 3.13.0 through
3.15.2 with no fixed release published, so every osv-scan run fails
with nothing to bump. Same treatment as the existing diskcache entry.
---
 osv-scanner.toml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/osv-scanner.toml b/osv-scanner.toml
index 7ab450945f5..5b0339bdcd0 100644
--- a/osv-scanner.toml
+++ b/osv-scanner.toml
@@ -2,3 +2,8 @@
 id = "GHSA-w8v5-vhqr-4h9v"
 ignoreUntil = 2026-09-09
 reason = "diskcache has no fixed release published; remove this entry once one exists"
+
+[[IgnoredVulns]]
+id = "GHSA-h7x2-h6g9-p789"
+ignoreUntil = 2026-09-14
+reason = "mlflow has no fixed release published; remove this entry once one exists"

From 40edeaaecb0c7b0d7c0fac06a74fa67d0b57400c Mon Sep 17 00:00:00 2001
From: "devin-ai-integration[bot]"
 <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Mon, 31 Aug 2026 13:59:59 -0700
Subject: [PATCH 35/37] fix(otel): emit cache token counts on OTel v2 LLM spans
 (#38716)

* fix(otel): emit cache token counts on OTel v2 LLM spans

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(otel): trim comment in LLMUsage adapter

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(otel): drop casts in LLMUsage cache token adapter

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(deps): bump restrictedpython to 8.3 for GHSA-ffg3-p8fm-mjx2

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/integrations/otel/mappers/genai.py    |  2 ++
 litellm/integrations/otel/model/payloads.py   | 22 ++++++++++++++-----
 litellm/integrations/otel/model/semconv.py    |  2 ++
 .../otel/test_otel_v2_components.py           | 22 +++++++++++++++++++
 .../otel/test_otel_v2_sources_of_truth.py     | 22 +++++++++++++++++++
 5 files changed, 65 insertions(+), 5 deletions(-)

diff --git a/litellm/integrations/otel/mappers/genai.py b/litellm/integrations/otel/mappers/genai.py
index b09498f9292..3ac92b04c27 100644
--- a/litellm/integrations/otel/mappers/genai.py
+++ b/litellm/integrations/otel/mappers/genai.py
@@ -62,6 +62,8 @@ class GenAIMapper:
         GenAI.RESPONSE_TIME_TO_FIRST_CHUNK: lambda d: d.time_to_first_chunk_seconds,
         GenAI.USAGE_INPUT_TOKENS: lambda d: d.usage.input_tokens,
         GenAI.USAGE_OUTPUT_TOKENS: lambda d: d.usage.output_tokens,
+        GenAI.USAGE_CACHE_CREATION_INPUT_TOKENS: lambda d: d.usage.cache_creation_input_tokens,
+        GenAI.USAGE_CACHE_READ_INPUT_TOKENS: lambda d: d.usage.cache_read_input_tokens,
         Error.TYPE: lambda d: d.error.error_type if d.error else None,
         Server.ADDRESS: lambda d: d.server.address if d.server else None,
         Server.PORT: lambda d: d.server.port if d.server else None,
diff --git a/litellm/integrations/otel/model/payloads.py b/litellm/integrations/otel/model/payloads.py
index f70c777e1a7..d35405538f6 100644
--- a/litellm/integrations/otel/model/payloads.py
+++ b/litellm/integrations/otel/model/payloads.py
@@ -95,6 +95,22 @@ class LLMUsage:
     input_tokens: int | None = None
     output_tokens: int | None = None
     total_tokens: int | None = None
+    cache_creation_input_tokens: int | None = None
+    cache_read_input_tokens: int | None = None
+
+    @classmethod
+    def from_standard_logging_payload(cls, payload: StandardLoggingPayload) -> LLMUsage:
+        # Cache token counts only exist on the raw provider usage object under metadata
+        metadata: Final[Mapping[str, object]] = payload.get("metadata") or {}
+        raw_usage: Final = metadata.get("usage_object")
+        usage_object: Final[Mapping[str, object]] = raw_usage if isinstance(raw_usage, Mapping) else {}
+        return cls(
+            input_tokens=as_int(payload.get("prompt_tokens")),
+            output_tokens=as_int(payload.get("completion_tokens")),
+            total_tokens=as_int(payload.get("total_tokens")),
+            cache_creation_input_tokens=as_int(usage_object.get("cache_creation_input_tokens")),
+            cache_read_input_tokens=as_int(usage_object.get("cache_read_input_tokens")),
+        )
 
 
 @dataclass(frozen=True)
@@ -363,11 +379,7 @@ class LLMCallSpanData:
             response_model=context.response_model,
             response_id=as_str(response.get("id")),
             request_params=LLMRequestParams.from_model_parameters(params),
-            usage=LLMUsage(
-                input_tokens=as_int(payload.get("prompt_tokens")),
-                output_tokens=as_int(payload.get("completion_tokens")),
-                total_tokens=as_int(payload.get("total_tokens")),
-            ),
+            usage=LLMUsage.from_standard_logging_payload(payload),
             finish_reasons=finish_reasons,
             error=_parse_error(payload),
             response_cost=as_float(payload.get("response_cost")),
diff --git a/litellm/integrations/otel/model/semconv.py b/litellm/integrations/otel/model/semconv.py
index 4ad0cb5d1b4..f7a6280f95b 100644
--- a/litellm/integrations/otel/model/semconv.py
+++ b/litellm/integrations/otel/model/semconv.py
@@ -110,6 +110,8 @@ class GenAI:
     # usage
     USAGE_INPUT_TOKENS: Final = "gen_ai.usage.input_tokens"
     USAGE_OUTPUT_TOKENS: Final = "gen_ai.usage.output_tokens"
+    USAGE_CACHE_CREATION_INPUT_TOKENS: Final = "gen_ai.usage.cache_creation.input_tokens"
+    USAGE_CACHE_READ_INPUT_TOKENS: Final = "gen_ai.usage.cache_read.input_tokens"
     # content (opt-in, gated by capture mode)
     INPUT_MESSAGES: Final = "gen_ai.input.messages"
     OUTPUT_MESSAGES: Final = "gen_ai.output.messages"
diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_components.py b/tests/test_litellm/integrations/otel/test_otel_v2_components.py
index 115e385eda4..4aa28b5abfd 100644
--- a/tests/test_litellm/integrations/otel/test_otel_v2_components.py
+++ b/tests/test_litellm/integrations/otel/test_otel_v2_components.py
@@ -3,6 +3,7 @@ baggage helpers, metrics, the typed coercion helpers, mapper branches, span-name
 builders, and the registry validator's failure paths. Needs the OTel SDK."""
 
 import json
+from dataclasses import replace
 
 import pytest
 
@@ -215,6 +216,27 @@ def test_genai_mapper_all_request_params():
     assert attrs["server.port"] == 443
 
 
+def test_genai_mapper_cache_token_attrs():
+    cached = replace(
+        _full_llm_call(),
+        usage=LLMUsage(
+            input_tokens=10,
+            output_tokens=5,
+            total_tokens=15,
+            cache_creation_input_tokens=7,
+            cache_read_input_tokens=3,
+        ),
+    )
+    attrs = GenAIMapper().map(cached)
+    assert attrs[GenAI.USAGE_CACHE_CREATION_INPUT_TOKENS] == 7
+    assert attrs[GenAI.USAGE_CACHE_READ_INPUT_TOKENS] == 3
+
+    # No cache usage keeps the span sparse: neither key present.
+    uncached = GenAIMapper().map(_full_llm_call())
+    assert GenAI.USAGE_CACHE_CREATION_INPUT_TOKENS not in uncached
+    assert GenAI.USAGE_CACHE_READ_INPUT_TOKENS not in uncached
+
+
 def test_genai_mapper_stamps_input_output_messages():
     data = LLMCallSpanData(
         operation=GenAIOperation.CHAT,
diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py b/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py
index baa72b5a7fe..ca628aa3405 100644
--- a/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py
+++ b/tests/test_litellm/integrations/otel/test_otel_v2_sources_of_truth.py
@@ -525,6 +525,28 @@ def test_llm_call_adapter_extracts_all_fields():
     assert data.identity.key_hash == "hsh"
 
 
+def test_llm_call_adapter_extracts_cache_tokens_from_usage_object():
+    payload = _sample_payload()
+    payload["metadata"] = {
+        **payload["metadata"],
+        "usage_object": {
+            "prompt_tokens": 10,
+            "completion_tokens": 5,
+            "cache_creation_input_tokens": 7,
+            "cache_read_input_tokens": 3,
+        },
+    }
+    data = LLMCallSpanData.from_standard_logging_payload(payload)
+    assert data.usage.cache_creation_input_tokens == 7
+    assert data.usage.cache_read_input_tokens == 3
+
+
+def test_llm_call_adapter_cache_tokens_none_without_usage_object():
+    data = LLMCallSpanData.from_standard_logging_payload(_sample_payload())
+    assert data.usage.cache_creation_input_tokens is None
+    assert data.usage.cache_read_input_tokens is None
+
+
 def test_llm_call_adapter_failure_path():
     payload = _sample_payload(
         status="failure",

From cc078edd1fb2a29e2c5c2826803ae399e6078143 Mon Sep 17 00:00:00 2001
From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Mon, 31 Aug 2026 21:56:57 +0000
Subject: [PATCH 36/37] test(mcp): isolate global MCP server registry in
 discoverable endpoints tests

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../mcp_server/test_discoverable_endpoints.py    | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py
index 3279c59acd4..598e9276423 100644
--- a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py
+++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py
@@ -35,6 +35,22 @@ def mock_mcp_client_ip():
         yield
 
 
+@pytest.fixture(autouse=True)
+def isolate_global_mcp_registry():
+    """Restore the module-global MCP server registry after each test.
+
+    Tests here register servers on ``global_mcp_server_manager`` directly; without a
+    restore, entries leak into other test modules sharing the same worker and break
+    assertions over the full registry contents.
+    """
+    from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager
+
+    snapshot = dict(global_mcp_server_manager.registry)
+    yield
+    global_mcp_server_manager.registry.clear()
+    global_mcp_server_manager.registry.update(snapshot)
+
+
 def _mock_callback_request(base_url: str = "http://localhost:3000/"):
     """Return a MagicMock Request for callback/authorize same-origin tests.
 

From 613d0ef3fa7153e5485698ae6fae47481f23b027 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Mon, 31 Aug 2026 15:04:07 -0700
Subject: [PATCH 37/37] test(gcs_pub_sub): expect router_metadata in the spend
 logs payload

The base added router_metadata to SpendLogsMetadata in #39001 without
updating this fixture, and its CI run never executed logging_testing,
so the job now fails on every branch merged with current staging.
---
 .../gcs_pub_sub_body/spend_logs_payload.json                    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/logging_callback_tests/gcs_pub_sub_body/spend_logs_payload.json b/tests/logging_callback_tests/gcs_pub_sub_body/spend_logs_payload.json
index 1838fb16e91..28912a27501 100644
--- a/tests/logging_callback_tests/gcs_pub_sub_body/spend_logs_payload.json
+++ b/tests/logging_callback_tests/gcs_pub_sub_body/spend_logs_payload.json
@@ -11,7 +11,7 @@
     "user": "",
     "team_id": "",
     "organization_id": "",
-    "metadata": "{\"applied_guardrails\": [], \"attempted_fallbacks\": null, \"original_model_group\": null, \"batch_models\": null, \"batch_successful_requests\": null, \"batch_failed_requests\": null, \"mcp_tool_call_metadata\": null, \"vector_store_request_metadata\": null, \"routing_decision\": null, \"internal_call_origin\": null, \"guardrail_information\": null, \"compression_savings\": null, \"litellm_gateway_injected_cache\": null, \"usage_object\": {\"completion_tokens\": 20, \"prompt_tokens\": 10, \"total_tokens\": 30, \"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"model_map_information\": {\"model_map_key\": \"gpt-4o\", \"model_map_value\": {\"key\": \"gpt-4o\", \"max_tokens\": 16384, \"max_input_tokens\": 128000, \"max_output_tokens\": 16384, \"input_cost_per_token\": 2.5e-06, \"cache_creation_input_token_cost\": null, \"cache_read_input_token_cost\": 1.25e-06, \"input_cost_per_character\": null, \"input_cost_per_token_above_128k_tokens\": null, \"input_cost_per_token_above_200k_tokens\": null, \"input_cost_per_query\": null, \"input_cost_per_second\": null, \"input_cost_per_audio_token\": null, \"input_cost_per_token_batches\": 1.25e-06, \"output_cost_per_token_batches\": 5e-06, \"output_cost_per_token\": 1e-05, \"output_cost_per_audio_token\": null, \"output_cost_per_character\": null, \"output_cost_per_token_above_128k_tokens\": null, \"output_cost_per_character_above_128k_tokens\": null, \"output_cost_per_token_above_200k_tokens\": null, \"output_cost_per_second\": null, \"output_cost_per_image\": null, \"output_vector_size\": null, \"litellm_provider\": \"openai\", \"mode\": \"chat\", \"supports_system_messages\": true, \"supports_response_schema\": true, \"supports_vision\": true, \"supports_function_calling\": true, \"supports_tool_choice\": true, \"supports_assistant_prefill\": false, \"supports_prompt_caching\": true, \"supports_audio_input\": false, \"supports_audio_output\": false, \"supports_pdf_input\": false, \"supports_embedding_image_input\": false, \"supports_native_streaming\": null, \"supports_web_search\": true, \"supports_reasoning\": false, \"search_context_cost_per_query\": {\"search_context_size_low\": 0.03, \"search_context_size_medium\": 0.035, \"search_context_size_high\": 0.05}, \"tpm\": null, \"rpm\": null, \"supported_openai_params\": [\"frequency_penalty\", \"logit_bias\", \"logprobs\", \"top_logprobs\", \"max_tokens\", \"max_completion_tokens\", \"modalities\", \"prediction\", \"n\", \"presence_penalty\", \"seed\", \"stop\", \"stream\", \"stream_options\", \"temperature\", \"top_p\", \"tools\", \"tool_choice\", \"function_call\", \"functions\", \"max_retries\", \"extra_headers\", \"parallel_tool_calls\", \"audio\", \"response_format\", \"user\"]}}, \"additional_usage_values\": {\"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"user_api_key\": null, \"user_api_key_alias\": null, \"user_api_key_team_id\": null, \"user_api_key_project_id\": null, \"user_api_key_project_alias\": null, \"user_api_key_org_id\": null, \"user_api_key_user_id\": null, \"user_api_key_team_alias\": null, \"spend_logs_metadata\": null, \"requester_ip_address\": null, \"status\": null, \"proxy_server_request\": null, \"error_information\": null, \"attempted_retries\": null, \"max_retries\": null}",
+    "metadata": "{\"applied_guardrails\": [], \"attempted_fallbacks\": null, \"original_model_group\": null, \"batch_models\": null, \"batch_successful_requests\": null, \"batch_failed_requests\": null, \"mcp_tool_call_metadata\": null, \"vector_store_request_metadata\": null, \"routing_decision\": null, \"internal_call_origin\": null, \"router_metadata\": null, \"guardrail_information\": null, \"compression_savings\": null, \"litellm_gateway_injected_cache\": null, \"usage_object\": {\"completion_tokens\": 20, \"prompt_tokens\": 10, \"total_tokens\": 30, \"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"model_map_information\": {\"model_map_key\": \"gpt-4o\", \"model_map_value\": {\"key\": \"gpt-4o\", \"max_tokens\": 16384, \"max_input_tokens\": 128000, \"max_output_tokens\": 16384, \"input_cost_per_token\": 2.5e-06, \"cache_creation_input_token_cost\": null, \"cache_read_input_token_cost\": 1.25e-06, \"input_cost_per_character\": null, \"input_cost_per_token_above_128k_tokens\": null, \"input_cost_per_token_above_200k_tokens\": null, \"input_cost_per_query\": null, \"input_cost_per_second\": null, \"input_cost_per_audio_token\": null, \"input_cost_per_token_batches\": 1.25e-06, \"output_cost_per_token_batches\": 5e-06, \"output_cost_per_token\": 1e-05, \"output_cost_per_audio_token\": null, \"output_cost_per_character\": null, \"output_cost_per_token_above_128k_tokens\": null, \"output_cost_per_character_above_128k_tokens\": null, \"output_cost_per_token_above_200k_tokens\": null, \"output_cost_per_second\": null, \"output_cost_per_image\": null, \"output_vector_size\": null, \"litellm_provider\": \"openai\", \"mode\": \"chat\", \"supports_system_messages\": true, \"supports_response_schema\": true, \"supports_vision\": true, \"supports_function_calling\": true, \"supports_tool_choice\": true, \"supports_assistant_prefill\": false, \"supports_prompt_caching\": true, \"supports_audio_input\": false, \"supports_audio_output\": false, \"supports_pdf_input\": false, \"supports_embedding_image_input\": false, \"supports_native_streaming\": null, \"supports_web_search\": true, \"supports_reasoning\": false, \"search_context_cost_per_query\": {\"search_context_size_low\": 0.03, \"search_context_size_medium\": 0.035, \"search_context_size_high\": 0.05}, \"tpm\": null, \"rpm\": null, \"supported_openai_params\": [\"frequency_penalty\", \"logit_bias\", \"logprobs\", \"top_logprobs\", \"max_tokens\", \"max_completion_tokens\", \"modalities\", \"prediction\", \"n\", \"presence_penalty\", \"seed\", \"stop\", \"stream\", \"stream_options\", \"temperature\", \"top_p\", \"tools\", \"tool_choice\", \"function_call\", \"functions\", \"max_retries\", \"extra_headers\", \"parallel_tool_calls\", \"audio\", \"response_format\", \"user\"]}}, \"additional_usage_values\": {\"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"user_api_key\": null, \"user_api_key_alias\": null, \"user_api_key_team_id\": null, \"user_api_key_project_id\": null, \"user_api_key_project_alias\": null, \"user_api_key_org_id\": null, \"user_api_key_user_id\": null, \"user_api_key_team_alias\": null, \"spend_logs_metadata\": null, \"requester_ip_address\": null, \"status\": null, \"proxy_server_request\": null, \"error_information\": null, \"attempted_retries\": null, \"max_retries\": null}",
     "cache_key": "Cache OFF",
     "spend": 0.00022500000000000002,
     "total_tokens": 30,