From b7a7754b0529721dc000cd1cadf68272da7e287d Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 28 Aug 2026 00:07:40 +0000 Subject: [PATCH 01/10] fix(bedrock): surface Nova Sonic user transcripts, speech events, and usage in realtime API Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../llms/bedrock/realtime/transformation.py | 154 ++++++++++++++-- ...odel_prices_and_context_window_backup.json | 20 +++ litellm/types/llms/openai.py | 39 ++++ model_prices_and_context_window.json | 20 +++ .../test_bedrock_realtime_transformation.py | 168 ++++++++++++++++++ 5 files changed, 383 insertions(+), 18 deletions(-) diff --git a/litellm/llms/bedrock/realtime/transformation.py b/litellm/llms/bedrock/realtime/transformation.py index 951bf636b2f..20996512295 100644 --- a/litellm/llms/bedrock/realtime/transformation.py +++ b/litellm/llms/bedrock/realtime/transformation.py @@ -7,7 +7,7 @@ Transforms between OpenAI Realtime API format and Bedrock Nova Sonic format. import base64 import json import uuid as uuid_lib -from typing import Any, Final +from typing import Any, Final, cast from pydantic import BaseModel @@ -20,16 +20,21 @@ from litellm.types.llms.openai import ( OpenAIRealtimeContentPartDone, OpenAIRealtimeDoneEvent, OpenAIRealtimeEvents, + OpenAIRealtimeInputAudioBufferSpeechEvent, + OpenAIRealtimeInputAudioTranscriptionCompleted, + OpenAIRealtimeInputAudioTranscriptionDelta, OpenAIRealtimeOutputItemDone, OpenAIRealtimeResponseAudioDone, OpenAIRealtimeResponseContentPartAdded, OpenAIRealtimeResponseDelta, OpenAIRealtimeResponseDoneObject, OpenAIRealtimeResponseTextDone, + OpenAIRealtimeResponseUsage, OpenAIRealtimeStreamResponseBaseObject, OpenAIRealtimeStreamResponseOutputItemAdded, OpenAIRealtimeStreamSession, OpenAIRealtimeStreamSessionEvents, + OpenAIRealtimeUsageTokenDetails, ) from litellm.types.realtime import ( ALL_DELTA_TYPES, @@ -43,6 +48,27 @@ class BedrockContentEnd(BaseModel): stopReason: str | None = None +class BedrockUsageTokenDetails(BaseModel): + speechTokens: int = 0 + textTokens: int = 0 + + +class BedrockUsageDetailsTotal(BaseModel): + input: BedrockUsageTokenDetails = BedrockUsageTokenDetails() + output: BedrockUsageTokenDetails = BedrockUsageTokenDetails() + + +class BedrockUsageDetails(BaseModel): + total: BedrockUsageDetailsTotal = BedrockUsageDetailsTotal() + + +class BedrockUsageEvent(BaseModel): + totalInputTokens: int = 0 + totalOutputTokens: int = 0 + totalTokens: int = 0 + details: BedrockUsageDetails = BedrockUsageDetails() + + TRIGGER_AUDIO_SAMPLE_RATE_HERTZ: Final = 16000 TRIGGER_AUDIO_BYTES_PER_SECOND: Final = TRIGGER_AUDIO_SAMPLE_RATE_HERTZ * 2 TRIGGER_LEADING_SILENCE: Final = bytes(TRIGGER_AUDIO_BYTES_PER_SECOND // 2) @@ -87,6 +113,12 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): # Text configuration self.text_media_type = "text/plain" + # Response-stream state (Bedrock events carry no role on textOutput, + # so the USER/ASSISTANT split from contentStart is tracked here) + self._user_transcript_active = False + self._user_transcript_generation_stage: str | None = None + self._latest_usage: OpenAIRealtimeResponseUsage | None = None + def validate_environment(self, headers: dict, model: str, api_key: str | None = None) -> dict: """Validate environment - no special validation needed for Bedrock.""" return headers @@ -691,6 +723,11 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): role: Final = content_start.get("role") if role != "ASSISTANT": + if role == "USER" and content_start.get("type") == "TEXT": + self._user_transcript_active = True + self._user_transcript_generation_stage = self._parse_generation_stage( + content_start.get("additionalModelFields") + ) return ( [], current_response_id, @@ -700,6 +737,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): ) verbose_logger.debug("Handling ASSISTANT contentStart") + is_new_response: Final = current_response_id is None # Initialize IDs if needed if not current_response_id: @@ -715,7 +753,8 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): returned_messages: Final[list[OpenAIRealtimeEvents]] = [] - # Send response.created + # Send response.created only once per response (a response can contain + # multiple content blocks, e.g. TEXT then AUDIO) response_created: Final = OpenAIRealtimeStreamResponseBaseObject( type="response.created", event_id=f"event_{uuid.uuid4()}", @@ -727,7 +766,8 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): "conversation_id": current_conversation_id, }, ) - returned_messages.append(response_created) + if is_new_response: + returned_messages.append(response_created) # Send response.output_item.added output_item_added: Final = OpenAIRealtimeStreamResponseOutputItemAdded( @@ -767,6 +807,70 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): current_delta_type, ) + @staticmethod + def _parse_generation_stage(additional_model_fields: object) -> str | None: + if not isinstance(additional_model_fields, str): + return None + try: + parsed: Final = json.loads(additional_model_fields) + except json.JSONDecodeError: + return None + stage: Final = parsed.get("generationStage") if isinstance(parsed, dict) else None + return stage if isinstance(stage, str) else None + + def transform_user_speech_event(self, is_speech_start: bool) -> tuple[OpenAIRealtimeEvents, ...]: + """Transform Bedrock userSpeechStart/userSpeechEnd to OpenAI speech boundary events.""" + verbose_logger.debug("Handling userSpeech%s", "Start" if is_speech_start else "End") + speech_event: Final[OpenAIRealtimeInputAudioBufferSpeechEvent] = { + "type": "input_audio_buffer.speech_started" if is_speech_start else "input_audio_buffer.speech_stopped", + "event_id": f"event_{uuid.uuid4()}", + "item_id": f"item_{uuid.uuid4()}", + } + return (speech_event,) + + def transform_usage_event(self, usage_event: BedrockUsageEvent) -> None: + """Record Bedrock usageEvent token totals for the next response.done.""" + verbose_logger.debug("Handling usageEvent") + input_details: Final[OpenAIRealtimeUsageTokenDetails] = { + "audio_tokens": usage_event.details.total.input.speechTokens, + "text_tokens": usage_event.details.total.input.textTokens, + "cached_tokens": 0, + } + output_details: Final[OpenAIRealtimeUsageTokenDetails] = { + "audio_tokens": usage_event.details.total.output.speechTokens, + "text_tokens": usage_event.details.total.output.textTokens, + } + latest_usage: Final[OpenAIRealtimeResponseUsage] = { + "input_tokens": usage_event.totalInputTokens, + "output_tokens": usage_event.totalOutputTokens, + "total_tokens": usage_event.totalTokens, + "input_token_details": input_details, + "output_token_details": output_details, + } + self._latest_usage = latest_usage + + def transform_user_transcript_event(self, transcript: str) -> tuple[OpenAIRealtimeEvents, ...]: + """Transform a USER-role Bedrock textOutput (ASR transcript) to OpenAI transcription events.""" + verbose_logger.debug("Handling USER textOutput (ASR transcript)") + item_id: Final = f"item_{uuid.uuid4()}" + delta_event: Final[OpenAIRealtimeInputAudioTranscriptionDelta] = { + "type": "conversation.item.input_audio_transcription.delta", + "event_id": f"event_{uuid.uuid4()}", + "item_id": item_id, + "content_index": 0, + "delta": transcript, + } + if self._user_transcript_generation_stage == "SPECULATIVE": + return (delta_event,) + completed_event: Final[OpenAIRealtimeInputAudioTranscriptionCompleted] = { + "type": "conversation.item.input_audio_transcription.completed", + "event_id": f"event_{uuid.uuid4()}", + "item_id": item_id, + "content_index": 0, + "transcript": transcript, + } + return (delta_event, completed_event) + def transform_text_output_event( self, event: dict, @@ -985,7 +1089,14 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): if not current_response_id or not current_conversation_id: return [], None, None, None - usage_obj: Final = get_empty_usage() + empty_usage: Final = get_empty_usage() + zero_usage: Final[OpenAIRealtimeResponseUsage] = { + "input_tokens": empty_usage.prompt_tokens, + "output_tokens": empty_usage.completion_tokens, + "total_tokens": empty_usage.total_tokens, + } + usage: Final = self._latest_usage or zero_usage + self._latest_usage = None response_done: Final = OpenAIRealtimeDoneEvent( type="response.done", event_id=f"event_{uuid.uuid4()}", @@ -995,11 +1106,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): status="completed", output=[], conversation_id=current_conversation_id, - usage={ - "prompt_tokens": usage_obj.prompt_tokens, - "completion_tokens": usage_obj.completion_tokens, - "total_tokens": usage_obj.total_tokens, - }, + usage=dict(usage), ), ) @@ -1042,8 +1149,6 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): # Create a function call arguments done event # This is a custom event format that matches what clients expect - from typing import cast - function_call_event: Final[dict[str, Any]] = { "type": "response.function_call_arguments.done", "event_id": f"event_{uuid.uuid4()}", @@ -1194,18 +1299,25 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): returned_messages.extend(events) elif "textOutput" in event: - events, current_delta_chunks = self.transform_text_output_event( - event, - current_output_item_id, - current_response_id, - current_delta_chunks, - ) - returned_messages.extend(events) + if self._user_transcript_active: + returned_messages.extend(self.transform_user_transcript_event(event["textOutput"].get("content", ""))) + else: + events, current_delta_chunks = self.transform_text_output_event( + event, + current_output_item_id, + current_response_id, + current_delta_chunks, + ) + returned_messages.extend(events) elif "audioOutput" in event: events = self.transform_audio_output_event(event, current_output_item_id, current_response_id) returned_messages.extend(events) + elif "contentEnd" in event and self._user_transcript_active: + self._user_transcript_active = False + self._user_transcript_generation_stage = None + elif "contentEnd" in event: events, current_delta_chunks = self.transform_content_end_event( event, @@ -1224,6 +1336,12 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): ) = self._response_done_events(current_response_id, current_conversation_id) returned_messages.extend(done_events) + elif "userSpeechStart" in event or "userSpeechEnd" in event: + returned_messages.extend(self.transform_user_speech_event("userSpeechStart" in event)) + + elif "usageEvent" in event: + self.transform_usage_event(BedrockUsageEvent.model_validate(event["usageEvent"])) + elif "toolUse" in event: events, tool_call_id, tool_name = self.transform_tool_use_event( event, current_output_item_id, current_response_id diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 77572f69b8b..8fb6108bdda 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -553,6 +553,26 @@ "supports_response_schema": true, "supports_vision": true }, + "amazon.nova-sonic-v1:0": { + "input_cost_per_audio_token": 3.4e-06, + "input_cost_per_token": 6e-08, + "litellm_provider": "bedrock", + "mode": "realtime", + "output_cost_per_audio_token": 1.36e-05, + "output_cost_per_token": 2.4e-07, + "supports_audio_input": true, + "supports_audio_output": true + }, + "amazon.nova-2-sonic-v1:0": { + "input_cost_per_audio_token": 3e-06, + "input_cost_per_token": 3.3e-07, + "litellm_provider": "bedrock", + "mode": "realtime", + "output_cost_per_audio_token": 1.2e-05, + "output_cost_per_token": 2.75e-06, + "supports_audio_input": true, + "supports_audio_output": true + }, "amazon.rerank-v1:0": { "input_cost_per_query": 0.001, "input_cost_per_token": 0.0, diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index a6115640d78..fcade835cce 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -2162,6 +2162,42 @@ class OpenAIRealtimeDoneEvent(TypedDict): type: Literal["response.done"] +class OpenAIRealtimeInputAudioBufferSpeechEvent(TypedDict): + type: ReadOnly[Literal["input_audio_buffer.speech_started", "input_audio_buffer.speech_stopped"]] + event_id: ReadOnly[str] + item_id: ReadOnly[str] + + +class OpenAIRealtimeInputAudioTranscriptionDelta(TypedDict): + type: ReadOnly[Literal["conversation.item.input_audio_transcription.delta"]] + event_id: ReadOnly[str] + item_id: ReadOnly[str] + content_index: ReadOnly[int] + delta: ReadOnly[str] + + +class OpenAIRealtimeInputAudioTranscriptionCompleted(TypedDict): + type: ReadOnly[Literal["conversation.item.input_audio_transcription.completed"]] + event_id: ReadOnly[str] + item_id: ReadOnly[str] + content_index: ReadOnly[int] + transcript: ReadOnly[str] + + +class OpenAIRealtimeUsageTokenDetails(TypedDict): + audio_tokens: ReadOnly[int] + text_tokens: ReadOnly[int] + cached_tokens: NotRequired[ReadOnly[int]] + + +class OpenAIRealtimeResponseUsage(TypedDict): + input_tokens: ReadOnly[int] + output_tokens: ReadOnly[int] + total_tokens: ReadOnly[int] + input_token_details: NotRequired[ReadOnly[OpenAIRealtimeUsageTokenDetails]] + output_token_details: NotRequired[ReadOnly[OpenAIRealtimeUsageTokenDetails]] + + class OpenAIRealtimeEventTypes(Enum): SESSION_CREATED = "session.created" # Beta delta event names @@ -2199,6 +2235,9 @@ OpenAIRealtimeEvents = ( | OpenAIRealtimeOutputItemDone | OpenAIRealtimeFunctionCallArgumentsDone | OpenAIRealtimeDoneEvent + | OpenAIRealtimeInputAudioBufferSpeechEvent + | OpenAIRealtimeInputAudioTranscriptionDelta + | OpenAIRealtimeInputAudioTranscriptionCompleted ) OpenAIRealtimeStreamList = list[OpenAIRealtimeEvents] diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 77572f69b8b..8fb6108bdda 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -553,6 +553,26 @@ "supports_response_schema": true, "supports_vision": true }, + "amazon.nova-sonic-v1:0": { + "input_cost_per_audio_token": 3.4e-06, + "input_cost_per_token": 6e-08, + "litellm_provider": "bedrock", + "mode": "realtime", + "output_cost_per_audio_token": 1.36e-05, + "output_cost_per_token": 2.4e-07, + "supports_audio_input": true, + "supports_audio_output": true + }, + "amazon.nova-2-sonic-v1:0": { + "input_cost_per_audio_token": 3e-06, + "input_cost_per_token": 3.3e-07, + "litellm_provider": "bedrock", + "mode": "realtime", + "output_cost_per_audio_token": 1.2e-05, + "output_cost_per_token": 2.75e-06, + "supports_audio_input": true, + "supports_audio_output": true + }, "amazon.rerank-v1:0": { "input_cost_per_query": 0.001, "input_cost_per_token": 0.0, diff --git a/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_transformation.py b/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_transformation.py index ae6b1febd6b..b910018e6c4 100644 --- a/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_transformation.py +++ b/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_transformation.py @@ -827,5 +827,173 @@ class TestBedrockRealtimeSessionEvents: assert event["session"]["modalities"] == ["text", "audio"] +class TestBedrockRealtimeUserEventsAndUsage: + """Regression tests for #38346: USER ASR transcripts, speech boundary events, + usage propagation, and duplicate response.created""" + + @staticmethod + def _run(config, messages): + logging_obj = MagicMock() + logging_obj.litellm_trace_id = "trace_123" + state = { + "session_configuration_request": json.dumps({"configured": True}), + "current_output_item_id": None, + "current_response_id": None, + "current_conversation_id": None, + "current_delta_chunks": [], + "current_item_chunks": [], + "current_delta_type": None, + } + all_events = [] + for msg in messages: + result = config.transform_realtime_response( + json.dumps(msg), + "amazon.nova-2-sonic-v1:0", + logging_obj, + realtime_response_transform_input=dict(state), + ) + all_events.extend(result["response"]) + state.update( + { + "current_output_item_id": result["current_output_item_id"], + "current_response_id": result["current_response_id"], + "current_conversation_id": result["current_conversation_id"], + "current_delta_chunks": result["current_delta_chunks"], + "current_delta_type": result["current_delta_type"], + } + ) + return all_events + + def test_user_speech_start_and_stop_events(self): + events = self._run( + BedrockRealtimeConfig(), + [{"event": {"userSpeechStart": {}}}, {"event": {"userSpeechEnd": {}}}], + ) + assert [e["type"] for e in events] == [ + "input_audio_buffer.speech_started", + "input_audio_buffer.speech_stopped", + ] + assert all(e["event_id"] and e["item_id"] for e in events) + + def test_user_transcript_emits_input_audio_transcription_events(self): + events = self._run( + BedrockRealtimeConfig(), + [ + { + "event": { + "contentStart": { + "role": "USER", + "type": "TEXT", + "additionalModelFields": json.dumps({"generationStage": "FINAL"}), + } + } + }, + {"event": {"textOutput": {"content": "ready"}}}, + {"event": {"contentEnd": {"stopReason": "PARTIAL_TURN"}}}, + ], + ) + deltas = [e for e in events if e["type"] == "conversation.item.input_audio_transcription.delta"] + completed = [e for e in events if e["type"] == "conversation.item.input_audio_transcription.completed"] + assert len(deltas) == 1 and deltas[0]["delta"] == "ready" + assert len(completed) == 1 and completed[0]["transcript"] == "ready" + assert deltas[0]["item_id"] == completed[0]["item_id"] + assert not any(e["type"] == "response.text.delta" for e in events) + + def test_speculative_user_transcript_emits_delta_only(self): + events = self._run( + BedrockRealtimeConfig(), + [ + { + "event": { + "contentStart": { + "role": "USER", + "type": "TEXT", + "additionalModelFields": json.dumps({"generationStage": "SPECULATIVE"}), + } + } + }, + {"event": {"textOutput": {"content": "rea"}}}, + ], + ) + assert [e["type"] for e in events] == ["conversation.item.input_audio_transcription.delta"] + + def test_user_transcript_state_resets_on_content_end(self): + events = self._run( + BedrockRealtimeConfig(), + [ + {"event": {"contentStart": {"role": "USER", "type": "TEXT"}}}, + {"event": {"contentEnd": {"stopReason": "PARTIAL_TURN"}}}, + {"event": {"contentStart": {"role": "ASSISTANT", "type": "TEXT"}}}, + {"event": {"textOutput": {"content": "Hi there"}}}, + ], + ) + text_deltas = [e for e in events if e["type"] == "response.text.delta"] + assert len(text_deltas) == 1 and text_deltas[0]["delta"] == "Hi there" + assert not any(e["type"].startswith("conversation.item.input_audio_transcription") for e in events) + + def test_response_created_emitted_once_per_response(self): + events = self._run( + BedrockRealtimeConfig(), + [ + {"event": {"contentStart": {"role": "ASSISTANT", "type": "TEXT"}}}, + {"event": {"textOutput": {"content": "Hi"}}}, + {"event": {"contentEnd": {"stopReason": "PARTIAL_TURN"}}}, + {"event": {"contentStart": {"role": "ASSISTANT", "type": "AUDIO"}}}, + ], + ) + assert sum(1 for e in events if e["type"] == "response.created") == 1 + + def test_usage_event_propagates_to_response_done(self): + events = self._run( + BedrockRealtimeConfig(), + [ + { + "event": { + "usageEvent": { + "totalInputTokens": 25, + "totalOutputTokens": 40, + "totalTokens": 65, + "details": { + "total": { + "input": {"speechTokens": 20, "textTokens": 5}, + "output": {"speechTokens": 30, "textTokens": 10}, + } + }, + } + } + }, + {"event": {"contentStart": {"role": "ASSISTANT", "type": "TEXT"}}}, + {"event": {"textOutput": {"content": "Hi"}}}, + {"event": {"contentEnd": {"stopReason": "END_TURN"}}}, + ], + ) + done_events = [e for e in events if e["type"] == "response.done"] + assert len(done_events) == 1 + usage = done_events[0]["response"]["usage"] + assert usage["input_tokens"] == 25 + assert usage["output_tokens"] == 40 + assert usage["total_tokens"] == 65 + assert usage["input_token_details"]["audio_tokens"] == 20 + assert usage["input_token_details"]["text_tokens"] == 5 + assert usage["output_token_details"]["audio_tokens"] == 30 + assert usage["output_token_details"]["text_tokens"] == 10 + + def test_response_done_without_usage_event_reports_zero_usage(self): + events = self._run( + BedrockRealtimeConfig(), + [ + {"event": {"contentStart": {"role": "ASSISTANT", "type": "TEXT"}}}, + {"event": {"textOutput": {"content": "Hi"}}}, + {"event": {"contentEnd": {"stopReason": "END_TURN"}}}, + ], + ) + done_events = [e for e in events if e["type"] == "response.done"] + assert len(done_events) == 1 + usage = done_events[0]["response"]["usage"] + assert usage["input_tokens"] == 0 + assert usage["output_tokens"] == 0 + assert usage["total_tokens"] == 0 + + if __name__ == "__main__": pytest.main([__file__, "-v"]) From 5eee3bd9f952caa50edfd42927945da86848ed8d Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 28 Aug 2026 00:28:22 +0000 Subject: [PATCH 02/10] fix(bedrock): dispatch success handlers for realtime sessions so spend is logged Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/bedrock/realtime/handler.py | 12 +++++ .../realtime/test_bedrock_realtime_handler.py | 48 +++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/litellm/llms/bedrock/realtime/handler.py b/litellm/llms/bedrock/realtime/handler.py index 3eeb3cb9fc6..b1b598039f8 100644 --- a/litellm/llms/bedrock/realtime/handler.py +++ b/litellm/llms/bedrock/realtime/handler.py @@ -13,6 +13,8 @@ from pydantic import TypeAdapter from litellm._logging import _redact_string, verbose_proxy_logger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging +from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER +from litellm.types.llms.openai import OpenAIRealtimeEvents from ..base_aws_llm import BaseAWSLLM from ..common_utils import BedrockError @@ -154,6 +156,7 @@ class BedrockRealtime(BaseAWSLLM): ) ) + logged_events: Final[list[OpenAIRealtimeEvents]] = [] # mutable-ok: events accumulate across stream loop iterations for spend logging bedrock_to_client_task: Final = asyncio.create_task( self._forward_bedrock_to_client( bedrock_stream, @@ -162,6 +165,7 @@ class BedrockRealtime(BaseAWSLLM): model, logging_obj, session_state, + logged_events, ) ) @@ -172,6 +176,11 @@ class BedrockRealtime(BaseAWSLLM): return_exceptions=True, ) + if logged_events: + GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue( + logging_obj.dispatch_success_handlers(logged_events, prefer_async_handlers=True) + ) + except Exception as e: verbose_proxy_logger.exception("Error in BedrockRealtime.async_realtime: %s", e) try: @@ -252,6 +261,7 @@ class BedrockRealtime(BaseAWSLLM): model: str, logging_obj: LiteLLMLogging, session_state: dict, + logged_events: "list[OpenAIRealtimeEvents] | None" = None, ): """Forward messages from Bedrock stream to client WebSocket.""" try: @@ -304,6 +314,8 @@ class BedrockRealtime(BaseAWSLLM): # Send transformed messages to client openai_messages = transformed_response.get("response", []) for openai_message in openai_messages: + if logged_events is not None and isinstance(openai_message, dict): + logged_events.append(openai_message) message_json = json.dumps(openai_message) await client_ws.send_text(message_json) verbose_proxy_logger.debug("Bedrock Realtime: Sent to client: %s", message_json[:200]) diff --git a/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_handler.py b/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_handler.py index 9efcee192b1..76a564845dd 100644 --- a/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_handler.py +++ b/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_handler.py @@ -104,6 +104,26 @@ class RealtimeClientWS: self.closed = True +class ScriptedBedrockReceiver: + def __init__(self, payloads): + self._payloads = list(payloads) + + async def receive(self): + if not self._payloads: + return None + payload = self._payloads.pop(0) + return SimpleNamespace(value=SimpleNamespace(bytes_=payload.encode("utf-8"))) + + +class ScriptedBedrockStream: + def __init__(self, payloads): + self.input_stream = FakeInputStream() + self._receiver = ScriptedBedrockReceiver(payloads) + + async def await_output(self): + return (None, self._receiver) + + class ImmediatelyEndingBedrockStream: def __init__(self): self.input_stream = FakeInputStream() @@ -271,6 +291,34 @@ class TestBedrockRealtimeHandler: assert "sessionEnd" in event_names assert stream.input_stream.closed + @pytest.mark.asyncio + async def test_forwarded_events_are_collected_for_spend_logging(self): + handler = BedrockRealtime() + stream = ScriptedBedrockStream( + [ + json.dumps({"event": {"userSpeechStart": {}}}), + json.dumps({"event": {"userSpeechEnd": {}}}), + ] + ) + client_ws = RealtimeClientWS() + logged_events = [] + + await handler._forward_bedrock_to_client( + stream, + client_ws, + BedrockRealtimeConfig(), + "amazon.nova-sonic-v1:0", + FakeLogging(), + {}, + logged_events, + ) + + assert [event["type"] for event in logged_events] == [ + "input_audio_buffer.speech_started", + "input_audio_buffer.speech_stopped", + ] + assert client_ws.closed + @pytest.mark.asyncio async def test_bedrock_stream_end_closes_client_websocket(self): handler = BedrockRealtime() From eee47dcdaa9a3531f5f78e3bb2a70315626ea175 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 28 Aug 2026 00:39:10 +0000 Subject: [PATCH 03/10] fix(bedrock): share one item_id across a user utterance's realtime events Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/bedrock/realtime/handler.py | 4 ++- .../llms/bedrock/realtime/transformation.py | 11 +++++-- .../realtime/test_bedrock_realtime_handler.py | 8 ++--- .../test_bedrock_realtime_transformation.py | 31 +++++++++++++++++++ 4 files changed, 45 insertions(+), 9 deletions(-) diff --git a/litellm/llms/bedrock/realtime/handler.py b/litellm/llms/bedrock/realtime/handler.py index b1b598039f8..0891379ef8c 100644 --- a/litellm/llms/bedrock/realtime/handler.py +++ b/litellm/llms/bedrock/realtime/handler.py @@ -156,7 +156,9 @@ class BedrockRealtime(BaseAWSLLM): ) ) - logged_events: Final[list[OpenAIRealtimeEvents]] = [] # mutable-ok: events accumulate across stream loop iterations for spend logging + logged_events: Final[ + list[OpenAIRealtimeEvents] + ] = [] # mutable-ok: events accumulate across stream loop iterations for spend logging bedrock_to_client_task: Final = asyncio.create_task( self._forward_bedrock_to_client( bedrock_stream, diff --git a/litellm/llms/bedrock/realtime/transformation.py b/litellm/llms/bedrock/realtime/transformation.py index 20996512295..ec5eff0a6fc 100644 --- a/litellm/llms/bedrock/realtime/transformation.py +++ b/litellm/llms/bedrock/realtime/transformation.py @@ -117,6 +117,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): # so the USER/ASSISTANT split from contentStart is tracked here) self._user_transcript_active = False self._user_transcript_generation_stage: str | None = None + self._user_item_id: str | None = None self._latest_usage: OpenAIRealtimeResponseUsage | None = None def validate_environment(self, headers: dict, model: str, api_key: str | None = None) -> dict: @@ -818,13 +819,19 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): stage: Final = parsed.get("generationStage") if isinstance(parsed, dict) else None return stage if isinstance(stage, str) else None + def _current_user_item_id(self, new_utterance: bool = False) -> str: + """Item id shared by all events of one user utterance (speech boundaries and transcript).""" + if new_utterance or self._user_item_id is None: + self._user_item_id = f"item_{uuid.uuid4()}" + return self._user_item_id + def transform_user_speech_event(self, is_speech_start: bool) -> tuple[OpenAIRealtimeEvents, ...]: """Transform Bedrock userSpeechStart/userSpeechEnd to OpenAI speech boundary events.""" verbose_logger.debug("Handling userSpeech%s", "Start" if is_speech_start else "End") speech_event: Final[OpenAIRealtimeInputAudioBufferSpeechEvent] = { "type": "input_audio_buffer.speech_started" if is_speech_start else "input_audio_buffer.speech_stopped", "event_id": f"event_{uuid.uuid4()}", - "item_id": f"item_{uuid.uuid4()}", + "item_id": self._current_user_item_id(new_utterance=is_speech_start), } return (speech_event,) @@ -852,7 +859,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): def transform_user_transcript_event(self, transcript: str) -> tuple[OpenAIRealtimeEvents, ...]: """Transform a USER-role Bedrock textOutput (ASR transcript) to OpenAI transcription events.""" verbose_logger.debug("Handling USER textOutput (ASR transcript)") - item_id: Final = f"item_{uuid.uuid4()}" + item_id: Final = self._current_user_item_id() delta_event: Final[OpenAIRealtimeInputAudioTranscriptionDelta] = { "type": "conversation.item.input_audio_transcription.delta", "event_id": f"event_{uuid.uuid4()}", diff --git a/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_handler.py b/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_handler.py index 76a564845dd..aa6573884e2 100644 --- a/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_handler.py +++ b/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_handler.py @@ -368,9 +368,7 @@ class TestBedrockRealtimeSessionLifecycle: [json.dumps({"type": "session.update", "session": {"instructions": "hi", "modalities": ["text"]}})] ) - await handler._forward_client_to_bedrock( - client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, FakeLogging() - ) + await handler._forward_client_to_bedrock(client_ws, stream, config, "amazon.nova-sonic-v1:0", {}, FakeLogging()) acked = [json.loads(message) for message in client_ws.sent_to_client] updated = [event for event in acked if event["type"] == "session.updated"] @@ -382,9 +380,7 @@ class TestBedrockRealtimeSessionLifecycle: handler = BedrockRealtime() config = BedrockRealtimeConfig() stream = FakeBedrockStream() - client_ws = DisconnectingClientWS( - [json.dumps({"type": "session.update", "session": {"instructions": "hi"}})] - ) + client_ws = DisconnectingClientWS([json.dumps({"type": "session.update", "session": {"instructions": "hi"}})]) await handler._forward_client_to_bedrock(client_ws, stream, config, "amazon.nova-sonic-v1:0", {}) diff --git a/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_transformation.py b/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_transformation.py index b910018e6c4..dbe84cacb29 100644 --- a/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_transformation.py +++ b/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_transformation.py @@ -874,6 +874,37 @@ class TestBedrockRealtimeUserEventsAndUsage: "input_audio_buffer.speech_stopped", ] assert all(e["event_id"] and e["item_id"] for e in events) + assert events[0]["item_id"] == events[1]["item_id"] + + def test_utterance_lifecycle_shares_one_item_id(self): + events = self._run( + BedrockRealtimeConfig(), + [ + {"event": {"userSpeechStart": {}}}, + {"event": {"userSpeechEnd": {}}}, + { + "event": { + "contentStart": { + "role": "USER", + "type": "TEXT", + "additionalModelFields": json.dumps({"generationStage": "FINAL"}), + } + } + }, + {"event": {"textOutput": {"content": "ready"}}}, + {"event": {"contentEnd": {"stopReason": "PARTIAL_TURN"}}}, + ], + ) + item_ids = {e["item_id"] for e in events if "item_id" in e} + assert len(item_ids) == 1 + + def test_new_utterance_gets_new_item_id(self): + config = BedrockRealtimeConfig() + first = self._run(config, [{"event": {"userSpeechStart": {}}}, {"event": {"userSpeechEnd": {}}}]) + second = self._run(config, [{"event": {"userSpeechStart": {}}}, {"event": {"userSpeechEnd": {}}}]) + assert first[0]["item_id"] == first[1]["item_id"] + assert second[0]["item_id"] == second[1]["item_id"] + assert first[0]["item_id"] != second[0]["item_id"] def test_user_transcript_emits_input_audio_transcription_events(self): events = self._run( From 6b2ada2a780e1ebd4f4899833fbc7be9a3214123 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 31 Aug 2026 11:49:09 -0700 Subject: [PATCH 04/10] fix(bedrock): per-response realtime usage deltas, spend-log event filter, single transcript completed --- litellm/llms/bedrock/realtime/handler.py | 70 +++++++--- .../llms/bedrock/realtime/transformation.py | 81 ++++++++---- .../realtime/test_bedrock_realtime_handler.py | 125 ++++++++++++++---- .../test_bedrock_realtime_transformation.py | 106 +++++++++++++++ 4 files changed, 313 insertions(+), 69 deletions(-) diff --git a/litellm/llms/bedrock/realtime/handler.py b/litellm/llms/bedrock/realtime/handler.py index ecd143dd087..42fe8941443 100644 --- a/litellm/llms/bedrock/realtime/handler.py +++ b/litellm/llms/bedrock/realtime/handler.py @@ -7,14 +7,17 @@ This uses aws_sdk_bedrock_runtime for bidirectional streaming with Nova Sonic. import asyncio import contextlib import json +from collections.abc import AsyncIterator, Mapping from typing import Final, Protocol from pydantic import JsonValue, TypeAdapter +import litellm from litellm._logging import _redact_string, verbose_proxy_logger from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER +from litellm.litellm_core_utils.realtime_streaming import DefaultLoggedRealTimeEventTypes from litellm.types.llms.openai import OpenAIRealtimeEvents from litellm.types.realtime import RealtimeResponseTransformInput @@ -34,6 +37,17 @@ def _json_str(value: JsonValue) -> str | None: return value if isinstance(value, str) else None +def _should_log_event(openai_message: Mapping[str, object]) -> bool: + logged_types: Final = ( + litellm.logged_real_time_event_types + if litellm.logged_real_time_event_types is not None + else DefaultLoggedRealTimeEventTypes + ) + if logged_types == "*": + return True + return openai_message.get("type") in logged_types + + class RealtimeClientWebSocket(Protocol): """The client-facing websocket surface the realtime bridge talks to.""" @@ -207,18 +221,22 @@ class BedrockRealtime(BaseAWSLLM): ) ) - logged_events: Final[list[OpenAIRealtimeEvents]] = [] # mutable-ok: filled across the stream loop - bedrock_to_client_task: Final = asyncio.create_task( - self._forward_bedrock_to_client( - bedrock_stream, - websocket, - transformation_config, - model, - logging_obj, - session_state, - logged_events, + async def forward_bedrock_and_collect_logged_events() -> tuple[OpenAIRealtimeEvents, ...]: + return tuple( + [ + event + async for event in self._forward_bedrock_to_client( + bedrock_stream, + websocket, + transformation_config, + model, + logging_obj, + session_state, + ) + ] ) - ) + + bedrock_to_client_task: Final = asyncio.create_task(forward_bedrock_and_collect_logged_events()) # Wait for both tasks to complete await asyncio.gather( @@ -227,9 +245,25 @@ class BedrockRealtime(BaseAWSLLM): return_exceptions=True, ) + forwarded_logged_events: Final = ( + bedrock_to_client_task.result() + if not bedrock_to_client_task.cancelled() and bedrock_to_client_task.exception() is None + else () + ) + logged_events: Final = ( + *forwarded_logged_events, + *( + leftover_event + for leftover_event in transformation_config.leftover_usage_done_events() + if _should_log_event(leftover_event) + ), + ) if logged_events: GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue( - logging_obj.dispatch_success_handlers(logged_events, prefer_async_handlers=True) + logging_obj.dispatch_success_handlers( + list(logged_events), # mutable-ok: realtime spend logging requires a list result + prefer_async_handlers=True, + ) ) except Exception as e: @@ -313,9 +347,8 @@ class BedrockRealtime(BaseAWSLLM): model: str, logging_obj: LiteLLMLogging, session_state: RealtimeResponseTransformInput, - logged_events: "list[OpenAIRealtimeEvents] | None" = None, # mutable-ok: caller-owned spend log accumulator - ): - """Forward messages from Bedrock stream to client WebSocket.""" + ) -> AsyncIterator[OpenAIRealtimeEvents]: + """Forward messages from Bedrock to the client, yielding the ones to record for spend logging.""" try: while True: # Receive from Bedrock @@ -363,13 +396,14 @@ class BedrockRealtime(BaseAWSLLM): ) # Send transformed messages to client - openai_messages = transformed_response.get("response", []) + response_value = transformed_response["response"] + openai_messages = response_value if isinstance(response_value, list) else (response_value,) for openai_message in openai_messages: - if logged_events is not None and isinstance(openai_message, dict): - logged_events.append(openai_message) message_json = json.dumps(openai_message) await client_ws.send_text(message_json) verbose_proxy_logger.debug("Bedrock Realtime: Sent to client: %s", message_json[:200]) + if _should_log_event(openai_message): + yield openai_message except Exception as e: verbose_proxy_logger.debug("Bedrock to client forwarding ended: %s", e, exc_info=True) diff --git a/litellm/llms/bedrock/realtime/transformation.py b/litellm/llms/bedrock/realtime/transformation.py index ec5eff0a6fc..28c2e446d10 100644 --- a/litellm/llms/bedrock/realtime/transformation.py +++ b/litellm/llms/bedrock/realtime/transformation.py @@ -41,7 +41,6 @@ from litellm.types.realtime import ( RealtimeResponseTransformInput, RealtimeResponseTypedDict, ) -from litellm.utils import get_empty_usage class BedrockContentEnd(BaseModel): @@ -118,7 +117,9 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): self._user_transcript_active = False self._user_transcript_generation_stage: str | None = None self._user_item_id: str | None = None - self._latest_usage: OpenAIRealtimeResponseUsage | None = None + self._user_transcript_buffer = "" + self._cumulative_usage = BedrockUsageEvent() + self._reported_usage = BedrockUsageEvent() def validate_environment(self, headers: dict, model: str, api_key: str | None = None) -> dict: """Validate environment - no special validation needed for Bedrock.""" @@ -836,47 +837,79 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): return (speech_event,) def transform_usage_event(self, usage_event: BedrockUsageEvent) -> None: - """Record Bedrock usageEvent token totals for the next response.done.""" + """Record Bedrock's session-cumulative usage totals for the next response.done.""" verbose_logger.debug("Handling usageEvent") + self._cumulative_usage = usage_event + + def _take_usage_delta(self) -> OpenAIRealtimeResponseUsage: + """Usage for the response now completing: cumulative totals minus what prior response.done events reported.""" + prior: Final = self._reported_usage + latest: Final = self._cumulative_usage + self._reported_usage = latest input_details: Final[OpenAIRealtimeUsageTokenDetails] = { - "audio_tokens": usage_event.details.total.input.speechTokens, - "text_tokens": usage_event.details.total.input.textTokens, + "audio_tokens": latest.details.total.input.speechTokens - prior.details.total.input.speechTokens, + "text_tokens": latest.details.total.input.textTokens - prior.details.total.input.textTokens, "cached_tokens": 0, } output_details: Final[OpenAIRealtimeUsageTokenDetails] = { - "audio_tokens": usage_event.details.total.output.speechTokens, - "text_tokens": usage_event.details.total.output.textTokens, + "audio_tokens": latest.details.total.output.speechTokens - prior.details.total.output.speechTokens, + "text_tokens": latest.details.total.output.textTokens - prior.details.total.output.textTokens, } - latest_usage: Final[OpenAIRealtimeResponseUsage] = { - "input_tokens": usage_event.totalInputTokens, - "output_tokens": usage_event.totalOutputTokens, - "total_tokens": usage_event.totalTokens, + usage_delta: Final[OpenAIRealtimeResponseUsage] = { + "input_tokens": latest.totalInputTokens - prior.totalInputTokens, + "output_tokens": latest.totalOutputTokens - prior.totalOutputTokens, + "total_tokens": latest.totalTokens - prior.totalTokens, "input_token_details": input_details, "output_token_details": output_details, } - self._latest_usage = latest_usage + return usage_delta + + def leftover_usage_done_events(self) -> tuple[OpenAIRealtimeEvents, ...]: + """Logged-only response.done for usage Bedrock reports after the final turn's contentEnd.""" + if self._cumulative_usage == self._reported_usage: + return () + usage: Final = self._take_usage_delta() + leftover_done: Final = OpenAIRealtimeDoneEvent( + type="response.done", + event_id=f"event_{uuid.uuid4()}", + response=OpenAIRealtimeResponseDoneObject( + object="realtime.response", + id=f"resp_{uuid.uuid4()}", + status="completed", + conversation_id=f"conv_{uuid.uuid4()}", + usage=dict(usage), # mutable-ok: OpenAIRealtimeResponseDoneObject types usage as plain dict + ), + ) + return (leftover_done,) def transform_user_transcript_event(self, transcript: str) -> tuple[OpenAIRealtimeEvents, ...]: - """Transform a USER-role Bedrock textOutput (ASR transcript) to OpenAI transcription events.""" + """Transform a USER-role Bedrock textOutput (ASR transcript) to an OpenAI transcription delta.""" verbose_logger.debug("Handling USER textOutput (ASR transcript)") - item_id: Final = self._current_user_item_id() delta_event: Final[OpenAIRealtimeInputAudioTranscriptionDelta] = { "type": "conversation.item.input_audio_transcription.delta", "event_id": f"event_{uuid.uuid4()}", - "item_id": item_id, + "item_id": self._current_user_item_id(), "content_index": 0, "delta": transcript, } - if self._user_transcript_generation_stage == "SPECULATIVE": - return (delta_event,) + if self._user_transcript_generation_stage != "SPECULATIVE": + self._user_transcript_buffer += transcript + return (delta_event,) + + def user_transcript_completed_events(self) -> tuple[OpenAIRealtimeEvents, ...]: + """One completed event with the full transcript once the FINAL user content block ends.""" + transcript: Final = self._user_transcript_buffer + if not transcript: + return () + self._user_transcript_buffer = "" completed_event: Final[OpenAIRealtimeInputAudioTranscriptionCompleted] = { "type": "conversation.item.input_audio_transcription.completed", "event_id": f"event_{uuid.uuid4()}", - "item_id": item_id, + "item_id": self._current_user_item_id(), "content_index": 0, "transcript": transcript, } - return (delta_event, completed_event) + return (completed_event,) def transform_text_output_event( self, @@ -1096,14 +1129,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): if not current_response_id or not current_conversation_id: return [], None, None, None - empty_usage: Final = get_empty_usage() - zero_usage: Final[OpenAIRealtimeResponseUsage] = { - "input_tokens": empty_usage.prompt_tokens, - "output_tokens": empty_usage.completion_tokens, - "total_tokens": empty_usage.total_tokens, - } - usage: Final = self._latest_usage or zero_usage - self._latest_usage = None + usage: Final = self._take_usage_delta() response_done: Final = OpenAIRealtimeDoneEvent( type="response.done", event_id=f"event_{uuid.uuid4()}", @@ -1324,6 +1350,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): elif "contentEnd" in event and self._user_transcript_active: self._user_transcript_active = False self._user_transcript_generation_stage = None + returned_messages.extend(self.user_transcript_completed_events()) elif "contentEnd" in event: events, current_delta_chunks = self.transform_content_end_event( diff --git a/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_handler.py b/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_handler.py index aa6573884e2..0ea5b7ad4a1 100644 --- a/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_handler.py +++ b/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_handler.py @@ -6,7 +6,7 @@ from unittest.mock import MagicMock import pytest - +import litellm from litellm.llms.bedrock.common_utils import BedrockError from litellm.llms.bedrock.realtime.handler import BedrockRealtime from litellm.llms.bedrock.realtime.transformation import BedrockRealtimeConfig @@ -124,14 +124,6 @@ class ScriptedBedrockStream: return (None, self._receiver) -class ImmediatelyEndingBedrockStream: - def __init__(self): - self.input_stream = FakeInputStream() - - async def await_output(self): - return (None, EndedBedrockReceiver()) - - class FakeStaticCredentialsResolver: pass @@ -171,7 +163,7 @@ def stub_aws_sdk_client(monkeypatch): async def invoke_model_with_bidirectional_stream(self, operation_input): captured["operation_input"] = operation_input - return ImmediatelyEndingBedrockStream() + return ScriptedBedrockStream(captured.get("scripted_payloads", [])) package = types.ModuleType("aws_sdk_bedrock_runtime") client_module = types.ModuleType("aws_sdk_bedrock_runtime.client") @@ -292,7 +284,40 @@ class TestBedrockRealtimeHandler: assert stream.input_stream.closed @pytest.mark.asyncio - async def test_forwarded_events_are_collected_for_spend_logging(self): + async def test_forwarded_events_are_filtered_to_logged_types_for_spend_logging(self): + handler = BedrockRealtime() + stream = ScriptedBedrockStream( + [ + json.dumps({"event": {"userSpeechStart": {}}}), + json.dumps({"event": {"contentStart": {"role": "ASSISTANT", "type": "TEXT"}}}), + json.dumps({"event": {"textOutput": {"content": "Hi"}}}), + json.dumps({"event": {"contentEnd": {"stopReason": "END_TURN"}}}), + ] + ) + client_ws = RealtimeClientWS() + + logged_events = [ + event + async for event in handler._forward_bedrock_to_client( + stream, + client_ws, + BedrockRealtimeConfig(), + "amazon.nova-sonic-v1:0", + FakeLogging(), + {}, + ) + ] + + assert [event["type"] for event in logged_events] == ["response.done"] + sent_types = [json.loads(message)["type"] for message in client_ws.sent_to_client] + assert "input_audio_buffer.speech_started" in sent_types + assert "response.text.delta" in sent_types + assert "response.done" in sent_types + assert client_ws.closed + + @pytest.mark.asyncio + async def test_logged_event_types_star_collects_every_forwarded_event(self, monkeypatch): + monkeypatch.setattr(litellm, "logged_real_time_event_types", "*") handler = BedrockRealtime() stream = ScriptedBedrockStream( [ @@ -301,37 +326,89 @@ class TestBedrockRealtimeHandler: ] ) client_ws = RealtimeClientWS() - logged_events = [] - await handler._forward_bedrock_to_client( - stream, - client_ws, - BedrockRealtimeConfig(), - "amazon.nova-sonic-v1:0", - FakeLogging(), - {}, - logged_events, - ) + logged_events = [ + event + async for event in handler._forward_bedrock_to_client( + stream, + client_ws, + BedrockRealtimeConfig(), + "amazon.nova-sonic-v1:0", + FakeLogging(), + {}, + ) + ] assert [event["type"] for event in logged_events] == [ "input_audio_buffer.speech_started", "input_audio_buffer.speech_stopped", ] - assert client_ws.closed + + @pytest.mark.asyncio + async def test_trailing_usage_after_last_done_is_dispatched_for_spend(self, stub_aws_sdk_client, monkeypatch): + import litellm.llms.bedrock.realtime.handler as handler_module + + dispatched = {} + + class RecordingLogging(FakeLogging): + async def dispatch_success_handlers(self, result=None, prefer_async_handlers=False, **kwargs): + dispatched["events"] = result + + class RecordingLoggingWorker: + def ensure_initialized_and_enqueue(self, coro): + dispatched["coro"] = coro + + monkeypatch.setattr(handler_module, "GLOBAL_LOGGING_WORKER", RecordingLoggingWorker()) + stub_aws_sdk_client["scripted_payloads"] = [ + json.dumps( + { + "event": { + "usageEvent": { + "totalInputTokens": 3, + "totalOutputTokens": 6, + "totalTokens": 9, + "details": { + "total": { + "input": {"speechTokens": 3, "textTokens": 0}, + "output": {"speechTokens": 0, "textTokens": 6}, + } + }, + } + } + } + ) + ] + + await BedrockRealtime().async_realtime( + model="amazon.nova-sonic-v1:0", + websocket=RealtimeClientWS(), + logging_obj=RecordingLogging(), + aws_region_name="us-east-1", + aws_access_key_id="k", + aws_secret_access_key="s", + ) + await dispatched["coro"] + + assert [event["type"] for event in dispatched["events"]] == ["response.done"] + usage = dispatched["events"][0]["response"]["usage"] + assert (usage["input_tokens"], usage["output_tokens"], usage["total_tokens"]) == (3, 6, 9) + assert usage["input_token_details"] == {"audio_tokens": 3, "text_tokens": 0, "cached_tokens": 0} + assert usage["output_token_details"] == {"audio_tokens": 0, "text_tokens": 6} @pytest.mark.asyncio async def test_bedrock_stream_end_closes_client_websocket(self): handler = BedrockRealtime() client_ws = ClosableClientWS() - await handler._forward_bedrock_to_client( + async for _ in handler._forward_bedrock_to_client( EndedBedrockStream(), client_ws, BedrockRealtimeConfig(), "amazon.nova-sonic-v1:0", MagicMock(), {}, - ) + ): + pass assert client_ws.closed diff --git a/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_transformation.py b/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_transformation.py index dbe84cacb29..a74f03449a1 100644 --- a/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_transformation.py +++ b/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_transformation.py @@ -1025,6 +1025,112 @@ class TestBedrockRealtimeUserEventsAndUsage: assert usage["output_tokens"] == 0 assert usage["total_tokens"] == 0 + @staticmethod + def _usage_event(total_input, total_output, in_speech, in_text, out_speech, out_text): + return { + "event": { + "usageEvent": { + "totalInputTokens": total_input, + "totalOutputTokens": total_output, + "totalTokens": total_input + total_output, + "details": { + "total": { + "input": {"speechTokens": in_speech, "textTokens": in_text}, + "output": {"speechTokens": out_speech, "textTokens": out_text}, + } + }, + } + } + } + + _ASSISTANT_TURN = ( + {"event": {"contentStart": {"role": "ASSISTANT", "type": "TEXT"}}}, + {"event": {"textOutput": {"content": "Hi"}}}, + {"event": {"contentEnd": {"stopReason": "END_TURN"}}}, + ) + + def test_multi_turn_usage_reports_per_response_deltas_not_cumulative_totals(self): + events = self._run( + BedrockRealtimeConfig(), + [ + self._usage_event(25, 40, in_speech=20, in_text=5, out_speech=30, out_text=10), + *self._ASSISTANT_TURN, + self._usage_event(40, 100, in_speech=30, in_text=10, out_speech=75, out_text=25), + *self._ASSISTANT_TURN, + ], + ) + usages = [e["response"]["usage"] for e in events if e["type"] == "response.done"] + assert len(usages) == 2 + assert (usages[0]["input_tokens"], usages[0]["output_tokens"], usages[0]["total_tokens"]) == (25, 40, 65) + assert (usages[1]["input_tokens"], usages[1]["output_tokens"], usages[1]["total_tokens"]) == (15, 60, 75) + assert usages[1]["input_token_details"] == {"audio_tokens": 10, "text_tokens": 5, "cached_tokens": 0} + assert usages[1]["output_token_details"] == {"audio_tokens": 45, "text_tokens": 15} + assert sum(u["total_tokens"] for u in usages) == 140 + + def test_usage_reported_after_last_response_done_flushes_as_logged_only_done(self): + config = BedrockRealtimeConfig() + self._run( + config, + [ + self._usage_event(25, 40, in_speech=20, in_text=5, out_speech=30, out_text=10), + *self._ASSISTANT_TURN, + ], + ) + assert config.leftover_usage_done_events() == () + + self._run(config, [self._usage_event(25, 46, in_speech=20, in_text=5, out_speech=30, out_text=16)]) + leftover = config.leftover_usage_done_events() + assert len(leftover) == 1 + assert leftover[0]["type"] == "response.done" + usage = leftover[0]["response"]["usage"] + assert (usage["input_tokens"], usage["output_tokens"], usage["total_tokens"]) == (0, 6, 6) + assert usage["output_token_details"] == {"audio_tokens": 0, "text_tokens": 6} + assert config.leftover_usage_done_events() == () + + def test_final_transcript_fragments_emit_one_completed_with_full_transcript(self): + events = self._run( + BedrockRealtimeConfig(), + [ + { + "event": { + "contentStart": { + "role": "USER", + "type": "TEXT", + "additionalModelFields": json.dumps({"generationStage": "FINAL"}), + } + } + }, + {"event": {"textOutput": {"content": "What is the "}}}, + {"event": {"textOutput": {"content": "capital of France?"}}}, + {"event": {"contentEnd": {"stopReason": "PARTIAL_TURN"}}}, + ], + ) + deltas = [e for e in events if e["type"] == "conversation.item.input_audio_transcription.delta"] + completed = [e for e in events if e["type"] == "conversation.item.input_audio_transcription.completed"] + assert [d["delta"] for d in deltas] == ["What is the ", "capital of France?"] + assert len(completed) == 1 + assert completed[0]["transcript"] == "What is the capital of France?" + assert {e["item_id"] for e in deltas + completed} == {completed[0]["item_id"]} + + def test_speculative_transcript_block_end_emits_no_completed(self): + events = self._run( + BedrockRealtimeConfig(), + [ + { + "event": { + "contentStart": { + "role": "USER", + "type": "TEXT", + "additionalModelFields": json.dumps({"generationStage": "SPECULATIVE"}), + } + } + }, + {"event": {"textOutput": {"content": "rea"}}}, + {"event": {"contentEnd": {"stopReason": "PARTIAL_TURN"}}}, + ], + ) + assert [e["type"] for e in events] == ["conversation.item.input_audio_transcription.delta"] + if __name__ == "__main__": pytest.main([__file__, "-v"]) From 666d8737aa8766ac293995e565d91c1faf82069e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 31 Aug 2026 12:19:41 -0700 Subject: [PATCH 05/10] fix(init): ignore pydantic ReadOnly TypedDict warning that floods proxy boot --- litellm/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/litellm/__init__.py b/litellm/__init__.py index c83e72a78b4..1447e05fdf7 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -7,6 +7,9 @@ warnings.filterwarnings("ignore", message=".*conflict with protected namespace.* # Suppress Pydantic 2.11+ deprecation warning about accessing model_fields on instances # This warning can accumulate during streaming and cause memory leaks warnings.filterwarnings("ignore", message=".*Accessing the.*attribute on the instance is deprecated.*") +# ReadOnly on TypedDict fields is repo-wide static discipline (LIT012); pydantic warns it +# cannot enforce it at runtime, which floods proxy boot once such a type is schema-walked +warnings.filterwarnings("ignore", message=".*`ReadOnly` qualifier.*") ### INIT VARIABLES ######################### import threading import os From 25c8d58400a48d1010e73d98987dd44b035764b5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 31 Aug 2026 13:09:39 -0700 Subject: [PATCH 06/10] fix(docker): install file and make in wolfi builders so the uvloop sdist can build --- Dockerfile | 2 ++ docker/Dockerfile.database | 2 ++ docker/Dockerfile.non_root | 2 ++ 3 files changed, 6 insertions(+) diff --git a/Dockerfile b/Dockerfile index 700b0d6525e..4688a733331 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,7 +39,9 @@ COPY --from=uvbin /uvx /usr/local/bin/uvx RUN apk add --no-cache \ bash \ + file \ gcc \ + make \ python3 \ python3-dev \ rust \ diff --git a/docker/Dockerfile.database b/docker/Dockerfile.database index f0d6d02fccf..9a243c68dca 100644 --- a/docker/Dockerfile.database +++ b/docker/Dockerfile.database @@ -38,7 +38,9 @@ COPY --from=uvbin /uvx /usr/local/bin/uvx RUN apk add --no-cache \ bash \ + file \ gcc \ + make \ python3 \ python3-dev \ openssl \ diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index 4a5df6ecd69..2cf70f5b01d 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -39,7 +39,9 @@ RUN for i in 1 2 3; do \ apk add --no-cache \ python3 \ python3-dev \ + file \ gcc \ + make \ rust \ bash \ coreutils \ From ebdb54d4f0c51c7c8df038dd5b189f4e84577203 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 31 Aug 2026 13:32:19 -0700 Subject: [PATCH 07/10] fix(docker): keep image venvs on the apk python and bump the wolfi digest Since the requires-python cap moved to <3.15, uv resolved the project python to 3.14, downloaded a managed interpreter under /root/.local/share/uv that the runtime stage never receives, and every layer-cache-miss image build broke: first at uvloop's cp314 sdist configure step, then, with file/make added, at the runtime stage where the copied venv's python symlink dangles and prisma imports fall through to the system python. UV_PYTHON_DOWNLOADS=0 (already the convention in migrations/backend/gateway) roots the venv on the apk python3. The wolfi-base digest bump is required alongside it: the pinned 08-22 base ships glibc-2.43 while the current apk python-3.13 needs GLIBC_2.44, and wolfi version-names glibc packages so apk upgrade cannot cross that boundary. With the venv on system 3.13 every dependency installs from wheels again, so the file and make packages added for the sdist build are reverted. --- Dockerfile | 11 +++++++---- backend/Dockerfile | 4 ++-- docker/Dockerfile.database | 11 +++++++---- docker/Dockerfile.non_root | 11 +++++++---- gateway/Dockerfile | 4 ++-- migrations/Dockerfile | 4 ++-- 6 files changed, 27 insertions(+), 18 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4688a733331..675a79b0686 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,10 @@ # syntax=docker/dockerfile:1.7 # Base image for building -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72 +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:57108e597a8cf3bd376b810f1c3539c21942daefa242cb9dddaae30f8aac735d # Runtime image -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72 +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:57108e597a8cf3bd376b810f1c3539c21942daefa242cb9dddaae30f8aac735d ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a # Pinned by digest like the other base images; bump explicitly on Node upgrades. ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43 @@ -39,9 +39,7 @@ COPY --from=uvbin /uvx /usr/local/bin/uvx RUN apk add --no-cache \ bash \ - file \ gcc \ - make \ python3 \ python3-dev \ rust \ @@ -51,8 +49,13 @@ RUN apk add --no-cache \ npm \ libsndfile +# UV_PYTHON_DOWNLOADS=0 keeps the venv on the apk python3 above. Without it, +# uv resolves requires-python to the newest allowed minor, downloads a managed +# interpreter under /root/.local/share/uv that the runtime stage never +# receives, and the copied venv's python symlink dangles at runtime. ENV UV_PROJECT_ENVIRONMENT=/app/.venv \ UV_LINK_MODE=copy \ + UV_PYTHON_DOWNLOADS=0 \ PATH="/app/.venv/bin:${PATH}" # Copy dependency metadata first for layer caching diff --git a/backend/Dockerfile b/backend/Dockerfile index 4ca40944606..8aea8312df9 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,5 +1,5 @@ -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72 -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72 +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:57108e597a8cf3bd376b810f1c3539c21942daefa242cb9dddaae30f8aac735d +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:57108e597a8cf3bd376b810f1c3539c21942daefa242cb9dddaae30f8aac735d ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a FROM $UV_IMAGE AS uvbin diff --git a/docker/Dockerfile.database b/docker/Dockerfile.database index 9a243c68dca..4243eea5796 100644 --- a/docker/Dockerfile.database +++ b/docker/Dockerfile.database @@ -1,10 +1,10 @@ # syntax=docker/dockerfile:1.7 # Base image for building -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72 +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:57108e597a8cf3bd376b810f1c3539c21942daefa242cb9dddaae30f8aac735d # Runtime image -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72 +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:57108e597a8cf3bd376b810f1c3539c21942daefa242cb9dddaae30f8aac735d ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a # Pinned by digest like the other base images; bump explicitly on Node upgrades. ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43 @@ -38,9 +38,7 @@ COPY --from=uvbin /uvx /usr/local/bin/uvx RUN apk add --no-cache \ bash \ - file \ gcc \ - make \ python3 \ python3-dev \ openssl \ @@ -49,8 +47,13 @@ RUN apk add --no-cache \ npm \ libsndfile +# UV_PYTHON_DOWNLOADS=0 keeps the venv on the apk python3 above. Without it, +# uv resolves requires-python to the newest allowed minor, downloads a managed +# interpreter under /root/.local/share/uv that the runtime stage never +# receives, and the copied venv's python symlink dangles at runtime. ENV UV_PROJECT_ENVIRONMENT=/app/.venv \ UV_LINK_MODE=copy \ + UV_PYTHON_DOWNLOADS=0 \ PATH="/app/.venv/bin:${PATH}" # Copy dependency metadata first for layer caching diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index 2cf70f5b01d..19ef97a4f46 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -1,8 +1,8 @@ # syntax=docker/dockerfile:1.7 # Base images -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72 -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72 +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:57108e597a8cf3bd376b810f1c3539c21942daefa242cb9dddaae30f8aac735d +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:57108e597a8cf3bd376b810f1c3539c21942daefa242cb9dddaae30f8aac735d ARG PROXY_EXTRAS_SOURCE=published ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a # Pinned by digest like the other base images; bump explicitly on Node upgrades. @@ -39,9 +39,7 @@ RUN for i in 1 2 3; do \ apk add --no-cache \ python3 \ python3-dev \ - file \ gcc \ - make \ rust \ bash \ coreutils \ @@ -52,8 +50,13 @@ RUN for i in 1 2 3; do \ npm && break || sleep 5; \ done +# UV_PYTHON_DOWNLOADS=0 keeps the venv on the apk python3 above. Without it, +# uv resolves requires-python to the newest allowed minor, downloads a managed +# interpreter under /root/.local/share/uv that the runtime stage never +# receives, and the copied venv's python symlink dangles at runtime. ENV UV_PROJECT_ENVIRONMENT=/app/.venv \ UV_LINK_MODE=copy \ + UV_PYTHON_DOWNLOADS=0 \ PATH="/app/.venv/bin:${PATH}" \ LITELLM_NON_ROOT=true \ XDG_CACHE_HOME=/app/.cache diff --git a/gateway/Dockerfile b/gateway/Dockerfile index 4a2e32e186e..235c535f9f1 100644 --- a/gateway/Dockerfile +++ b/gateway/Dockerfile @@ -1,5 +1,5 @@ -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72 -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72 +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:57108e597a8cf3bd376b810f1c3539c21942daefa242cb9dddaae30f8aac735d +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:57108e597a8cf3bd376b810f1c3539c21942daefa242cb9dddaae30f8aac735d ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a FROM $UV_IMAGE AS uvbin diff --git a/migrations/Dockerfile b/migrations/Dockerfile index 6335e6f6bd8..524450024e0 100644 --- a/migrations/Dockerfile +++ b/migrations/Dockerfile @@ -1,5 +1,5 @@ -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72 -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72 +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:57108e597a8cf3bd376b810f1c3539c21942daefa242cb9dddaae30f8aac735d +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:57108e597a8cf3bd376b810f1c3539c21942daefa242cb9dddaae30f8aac735d ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a FROM $UV_IMAGE AS uvbin From db7eb641cbc0a81dcd640350501b357df53abf94 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 31 Aug 2026 13:58:53 -0700 Subject: [PATCH 08/10] ci(osv): ignore GHSA-h7x2-h6g9-p789 until mlflow ships a fix The advisory was modified 2026-08-31 and flags mlflow 3.13.0 through 3.15.2 with no fixed release published, so every osv-scan run fails with nothing to bump. Same treatment as the existing diskcache entry. --- osv-scanner.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/osv-scanner.toml b/osv-scanner.toml index 7ab450945f5..5b0339bdcd0 100644 --- a/osv-scanner.toml +++ b/osv-scanner.toml @@ -2,3 +2,8 @@ id = "GHSA-w8v5-vhqr-4h9v" ignoreUntil = 2026-09-09 reason = "diskcache has no fixed release published; remove this entry once one exists" + +[[IgnoredVulns]] +id = "GHSA-h7x2-h6g9-p789" +ignoreUntil = 2026-09-14 +reason = "mlflow has no fixed release published; remove this entry once one exists" From cc078edd1fb2a29e2c5c2826803ae399e6078143 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 21:56:57 +0000 Subject: [PATCH 09/10] test(mcp): isolate global MCP server registry in discoverable endpoints tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../mcp_server/test_discoverable_endpoints.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py index 3279c59acd4..598e9276423 100644 --- a/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py @@ -35,6 +35,22 @@ def mock_mcp_client_ip(): yield +@pytest.fixture(autouse=True) +def isolate_global_mcp_registry(): + """Restore the module-global MCP server registry after each test. + + Tests here register servers on ``global_mcp_server_manager`` directly; without a + restore, entries leak into other test modules sharing the same worker and break + assertions over the full registry contents. + """ + from litellm.proxy._experimental.mcp_server.mcp_server_manager import global_mcp_server_manager + + snapshot = dict(global_mcp_server_manager.registry) + yield + global_mcp_server_manager.registry.clear() + global_mcp_server_manager.registry.update(snapshot) + + def _mock_callback_request(base_url: str = "http://localhost:3000/"): """Return a MagicMock Request for callback/authorize same-origin tests. From 613d0ef3fa7153e5485698ae6fae47481f23b027 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 31 Aug 2026 15:04:07 -0700 Subject: [PATCH 10/10] test(gcs_pub_sub): expect router_metadata in the spend logs payload The base added router_metadata to SpendLogsMetadata in #39001 without updating this fixture, and its CI run never executed logging_testing, so the job now fails on every branch merged with current staging. --- .../gcs_pub_sub_body/spend_logs_payload.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/logging_callback_tests/gcs_pub_sub_body/spend_logs_payload.json b/tests/logging_callback_tests/gcs_pub_sub_body/spend_logs_payload.json index 1838fb16e91..28912a27501 100644 --- a/tests/logging_callback_tests/gcs_pub_sub_body/spend_logs_payload.json +++ b/tests/logging_callback_tests/gcs_pub_sub_body/spend_logs_payload.json @@ -11,7 +11,7 @@ "user": "", "team_id": "", "organization_id": "", - "metadata": "{\"applied_guardrails\": [], \"attempted_fallbacks\": null, \"original_model_group\": null, \"batch_models\": null, \"batch_successful_requests\": null, \"batch_failed_requests\": null, \"mcp_tool_call_metadata\": null, \"vector_store_request_metadata\": null, \"routing_decision\": null, \"internal_call_origin\": null, \"guardrail_information\": null, \"compression_savings\": null, \"litellm_gateway_injected_cache\": null, \"usage_object\": {\"completion_tokens\": 20, \"prompt_tokens\": 10, \"total_tokens\": 30, \"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"model_map_information\": {\"model_map_key\": \"gpt-4o\", \"model_map_value\": {\"key\": \"gpt-4o\", \"max_tokens\": 16384, \"max_input_tokens\": 128000, \"max_output_tokens\": 16384, \"input_cost_per_token\": 2.5e-06, \"cache_creation_input_token_cost\": null, \"cache_read_input_token_cost\": 1.25e-06, \"input_cost_per_character\": null, \"input_cost_per_token_above_128k_tokens\": null, \"input_cost_per_token_above_200k_tokens\": null, \"input_cost_per_query\": null, \"input_cost_per_second\": null, \"input_cost_per_audio_token\": null, \"input_cost_per_token_batches\": 1.25e-06, \"output_cost_per_token_batches\": 5e-06, \"output_cost_per_token\": 1e-05, \"output_cost_per_audio_token\": null, \"output_cost_per_character\": null, \"output_cost_per_token_above_128k_tokens\": null, \"output_cost_per_character_above_128k_tokens\": null, \"output_cost_per_token_above_200k_tokens\": null, \"output_cost_per_second\": null, \"output_cost_per_image\": null, \"output_vector_size\": null, \"litellm_provider\": \"openai\", \"mode\": \"chat\", \"supports_system_messages\": true, \"supports_response_schema\": true, \"supports_vision\": true, \"supports_function_calling\": true, \"supports_tool_choice\": true, \"supports_assistant_prefill\": false, \"supports_prompt_caching\": true, \"supports_audio_input\": false, \"supports_audio_output\": false, \"supports_pdf_input\": false, \"supports_embedding_image_input\": false, \"supports_native_streaming\": null, \"supports_web_search\": true, \"supports_reasoning\": false, \"search_context_cost_per_query\": {\"search_context_size_low\": 0.03, \"search_context_size_medium\": 0.035, \"search_context_size_high\": 0.05}, \"tpm\": null, \"rpm\": null, \"supported_openai_params\": [\"frequency_penalty\", \"logit_bias\", \"logprobs\", \"top_logprobs\", \"max_tokens\", \"max_completion_tokens\", \"modalities\", \"prediction\", \"n\", \"presence_penalty\", \"seed\", \"stop\", \"stream\", \"stream_options\", \"temperature\", \"top_p\", \"tools\", \"tool_choice\", \"function_call\", \"functions\", \"max_retries\", \"extra_headers\", \"parallel_tool_calls\", \"audio\", \"response_format\", \"user\"]}}, \"additional_usage_values\": {\"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"user_api_key\": null, \"user_api_key_alias\": null, \"user_api_key_team_id\": null, \"user_api_key_project_id\": null, \"user_api_key_project_alias\": null, \"user_api_key_org_id\": null, \"user_api_key_user_id\": null, \"user_api_key_team_alias\": null, \"spend_logs_metadata\": null, \"requester_ip_address\": null, \"status\": null, \"proxy_server_request\": null, \"error_information\": null, \"attempted_retries\": null, \"max_retries\": null}", + "metadata": "{\"applied_guardrails\": [], \"attempted_fallbacks\": null, \"original_model_group\": null, \"batch_models\": null, \"batch_successful_requests\": null, \"batch_failed_requests\": null, \"mcp_tool_call_metadata\": null, \"vector_store_request_metadata\": null, \"routing_decision\": null, \"internal_call_origin\": null, \"router_metadata\": null, \"guardrail_information\": null, \"compression_savings\": null, \"litellm_gateway_injected_cache\": null, \"usage_object\": {\"completion_tokens\": 20, \"prompt_tokens\": 10, \"total_tokens\": 30, \"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"model_map_information\": {\"model_map_key\": \"gpt-4o\", \"model_map_value\": {\"key\": \"gpt-4o\", \"max_tokens\": 16384, \"max_input_tokens\": 128000, \"max_output_tokens\": 16384, \"input_cost_per_token\": 2.5e-06, \"cache_creation_input_token_cost\": null, \"cache_read_input_token_cost\": 1.25e-06, \"input_cost_per_character\": null, \"input_cost_per_token_above_128k_tokens\": null, \"input_cost_per_token_above_200k_tokens\": null, \"input_cost_per_query\": null, \"input_cost_per_second\": null, \"input_cost_per_audio_token\": null, \"input_cost_per_token_batches\": 1.25e-06, \"output_cost_per_token_batches\": 5e-06, \"output_cost_per_token\": 1e-05, \"output_cost_per_audio_token\": null, \"output_cost_per_character\": null, \"output_cost_per_token_above_128k_tokens\": null, \"output_cost_per_character_above_128k_tokens\": null, \"output_cost_per_token_above_200k_tokens\": null, \"output_cost_per_second\": null, \"output_cost_per_image\": null, \"output_vector_size\": null, \"litellm_provider\": \"openai\", \"mode\": \"chat\", \"supports_system_messages\": true, \"supports_response_schema\": true, \"supports_vision\": true, \"supports_function_calling\": true, \"supports_tool_choice\": true, \"supports_assistant_prefill\": false, \"supports_prompt_caching\": true, \"supports_audio_input\": false, \"supports_audio_output\": false, \"supports_pdf_input\": false, \"supports_embedding_image_input\": false, \"supports_native_streaming\": null, \"supports_web_search\": true, \"supports_reasoning\": false, \"search_context_cost_per_query\": {\"search_context_size_low\": 0.03, \"search_context_size_medium\": 0.035, \"search_context_size_high\": 0.05}, \"tpm\": null, \"rpm\": null, \"supported_openai_params\": [\"frequency_penalty\", \"logit_bias\", \"logprobs\", \"top_logprobs\", \"max_tokens\", \"max_completion_tokens\", \"modalities\", \"prediction\", \"n\", \"presence_penalty\", \"seed\", \"stop\", \"stream\", \"stream_options\", \"temperature\", \"top_p\", \"tools\", \"tool_choice\", \"function_call\", \"functions\", \"max_retries\", \"extra_headers\", \"parallel_tool_calls\", \"audio\", \"response_format\", \"user\"]}}, \"additional_usage_values\": {\"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"user_api_key\": null, \"user_api_key_alias\": null, \"user_api_key_team_id\": null, \"user_api_key_project_id\": null, \"user_api_key_project_alias\": null, \"user_api_key_org_id\": null, \"user_api_key_user_id\": null, \"user_api_key_team_alias\": null, \"spend_logs_metadata\": null, \"requester_ip_address\": null, \"status\": null, \"proxy_server_request\": null, \"error_information\": null, \"attempted_retries\": null, \"max_retries\": null}", "cache_key": "Cache OFF", "spend": 0.00022500000000000002, "total_tokens": 30,