From 6518decca7c6bf236050b75802a384c58b75a3e1 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Mon, 24 Aug 2026 13:34:38 -0400 Subject: [PATCH] fix(bedrock/realtime): open the response a tool-first Nova Sonic turn closes Nova Sonic starts a tool turn with contentStart role TOOL, which emits nothing, so when the tool call is the first thing in a turn there is no open response. The tool lifecycle still emitted response.done for the id it minted, handing the client a close for a response it never saw created: the mirror of the orphan just fixed. Emit response.created ahead of the tool events whenever the response id is minted, matching what the Gemini realtime config does on its toolCall branch. --- .../llms/bedrock/realtime/transformation.py | 31 +++++++++-- .../test_bedrock_realtime_transformation.py | 55 ++++++++++++++++--- 2 files changed, 75 insertions(+), 11 deletions(-) diff --git a/litellm/llms/bedrock/realtime/transformation.py b/litellm/llms/bedrock/realtime/transformation.py index d0936231d52..df3d992ccd1 100644 --- a/litellm/llms/bedrock/realtime/transformation.py +++ b/litellm/llms/bedrock/realtime/transformation.py @@ -1155,10 +1155,11 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): Nova Sonic delivers one tool call, fully formed, in a single event, and starts the block with ``contentStart`` role ``TOOL``, which opens no OpenAI response. Mint the - response/item ids when they are missing, emit the item added/delta/done trio the - OpenAI realtime protocol requires around ``function_call_arguments.done``, then close - the response so no in-progress response is left orphaned and downstream spend logging - can harvest the call from ``response.done`` output. + response/item ids when they are missing, opening the response first so its + ``response.done`` is never unmatched, emit the item added/delta/done trio the OpenAI + realtime protocol requires around ``function_call_arguments.done``, then close the + response so nothing is left in progress and downstream spend logging can harvest the + call from ``response.done`` output. Returns: Tuple of (events, tool_call_id, tool_name). The caller clears the response and @@ -1167,6 +1168,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): verbose_logger.debug("Handling toolUse") tool_use: Final = BedrockToolUse.model_validate(event["toolUse"]) + is_new_response: Final = not current_response_id response_id: Final = current_response_id or f"resp_{uuid.uuid4()}" item_id: Final = current_output_item_id or f"item_{uuid.uuid4()}" tool_call_id: Final = tool_use.toolUseId @@ -1186,7 +1188,28 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): {**function_call_item, "status": "in_progress", "arguments": ""} ) + # A tool turn that Bedrock opens with contentStart role TOOL has no response yet, so the + # response.done below would close an id the client never saw opened. + response_created: Final[tuple[OpenAIRealtimeEvents, ...]] = ( + ( + OpenAIRealtimeStreamResponseBaseObject( + type="response.created", + event_id=f"event_{uuid.uuid4()}", + response={ + "object": "realtime.response", + "id": response_id, + "status": "in_progress", + "output": [], + "conversation_id": conversation_id, + }, + ), + ) + if is_new_response + else () + ) + events: Final[list[OpenAIRealtimeEvents]] = [ + *response_created, OpenAIRealtimeStreamResponseOutputItemAdded( type="response.output_item.added", event_id=f"event_{uuid.uuid4()}", diff --git a/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_transformation.py b/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_transformation.py index 17bf5ae46cd..7d8187a4540 100644 --- a/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_transformation.py +++ b/tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_transformation.py @@ -16,7 +16,8 @@ from litellm.llms.bedrock.realtime.transformation import ( ) from litellm.llms.bedrock.realtime.trigger_audio import ready_trigger_pcm -# The OpenAI realtime function-call lifecycle a single Nova Sonic toolUse must expand into. +# The OpenAI realtime function-call lifecycle a single Nova Sonic toolUse must expand into, +# when an assistant block already opened the response the tool call belongs to. _TOOL_CALL_EVENT_SEQUENCE = [ "response.output_item.added", "conversation.item.added", @@ -26,6 +27,10 @@ _TOOL_CALL_EVENT_SEQUENCE = [ "response.done", ] +# Nova Sonic opens tool turns with contentStart role TOOL, which opens no response, so the +# tool call has to open one itself before it can close it. +_TOOL_CALL_EVENT_SEQUENCE_NEW_RESPONSE = ["response.created", *_TOOL_CALL_EVENT_SEQUENCE] + def _only(events, event_type): matches = [event for event in events if event["type"] == event_type] @@ -34,11 +39,10 @@ def _only(events, event_type): def _response_id_of(event): - if event["type"] == "response.done": + """The response id an event is bound to, or None for events that carry no response id.""" + if event["type"] in ("response.created", "response.done"): return event["response"]["id"] - if event["type"] == "conversation.item.added": - return None - return event["response_id"] + return event.get("response_id") class TestBedrockRealtimeConfig: @@ -659,8 +663,10 @@ class TestBedrockRealtimeResponseTransformation: None, "conv_1", ) - assert [event["type"] for event in events] == _TOOL_CALL_EVENT_SEQUENCE + assert [event["type"] for event in events] == _TOOL_CALL_EVENT_SEQUENCE_NEW_RESPONSE function_call = _only(events, "response.function_call_arguments.done") + assert _only(events, "response.created")["response"]["id"] == function_call["response_id"] + assert _only(events, "response.created")["response"]["conversation_id"] == "conv_1" assert function_call["call_id"] == "tool_call_no_ids" assert function_call["name"] == "get_weather" assert function_call["response_id"].startswith("resp_") @@ -790,8 +796,9 @@ class TestBedrockRealtimeResponseTransformation: realtime_response_transform_input=state, ) - assert [msg["type"] for msg in result["response"]] == _TOOL_CALL_EVENT_SEQUENCE + assert [msg["type"] for msg in result["response"]] == _TOOL_CALL_EVENT_SEQUENCE_NEW_RESPONSE function_call = _only(result["response"], "response.function_call_arguments.done") + assert _only(result["response"], "response.created")["response"]["id"] == function_call["response_id"] assert function_call["response_id"].startswith("resp_") assert function_call["item_id"].startswith("item_") assert json.loads(function_call["arguments"]) == {"location": "Seattle"} @@ -1327,6 +1334,40 @@ class TestBedrockRealtimeContentBlockLifecycle: assert created == done assert state["current_response_id"] is None + def test_every_response_is_opened_and_closed_on_a_tool_first_turn(self): + """ + Nova Sonic opens tool turns with contentStart role TOOL, which emits nothing, so the + tool call is the first thing in the session. It has to open the response it closes, + or the client sees a response.done for an id it never saw created. + """ + config = BedrockRealtimeConfig() + logging_obj = MagicMock() + logging_obj.litellm_trace_id = "trace_123" + state = self._state(current_conversation_id=None) + + turn = [ + {"event": {"contentStart": {"role": "TOOL", "type": "TOOL"}}}, + { + "event": { + "toolUse": { + "toolUseId": "tool_1", + "toolName": "get_weather", + "content": json.dumps({"location": "Seattle"}), + } + } + }, + {"event": {"contentEnd": {"stopReason": "TOOL_USE", "type": "TOOL"}}}, + ] + emitted = [msg for event in turn for msg in self._apply(config, logging_obj, state, event)["response"]] + + created = [msg["response"]["id"] for msg in emitted if msg["type"] == "response.created"] + done = [msg["response"]["id"] for msg in emitted if msg["type"] == "response.done"] + assert len(created) == 1 + assert created == done + # Every event in the turn is bound to that one response. + assert {_response_id_of(msg) for msg in emitted} - {None} == set(created) + assert state["current_response_id"] is None + def test_second_assistant_content_block_reuses_response_not_item(self): config = BedrockRealtimeConfig() logging_obj = MagicMock()