scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-07 02:59:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 103

fix(bedrock/realtime): open the response a tool-first Nova Sonic turn closes

Browse source 
Nova Sonic starts a tool turn with contentStart role TOOL, which emits nothing, so
when the tool call is the first thing in a turn there is no open response. The tool
lifecycle still emitted response.done for the id it minted, handing the client a
close for a response it never saw created: the mirror of the orphan just fixed.

Emit response.created ahead of the tool events whenever the response id is minted,
matching what the Gemini realtime config does on its toolCall branch.
This commit is contained in:
mubashir1osmani 2026-08-24 13:34:38 -04:00
parent 06e5e7d0da
commit 6518decca7
2 changed files with 75 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
litellm/llms/bedrock/realtime/transformation.py
View file

@ -1155,10 +1155,11 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
Nova Sonic delivers one tool call, fully formed, in a single event, and starts the
block with ``contentStart`` role ``TOOL``, which opens no OpenAI response. Mint the
response/item ids when they are missing, emit the item added/delta/done trio the
OpenAI realtime protocol requires around ``function_call_arguments.done``, then close
the response so no in-progress response is left orphaned and downstream spend logging
can harvest the call from ``response.done`` output.
response/item ids when they are missing, opening the response first so its
``response.done`` is never unmatched, emit the item added/delta/done trio the OpenAI
realtime protocol requires around ``function_call_arguments.done``, then close the
response so nothing is left in progress and downstream spend logging can harvest the
call from ``response.done`` output.
Returns:
Tuple of (events, tool_call_id, tool_name). The caller clears the response and
@ -1167,6 +1168,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
verbose_logger.debug("Handling toolUse")
tool_use: Final = BedrockToolUse.model_validate(event["toolUse"])
is_new_response: Final = not current_response_id
response_id: Final = current_response_id or f"resp_{uuid.uuid4()}"
item_id: Final = current_output_item_id or f"item_{uuid.uuid4()}"
tool_call_id: Final = tool_use.toolUseId
@ -1186,7 +1188,28 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
{**function_call_item, "status": "in_progress", "arguments": ""}
)
# A tool turn that Bedrock opens with contentStart role TOOL has no response yet, so the
# response.done below would close an id the client never saw opened.
response_created: Final[tuple[OpenAIRealtimeEvents, ...]] = (
(
OpenAIRealtimeStreamResponseBaseObject(
type="response.created",
event_id=f"event_{uuid.uuid4()}",
response={
"object": "realtime.response",
"id": response_id,
"status": "in_progress",
"output": [],
"conversation_id": conversation_id,
},
),
)
if is_new_response
else ()
)
events: Final[list[OpenAIRealtimeEvents]] = [
*response_created,
OpenAIRealtimeStreamResponseOutputItemAdded(
type="response.output_item.added",
event_id=f"event_{uuid.uuid4()}",

55
tests/test_litellm/llms/bedrock/realtime/test_bedrock_realtime_transformation.py
View file

@ -16,7 +16,8 @@ from litellm.llms.bedrock.realtime.transformation import (
)
from litellm.llms.bedrock.realtime.trigger_audio import ready_trigger_pcm
# The OpenAI realtime function-call lifecycle a single Nova Sonic toolUse must expand into.
# The OpenAI realtime function-call lifecycle a single Nova Sonic toolUse must expand into,
# when an assistant block already opened the response the tool call belongs to.
_TOOL_CALL_EVENT_SEQUENCE = [
"response.output_item.added",
"conversation.item.added",
@ -26,6 +27,10 @@ _TOOL_CALL_EVENT_SEQUENCE = [
"response.done",
]
# Nova Sonic opens tool turns with contentStart role TOOL, which opens no response, so the
# tool call has to open one itself before it can close it.
_TOOL_CALL_EVENT_SEQUENCE_NEW_RESPONSE = ["response.created", *_TOOL_CALL_EVENT_SEQUENCE]
def _only(events, event_type):
matches = [event for event in events if event["type"] == event_type]
@ -34,11 +39,10 @@ def _only(events, event_type):
def _response_id_of(event):
if event["type"] == "response.done":
"""The response id an event is bound to, or None for events that carry no response id."""
if event["type"] in ("response.created", "response.done"):
return event["response"]["id"]
if event["type"] == "conversation.item.added":
return None
return event["response_id"]
return event.get("response_id")
class TestBedrockRealtimeConfig:
@ -659,8 +663,10 @@ class TestBedrockRealtimeResponseTransformation:
None,
"conv_1",
)
assert [event["type"] for event in events] == _TOOL_CALL_EVENT_SEQUENCE
assert [event["type"] for event in events] == _TOOL_CALL_EVENT_SEQUENCE_NEW_RESPONSE
function_call = _only(events, "response.function_call_arguments.done")
assert _only(events, "response.created")["response"]["id"] == function_call["response_id"]
assert _only(events, "response.created")["response"]["conversation_id"] == "conv_1"
assert function_call["call_id"] == "tool_call_no_ids"
assert function_call["name"] == "get_weather"
assert function_call["response_id"].startswith("resp_")
@ -790,8 +796,9 @@ class TestBedrockRealtimeResponseTransformation:
realtime_response_transform_input=state,
)
assert [msg["type"] for msg in result["response"]] == _TOOL_CALL_EVENT_SEQUENCE
assert [msg["type"] for msg in result["response"]] == _TOOL_CALL_EVENT_SEQUENCE_NEW_RESPONSE
function_call = _only(result["response"], "response.function_call_arguments.done")
assert _only(result["response"], "response.created")["response"]["id"] == function_call["response_id"]
assert function_call["response_id"].startswith("resp_")
assert function_call["item_id"].startswith("item_")
assert json.loads(function_call["arguments"]) == {"location": "Seattle"}
@ -1327,6 +1334,40 @@ class TestBedrockRealtimeContentBlockLifecycle:
assert created == done
assert state["current_response_id"] is None
def test_every_response_is_opened_and_closed_on_a_tool_first_turn(self):
"""
Nova Sonic opens tool turns with contentStart role TOOL, which emits nothing, so the
tool call is the first thing in the session. It has to open the response it closes,
or the client sees a response.done for an id it never saw created.
"""
config = BedrockRealtimeConfig()
logging_obj = MagicMock()
logging_obj.litellm_trace_id = "trace_123"
state = self._state(current_conversation_id=None)
turn = [
{"event": {"contentStart": {"role": "TOOL", "type": "TOOL"}}},
{
"event": {
"toolUse": {
"toolUseId": "tool_1",
"toolName": "get_weather",
"content": json.dumps({"location": "Seattle"}),
}
}
},
{"event": {"contentEnd": {"stopReason": "TOOL_USE", "type": "TOOL"}}},
]
emitted = [msg for event in turn for msg in self._apply(config, logging_obj, state, event)["response"]]
created = [msg["response"]["id"] for msg in emitted if msg["type"] == "response.created"]
done = [msg["response"]["id"] for msg in emitted if msg["type"] == "response.done"]
assert len(created) == 1
assert created == done
# Every event in the turn is bound to that one response.
assert {_response_id_of(msg) for msg in emitted} - {None} == set(created)
assert state["current_response_id"] is None
def test_second_assistant_content_block_reuses_response_not_item(self):
config = BedrockRealtimeConfig()
logging_obj = MagicMock()