From ba75de7848e28c59979099e6fb715e392227b6c5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 22 May 2026 21:16:29 +0000 Subject: [PATCH] fix(realtime): preserve client session.update fields on follow-up Gemini setup In non-deferred mode the auto-setup pre-populates session_configuration_request, so a later client session.update carrying tools or instructions used to fall into the subsequent path and only forward turn_detection. Rebuild a merged follow-up setup that overlays the new client fields on top of the original setup so tools/instructions/etc. are no longer silently dropped. --- .../llms/gemini/realtime/transformation.py | 110 ++++---- .../test_gemini_realtime_transformation.py | 247 +++++++++++++----- 2 files changed, 225 insertions(+), 132 deletions(-) diff --git a/litellm/llms/gemini/realtime/transformation.py b/litellm/llms/gemini/realtime/transformation.py index af3cf5579a7..b69f65a8cdb 100644 --- a/litellm/llms/gemini/realtime/transformation.py +++ b/litellm/llms/gemini/realtime/transformation.py @@ -297,84 +297,62 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): On the FIRST session.update (when session_configuration_request is None), the full setup with all configuration is sent. - Subsequent session.update messages are normally ignored because Gemini - doesn't support dynamic updates of arbitrary fields. As an exception, - a session.update that carries a ``turn_detection`` change (used by the - guardrail layer to disable VAD auto-response so transcription guardrails - can gate replies) is forwarded as a follow-up setup carrying just the - ``realtimeInputConfig`` change. + Subsequent session.update messages are forwarded as a follow-up setup + with the new fields merged into the original setup. Gemini Live treats + a follow-up BidiGenerateContentSetup as a full session replacement + rather than a partial merge, so we carry forward the previous setup + (tools, generationConfig, inputAudioTranscription, systemInstruction, + ...) and overlay the new fields on top. This preserves the old + behavior where clients could refine the session via session.update + (e.g. add tools after the auto-setup on connect), and also keeps the + guardrail-driven turn_detection update working. """ session_payload = json_message.get("session") or {} + # Normalize GA-remapped fields (``output_modalities``, + # nested ``audio.input.transcription``, + # ``audio.input.turn_detection``) back to their flat beta keys so + # ``map_openai_params`` picks them up. Without this, GA clients' + # explicit modality / transcription / turn-detection settings + # would be silently dropped because ``map_openai_params`` only + # recognises the flat OpenAI-beta key names. + session_payload = self._normalize_session_payload_for_mapping(session_payload) + new_overrides = self.map_openai_params( + optional_params={}, non_default_params=session_payload + ) + if session_configuration_request is None: - # First session.update - send the setup with all configuration. - # Normalize GA-remapped fields (``output_modalities``, - # nested ``audio.input.transcription``, - # ``audio.input.turn_detection``) back to their flat beta keys so - # ``map_openai_params`` picks them up. Without this, GA clients' - # explicit modality / transcription / turn-detection settings - # would be silently dropped because ``map_openai_params`` only - # recognises the flat OpenAI-beta key names. - session_payload = self._normalize_session_payload_for_mapping( - session_payload - ) - client_session_configuration_request = self.map_openai_params( - optional_params={}, non_default_params=session_payload - ) - generation_config = client_session_configuration_request.setdefault( - "generationConfig", {} - ) + generation_config = new_overrides.setdefault("generationConfig", {}) generation_config.setdefault("responseModalities", ["AUDIO"]) - client_session_configuration_request.setdefault( - "inputAudioTranscription", {} - ) - client_session_configuration_request["model"] = f"models/{model}" - gemini_setup_msg = json.dumps( - {"setup": client_session_configuration_request} - ) + new_overrides.setdefault("inputAudioTranscription", {}) + new_overrides["model"] = f"models/{model}" verbose_logger.debug( "Gemini Realtime: Sending initial setup with tools to backend" ) - return [gemini_setup_msg] + return [json.dumps({"setup": new_overrides})] - # Subsequent session.update: only forward if it carries a - # turn_detection change (e.g. guardrail-injected disable of VAD - # auto-response). Anything else is dropped because Gemini doesn't - # support dynamic updates. - turn_detection = self._extract_turn_detection(session_payload) - if turn_detection is not None: - transformed_audio_activity_config = self.map_automatic_turn_detection( - cast(OpenAIRealtimeTurnDetection, turn_detection) + if not new_overrides: + verbose_logger.debug( + "Gemini Realtime: Ignoring session.update (no mappable fields)" ) - if len(transformed_audio_activity_config) > 0: - # Gemini Live treats a subsequent BidiGenerateContentSetup as a - # full session replacement rather than a partial merge. Carry - # forward the original setup (tools, generationConfig, - # inputAudioTranscription, systemInstruction, ...) and only - # override realtimeInputConfig so the guardrail-driven VAD - # change doesn't silently drop tools or other config. - try: - original_setup = cast( - BidiGenerateContentSetup, - json.loads(session_configuration_request).get("setup", {}), - ) - except (json.JSONDecodeError, AttributeError): - original_setup = {} - follow_up_setup: BidiGenerateContentSetup = { - **original_setup, - "model": f"models/{model}", - "realtimeInputConfig": BidiGenerateContentRealtimeInputConfig( - automaticActivityDetection=transformed_audio_activity_config - ), - } - verbose_logger.debug( - "Gemini Realtime: Forwarding turn_detection-only session.update as setup" - ) - return [json.dumps({"setup": follow_up_setup})] + return [] + try: + original_setup = cast( + BidiGenerateContentSetup, + json.loads(session_configuration_request).get("setup", {}), + ) + except (json.JSONDecodeError, AttributeError): + original_setup = {} + + follow_up_setup: BidiGenerateContentSetup = { + **original_setup, + **new_overrides, + "model": f"models/{model}", + } verbose_logger.debug( - "Gemini Realtime: Ignoring session.update (setup already sent)" + "Gemini Realtime: Forwarding session.update as follow-up setup" ) - return [] + return [json.dumps({"setup": follow_up_setup})] def _handle_conversation_item(self, json_message: dict) -> List[str]: """ diff --git a/tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py b/tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py index 409e0734ddc..3c51c209e92 100644 --- a/tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py +++ b/tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py @@ -81,7 +81,10 @@ def test_session_created_does_not_overwrite_session_configuration_request(): ) # Must keep original setup payload (with "setup"), not overwrite with session.created event. - assert transformed["session_configuration_request"] == session_configuration_request_str + assert ( + transformed["session_configuration_request"] + == session_configuration_request_str + ) # Also verify emitted session.created reflects audio modality from setup payload. session_created = transformed["response"][0] @@ -287,7 +290,7 @@ def test_gemini_3_1_flash_live_preview_model_cost_map_entry(): def test_gemini_realtime_tool_call_transformation(): """Test transformation of Gemini toolCall to OpenAI function_call_arguments.done format.""" config = GeminiRealtimeConfig() - + # Gemini toolCall message format gemini_tool_call = { "toolCall": { @@ -300,11 +303,11 @@ def test_gemini_realtime_tool_call_transformation(): ] } } - + gemini_tool_call_str = json.dumps(gemini_tool_call) logging_obj = MagicMock() logging_obj.litellm_trace_id = "test-trace-123" - + # Transform the toolCall message result = config.transform_realtime_response( gemini_tool_call_str, @@ -320,27 +323,29 @@ def test_gemini_realtime_tool_call_transformation(): "current_delta_type": None, }, ) - + print("Tool call transformation result:", json.dumps(result, indent=2)) - + # Verify the transformation responses = result["response"] assert len(responses) > 0, "Expected at least one response event" - + # Find the function_call_arguments.done event function_call_event = None for event in responses: if event.get("type") == "response.function_call_arguments.done": function_call_event = event break - - assert function_call_event is not None, "Expected function_call_arguments.done event" + + assert ( + function_call_event is not None + ), "Expected function_call_arguments.done event" assert function_call_event["call_id"] == "call_123" assert function_call_event["name"] == "get_weather" assert function_call_event["response_id"] == "resp_123" assert function_call_event["item_id"] == "item_123_tool_0" assert function_call_event["output_index"] == 0 - + # Verify arguments are properly serialized as JSON string args = json.loads(function_call_event["arguments"]) assert args["location"] == "San Francisco" @@ -350,7 +355,7 @@ def test_gemini_realtime_tool_call_transformation(): def test_gemini_realtime_session_update_with_tools(): """Test transformation of OpenAI session.update with tools to Gemini setup format.""" config = GeminiRealtimeConfig() - + # OpenAI format session update with tools session_update = { "type": "session.update", @@ -384,28 +389,30 @@ def test_gemini_realtime_session_update_with_tools(): ], }, } - + # Transform to Gemini format (first session.update, so setup should be sent) messages = config.transform_realtime_request( - json.dumps(session_update), "gemini-2.5-flash", session_configuration_request=None + json.dumps(session_update), + "gemini-2.5-flash", + session_configuration_request=None, ) - + assert len(messages) == 1, "Expected one setup message" - + gemini_setup = json.loads(messages[0]) assert "setup" in gemini_setup - + setup_config = gemini_setup["setup"] - + # Verify tools are at top level, not in generationConfig assert "tools" in setup_config assert "tools" not in setup_config.get("generationConfig", {}) - + # Verify tool structure matches Gemini format tools = setup_config["tools"] assert len(tools) == 1 assert "function_declarations" in tools[0] - + function_decl = tools[0]["function_declarations"][0] assert function_decl["name"] == "get_weather" assert "Get the current weather" in function_decl["description"] @@ -424,7 +431,9 @@ def test_gemini_session_update_defaults_to_audio_modality(): } messages = config.transform_realtime_request( - json.dumps(session_update), "gemini-2.5-flash", session_configuration_request=None + json.dumps(session_update), + "gemini-2.5-flash", + session_configuration_request=None, ) assert len(messages) == 1 @@ -458,17 +467,19 @@ def test_gemini_realtime_function_call_output_transformation(): logging_obj = MagicMock() logging_obj.litellm_trace_id = "trace_func_output" config.transform_realtime_response( - json.dumps({ - "toolCall": { - "functionCalls": [ - { - "id": "call_123", - "name": "get_weather", - "args": {"location": "San Francisco"}, - } - ] + json.dumps( + { + "toolCall": { + "functionCalls": [ + { + "id": "call_123", + "name": "get_weather", + "args": {"location": "San Francisco"}, + } + ] + } } - }), + ), "gemini-2.5-flash", logging_obj, realtime_response_transform_input={ @@ -489,29 +500,33 @@ def test_gemini_realtime_function_call_output_transformation(): "item": { "type": "function_call_output", "call_id": "call_123", - "output": json.dumps({ - "location": "San Francisco", - "temperature": 72, - "unit": "fahrenheit", - "conditions": "sunny", - }), + "output": json.dumps( + { + "location": "San Francisco", + "temperature": 72, + "unit": "fahrenheit", + "conditions": "sunny", + } + ), }, } - + # Transform to Gemini format messages = config.transform_realtime_request( - json.dumps(function_output), "gemini-2.5-flash", session_configuration_request="existing" + json.dumps(function_output), + "gemini-2.5-flash", + session_configuration_request="existing", ) - + assert len(messages) == 1, "Expected one toolResponse message" - + gemini_response = json.loads(messages[0]) assert "toolResponse" in gemini_response - + tool_response = gemini_response["toolResponse"] assert "functionResponses" in tool_response assert len(tool_response["functionResponses"]) == 1 - + func_response = tool_response["functionResponses"][0] assert func_response["id"] == "call_123" assert func_response["name"] == "get_weather" @@ -523,31 +538,35 @@ def test_gemini_realtime_function_call_output_transformation(): def test_gemini_realtime_user_text_transformation(): """Test transformation of OpenAI user message to Gemini clientContent format.""" config = GeminiRealtimeConfig() - + # OpenAI format user message user_message = { "type": "conversation.item.create", "item": { "type": "message", "role": "user", - "content": [{"type": "input_text", "text": "What's the weather in London?"}], + "content": [ + {"type": "input_text", "text": "What's the weather in London?"} + ], }, } - + # Transform to Gemini format messages = config.transform_realtime_request( - json.dumps(user_message), "gemini-2.5-flash", session_configuration_request="existing" + json.dumps(user_message), + "gemini-2.5-flash", + session_configuration_request="existing", ) - + assert len(messages) == 1, "Expected one clientContent message" - + gemini_message = json.loads(messages[0]) assert "clientContent" in gemini_message - + client_content = gemini_message["clientContent"] assert "turns" in client_content assert len(client_content["turns"]) == 1 - + turn = client_content["turns"][0] assert turn["role"] == "user" assert len(turn["parts"]) == 1 @@ -661,19 +680,19 @@ def test_gemini_tool_call_emits_response_created_preamble(): config = GeminiRealtimeConfig() logging_obj = MagicMock() logging_obj.litellm_trace_id = "trace_123" - + gemini_tool_call = { "toolCall": { "functionCalls": [ { "id": "call_123", "name": "get_weather", - "args": {"location": "San Francisco", "unit": "fahrenheit"} + "args": {"location": "San Francisco", "unit": "fahrenheit"}, } ] } } - + # Transform with current_response_id=None to trigger preamble emission result = config.transform_realtime_response( json.dumps(gemini_tool_call), @@ -689,7 +708,7 @@ def test_gemini_tool_call_emits_response_created_preamble(): "current_delta_type": None, }, ) - + responses = result["response"] # Should have: response.created, output_item.added, function_call_arguments.done, output_item.done, conversation.item.created, response.done assert len(responses) >= 6 @@ -782,13 +801,15 @@ def test_gemini_tool_call_resets_ids_for_post_tool_model_turn(): post_tool_events = post_tool_result["response"] assert post_tool_events[0]["type"] == "response.created" assert post_tool_events[0]["response"]["id"] != tool_response_id - assert post_tool_result["current_response_id"] == post_tool_events[0]["response"]["id"] + assert ( + post_tool_result["current_response_id"] == post_tool_events[0]["response"]["id"] + ) def test_gemini_function_call_output_includes_name(): """Verify function_call_output includes name field from stored mapping.""" config = GeminiRealtimeConfig() - + # First, receive a toolCall from Gemini (this stores the call_id → name mapping) gemini_tool_call = { "toolCall": { @@ -796,15 +817,15 @@ def test_gemini_function_call_output_includes_name(): { "id": "call_123", "name": "get_weather", - "args": {"location": "San Francisco"} + "args": {"location": "San Francisco"}, } ] } } - + logging_obj = MagicMock() logging_obj.litellm_trace_id = "trace_123" - + config.transform_realtime_response( json.dumps(gemini_tool_call), "gemini-2.5-flash", @@ -819,34 +840,128 @@ def test_gemini_function_call_output_includes_name(): "current_delta_type": None, }, ) - + # Verify mapping was stored assert "call_123" in config._tool_call_id_to_name assert config._tool_call_id_to_name["call_123"] == "get_weather" - + # Now send a function_call_output back (this should include the name) function_output = { "type": "conversation.item.create", "item": { "type": "function_call_output", "call_id": "call_123", - "output": json.dumps({"result": "72 degrees"}) - } + "output": json.dumps({"result": "72 degrees"}), + }, } - + result = config.transform_realtime_request( json.dumps(function_output), "gemini-2.5-flash", session_configuration_request="{}", ) - + assert len(result) == 1 tool_response = json.loads(result[0]) assert "toolResponse" in tool_response assert "functionResponses" in tool_response["toolResponse"] assert len(tool_response["toolResponse"]["functionResponses"]) == 1 - + function_response = tool_response["toolResponse"]["functionResponses"][0] assert function_response["id"] == "call_123" assert function_response["name"] == "get_weather" # ✅ Name is included assert "response" in function_response + + +def test_gemini_subsequent_session_update_forwards_tools_merged_with_original_setup(): + """A client session.update sent after the auto-setup must forward tools/ + instructions as a follow-up setup, merged with the original setup so we + don't drop the pre-existing config (model, generationConfig, etc.).""" + config = GeminiRealtimeConfig() + + original_setup = { + "setup": { + "model": "models/gemini-2.5-flash-native-audio", + "generationConfig": {"responseModalities": ["AUDIO"]}, + "inputAudioTranscription": {}, + "systemInstruction": {"role": "user", "parts": [{"text": "original"}]}, + } + } + + session_update = { + "type": "session.update", + "session": { + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather.", + "parameters": { + "type": "object", + "properties": {"location": {"type": "string"}}, + "required": ["location"], + }, + }, + } + ], + "instructions": "Be concise.", + }, + } + + messages = config.transform_realtime_request( + json.dumps(session_update), + "gemini-2.5-flash-native-audio", + session_configuration_request=json.dumps(original_setup), + ) + + assert len(messages) == 1 + follow_up = json.loads(messages[0])["setup"] + assert "tools" in follow_up + assert follow_up["tools"][0]["function_declarations"][0]["name"] == "get_weather" + # systemInstruction overwritten by client's instructions + assert follow_up["systemInstruction"]["parts"][0]["text"] == "Be concise." + # Original generationConfig / model / inputAudioTranscription preserved + assert follow_up["generationConfig"]["responseModalities"] == ["AUDIO"] + assert follow_up["model"] == "models/gemini-2.5-flash-native-audio" + assert follow_up["inputAudioTranscription"] == {} + + +def test_gemini_subsequent_session_update_with_turn_detection_only_preserves_original_tools(): + """A subsequent session.update carrying only turn_detection (the + guardrail-injected disable) must keep the original tools/generationConfig.""" + config = GeminiRealtimeConfig() + + original_setup = { + "setup": { + "model": "models/gemini-2.5-flash-native-audio", + "generationConfig": {"responseModalities": ["AUDIO"]}, + "inputAudioTranscription": {}, + "tools": [ + { + "function_declarations": [ + {"name": "lookup", "description": "x", "parameters": {}} + ] + } + ], + } + } + + session_update = { + "type": "session.update", + "session": {"turn_detection": {"create_response": False}}, + } + + messages = config.transform_realtime_request( + json.dumps(session_update), + "gemini-2.5-flash-native-audio", + session_configuration_request=json.dumps(original_setup), + ) + + assert len(messages) == 1 + follow_up = json.loads(messages[0])["setup"] + assert follow_up["tools"] == original_setup["setup"]["tools"] + assert ( + follow_up["realtimeInputConfig"]["automaticActivityDetection"]["disabled"] + is True + )