From 58089dfcb742171c8bb08199a805035b30bfda60 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 17:53:15 -0700 Subject: [PATCH] fix(realtime): ack unchanged Gemini Live session.update resends and honor null turn_detection An update that leaves the applied setup unchanged now gets session.updated instead of session_update_not_applied, so clients that resend their session with the first response keep working. Deferred setup mode merges held updates like the default mode, turn_detection null turns Gemini's activity detection off, drop_params drops a voice outside the documented Live voices, and the lazy setup deadline moves to 240 seconds --- litellm/constants.py | 2 +- .../litellm_core_utils/realtime_streaming.py | 56 ++++++++--- .../llms/gemini/realtime/transformation.py | 98 ++++++++++++++----- .../test_realtime_streaming.py | 50 ++++++++-- .../test_gemini_realtime_transformation.py | 45 ++++++++- 5 files changed, 204 insertions(+), 47 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index a4ec377c707..67ce2c806de 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -416,7 +416,7 @@ REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS: Final = get_env_float( "REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS", 20.0 ) BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS: Final = get_env_float("BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS", 5.0) -REALTIME_LAZY_SETUP_DEADLINE_SECONDS: Final = 60.0 +REALTIME_LAZY_SETUP_DEADLINE_SECONDS: Final = 240.0 # RFC 6455 caps the close frame payload at 125 bytes, 2 of which carry the status code WEBSOCKET_CLOSE_REASON_MAX_BYTES: Final = 123 diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index a9aba6d460c..897bd53c14d 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -350,6 +350,7 @@ class RealTimeStreaming: # content (realtimeInput / clientContent / toolResponse) has been sent. self._content_sent_after_setup: bool = False self._held_session_update: Mapping[str, object] | None = None + self._applied_session_fields: Mapping[str, object] | None = None self._held_session_update_bytes: int = 0 # Whether this is a transcription-only session (session.type == "transcription", # e.g. gpt-realtime-whisper). Such sessions must not be sent response.create and @@ -661,7 +662,7 @@ class RealTimeStreaming: def _builds_setup_lazily(self) -> bool: if self.provider_config is None: return False - return self.provider_config.builds_setup_from_session_update() and not self._uses_deferred_backend_setup() + return self.provider_config.builds_setup_from_session_update() def _holds_session_updates_until_first_content(self) -> bool: return self.session_configuration_request is None and self._builds_setup_lazily() @@ -672,10 +673,12 @@ class RealTimeStreaming: return False client_event: Final = _decode_json_object(message) session: Final = client_event.get("session") + update: Final = _SESSION_FIELDS.validate_python(session) if isinstance(session, Mapping) else _NO_SESSION_FIELDS + if self._leaves_applied_setup_unchanged(update): + await self._acknowledge_session_setup(self._applied_session_fields or _NO_SESSION_FIELDS) + return True rejection: Final = provider_config.setup_session_update_rejection( - self.model, - _SESSION_FIELDS.validate_python(session) if isinstance(session, Mapping) else _NO_SESSION_FIELDS, - setup_sent=self.session_configuration_request is not None, + self.model, update, setup_sent=self.session_configuration_request is not None ) if rejection is None: return False @@ -694,18 +697,37 @@ class RealTimeStreaming: return session: Final = client_event.get("session") update: Final = _SESSION_FIELDS.validate_python(session) if isinstance(session, Mapping) else _NO_SESSION_FIELDS - self._held_session_update = _merge_session_fields(self._held_session_update or _NO_SESSION_FIELDS, update) - held_message: Final = self._held_session_update_message() - self._held_session_update_bytes = len(held_message.encode("utf-8")) - await self._acknowledge_held_session_update(held_message) + held: Final = _merge_session_fields(self._held_session_update or _NO_SESSION_FIELDS, update) + self._held_session_update = held + self._held_session_update_bytes = len(_session_update_frame(held).encode("utf-8")) + await self._acknowledge_session_setup(held) - async def _acknowledge_held_session_update(self, held_message: str) -> None: + def _setup_preview(self, session: Mapping[str, object]) -> str | None: if self.provider_config is None: - return - preview: Final = self.provider_config.transform_realtime_request(held_message, self.model, None) - setup: Final = next( + return None + preview: Final = self.provider_config.transform_realtime_request( + _session_update_frame(session), self.model, None + ) + return next( (frame for frame in preview if isinstance(frame, str) and "setup" in _decode_json_object(frame)), None ) + + def _leaves_applied_setup_unchanged(self, update: Mapping[str, object]) -> bool: + applied: Final = self._applied_session_fields + if self.session_configuration_request is None or applied is None: + return False + applied_setup: Final = self._setup_preview(applied) + merged_setup: Final = self._setup_preview(_merge_session_fields(applied, update)) + return ( + applied_setup is not None + and merged_setup is not None + and _decode_json_object(applied_setup) == _decode_json_object(merged_setup) + ) + + async def _acknowledge_session_setup(self, session: Mapping[str, object]) -> None: + if self.provider_config is None: + return + setup: Final = self._setup_preview(session) if setup is None: return acknowledged: Final = self.provider_config.transform_session_created_event( @@ -719,12 +741,14 @@ class RealTimeStreaming: await self._send_event_to_client(event, event_str) async def _send_held_session_update_as_setup(self) -> None: - if self._held_session_update is None: + held: Final = self._held_session_update + if held is None: return message: Final = self._held_session_update_message() self._held_session_update = None self._held_session_update_bytes = 0 - await self._send_to_backend(message) + if await self._send_to_backend(message): + self._applied_session_fields = held async def _send_setup_before_content(self, msg_type: str | None) -> None: if self._holds_session_updates_until_first_content(): @@ -754,6 +778,7 @@ class RealTimeStreaming: setup: Final = self.maybe_inject_guardrail_auto_response_disable(default_setup) await self.backend_ws.send(setup) self._cache_session_configuration_request(setup) + self._applied_session_fields = _NO_SESSION_FIELDS def _enforce_transcription_session_model(self, message: str) -> str: """Force client transcription session updates to the authorized model. @@ -1817,6 +1842,9 @@ class RealTimeStreaming: # actually forward to the backend. self.store_input(message=message) + if msg_type != "session.update" and self._uses_deferred_backend_setup(): + await self._send_setup_before_content(msg_type) + if self._should_buffer_client_message_until_setup(message): self._buffer_pending_message_until_setup(message) continue diff --git a/litellm/llms/gemini/realtime/transformation.py b/litellm/llms/gemini/realtime/transformation.py index 508cbebdd6d..06965fcc4e4 100644 --- a/litellm/llms/gemini/realtime/transformation.py +++ b/litellm/llms/gemini/realtime/transformation.py @@ -84,21 +84,67 @@ OPENAI_STOCK_REALTIME_VOICES: Final[frozenset[str]] = frozenset( {"alloy", "ash", "ballad", "cedar", "coral", "echo", "marin", "sage", "shimmer", "verse"} ) +# Per https://docs.cloud.google.com/vertex-ai/generative-ai/docs/live-api/configure-language-voice (2026-10-28) +GEMINI_LIVE_DOCUMENTED_VOICES: Final[frozenset[str]] = frozenset( + { + "achernar", + "achird", + "algenib", + "algieba", + "alnilam", + "aoede", + "autonoe", + "callirrhoe", + "charon", + "despina", + "enceladus", + "erinome", + "fenrir", + "gacrux", + "iapetus", + "kore", + "laomedeia", + "leda", + "orus", + "puck", + "pulcherrima", + "rasalgethi", + "sadachbia", + "sadaltager", + "schedar", + "sulafat", + "umbriel", + "vindemiatrix", + "zephyr", + "zubenelgenubi", + } +) -def _gemini_live_speech_config(voice: object) -> Mapping[str, object] | None: + +def _gemini_live_speech_config(voice: object, drop_params: bool) -> Mapping[str, object] | None: """Build the Gemini Live speechConfig for a client-requested voice. - OpenAI stock voice names have no Gemini equivalent and Gemini Live closes - the session on an unknown voice, so they are dropped with a warning and - the model keeps its default voice. Every other name is forwarded verbatim. + Gemini Live closes the session on a voice name it does not know. OpenAI + stock voice names are always dropped, and with drop_params so is any name + outside Google's documented Live voices, keeping the model's default voice. + Every other name is forwarded verbatim for Gemini to accept or refuse. """ - if isinstance(voice, str) and voice.lower() in OPENAI_STOCK_REALTIME_VOICES: + if not isinstance(voice, str): + return VertexGeminiConfig().map_audio_params({"voice": voice}) + if voice.lower() in OPENAI_STOCK_REALTIME_VOICES: verbose_logger.warning( "Gemini Realtime: voice %s is an OpenAI voice with no Gemini equivalent; " "dropping it so the session keeps the model's default voice.", voice, ) return None + if drop_params and voice.lower() not in GEMINI_LIVE_DOCUMENTED_VOICES: + verbose_logger.warning( + "Gemini Realtime: voice %s is not a documented Gemini Live voice; " + "drop_params is set, so it is dropped and the session keeps the model's default voice.", + voice, + ) + return None return VertexGeminiConfig().map_audio_params({"voice": voice}) @@ -373,30 +419,36 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): ) elif key == "input_audio_transcription" and value is not None: optional_params["inputAudioTranscription"] = {} - elif key == "turn_detection" and value is not None: - value_typed = cast(OpenAIRealtimeTurnDetection, value) - if ( - isinstance(value_typed, dict) - and value_typed.get("type") == "semantic_vad" - and "create_response" not in value_typed - ): - # Pipecat/OpenAI GA semantic VAD — skip; Gemini uses its own VAD. - # Only skip when there is no create_response override so that - # a guardrail-injected create_response:false is not dropped. - continue - transformed_audio_activity_config = self.map_automatic_turn_detection(value_typed) - if transformed_audio_activity_config: - optional_params["realtimeInputConfig"] = BidiGenerateContentRealtimeInputConfig( - automaticActivityDetection=transformed_audio_activity_config - ) + elif key == "turn_detection": + realtime_input_config = self._realtime_input_config(cast(object, value)) + if realtime_input_config is not None: + optional_params["realtimeInputConfig"] = realtime_input_config elif key == "voice": - speech_config = _gemini_live_speech_config(value) + speech_config = _gemini_live_speech_config(value, self.drop_params) if speech_config: optional_params["generationConfig"]["speechConfig"] = speech_config if len(optional_params["generationConfig"]) == 0: optional_params.pop("generationConfig") return optional_params + def _realtime_input_config(self, turn_detection: object) -> BidiGenerateContentRealtimeInputConfig | None: + if turn_detection is None: + return BidiGenerateContentRealtimeInputConfig(automaticActivityDetection={"disabled": True}) + value_typed: Final = cast(OpenAIRealtimeTurnDetection, turn_detection) + if ( + isinstance(value_typed, dict) + and value_typed.get("type") == "semantic_vad" + and "create_response" not in value_typed + ): + # Pipecat/OpenAI GA semantic VAD — skip; Gemini uses its own VAD. + # Only skip when there is no create_response override so that + # a guardrail-injected create_response:false is not dropped. + return None + transformed_audio_activity_config: Final = self.map_automatic_turn_detection(value_typed) + if not transformed_audio_activity_config: + return None + return BidiGenerateContentRealtimeInputConfig(automaticActivityDetection=transformed_audio_activity_config) + @staticmethod def _extract_turn_detection(session: dict) -> dict | None: """Extract turn_detection from a session.update payload. @@ -445,6 +497,8 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): if isinstance(input_cfg, dict): if "input_audio_transcription" not in normalized and "transcription" in input_cfg: normalized["input_audio_transcription"] = input_cfg["transcription"] + if "turn_detection" not in normalized and "turn_detection" in input_cfg: + normalized["turn_detection"] = input_cfg["turn_detection"] output_cfg: Final = audio.get("output") if isinstance(output_cfg, dict) and output_cfg.get("voice"): normalized["voice"] = output_cfg["voice"] diff --git a/tests/unit/litellm_core_utils/test_realtime_streaming.py b/tests/unit/litellm_core_utils/test_realtime_streaming.py index 41a5ded28aa..c530df79c89 100644 --- a/tests/unit/litellm_core_utils/test_realtime_streaming.py +++ b/tests/unit/litellm_core_utils/test_realtime_streaming.py @@ -2856,7 +2856,9 @@ async def test_default_mode_sends_the_held_setup_when_no_content_arrives_before_ async def late_update_once_the_setup_is_out() -> str: await setup_sent.wait() - return _LATE_INSTRUCTIONS_UPDATE + return json.dumps( + {"type": "session.update", "event_id": "evt_late", "session": {"instructions": "Answer in French."}} + ) async def disconnect() -> str: raise ConnectionClosed(None, None) @@ -2909,9 +2911,29 @@ async def test_default_mode_refuses_held_session_updates_past_the_size_cap(monke @pytest.mark.asyncio -async def test_deferred_setup_answers_a_second_session_update_with_an_error(monkeypatch): - """Gemini Live ignores a second setup, so with deferred setup the second - session.update gets the same not-applied error as in the default mode.""" +async def test_default_mode_acknowledges_an_unchanged_session_update_after_content(monkeypatch): + """Pipecat resends its unchanged session.update right after its first + conversation items and treats any error event as fatal.""" + session_update: Final = json.dumps( + {"type": "session.update", "session": {"modalities": ["text"], "instructions": "Be brief."}} + ) + streaming, backend_ws = _gemini_default_mode_streaming( + monkeypatch, [session_update, _TEXT_ITEM_CREATE, session_update], model="gemini-live-2.5-flash" + ) + + await streaming.client_ack_messages() + + frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] + assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"] + events: Final = _events_sent_to_client(streaming) + assert [event["type"] for event in events] == ["session.updated", "session.updated"] + assert events[0]["session"] == events[1]["session"] + + +@pytest.mark.asyncio +async def test_deferred_setup_merges_session_updates_until_the_first_content(monkeypatch): + """Deferred setup merges every session.update sent before the first content, + the same as the default mode, and answers a changed one after it as late.""" monkeypatch.setattr(litellm, "gemini_live_defer_setup", True, raising=False) from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig @@ -2919,7 +2941,11 @@ async def test_deferred_setup_answers_a_second_session_update_with_an_error(monk client_ws.receive_text = AsyncMock( side_effect=[ json.dumps({"type": "session.update", "session": {"modalities": ["text"]}}), - _LATE_INSTRUCTIONS_UPDATE, + json.dumps({"type": "session.update", "session": {"instructions": "Be brief."}}), + _TEXT_ITEM_CREATE, + json.dumps( + {"type": "session.update", "event_id": "evt_late", "session": {"instructions": "Answer in French."}} + ), ConnectionClosed(None, None), ] ) @@ -2933,7 +2959,9 @@ async def test_deferred_setup_answers_a_second_session_update_with_an_error(monk await streaming.client_ack_messages() frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] - assert [next(iter(frame)) for frame in frames] == ["setup"] + assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"] + assert frames[0]["setup"]["generationConfig"]["responseModalities"] == ["TEXT"] + assert frames[0]["setup"]["systemInstruction"]["parts"][0]["text"] == "Be brief." errors: Final = [event["error"] for event in _events_sent_to_client(streaming) if event["type"] == "error"] assert [(error["code"], error["event_id"]) for error in errors] == [("session_update_not_applied", "evt_late")] @@ -2973,14 +3001,18 @@ async def test_deferred_setup_buffers_audio_until_backend_setup_complete(monkeyp @pytest.mark.asyncio -async def test_deferred_setup_sends_session_update_before_buffered_audio(monkeypatch): +async def test_deferred_setup_sends_the_held_session_update_when_more_audio_arrives(monkeypatch): monkeypatch.setattr(litellm, "gemini_live_defer_setup", True, raising=False) from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig client_ws = MagicMock() + client_ws.send_text = AsyncMock() audio_msg = json.dumps({"type": "input_audio_buffer.append", "audio": "AA=="}) session_update = json.dumps({"type": "session.update", "session": {"modalities": ["audio"]}}) - client_ws.receive_text = AsyncMock(side_effect=[audio_msg, session_update, ConnectionClosed(None, None)]) + later_audio_msg = json.dumps({"type": "input_audio_buffer.append", "audio": "AQ=="}) + client_ws.receive_text = AsyncMock( + side_effect=[audio_msg, session_update, later_audio_msg, ConnectionClosed(None, None)] + ) backend_ws = MagicMock() backend_ws.send = AsyncMock() logging_obj: Final = _mock_logging() @@ -3000,7 +3032,7 @@ async def test_deferred_setup_sends_session_update_before_buffered_audio(monkeyp sent_payload = json.loads(backend_ws.send.await_args_list[0].args[0]) assert "setup" in sent_payload assert "realtimeInput" not in sent_payload - assert streaming._pending_messages_until_setup == [audio_msg] + assert streaming._pending_messages_until_setup == [audio_msg, later_audio_msg] @pytest.mark.asyncio diff --git a/tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py b/tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py index 141c816bf35..9e35e07fd54 100644 --- a/tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py +++ b/tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py @@ -8,7 +8,7 @@ import pytest import litellm -from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig +from litellm.llms.gemini.realtime.transformation import GEMINI_LIVE_DOCUMENTED_VOICES, GeminiRealtimeConfig from litellm.types.llms.gemini import BidiGenerateContentServerMessage @@ -1461,6 +1461,27 @@ def test_gemini_manual_vad_activity_markers_are_empty_messages(): assert [json.loads(frame) for frame in turn_end] == [{"realtimeInput": {"activityEnd": {}}}] +@pytest.mark.parametrize( + ("session", "expected_keys"), + [ + ({"turn_detection": None}, [["activityStart", "audio"], ["activityEnd"]]), + ({"type": "realtime", "audio": {"input": {"turn_detection": None}}}, [["activityStart", "audio"], ["activityEnd"]]), + ({"type": "realtime", "audio": {"input": {}}}, [["audio"], ["audioStreamEnd"]]), + ], +) +def test_gemini_null_turn_detection_makes_the_client_mark_each_turn(session, expected_keys): + """OpenAI's turn_detection null turns server VAD off, so the client commits each turn itself.""" + config = GeminiRealtimeConfig() + setup = config.transform_realtime_request( + json.dumps({"type": "session.update", "session": session}), "gemini-live-2.5-flash-native-audio", None + ) + commit = json.dumps({"type": "input_audio_buffer.commit"}) + + keys = [_realtime_input_keys(config, event, setup[0]) for event in (_APPEND, commit)] + + assert keys == expected_keys + + def test_gemini_automatic_vad_sends_no_activity_markers(): config = GeminiRealtimeConfig() commit = json.dumps({"type": "input_audio_buffer.commit"}) @@ -1948,6 +1969,28 @@ def test_map_openai_params_drops_stock_voice_case_insensitively(): assert passthrough["generationConfig"]["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Kore" +_DOCUMENTED_VOICE: Final = sorted(GEMINI_LIVE_DOCUMENTED_VOICES)[0].capitalize() + + +@pytest.mark.parametrize( + ("voice", "drop_params", "forwarded"), + [ + ("Notavoice", True, None), + ("Notavoice", False, "Notavoice"), + (_DOCUMENTED_VOICE, True, _DOCUMENTED_VOICE), + ], +) +def test_map_openai_params_drops_an_undocumented_voice_only_with_drop_params(voice, drop_params, forwarded): + """Gemini Live closes the session on a voice it does not know, and AI Studio knows + names Vertex AI refuses, so only drop_params narrows to the documented voices.""" + mapped = GeminiRealtimeConfig(drop_params=drop_params).map_openai_params( + optional_params={}, non_default_params={"voice": voice} + ) + + speech_config = mapped.get("generationConfig", {}).get("speechConfig") + assert (speech_config["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] if speech_config else None) == forwarded + + @pytest.fixture(autouse=False) def patch_gemini_transcribe_live_cost_map_entry(monkeypatch): """Inject the gemini-3.5-transcribe-live registry entry locally.