From 4f89cbd17417cd03e0002429e4d21f2f65fd6b2c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 4 Sep 2026 21:47:43 -0700 Subject: [PATCH 01/10] fix(realtime): send Gemini Live setup on the client's first message so session.update modalities apply Gemini and Vertex AI Live accept exactly one setup per session, and the proxy sent its default audio setup at connect, so a client's session.update (GA output_modalities or beta modalities) was dropped and every answer came back as audio. The setup now goes out on the client's first message: a session.update becomes that setup, anything else gets the model default. Bedrock Nova Sonic keeps its connect-time setup through a provider hook. Text setups label deltas response.output_text.delta, and downgrading TEXT to AUDIO on an audio-only model logs a warning --- .../litellm_core_utils/realtime_streaming.py | 13 ++ .../llms/base_llm/realtime/transformation.py | 3 + litellm/llms/custom_httpx/llm_http_handler.py | 15 +- .../llms/gemini/realtime/transformation.py | 7 + .../test_realtime_streaming.py | 55 ++++++ .../custom_httpx/test_llm_http_handler.py | 184 ++++++++++++++++++ 6 files changed, 272 insertions(+), 5 deletions(-) diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index 8479e108d17..bde3f1abe57 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -440,6 +440,18 @@ class RealTimeStreaming: await self.backend_ws.send(message) return True + async def _send_default_setup_before_first_client_message(self, msg_type: str | None) -> None: + if self.session_configuration_request is not None or self.provider_config is None: + return + if msg_type == "session.update" or self._uses_deferred_backend_setup(): + return + default_setup: Final = self.provider_config.session_configuration_request(self.model) + if not default_setup: + return + setup: Final = self._maybe_inject_guardrail_auto_response_disable(default_setup) + await self.backend_ws.send(setup) + self._cache_session_configuration_request(setup) + def _enforce_transcription_session_model(self, message: str) -> str: """Force client transcription session updates to the authorized model. @@ -1477,6 +1489,7 @@ class RealTimeStreaming: # would permanently disable the injection if ``_send_to_backend`` # raised — neither this loop nor # ``_maybe_send_guardrail_turn_detection_update`` would retry. + await self._send_default_setup_before_first_client_message(msg_type) sent = await self._send_to_backend(message) if guardrail_turn_detection_injected and sent: self._guardrail_turn_detection_update_sent = True diff --git a/litellm/llms/base_llm/realtime/transformation.py b/litellm/llms/base_llm/realtime/transformation.py index cfcde7c6e9e..f6e8a8d848e 100644 --- a/litellm/llms/base_llm/realtime/transformation.py +++ b/litellm/llms/base_llm/realtime/transformation.py @@ -71,6 +71,9 @@ class BaseRealtimeConfig(ABC): def session_configuration_request(self, model: str) -> str | None: # message sent to setup the realtime session return None + def builds_setup_from_session_update(self) -> bool: + return False + def unbilled_usage_on_session_close(self, model: str) -> RealtimeInputAudioTranscriptionUsage | None: return None diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index f281c249c72..76793d2526a 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -6134,14 +6134,19 @@ class BaseLLMHTTPHandler: ), ) - # Auto-send session setup if the provider requires it (e.g. - # Gemini/Vertex AI Live needs a `setup` before any realtime_input). + # Auto-send session setup at connect unless the provider builds it from the + # client's first session.update (Gemini/Vertex AI Live accept exactly one + # setup, so RealTimeStreaming sends theirs on the first client message). # Build the streaming handler first so a transcription guardrail's # auto-response disable can be folded into this one setup: Gemini # rejects a second setup, so a follow-up disable would be dropped # and the guardrail bypassed. _session_config: str | None = None - if provider_config.requires_session_configuration(): + sends_setup_at_connect: Final = ( + provider_config.requires_session_configuration() + and not provider_config.builds_setup_from_session_update() + ) + if sends_setup_at_connect: _session_config = provider_config.session_configuration_request(model) if _session_config: _session_config = realtime_streaming._maybe_inject_guardrail_auto_response_disable( @@ -6150,9 +6155,9 @@ class BaseLLMHTTPHandler: await backend_ws.send(_session_config) realtime_streaming.session_configuration_request = _session_config - # For providers that defer setup until client session.update, optionally + # For providers that defer setup until the client's first message, optionally # send synthetic session.created to unblock clients waiting on connect. - if not provider_config.requires_session_configuration(): + if not sends_setup_at_connect: synthetic_session: Final = provider_config.transform_session_created_event( model=model, logging_session_id=logging_obj.litellm_trace_id, diff --git a/litellm/llms/gemini/realtime/transformation.py b/litellm/llms/gemini/realtime/transformation.py index c92af7de145..f155b048655 100644 --- a/litellm/llms/gemini/realtime/transformation.py +++ b/litellm/llms/gemini/realtime/transformation.py @@ -434,6 +434,10 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): modality.upper() if isinstance(modality, str) else str(modality).upper() for modality in modalities ) if GeminiRealtimeConfig._is_audio_only_live_model(model) and "TEXT" in normalized: + verbose_logger.warning( + "Gemini Live: %s only produces audio; downgrading the requested TEXT response modality to AUDIO", + model, + ) return tuple(modality for modality in normalized if modality != "TEXT") or ("AUDIO",) if GeminiRealtimeConfig._is_text_only_live_model(model) and "AUDIO" in normalized: return tuple(modality for modality in normalized if modality != "AUDIO") or ("TEXT",) @@ -1629,6 +1633,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): # Deferred setup opt-in: litellm.gemini_live_defer_setup = True return not litellm.gemini_live_defer_setup + def builds_setup_from_session_update(self) -> bool: + return True + def session_configuration_request(self, model: str) -> str: """ diff --git a/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py b/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py index 52e88db753a..e262c163612 100644 --- a/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py +++ b/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py @@ -2377,6 +2377,61 @@ async def test_follow_up_setup_updates_cached_session_configuration_request(): assert streaming.session_configuration_request == follow_up_setup +def _gemini_default_mode_streaming(monkeypatch, client_messages, model="gemini-live-2.5-flash-native-audio"): + monkeypatch.setattr(litellm, "gemini_live_defer_setup", False, raising=False) + from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig + + client_ws = MagicMock() + client_ws.receive_text = AsyncMock(side_effect=[*client_messages, ConnectionClosed(None, None)]) + backend_ws = MagicMock() + backend_ws.send = AsyncMock() + streaming = RealTimeStreaming( + client_ws, + backend_ws, + MagicMock(), + provider_config=GeminiRealtimeConfig(), + model=model, + ) + return streaming, backend_ws + + +_TEXT_ITEM_CREATE = json.dumps( + { + "type": "conversation.item.create", + "item": {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hi"}]}, + } +) + + +@pytest.mark.asyncio +async def test_default_mode_sends_default_setup_before_first_non_session_client_message(monkeypatch): + """Gemini Live setup is no longer pre-sent at connect, so a client that never + sends session.update still gets the provider default setup ahead of its + first content frame.""" + streaming, backend_ws = _gemini_default_mode_streaming(monkeypatch, [_TEXT_ITEM_CREATE]) + + await streaming.client_ack_messages() + + frames = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] + assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"] + assert frames[0]["setup"]["generationConfig"]["responseModalities"] == ["AUDIO"] + assert streaming.session_configuration_request == backend_ws.send.await_args_list[0].args[0] + + +@pytest.mark.asyncio +async def test_default_mode_session_update_first_builds_the_only_setup(monkeypatch): + session_update = json.dumps({"type": "session.update", "session": {"modalities": ["text"]}}) + streaming, backend_ws = _gemini_default_mode_streaming( + monkeypatch, [session_update, _TEXT_ITEM_CREATE], model="gemini-live-2.5-flash" + ) + + await streaming.client_ack_messages() + + frames = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] + assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"] + assert frames[0]["setup"]["generationConfig"]["responseModalities"] == ["TEXT"] + + @pytest.mark.asyncio async def test_deferred_setup_buffers_audio_until_backend_setup_complete(monkeypatch): """Pipecat may send audio before session.update when setup is deferred.""" diff --git a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py index 023e2d8843f..3708cee094f 100644 --- a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py @@ -6,6 +6,7 @@ from unittest.mock import AsyncMock, Mock, patch import httpx import pytest +import websockets.exceptions import litellm from litellm._logging import verbose_logger @@ -1887,6 +1888,189 @@ async def test_async_realtime_error_event_send_failure_still_closes(): assert client_ws.events == [("close", (1011, "Internal server error: vertex token refresh exploded"))] +_GEMINI_LIVE_SETUP_COMPLETE = json.dumps({"setupComplete": {}}) +_GEMINI_LIVE_TEXT_TURN = json.dumps({"serverContent": {"modelTurn": {"parts": [{"text": "Blue."}]}}}) +_GEMINI_LIVE_TURN_COMPLETE = json.dumps({"serverContent": {"turnComplete": True}}) + + +class _ScriptedRealtimeBackend: + """Backend socket that answers the first setup frame with the scripted replies + and closes once the client is done, so both bridge loops run to completion.""" + + def __init__(self, replies_after_setup): + self.sent = [] + self.closed = asyncio.Event() + self._replies_after_setup = list(replies_after_setup) + self._inbox = asyncio.Queue() + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, tb): + return None + + async def send(self, payload): + self.sent.append(payload) + if "setup" in json.loads(payload): + for reply in self._replies_after_setup: + self._inbox.put_nowait(reply) + + def close_after_replies(self): + self._inbox.put_nowait(None) + + async def recv(self, decode=True): + reply = await self._inbox.get() + if reply is None: + self.closed.set() + raise websockets.exceptions.ConnectionClosedOK(None, None) + return reply + + +class _ScriptedRealtimeClient: + def __init__(self, messages, backend): + self.scope = {"headers": []} + self.exceptions = websockets.exceptions + self.sent = [] + self._messages = list(messages) + self._backend = backend + + async def send_text(self, payload): + self.sent.append(payload) + + async def receive_text(self): + if self._messages: + return self._messages.pop(0) + self._backend.close_after_replies() + await self._backend.closed.wait() + raise websockets.exceptions.ConnectionClosedOK(None, None) + + async def close(self, code=None, reason=None): + return None + + +def _realtime_logging_obj(): + logging_obj = Mock() + logging_obj.litellm_trace_id = "trace-lit-6974" + logging_obj.async_success_handler = AsyncMock() + return logging_obj + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "model, session, backend_modality, client_modality, backend_turn", + [ + ( + "gemini-live-2.5-flash", + {"type": "realtime", "output_modalities": ["text"]}, + "TEXT", + "text", + [_GEMINI_LIVE_TEXT_TURN], + ), + ("gemini-live-2.5-flash", {"modalities": ["text"]}, "TEXT", "text", [_GEMINI_LIVE_TEXT_TURN]), + ( + "gemini-live-2.5-flash-native-audio", + {"type": "realtime", "output_modalities": ["text"]}, + "AUDIO", + "audio", + [], + ), + ], +) +async def test_async_realtime_vertex_first_session_update_builds_the_only_setup( + model, session, backend_modality, client_modality, backend_turn, monkeypatch +): + """Regression for text-only Vertex Live sessions answered in audio: the + connect-time default setup (AUDIO) used to go out before the client's + session.update, and Gemini accepts only one setup per connection, so the + requested modality was dropped and billed at the audio rate. The setup now + goes out on the client's first message, built from session.update when that + comes first. An audio-only model still gets AUDIO, and the client learns the + downgrade from session.updated.""" + from litellm.llms.vertex_ai.realtime.transformation import VertexAIRealtimeConfig + + monkeypatch.setattr(litellm, "gemini_live_defer_setup", False, raising=False) + backend = _ScriptedRealtimeBackend([_GEMINI_LIVE_SETUP_COMPLETE, *backend_turn, _GEMINI_LIVE_TURN_COMPLETE]) + client_ws = _ScriptedRealtimeClient( + [ + json.dumps({"type": "session.update", "session": session}), + json.dumps( + { + "type": "conversation.item.create", + "item": { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "What color is the sky? One word."}], + }, + } + ), + json.dumps({"type": "response.create"}), + ], + backend, + ) + handler = BaseLLMHTTPHandler() + + with patch.object(handler, "_open_realtime_backend_ws", AsyncMock(return_value=backend)): + await asyncio.wait_for( + handler.async_realtime( + model=model, + websocket=client_ws, + logging_obj=_realtime_logging_obj(), + provider_config=VertexAIRealtimeConfig( + access_token="fake-token", project="fake-project", location="us-central1" + ), + headers={}, + ), + timeout=10, + ) + + backend_frames = [json.loads(frame) for frame in backend.sent] + setups = [frame["setup"] for frame in backend_frames if "setup" in frame] + assert len(setups) == 1 + assert "setup" in backend_frames[0] + assert setups[0]["generationConfig"]["responseModalities"] == [backend_modality] + assert any("clientContent" in frame for frame in backend_frames[1:]) + + client_events = [json.loads(frame) for frame in client_ws.sent] + event_types = [event["type"] for event in client_events] + assert event_types[:2] == ["session.created", "session.updated"] + assert client_events[1]["session"]["modalities"] == [client_modality] + if backend_turn: + assert "response.output_text.delta" in event_types + assert "response.output_audio_transcript.delta" not in event_types + + +@pytest.mark.asyncio +async def test_async_realtime_sends_setup_at_connect_when_provider_cannot_build_it_from_session_update(): + """Bedrock Nova Sonic style providers keep the connect-time setup and get no + synthetic session.created.""" + setup = json.dumps({"event": {"sessionStart": {}}}) + backend = _ScriptedRealtimeBackend([]) + client_ws = _ScriptedRealtimeClient([], backend) + provider_config = Mock() + provider_config.get_complete_url.return_value = "wss://backend.example/live" + provider_config.validate_environment.return_value = {} + provider_config.requires_session_configuration.return_value = True + provider_config.builds_setup_from_session_update.return_value = False + provider_config.session_configuration_request.return_value = setup + provider_config.unbilled_usage_on_session_close.return_value = None + handler = BaseLLMHTTPHandler() + + with patch.object(handler, "_open_realtime_backend_ws", AsyncMock(return_value=backend)): + await asyncio.wait_for( + handler.async_realtime( + model="amazon.nova-sonic-v1:0", + websocket=client_ws, + logging_obj=_realtime_logging_obj(), + provider_config=provider_config, + headers={}, + ), + timeout=10, + ) + + assert backend.sent == [setup] + assert client_ws.sent == [] + + class _JSONBodyAudioTranscriptionConfig(BaseAudioTranscriptionConfig): def get_supported_openai_params(self, model): return [] From 5d8c2141c7e469a944aa42df5e3eb46c0163f3a5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 4 Sep 2026 22:40:56 -0700 Subject: [PATCH 02/10] fix(realtime): keep one Gemini Live response modality and skip the "inf" token limit in the setup --- .../llms/gemini/realtime/transformation.py | 23 ++++++++++- .../test_gemini_realtime_transformation.py | 41 +++++++++++++++++++ .../test_vertex_ai_realtime_transformation.py | 19 +++++++++ 3 files changed, 81 insertions(+), 2 deletions(-) diff --git a/litellm/llms/gemini/realtime/transformation.py b/litellm/llms/gemini/realtime/transformation.py index f155b048655..012f92069b7 100644 --- a/litellm/llms/gemini/realtime/transformation.py +++ b/litellm/llms/gemini/realtime/transformation.py @@ -305,7 +305,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): optional_params["systemInstruction"] = HttpxContentType(role="user", parts=[{"text": value}]) elif key == "temperature": optional_params["generationConfig"]["temperature"] = value - elif key == "max_response_output_tokens": + elif key == "max_response_output_tokens" and isinstance(value, int): optional_params["generationConfig"]["maxOutputTokens"] = value elif key == "modalities": optional_params["generationConfig"]["responseModalities"] = [ @@ -429,10 +429,17 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): @staticmethod def _coerce_response_modalities(model: str, modalities: Sequence[object]) -> tuple[str, ...]: """Swap responseModalities a Live model cannot produce: TEXT to AUDIO for - audio-only models, AUDIO to TEXT for text-only ones (e.g. transcribe-live).""" + audio-only models, AUDIO to TEXT for text-only ones (e.g. transcribe-live), + then keep a single modality because Gemini Live rejects a setup naming two.""" normalized: Final = tuple( modality.upper() if isinstance(modality, str) else str(modality).upper() for modality in modalities ) + return GeminiRealtimeConfig._single_response_modality( + GeminiRealtimeConfig._swap_unsupported_response_modalities(model, normalized) + ) + + @staticmethod + def _swap_unsupported_response_modalities(model: str, normalized: tuple[str, ...]) -> tuple[str, ...]: if GeminiRealtimeConfig._is_audio_only_live_model(model) and "TEXT" in normalized: verbose_logger.warning( "Gemini Live: %s only produces audio; downgrading the requested TEXT response modality to AUDIO", @@ -443,6 +450,18 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): return tuple(modality for modality in normalized if modality != "AUDIO") or ("TEXT",) return normalized + @staticmethod + def _single_response_modality(modalities: tuple[str, ...]) -> tuple[str, ...]: + if len(modalities) <= 1: + return modalities + chosen: Final = "AUDIO" if "AUDIO" in modalities else modalities[0] + verbose_logger.warning( + "Gemini Live: at most one response modality is allowed per session; using %s for the requested %s", + chosen, + ", ".join(modalities), + ) + return (chosen,) + @staticmethod def _finalize_gemini_live_setup(model: str, setup: dict[str, Any]) -> dict[str, Any]: generation_config: Final = setup.get("generationConfig") diff --git a/tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py b/tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py index 3d8200bc474..ea26169ebc7 100644 --- a/tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py +++ b/tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py @@ -2207,3 +2207,44 @@ def test_unbilled_usage_on_session_close_flushes_trailing_audio(patch_gemini_tra } assert usage == expected assert config.unbilled_usage_on_session_close("gemini-3.5-transcribe-live") is None + + +def test_gemini_realtime_two_output_modalities_collapse_to_audio(): + """Gemini Live closes the socket with 1007 when the setup names two response + modalities, so a GA session.update asking for ["text", "audio"] must reach the + backend as AUDIO only, the same choice the beta shim makes.""" + config = GeminiRealtimeConfig() + session_update = { + "type": "session.update", + "session": {"output_modalities": ["text", "audio"], "instructions": "Be brief."}, + } + + messages = config.transform_realtime_request( + json.dumps(session_update), + "gemini-live-2.5-flash", + session_configuration_request=None, + ) + + assert len(messages) == 1 + assert json.loads(messages[0])["setup"]["generationConfig"]["responseModalities"] == ["AUDIO"] + + +def test_gemini_realtime_inf_max_response_output_tokens_is_not_forwarded(): + """OpenAI's default max_response_output_tokens is the string "inf", which Gemini + rejects with 1007 (TYPE_INT32), so only integer limits map to maxOutputTokens.""" + config = GeminiRealtimeConfig() + + def generation_config_for(limit): + session_update = { + "type": "session.update", + "session": {"modalities": ["text"], "max_response_output_tokens": limit}, + } + messages = config.transform_realtime_request( + json.dumps(session_update), + "gemini-live-2.5-flash", + session_configuration_request=None, + ) + return json.loads(messages[0])["setup"]["generationConfig"] + + assert "maxOutputTokens" not in generation_config_for("inf") + assert generation_config_for(200)["maxOutputTokens"] == 200 diff --git a/tests/test_litellm/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py b/tests/test_litellm/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py index d4cf58bc0b4..6406cbbb40f 100644 --- a/tests/test_litellm/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py @@ -556,3 +556,22 @@ def test_vertex_native_audio_unmapped_voice_passes_through(patch_native_audio_co generation_config = json.loads(messages[0])["setup"]["generationConfig"] assert generation_config["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Kore" + + +def test_vertex_session_update_two_output_modalities_collapse_to_audio(): + """Vertex Live rejects a setup naming two response modalities, so a GA + ["text", "audio"] request must be sent as AUDIO only.""" + cfg = VertexAIRealtimeConfig(access_token="tok", project="my-proj", location="global") + session_update = { + "type": "session.update", + "session": {"output_modalities": ["text", "audio"], "instructions": "Be brief."}, + } + + messages = cfg.transform_realtime_request( + json.dumps(session_update), + "gemini-live-2.5-flash", + session_configuration_request=None, + ) + + assert len(messages) == 1 + assert json.loads(messages[0])["setup"]["generationConfig"]["responseModalities"] == ["AUDIO"] From 4b75d5584824c89556707fb7f718fd990ccd2d8c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 4 Sep 2026 23:07:27 -0700 Subject: [PATCH 03/10] fix(realtime): merge every session.update sent before the first content frame into the Gemini Live setup Gemini and Vertex Live accept one setup per connection, so the bridge used to turn only the first session.update into that setup and drop the rest without telling the client. Now every session.update that arrives before the first content frame is held, deep-merged (later values win per field), acknowledged with a session.updated built from the setup it would produce, and the merged session becomes the one setup sent with the first conversation item, audio frame, or tool result. The session events also surface the systemInstruction text as instructions, which the old string check never matched because the setup carries it as a Content object. --- .../litellm_core_utils/realtime_streaming.py | 77 +++++++++++++ .../llms/gemini/realtime/transformation.py | 20 +++- .../test_realtime_streaming.py | 101 ++++++++++++++---- 3 files changed, 172 insertions(+), 26 deletions(-) diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index bde3f1abe57..a09b6701e8d 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -1,8 +1,10 @@ import asyncio import json from collections.abc import Mapping, Sequence +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Protocol, TypedDict, cast +from pydantic import TypeAdapter from typing_extensions import ReadOnly import litellm @@ -75,6 +77,35 @@ class _ClientWebSocket(_ScopedWebSocket, Protocol): async def receive_text(self) -> str: ... +_SESSION_FIELDS: Final = TypeAdapter(dict[str, object]) +_NO_SESSION_FIELDS: Final[Mapping[str, object]] = MappingProxyType({}) + + +class _SessionUpdateFrame(TypedDict): + type: ReadOnly[str] + session: ReadOnly[Mapping[str, object]] + + +def _merge_session_fields(base: Mapping[str, object], update: Mapping[str, object]) -> Mapping[str, object]: + def merged_value(key: str) -> object: + if key not in update: + return base[key] + value: Final = update[key] + base_value: Final = base.get(key) + if isinstance(value, Mapping) and isinstance(base_value, Mapping): + return _merge_session_fields( + _SESSION_FIELDS.validate_python(base_value), _SESSION_FIELDS.validate_python(value) + ) + return value + + return MappingProxyType({key: merged_value(key) for key in (*base, *update)}) + + +def _session_update_frame(session: Mapping[str, object]) -> str: + frame: Final[_SessionUpdateFrame] = {"type": "session.update", "session": session} + return json.dumps(frame, default=_SESSION_FIELDS.validate_python) + + def _decode_json_object(payload: str) -> Mapping[str, object]: """Decode a realtime frame into its top-level field mapping.""" return json.loads(payload) @@ -160,6 +191,7 @@ class RealTimeStreaming: # Gemini Live rejects a follow-up BidiGenerateContentSetup once any # content (realtimeInput / clientContent / toolResponse) has been sent. self._content_sent_after_setup: bool = False + self._held_session_update: Mapping[str, object] | None = None # Whether this is a transcription-only session (session.type == "transcription", # e.g. gpt-realtime-whisper). Such sessions must not be sent response.create and # their input_audio_transcription.completed usage drives duration-based cost. @@ -440,6 +472,46 @@ class RealTimeStreaming: await self.backend_ws.send(message) return True + def _holds_session_updates_until_first_content(self) -> bool: + if self.provider_config is None or self.session_configuration_request is not None: + return False + return self.provider_config.builds_setup_from_session_update() and not self._uses_deferred_backend_setup() + + def _held_session_update_message(self) -> str: + return _session_update_frame(self._held_session_update or _NO_SESSION_FIELDS) + + async def _hold_session_update(self, message: str) -> None: + session: Final = _decode_json_object(message).get("session") + update: Final = _SESSION_FIELDS.validate_python(session) if isinstance(session, Mapping) else _NO_SESSION_FIELDS + self._held_session_update = _merge_session_fields(self._held_session_update or _NO_SESSION_FIELDS, update) + await self._acknowledge_held_session_update() + + async def _acknowledge_held_session_update(self) -> None: + if self.provider_config is None: + return + preview: Final = self.provider_config.transform_realtime_request( + self._held_session_update_message(), self.model, None + ) + setup: Final = next((frame for frame in preview if "setup" in _decode_json_object(frame)), None) + if setup is None: + return + acknowledged: Final = self.provider_config.transform_session_created_event( + self.model, str(self.logging_obj.litellm_trace_id), setup + ) + if acknowledged is None: + return + event: Final[OpenAIRealtimeStreamSessionEvents] = {**acknowledged, "type": "session.updated"} + event_str: Final = json.dumps(event) + self.store_message(event_str) + await self._send_event_to_client(event, event_str) + + async def _send_held_session_update_as_setup(self) -> None: + if self._held_session_update is None: + return + message: Final = self._held_session_update_message() + self._held_session_update = None + await self._send_to_backend(message) + async def _send_default_setup_before_first_client_message(self, msg_type: str | None) -> None: if self.session_configuration_request is not None or self.provider_config is None: return @@ -1489,6 +1561,11 @@ class RealTimeStreaming: # would permanently disable the injection if ``_send_to_backend`` # raised — neither this loop nor # ``_maybe_send_guardrail_turn_detection_update`` would retry. + if self._holds_session_updates_until_first_content(): + if msg_type == "session.update": + await self._hold_session_update(message) + continue + await self._send_held_session_update_as_setup() await self._send_default_setup_before_first_client_message(msg_type) sent = await self._send_to_backend(message) if guardrail_turn_detection_injected and sent: diff --git a/litellm/llms/gemini/realtime/transformation.py b/litellm/llms/gemini/realtime/transformation.py index 012f92069b7..d7832965d7e 100644 --- a/litellm/llms/gemini/realtime/transformation.py +++ b/litellm/llms/gemini/realtime/transformation.py @@ -127,6 +127,20 @@ def _base64_decoded_byte_count(data: str) -> int: return max(len(data) * 3 // 4 - padding, 0) +def _system_instruction_text(system_instruction: object) -> str | None: + if isinstance(system_instruction, str): + return system_instruction + if not isinstance(system_instruction, Mapping): + return None + parts: Final = system_instruction.get("parts") + if not isinstance(parts, Sequence): + return None + texts: Final = tuple( + part["text"] for part in parts if isinstance(part, Mapping) and isinstance(part.get("text"), str) + ) + return "\n".join(texts) if texts else None + + class GeminiRealtimeConfig(BaseRealtimeConfig): _TOOL_CALL_ID_TO_NAME_MAX = 256 # LRU cap for call_id→name mapping @@ -664,13 +678,13 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): generation_config: Final = session_configuration_request_dict.get("generationConfig", {}) or {} gemini_modalities: Final = generation_config.get("responseModalities", ["AUDIO"]) _modalities: Final = [modality.lower() for modality in cast(list[str], gemini_modalities)] - _system_instruction: Final = session_configuration_request_dict.get("systemInstruction") + instructions: Final = _system_instruction_text(session_configuration_request_dict.get("systemInstruction")) session: Final = OpenAIRealtimeStreamSession( id=logging_session_id, modalities=_modalities, ) - if _system_instruction is not None and isinstance(_system_instruction, str): - session["instructions"] = _system_instruction + if instructions is not None: + session["instructions"] = instructions if _model is not None and isinstance(_model, str): # Strip Vertex/AI Studio path prefixes to expose the bare model name. if "/models/" in _model: diff --git a/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py b/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py index e262c163612..859afa01e96 100644 --- a/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py +++ b/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py @@ -1121,9 +1121,11 @@ async def test_client_ack_caches_setup_to_prevent_duplicate_session_update_setup side_effect=[ json.dumps({"type": "session.update", "session": {"tools": []}}), json.dumps({"type": "session.update", "session": {"tools": []}}), + _TEXT_ITEM_CREATE, Exception("client done"), ] ) + websocket.send_text = AsyncMock() provider_config = MagicMock() @@ -1403,7 +1405,6 @@ async def test_realtime_guardrail_blocks_prompt_injection(monkeypatch: pytest.Mo ) - @pytest.mark.asyncio async def test_realtime_guardrail_allows_clean_transcript(monkeypatch: pytest.MonkeyPatch): """ @@ -1460,7 +1461,6 @@ async def test_realtime_guardrail_allows_clean_transcript(monkeypatch: pytest.Mo assert len(response_creates) == 1, f"Clean transcript should trigger response.create, got: {sent_to_backend}" - @pytest.mark.asyncio async def test_realtime_text_input_guardrail_blocks_and_returns_error(monkeypatch: pytest.MonkeyPatch): """ @@ -1554,7 +1554,6 @@ async def test_realtime_text_input_guardrail_blocks_and_returns_error(monkeypatc assert len(original_items) == 0, f"Blocked item should not be forwarded to backend, got: {original_items}" - @pytest.mark.asyncio async def test_realtime_function_call_output_guardrail_blocks_and_returns_error(monkeypatch: pytest.MonkeyPatch): """ @@ -1643,7 +1642,6 @@ async def test_realtime_function_call_output_guardrail_blocks_and_returns_error( assert "test@example.com" not in sanitized_item["output"] - @pytest.mark.asyncio async def test_realtime_function_call_output_guardrail_allows_clean_output(monkeypatch: pytest.MonkeyPatch): """ @@ -1708,7 +1706,6 @@ async def test_realtime_function_call_output_guardrail_allows_clean_output(monke assert len(forwarded) == 1, f"Clean function_call_output should be forwarded, got: {forwarded}" - @pytest.mark.asyncio async def test_realtime_text_input_guardrail_uses_pre_call_mode(monkeypatch: pytest.MonkeyPatch): """ @@ -1744,7 +1741,6 @@ async def test_realtime_text_input_guardrail_uses_pre_call_mode(monkeypatch: pyt ) - @pytest.mark.asyncio async def test_realtime_session_created_injects_session_update_for_audio_guardrail(monkeypatch: pytest.MonkeyPatch): """ @@ -1801,7 +1797,6 @@ async def test_realtime_session_created_injects_session_update_for_audio_guardra ) - @pytest.mark.asyncio async def test_realtime_session_created_does_not_inject_session_update_for_pre_call_only( monkeypatch: pytest.MonkeyPatch, @@ -1846,7 +1841,6 @@ async def test_realtime_session_created_does_not_inject_session_update_for_pre_c assert len(session_updates) == 0, f"pre_call-only guardrail must not inject session.update, got: {sent_to_backend}" - @pytest.mark.asyncio async def test_pre_call_and_post_call_guardrails_do_not_disable_server_vad(monkeypatch: pytest.MonkeyPatch): """Model Armor-style pre_call + post_call must not gate audio VAD.""" @@ -1862,17 +1856,17 @@ async def test_pre_call_and_post_call_guardrails_do_not_disable_server_vad(monke litellm, "callbacks", [ - ModelArmorStyleGuardrail( - guardrail_name="model_armor_all_pre_call", - event_hook=GuardrailEventHooks.pre_call, - default_on=False, - ), - ModelArmorStyleGuardrail( - guardrail_name="model_armor_all_post_call", - event_hook=GuardrailEventHooks.post_call, - default_on=False, - ), - ], + ModelArmorStyleGuardrail( + guardrail_name="model_armor_all_pre_call", + event_hook=GuardrailEventHooks.pre_call, + default_on=False, + ), + ModelArmorStyleGuardrail( + guardrail_name="model_armor_all_post_call", + event_hook=GuardrailEventHooks.post_call, + default_on=False, + ), + ], ) client_ws = MagicMock() @@ -1896,7 +1890,6 @@ async def test_pre_call_and_post_call_guardrails_do_not_disable_server_vad(monke assert streaming._has_audio_transcription_guardrails() is False - @pytest.mark.asyncio async def test_end_session_after_n_fails_closes_connection(monkeypatch: pytest.MonkeyPatch): """ @@ -1943,7 +1936,6 @@ async def test_end_session_after_n_fails_closes_connection(monkeypatch: pytest.M assert streaming._violation_count == 2 - @pytest.mark.asyncio async def test_on_violation_end_session_closes_on_first_fail(monkeypatch: pytest.MonkeyPatch): """ @@ -1989,7 +1981,6 @@ async def test_on_violation_end_session_closes_on_first_fail(monkeypatch: pytest assert streaming._violation_count == 1 - @pytest.mark.asyncio async def test_provider_path_suppresses_duplicate_session_created_after_synthetic(): client_ws = MagicMock() @@ -2162,6 +2153,7 @@ async def test_guardrail_turn_detection_injected_into_first_session_update_defer logging_obj.success_handler = MagicMock() provider_config = MagicMock() + provider_config.builds_setup_from_session_update = MagicMock(return_value=False) transformed_messages = [] def mock_transform(msg, model, session_config): @@ -2228,6 +2220,7 @@ async def test_guardrail_turn_detection_injection_tolerates_non_dict_value( logging_obj.success_handler = MagicMock() provider_config = MagicMock() + provider_config.builds_setup_from_session_update = MagicMock(return_value=False) transformed_messages = [] def mock_transform(msg, model, session_config): @@ -2383,6 +2376,7 @@ def _gemini_default_mode_streaming(monkeypatch, client_messages, model="gemini-l client_ws = MagicMock() client_ws.receive_text = AsyncMock(side_effect=[*client_messages, ConnectionClosed(None, None)]) + client_ws.send_text = AsyncMock() backend_ws = MagicMock() backend_ws.send = AsyncMock() streaming = RealTimeStreaming( @@ -2432,6 +2426,65 @@ async def test_default_mode_session_update_first_builds_the_only_setup(monkeypat assert frames[0]["setup"]["generationConfig"]["responseModalities"] == ["TEXT"] +@pytest.mark.asyncio +async def test_default_mode_merges_every_session_update_sent_before_the_first_content_frame(monkeypatch): + """Clients configure a session over several session.update messages. Gemini + Live takes one setup, so every update sent before the first content frame is + merged into that setup instead of only the first one applying.""" + instructions_update = json.dumps({"type": "session.update", "session": {"instructions": "Be brief."}}) + modalities_update = json.dumps({"type": "session.update", "session": {"modalities": ["text"]}}) + streaming, backend_ws = _gemini_default_mode_streaming( + monkeypatch, [instructions_update, modalities_update, _TEXT_ITEM_CREATE], model="gemini-live-2.5-flash" + ) + + await streaming.client_ack_messages() + + frames = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] + assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"] + assert frames[0]["setup"]["generationConfig"]["responseModalities"] == ["TEXT"] + assert frames[0]["setup"]["systemInstruction"]["parts"][0]["text"] == "Be brief." + + +@pytest.mark.asyncio +async def test_default_mode_merges_nested_audio_settings_across_session_updates(monkeypatch): + vad_update = json.dumps( + {"type": "session.update", "session": {"audio": {"input": {"turn_detection": {"silence_duration_ms": 1500}}}}} + ) + voice_update = json.dumps({"type": "session.update", "session": {"audio": {"output": {"voice": "Kore"}}}}) + streaming, backend_ws = _gemini_default_mode_streaming( + monkeypatch, [vad_update, voice_update, _TEXT_ITEM_CREATE], model="gemini-live-2.5-flash" + ) + + await streaming.client_ack_messages() + + setup = json.loads(backend_ws.send.await_args_list[0].args[0])["setup"] + assert setup["realtimeInputConfig"]["automaticActivityDetection"]["silenceDurationMs"] == 1500 + assert setup["generationConfig"]["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Kore" + + +@pytest.mark.asyncio +async def test_default_mode_acknowledges_each_held_session_update_before_any_backend_frame(monkeypatch): + """A client that waits for session.updated before sending content must not + hang while its updates are held, so each one is acknowledged right away + with the modalities the merged setup will carry.""" + instructions_update = json.dumps({"type": "session.update", "session": {"instructions": "Be brief."}}) + modalities_update = json.dumps({"type": "session.update", "session": {"modalities": ["text"]}}) + streaming, backend_ws = _gemini_default_mode_streaming( + monkeypatch, [instructions_update, modalities_update], model="gemini-live-2.5-flash" + ) + + await streaming.client_ack_messages() + + acks = [json.loads(call.args[0]) for call in streaming.websocket.send_text.await_args_list] + assert [ack["type"] for ack in acks] == ["session.updated", "session.updated"] + assert [ack["session"].get("output_modalities") or ack["session"].get("modalities") for ack in acks] == [ + ["audio"], + ["text"], + ] + assert acks[1]["session"]["instructions"] == "Be brief." + backend_ws.send.assert_not_awaited() + + @pytest.mark.asyncio async def test_deferred_setup_buffers_audio_until_backend_setup_complete(monkeypatch): """Pipecat may send audio before session.update when setup is deferred.""" @@ -3007,7 +3060,9 @@ async def test_log_messages_routes_async_logging_through_bounded_worker(): mock_worker.ensure_initialized_and_enqueue.assert_called_once() enqueued = mock_worker.ensure_initialized_and_enqueue.call_args - assert (enqueued.args or tuple(enqueued.kwargs.values()))[0] is logging_obj.dispatch_success_handlers.return_value + assert (enqueued.args or tuple(enqueued.kwargs.values()))[ + 0 + ] is logging_obj.dispatch_success_handlers.return_value logging_obj.dispatch_success_handlers.assert_called_once_with(streaming.messages, prefer_async_handlers=True) logging_obj.success_handler.assert_not_called() # the bare create_task path must no longer be used for success logging From 20217bd2e0fd78ee068c846c837198168d0e2d35 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 4 Sep 2026 23:21:18 -0700 Subject: [PATCH 04/10] fix(realtime): cap the session.update merge depth so the recursive detector accepts it _merge_session_fields recurses once per nested mapping level, so CI's recursive_detector flagged it. Bound the descent at eight levels (the OpenAI session object nests four deep at most), replace anything deeper wholesale with the later session.update's value, and list the helper with the other depth-capped functions the detector allows. --- litellm/litellm_core_utils/realtime_streaming.py | 9 ++++++--- tests/code_coverage_tests/recursive_detector.py | 1 + .../litellm_core_utils/test_realtime_streaming.py | 15 +++++++++++++++ 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index a09b6701e8d..4ae17785804 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -79,6 +79,7 @@ class _ClientWebSocket(_ScopedWebSocket, Protocol): _SESSION_FIELDS: Final = TypeAdapter(dict[str, object]) _NO_SESSION_FIELDS: Final[Mapping[str, object]] = MappingProxyType({}) +_SESSION_MERGE_MAX_DEPTH: Final = 8 class _SessionUpdateFrame(TypedDict): @@ -86,15 +87,17 @@ class _SessionUpdateFrame(TypedDict): session: ReadOnly[Mapping[str, object]] -def _merge_session_fields(base: Mapping[str, object], update: Mapping[str, object]) -> Mapping[str, object]: +def _merge_session_fields( + base: Mapping[str, object], update: Mapping[str, object], depth: int = 0 +) -> Mapping[str, object]: def merged_value(key: str) -> object: if key not in update: return base[key] value: Final = update[key] base_value: Final = base.get(key) - if isinstance(value, Mapping) and isinstance(base_value, Mapping): + if depth < _SESSION_MERGE_MAX_DEPTH and isinstance(value, Mapping) and isinstance(base_value, Mapping): return _merge_session_fields( - _SESSION_FIELDS.validate_python(base_value), _SESSION_FIELDS.validate_python(value) + _SESSION_FIELDS.validate_python(base_value), _SESSION_FIELDS.validate_python(value), depth + 1 ) return value diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index e9f87ba6cae..4ac150d788b 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -67,6 +67,7 @@ IGNORE_FUNCTIONS = [ "_redact_agent_params_tree", # max depth set (default 10), same shape as _redact_sensitive_litellm_params. "_restore_redacted_nested_value", # max depth set (default 10), mirrors _redact_agent_params_tree on the write side. "_unqualified", # bounded by the qualifier depth of a static TypedDict annotation (Annotated, Required/NotRequired, ReadOnly around one type, no cycles possible). + "_merge_session_fields", # max depth set (_SESSION_MERGE_MAX_DEPTH); deeper mappings are replaced wholesale by the later session.update. ] diff --git a/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py b/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py index 859afa01e96..a4ed017d7df 100644 --- a/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py +++ b/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py @@ -10,6 +10,7 @@ import litellm from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.litellm_core_utils.realtime_streaming import ( RealTimeStreaming, + _merge_session_fields, client_sent_openai_beta_realtime_header, ) from litellm.llms.xai.realtime.transformation import XAIRealtimeNormalizer @@ -2462,6 +2463,20 @@ async def test_default_mode_merges_nested_audio_settings_across_session_updates( assert setup["generationConfig"]["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Kore" +def test_merge_session_fields_replaces_nested_mappings_wholesale_past_the_depth_cap(): + def nested(levels: int, key: str) -> dict: + return {key: levels, **({"child": nested(levels - 1, key)} if levels else {})} + + def level(node, depth: int): + return node if depth == 0 else level(node["child"], depth - 1) + + merged = _merge_session_fields(nested(10, "keep"), nested(10, "new")) + + assert all({"keep", "new"} <= set(level(merged, depth)) for depth in range(9)) + assert "keep" not in level(merged, 9) + assert level(merged, 9)["new"] == 1 + + @pytest.mark.asyncio async def test_default_mode_acknowledges_each_held_session_update_before_any_backend_frame(monkeypatch): """A client that waits for session.updated before sending content must not From 60f2e091b425588e08bce4be7e252fee251ff445 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 5 Sep 2026 00:43:36 -0700 Subject: [PATCH 05/10] fix(gemini-live): accept a string output modality and drop values Live cannot produce A bare string output_modalities is read as a one-element list and modality values Gemini Live cannot produce are dropped with a warning instead of reaching the backend, so a malformed session.update no longer costs the session. Transcription guardrails now fold their VAD disable into the first held session.update only, so the merged setup carries it exactly once. --- .../litellm_core_utils/realtime_streaming.py | 1 + .../llms/gemini/realtime/transformation.py | 41 ++++++++++++++----- .../test_realtime_streaming.py | 29 +++++++++++++ .../test_gemini_realtime_transformation.py | 31 ++++++++++++++ 4 files changed, 92 insertions(+), 10 deletions(-) diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index 4ae17785804..88403f76435 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -1444,6 +1444,7 @@ class RealTimeStreaming: if ( msg_type == "session.update" and self.session_configuration_request is None + and self._held_session_update is None and not self._guardrail_turn_detection_update_sent and self._has_audio_transcription_guardrails() ): diff --git a/litellm/llms/gemini/realtime/transformation.py b/litellm/llms/gemini/realtime/transformation.py index d7832965d7e..40095b4b1e1 100644 --- a/litellm/llms/gemini/realtime/transformation.py +++ b/litellm/llms/gemini/realtime/transformation.py @@ -74,6 +74,7 @@ MAP_GEMINI_FIELD_TO_OPENAI_EVENT: Final[dict[str, OpenAIRealtimeEventTypes | Res # Keys the main transform loop handles; siblings like ``usageMetadata`` are skipped. _KNOWN_GEMINI_TOP_LEVEL_KEYS: Final[set] = {map_key.split(".", 1)[0] for map_key in MAP_GEMINI_FIELD_TO_OPENAI_EVENT} +_GEMINI_LIVE_RESPONSE_MODALITIES: Final = frozenset({"TEXT", "AUDIO"}) OPENAI_STOCK_REALTIME_VOICES: Final[frozenset[str]] = frozenset( @@ -322,9 +323,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): elif key == "max_response_output_tokens" and isinstance(value, int): optional_params["generationConfig"]["maxOutputTokens"] = value elif key == "modalities": - optional_params["generationConfig"]["responseModalities"] = [ - modality.upper() for modality in cast(list[str], value) - ] + optional_params["generationConfig"]["responseModalities"] = list( + GeminiRealtimeConfig._requested_response_modalities(value) + ) elif key == "tools": from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( VertexGeminiConfig, @@ -440,18 +441,38 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): def _default_response_modality(model: str) -> GeminiResponseModalities: return "TEXT" if GeminiRealtimeConfig._is_text_only_live_model(model) else "AUDIO" + @staticmethod + def _requested_response_modalities(value: object) -> tuple[str, ...]: + if isinstance(value, str): + return (value.upper(),) + if not isinstance(value, (list, tuple)): + return () + return tuple(str(modality).upper() for modality in value) + @staticmethod def _coerce_response_modalities(model: str, modalities: Sequence[object]) -> tuple[str, ...]: - """Swap responseModalities a Live model cannot produce: TEXT to AUDIO for - audio-only models, AUDIO to TEXT for text-only ones (e.g. transcribe-live), - then keep a single modality because Gemini Live rejects a setup naming two.""" - normalized: Final = tuple( - modality.upper() if isinstance(modality, str) else str(modality).upper() for modality in modalities - ) + """Drop modalities Gemini Live cannot produce at all, swap the ones this + model cannot (TEXT to AUDIO for audio-only models, AUDIO to TEXT for + text-only ones such as transcribe-live), then keep a single modality + because Gemini Live rejects a setup naming two.""" return GeminiRealtimeConfig._single_response_modality( - GeminiRealtimeConfig._swap_unsupported_response_modalities(model, normalized) + GeminiRealtimeConfig._swap_unsupported_response_modalities( + model, GeminiRealtimeConfig._known_response_modalities(model, modalities) + ) ) + @staticmethod + def _known_response_modalities(model: str, modalities: Sequence[object]) -> tuple[str, ...]: + normalized: Final = tuple(str(modality).upper() for modality in modalities) + known: Final = tuple(modality for modality in normalized if modality in _GEMINI_LIVE_RESPONSE_MODALITIES) + if len(known) < len(normalized): + verbose_logger.warning( + "Gemini Live: %s only produces TEXT or AUDIO; ignoring the requested %s", + model, + ", ".join(modality for modality in normalized if modality not in _GEMINI_LIVE_RESPONSE_MODALITIES), + ) + return known or (GeminiRealtimeConfig._default_response_modality(model),) + @staticmethod def _swap_unsupported_response_modalities(model: str, normalized: tuple[str, ...]) -> tuple[str, ...]: if GeminiRealtimeConfig._is_audio_only_live_model(model) and "TEXT" in normalized: diff --git a/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py b/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py index a4ed017d7df..f8528b68c62 100644 --- a/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py +++ b/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py @@ -2500,6 +2500,35 @@ async def test_default_mode_acknowledges_each_held_session_update_before_any_bac backend_ws.send.assert_not_awaited() +@pytest.mark.asyncio +async def test_default_mode_folds_transcription_guardrail_vad_disable_into_the_merged_setup(monkeypatch): + """A transcription guardrail gates every turn, so its auto-response disable + has to land in the one setup built from the held session.updates, injected + once rather than into every update that gets merged.""" + monkeypatch.setattr(litellm, "callbacks", [_transcription_guardrail()]) + debug = MagicMock() + monkeypatch.setattr("litellm.litellm_core_utils.realtime_streaming.verbose_logger.debug", debug) + instructions_update = json.dumps({"type": "session.update", "session": {"instructions": "Be brief."}}) + vad_update = json.dumps( + {"type": "session.update", "session": {"turn_detection": {"type": "server_vad", "silence_duration_ms": 900}}} + ) + streaming, backend_ws = _gemini_default_mode_streaming( + monkeypatch, [instructions_update, vad_update, _TEXT_ITEM_CREATE], model="gemini-live-2.5-flash" + ) + + await streaming.client_ack_messages() + + frames = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] + assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"] + activity = frames[0]["setup"]["realtimeInputConfig"]["automaticActivityDetection"] + assert activity["disabled"] is True + assert activity["silenceDurationMs"] == 900 + assert frames[0]["setup"]["systemInstruction"]["parts"][0]["text"] == "Be brief." + assert streaming._guardrail_turn_detection_update_sent is True + injected = [call for call in debug.call_args_list if str(call.args[0]).startswith("Injected turn_detection")] + assert len(injected) == 1 + + @pytest.mark.asyncio async def test_deferred_setup_buffers_audio_until_backend_setup_complete(monkeypatch): """Pipecat may send audio before session.update when setup is deferred.""" diff --git a/tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py b/tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py index ea26169ebc7..e9b76f0696e 100644 --- a/tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py +++ b/tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py @@ -626,6 +626,35 @@ def test_gemini_audio_only_live_models_drop_text_from_text_audio_combo(patch_gem assert setup["generationConfig"]["responseModalities"] == ["AUDIO"] +@pytest.mark.parametrize("output_modalities", ["text", ["text"]]) +def test_gemini_session_update_accepts_a_single_string_output_modality(output_modalities): + """Regression: a bare string was iterated per character, so output_modalities + "text" reached Vertex as responseModalities ["T"], which it rejects with 1007.""" + config = GeminiRealtimeConfig() + session_update = {"type": "session.update", "session": {"type": "realtime", "output_modalities": output_modalities}} + + messages = config.transform_realtime_request( + json.dumps(session_update), "gemini-live-2.5-flash", session_configuration_request=None + ) + + assert json.loads(messages[0])["setup"]["generationConfig"]["responseModalities"] == ["TEXT"] + + +@pytest.mark.parametrize( + ("output_modalities", "expected"), + [(["video", "text"], ["TEXT"]), (["video"], ["AUDIO"]), (None, ["AUDIO"])], +) +def test_gemini_session_update_drops_modalities_live_cannot_produce(output_modalities, expected): + config = GeminiRealtimeConfig() + session_update = {"type": "session.update", "session": {"type": "realtime", "output_modalities": output_modalities}} + + messages = config.transform_realtime_request( + json.dumps(session_update), "gemini-live-2.5-flash", session_configuration_request=None + ) + + assert json.loads(messages[0])["setup"]["generationConfig"]["responseModalities"] == expected + + def test_gemini_non_live_model_preserves_text_modality(): config = GeminiRealtimeConfig() session_update = { @@ -1930,6 +1959,8 @@ def test_gemini_response_done_bills_audio_output_tokens_at_audio_rate(monkeypatc litellm_model_name="gemini-2.5-flash-native-audio-preview-12-2025", ) assert cost == pytest.approx(377 * 5e-07 + 51 * 1.2e-05 + 37 * 2e-06) + + @pytest.fixture(autouse=False) def patch_gemini_transcribe_live_cost_map_entry(monkeypatch): """Inject the gemini-3.5-transcribe-live registry entry locally. From f34e5aa189556c8ff4dd57e54b68e068ff3c8a5d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 21 Oct 2026 00:26:40 -0700 Subject: [PATCH 06/10] fix(gemini realtime): answer an error for session updates Gemini Live cannot apply An output modality the model cannot produce, or a session.update sent after the first content, now gets an invalid_request_error event instead of a silent swap or drop, unless drop_params is set. The synthetic session.created reports the model's default modality, setupComplete no longer adds an extra session.updated, and merging nested session updates is linear in their size --- .../litellm_core_utils/realtime_streaming.py | 30 +++- .../llms/base_llm/realtime/transformation.py | 29 +++- litellm/llms/bedrock/realtime/handler.py | 20 +-- .../llms/bedrock/realtime/transformation.py | 19 +-- .../llms/gemini/realtime/transformation.py | 53 ++++++- .../llms/vertex_ai/realtime/transformation.py | 5 +- litellm/utils.py | 2 +- .../test_realtime_streaming.py | 136 +++++++++++++++++- .../custom_httpx/test_llm_http_handler.py | 14 +- .../test_gemini_realtime_transformation.py | 36 +++++ .../test_vertex_ai_realtime_transformation.py | 19 +++ 11 files changed, 313 insertions(+), 50 deletions(-) diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index c56edd7778d..ed75031eb9e 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -148,7 +148,7 @@ def _merge_session_fields( ) return update[key] - return MappingProxyType({key: merged_value(key) for key in (*base, *update)}) + return MappingProxyType({key: merged_value(key) for key in dict.fromkeys((*base, *update))}) def _session_update_frame(session: Mapping[str, object]) -> str: @@ -639,11 +639,30 @@ class RealTimeStreaming: self._usage_meter.observe_client_event(sent_event) return True - def _holds_session_updates_until_first_content(self) -> bool: - if self.provider_config is None or self.session_configuration_request is not None: + def _builds_setup_lazily(self) -> bool: + if self.provider_config is None: return False return self.provider_config.builds_setup_from_session_update() and not self._uses_deferred_backend_setup() + def _holds_session_updates_until_first_content(self) -> bool: + return self.session_configuration_request is None and self._builds_setup_lazily() + + async def _rejected_setup_session_update(self, message: str) -> bool: + provider_config: Final = self.provider_config + if provider_config is None or not self._builds_setup_lazily(): + return False + client_event: Final = _decode_json_object(message) + session: Final = client_event.get("session") + rejection: Final = provider_config.setup_session_update_rejection( + self.model, + _SESSION_FIELDS.validate_python(session) if isinstance(session, Mapping) else _NO_SESSION_FIELDS, + setup_sent=self.session_configuration_request is not None, + ) + if rejection is None: + return False + await self.websocket.send_text(json.dumps(rejection.error_event(client_event.get("event_id")))) + return True + def _held_session_update_message(self) -> str: return _session_update_frame(self._held_session_update or _NO_SESSION_FIELDS) @@ -1222,6 +1241,9 @@ class RealTimeStreaming: break finally: self._flushing_pending_messages_until_setup = False + if self.session_created_sent_to_client and self._builds_setup_lazily(): + await self._maybe_send_guardrail_turn_detection_update() + continue if self.session_created_sent_to_client: # A synthetic session.created (with placeholder defaults) was # already forwarded to the client when we connected. The @@ -1779,6 +1801,8 @@ class RealTimeStreaming: # ``_maybe_send_guardrail_turn_detection_update`` would retry. if msg_type == "session.update" and self.provider_config is None: await self._send_session_gate_once() + if msg_type == "session.update" and await self._rejected_setup_session_update(message): + continue if self._holds_session_updates_until_first_content(): if msg_type == "session.update": await self._hold_session_update(message) diff --git a/litellm/llms/base_llm/realtime/transformation.py b/litellm/llms/base_llm/realtime/transformation.py index 51d34982959..47054b66187 100644 --- a/litellm/llms/base_llm/realtime/transformation.py +++ b/litellm/llms/base_llm/realtime/transformation.py @@ -1,11 +1,13 @@ +import dataclasses from abc import ABC, abstractmethod from collections.abc import Mapping, Sequence from types import TracebackType -from typing import TYPE_CHECKING, Any, Protocol +from typing import TYPE_CHECKING, Any, Literal, Protocol import httpx from typing_extensions import Self +from litellm._uuid import uuid from litellm.types.llms.openai import OpenAIRealtimeStreamSessionEvents from litellm.types.realtime import ( RealtimeInputAudioTranscriptionUsage, @@ -23,6 +25,26 @@ else: LiteLLMLoggingObj = Any +@dataclasses.dataclass(frozen=True, slots=True) +class RealtimeSessionUpdateRejection: + code: Literal["invalid_type", "invalid_value", "session_update_not_applied"] + message: str + param: str + + def error_event(self, client_event_id: object) -> dict[str, object]: + return { + "type": "error", + "event_id": f"event_{uuid.uuid4()}", + "error": { + "type": "invalid_request_error", + "code": self.code, + "message": self.message, + "param": self.param, + "event_id": client_event_id if isinstance(client_event_id, str) else None, + }, + } + + class RealtimeBackend(Protocol): async def __aenter__(self) -> Self: ... @@ -97,6 +119,11 @@ class BaseRealtimeConfig(ABC): def builds_setup_from_session_update(self) -> bool: return False + def setup_session_update_rejection( + self, model: str, session: Mapping[str, object], *, setup_sent: bool + ) -> RealtimeSessionUpdateRejection | None: + return None + def unbilled_usage_on_session_close(self, model: str) -> RealtimeInputAudioTranscriptionUsage | None: return None diff --git a/litellm/llms/bedrock/realtime/handler.py b/litellm/llms/bedrock/realtime/handler.py index e9b2f26e977..00622a1e287 100644 --- a/litellm/llms/bedrock/realtime/handler.py +++ b/litellm/llms/bedrock/realtime/handler.py @@ -17,7 +17,6 @@ from pydantic import JsonValue, TypeAdapter import litellm from litellm._logging import redact_string, verbose_proxy_logger -from litellm._uuid import uuid from litellm.constants import ( BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS, BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY, @@ -31,13 +30,14 @@ from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.litellm_core_utils.realtime_streaming import DefaultLoggedRealTimeEventTypes +from litellm.llms.base_llm.realtime.transformation import RealtimeSessionUpdateRejection from litellm.types.llms.bedrock import AwsAuthParams from litellm.types.llms.openai import OpenAIRealtimeEvents from litellm.types.realtime import RealtimeResponseTransformInput from ..base_aws_llm import BaseAWSLLM, run_aws_signing from ..common_utils import BedrockError -from .transformation import BedrockRealtimeConfig, BedrockSessionUpdateRejection, requested_modalities +from .transformation import BedrockRealtimeConfig, requested_modalities _CLIENT_MESSAGE_ADAPTER: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) @@ -256,23 +256,11 @@ async def _ack_session_update( async def _reject_session_update( client_ws: RealtimeClientWebSocket, parsed_client_message: Mapping[str, JsonValue], - rejection: BedrockSessionUpdateRejection, + rejection: RealtimeSessionUpdateRejection, ) -> bool: """Send an error event for a session.update Bedrock cannot apply; False means the client is gone.""" - client_event_id: Final = parsed_client_message.get("event_id") - error_event: Final = { - "type": "error", - "event_id": f"event_{uuid.uuid4()}", - "error": { - "type": "invalid_request_error", - "code": rejection.code, - "message": rejection.message, - "param": rejection.param, - "event_id": client_event_id if isinstance(client_event_id, str) else None, - }, - } try: - await client_ws.send_text(json.dumps(error_event)) + await client_ws.send_text(json.dumps(rejection.error_event(parsed_client_message.get("event_id")))) except Exception as e: # noqa: BLE001 # any send failure means the client is gone verbose_proxy_logger.debug("Client to Bedrock forwarding ended: %s", e, exc_info=True) return False diff --git a/litellm/llms/bedrock/realtime/transformation.py b/litellm/llms/bedrock/realtime/transformation.py index 9cb38822881..97eda5fdbc9 100644 --- a/litellm/llms/bedrock/realtime/transformation.py +++ b/litellm/llms/bedrock/realtime/transformation.py @@ -11,7 +11,7 @@ import math import uuid as uuid_lib from collections.abc import Mapping, Sequence from types import MappingProxyType -from typing import Final, Literal +from typing import Final import httpx from pydantic import ConfigDict, JsonValue, TypeAdapter @@ -19,7 +19,7 @@ from pydantic import ConfigDict, JsonValue, TypeAdapter from litellm._logging import verbose_logger from litellm._uuid import uuid from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig +from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeSessionUpdateRejection from litellm.llms.bedrock.common_utils import BedrockError from litellm.llms.bedrock.realtime.trigger_audio import ready_trigger_pcm from litellm.types.llms.base import LiteLLMBaseModel @@ -195,13 +195,6 @@ def _malformed_settings(session_config: Mapping[str, JsonValue]) -> tuple[tuple[ return tuple((name, expected) for name, expected, is_malformed in malformed if is_malformed) -@dataclasses.dataclass(frozen=True, slots=True) -class BedrockSessionUpdateRejection: - code: Literal["invalid_type", "invalid_value", "session_update_not_applied"] - message: str - param: str - - @dataclasses.dataclass(frozen=True, slots=True) class BedrockLockedSessionSettings: """The session settings Bedrock Nova Sonic locks in once the prompt starts.""" @@ -532,7 +525,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): max_tokens=max_tokens, ) - def session_update_rejection(self, json_message: Mapping[str, JsonValue]) -> BedrockSessionUpdateRejection | None: + def session_update_rejection(self, json_message: Mapping[str, JsonValue]) -> RealtimeSessionUpdateRejection | None: """The error for a session.update this session cannot apply, or None when it applies.""" session_config: Final = json_message.get("session") if not isinstance(session_config, Mapping): @@ -540,7 +533,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): malformed: Final = () if self.drop_params else _malformed_settings(session_config) if malformed: name, expected = malformed[0] - return BedrockSessionUpdateRejection( + return RealtimeSessionUpdateRejection( code="invalid_type", message=( f"Invalid type for 'session.{name}': Bedrock Nova Sonic needs {expected} there. " @@ -550,7 +543,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): ) unplayable_voice: Final = None if self.drop_params else _unplayable_voice(session_config) if unplayable_voice is not None: - return BedrockSessionUpdateRejection( + return RealtimeSessionUpdateRejection( code="invalid_value", message=( f"Invalid value for 'session.voice': '{unplayable_voice}' is not a Bedrock Nova Sonic voice. " @@ -561,7 +554,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig): if not self.prompt_started: return None if self._apply_locked_settings(session_config, self._started_settings) != self._started_settings: - return BedrockSessionUpdateRejection( + return RealtimeSessionUpdateRejection( code="session_update_not_applied", message="Bedrock Nova Sonic cannot change tools, instructions, voice, output audio format, temperature or max output tokens after the session has started, so this session.update was not applied", param="session", diff --git a/litellm/llms/gemini/realtime/transformation.py b/litellm/llms/gemini/realtime/transformation.py index 1cde5f4956e..a8f11ba5484 100644 --- a/litellm/llms/gemini/realtime/transformation.py +++ b/litellm/llms/gemini/realtime/transformation.py @@ -14,7 +14,7 @@ import litellm from litellm import verbose_logger from litellm._uuid import uuid from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig +from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeSessionUpdateRejection from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( VertexGeminiConfig, ) @@ -154,8 +154,9 @@ def _system_instruction_text(system_instruction: HttpxContentType | None) -> str class GeminiRealtimeConfig(BaseRealtimeConfig): _TOOL_CALL_ID_TO_NAME_MAX = 256 # LRU cap for call_id→name mapping - def __init__(self): + def __init__(self, drop_params: bool = False): super().__init__() + self.drop_params = drop_params self._tool_call_id_to_name: OrderedDict[str, str] = OrderedDict() # Gemini Live sometimes emits usageMetadata in a standalone frame between # turns; buffer it here so the next response.done carries the token counts. @@ -467,6 +468,47 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): return () return tuple(str(modality).upper() for modality in _JSON_ARRAY.validate_python(value)) + @staticmethod + def _unsupported_response_modalities(model: str, requested: tuple[str, ...]) -> tuple[str, ...]: + unknown: Final = tuple(modality for modality in requested if modality not in _GEMINI_LIVE_RESPONSE_MODALITIES) + if unknown: + return unknown + if GeminiRealtimeConfig._is_audio_only_live_model(model) and frozenset(requested) == frozenset(("TEXT",)): + return ("TEXT",) + if GeminiRealtimeConfig._is_text_only_live_model(model) and "AUDIO" in requested: + return ("AUDIO",) + return () + + def setup_session_update_rejection( + self, model: str, session: Mapping[str, object], *, setup_sent: bool + ) -> RealtimeSessionUpdateRejection | None: + if self.drop_params: + return None + if setup_sent: + return RealtimeSessionUpdateRejection( + code="session_update_not_applied", + message=( + "Gemini Live takes its session configuration once, with the first conversation content, " + "so this session.update was not applied. Send session.update before the first content, " + "or set drop_params to drop it instead" + ), + param="session", + ) + param: Final = "output_modalities" if "output_modalities" in session else "modalities" + unsupported: Final = GeminiRealtimeConfig._unsupported_response_modalities( + model, GeminiRealtimeConfig._requested_response_modalities(session.get(param)) + ) + if not unsupported: + return None + return RealtimeSessionUpdateRejection( + code="invalid_value", + message=( + f"Invalid value for 'session.{param}': {model} cannot produce " + f"{', '.join(modality.lower() for modality in unsupported)} output. Set drop_params to drop it instead" + ), + param=f"session.{param}", + ) + @staticmethod def _coerce_response_modalities(model: str, modalities: Sequence[object]) -> tuple[str, ...]: """Drop modalities Gemini Live cannot produce at all, swap the ones this @@ -708,10 +750,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): logging_session_id: str, session_configuration_request: str | None = None, ) -> OpenAIRealtimeStreamSessionEvents: - if session_configuration_request: - session_configuration_request_dict: BidiGenerateContentSetup = _parse_setup(session_configuration_request) - else: - session_configuration_request_dict = {} + session_configuration_request_dict: Final = _parse_setup( + session_configuration_request or self.session_configuration_request(model) + ) _model: Final = session_configuration_request_dict.get("model") or model generation_config: Final = session_configuration_request_dict.get("generationConfig", {}) or {} diff --git a/litellm/llms/vertex_ai/realtime/transformation.py b/litellm/llms/vertex_ai/realtime/transformation.py index 55f1cf58cbe..421e324ede6 100644 --- a/litellm/llms/vertex_ai/realtime/transformation.py +++ b/litellm/llms/vertex_ai/realtime/transformation.py @@ -38,8 +38,8 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig): (they require async I/O) and injected at construction time. """ - def __init__(self, access_token: str, project: str, location: str) -> None: - super().__init__() + def __init__(self, access_token: str, project: str, location: str, drop_params: bool = False) -> None: + super().__init__(drop_params=drop_params) self._access_token = access_token self._project = project self._location = location @@ -261,4 +261,5 @@ def vertex_realtime_config( access_token=access_token, project=project, location=VertexBase.get_vertex_region(vertex_region=location, model=model), + drop_params=drop_params, ) diff --git a/litellm/utils.py b/litellm/utils.py index 379eddb3155..0c0f7019854 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -10487,7 +10487,7 @@ class ProviderConfigManager: if LlmProviders.GEMINI == provider: from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig - return GeminiRealtimeConfig() + return GeminiRealtimeConfig(drop_params=drop_params) if LlmProviders.META == provider: from litellm.llms.meta.realtime.transformation import MetaRealtimeConfig diff --git a/tests/unit/litellm_core_utils/test_realtime_streaming.py b/tests/unit/litellm_core_utils/test_realtime_streaming.py index 6b915316d6c..1f78823dac7 100644 --- a/tests/unit/litellm_core_utils/test_realtime_streaming.py +++ b/tests/unit/litellm_core_utils/test_realtime_streaming.py @@ -1,6 +1,7 @@ import asyncio import base64 import json +import time from collections.abc import Coroutine, Mapping, Sequence from dataclasses import dataclass from typing import Final, Literal @@ -1264,6 +1265,7 @@ async def test_client_ack_caches_setup_to_prevent_duplicate_session_update_setup websocket.send_text = AsyncMock() provider_config = MagicMock() + provider_config.setup_session_update_rejection = MagicMock(return_value=None) def _transform(message: str, model: str, session_configuration_request=None): if session_configuration_request is None: @@ -2419,6 +2421,7 @@ async def test_subsequent_session_update_cannot_reenable_vad_when_guardrails_act logging_obj.success_handler = MagicMock() provider_config = MagicMock() + provider_config.setup_session_update_rejection = MagicMock(return_value=None) transformed_messages = [] def mock_transform(msg, model, session_config): @@ -2471,6 +2474,7 @@ async def test_follow_up_setup_updates_cached_session_configuration_request(): logging_obj.success_handler = MagicMock() provider_config = MagicMock() + provider_config.setup_session_update_rejection = MagicMock(return_value=None) follow_up_setup = json.dumps( { "setup": { @@ -2504,7 +2508,9 @@ async def test_follow_up_setup_updates_cached_session_configuration_request(): assert streaming.session_configuration_request == follow_up_setup -def _gemini_default_mode_streaming(monkeypatch, client_messages, model="gemini-live-2.5-flash-native-audio"): +def _gemini_default_mode_streaming( + monkeypatch, client_messages, model="gemini-live-2.5-flash-native-audio", drop_params=False +): monkeypatch.setattr(litellm, "gemini_live_defer_setup", False, raising=False) from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig @@ -2517,12 +2523,16 @@ def _gemini_default_mode_streaming(monkeypatch, client_messages, model="gemini-l client_ws, backend_ws, MagicMock(), - provider_config=GeminiRealtimeConfig(), + provider_config=GeminiRealtimeConfig(drop_params=drop_params), model=model, ) return streaming, backend_ws +def _events_sent_to_client(streaming: RealTimeStreaming) -> list[dict]: + return [json.loads(call.args[0]) for call in streaming.websocket.send_text.await_args_list] + + _TEXT_ITEM_CREATE = json.dumps( { "type": "conversation.item.create", @@ -2610,6 +2620,26 @@ def test_merge_session_fields_replaces_nested_mappings_wholesale_past_the_depth_ assert level(merged, 9)["new"] == 1 + +def _merge_cost(levels: int, leaves: int) -> float: + def nested(depth: int) -> dict[str, object]: + return {"child": nested(depth - 1)} if depth else {f"k{index}": index for index in range(leaves)} + + session: Final = nested(levels) + + def once() -> float: + started: Final = time.perf_counter() + _merge_session_fields(session, session) + return time.perf_counter() - started + + return min(once() for _ in range(3)) + + +def test_merge_session_fields_cost_stays_flat_as_matching_updates_nest_deeper(): + growth: Final = _merge_cost(7, 2000) / _merge_cost(1, 2000) + + assert growth < 8.0, f"merging seven nested levels cost {growth:.1f}x one level" + @pytest.mark.asyncio async def test_default_mode_acknowledges_each_held_session_update_before_any_backend_frame(monkeypatch): """A client that waits for session.updated before sending content must not @@ -2633,6 +2663,108 @@ async def test_default_mode_acknowledges_each_held_session_update_before_any_bac backend_ws.send.assert_not_awaited() +_LATE_INSTRUCTIONS_UPDATE = json.dumps( + {"type": "session.update", "event_id": "evt_late", "session": {"instructions": "Be brief."}} +) + + +@pytest.mark.asyncio +async def test_default_mode_answers_a_session_update_after_the_first_content_with_an_error(monkeypatch): + streaming, backend_ws = _gemini_default_mode_streaming(monkeypatch, [_TEXT_ITEM_CREATE, _LATE_INSTRUCTIONS_UPDATE]) + + await streaming.client_ack_messages() + + errors: Final = [event["error"] for event in _events_sent_to_client(streaming) if event["type"] == "error"] + assert [(error["code"], error["event_id"]) for error in errors] == [("session_update_not_applied", "evt_late")] + frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] + assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"] + + +@pytest.mark.asyncio +async def test_default_mode_drops_a_late_session_update_quietly_under_drop_params(monkeypatch): + streaming, backend_ws = _gemini_default_mode_streaming( + monkeypatch, [_TEXT_ITEM_CREATE, _LATE_INSTRUCTIONS_UPDATE], drop_params=True + ) + + await streaming.client_ack_messages() + + assert _events_sent_to_client(streaming) == [] + assert backend_ws.send.await_count == 2 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("model", "requested_output"), + [ + ("gemini-live-2.5-flash-native-audio", {"modalities": ["text"]}), + ("gemini-live-2.5-flash", {"output_modalities": ["image"]}), + ], +) +async def test_default_mode_rejects_an_output_modality_the_model_cannot_produce(monkeypatch, model, requested_output): + update: Final = json.dumps( + {"type": "session.update", "session": {**requested_output, "instructions": "Be brief."}} + ) + streaming, backend_ws = _gemini_default_mode_streaming(monkeypatch, [update, _TEXT_ITEM_CREATE], model=model) + + await streaming.client_ack_messages() + + events: Final = _events_sent_to_client(streaming) + assert [(event["type"], event["error"]["code"]) for event in events] == [("error", "invalid_value")] + setup: Final = json.loads(backend_ws.send.await_args_list[0].args[0])["setup"] + assert "systemInstruction" not in setup + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("modalities", "drop_params"), + [(["text", "audio"], False), (["text"], True)], + ids=["openai-default-pair", "text-dropped-under-drop-params"], +) +async def test_default_mode_applies_the_update_with_audio_output_on_an_audio_only_model( + monkeypatch, modalities, drop_params +): + update: Final = json.dumps( + {"type": "session.update", "session": {"modalities": modalities, "instructions": "Be brief."}} + ) + streaming, backend_ws = _gemini_default_mode_streaming( + monkeypatch, [update, _TEXT_ITEM_CREATE], drop_params=drop_params + ) + + await streaming.client_ack_messages() + + assert [event["type"] for event in _events_sent_to_client(streaming)] == ["session.updated"] + setup: Final = json.loads(backend_ws.send.await_args_list[0].args[0])["setup"] + assert setup["generationConfig"]["responseModalities"] == ["AUDIO"] + assert setup["systemInstruction"]["parts"][0]["text"] == "Be brief." + + +@pytest.mark.asyncio +async def test_default_mode_sends_no_extra_session_updated_when_gemini_confirms_the_setup(monkeypatch): + """Each held session.update was already acknowledged, so Gemini's setupComplete + must not reach the client as one more session.updated nobody asked for.""" + monkeypatch.setattr(litellm, "gemini_live_defer_setup", False, raising=False) + from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig + + model: Final = "gemini-live-2.5-flash-native-audio" + config: Final = GeminiRealtimeConfig() + client_ws: Final = MagicMock() + client_ws.send_text = AsyncMock() + backend_ws: Final = MagicMock() + backend_ws.send = AsyncMock() + backend_ws.recv = AsyncMock(side_effect=[json.dumps({"setupComplete": {}}).encode(), ConnectionClosed(None, None)]) + logging_obj: Final = _mock_logging() + logging_obj.litellm_trace_id = "trace_lazy" + logging_obj.async_success_handler = AsyncMock() + logging_obj.success_handler = MagicMock() + streaming: Final = RealTimeStreaming(client_ws, backend_ws, logging_obj, provider_config=config, model=model) + streaming.session_created_sent_to_client = True + streaming.session_configuration_request = config.session_configuration_request(model) + + await streaming.backend_to_client_send_messages() + + assert [json.loads(call.args[0])["type"] for call in client_ws.send_text.await_args_list] == [] + + @pytest.mark.asyncio async def test_default_mode_folds_transcription_guardrail_vad_disable_into_the_merged_setup(monkeypatch): """A transcription guardrail gates every turn, so its auto-response disable diff --git a/tests/unit/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py index 42ec2d0bcec..0cdccb528fa 100644 --- a/tests/unit/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py @@ -2766,7 +2766,7 @@ def _realtime_logging_obj(): @pytest.mark.asyncio @pytest.mark.parametrize( - "model, session, backend_modality, client_modality, backend_turn", + "model, session, backend_modality, client_modality, backend_turn, drop_params", [ ( "gemini-live-2.5-flash", @@ -2774,27 +2774,29 @@ def _realtime_logging_obj(): "TEXT", "text", [_GEMINI_LIVE_TEXT_TURN], + False, ), - ("gemini-live-2.5-flash", {"modalities": ["text"]}, "TEXT", "text", [_GEMINI_LIVE_TEXT_TURN]), + ("gemini-live-2.5-flash", {"modalities": ["text"]}, "TEXT", "text", [_GEMINI_LIVE_TEXT_TURN], False), ( "gemini-live-2.5-flash-native-audio", {"type": "realtime", "output_modalities": ["text"]}, "AUDIO", "audio", [], + True, ), ], ) async def test_async_realtime_vertex_first_session_update_builds_the_only_setup( - model, session, backend_modality, client_modality, backend_turn, monkeypatch + model, session, backend_modality, client_modality, backend_turn, drop_params: bool, monkeypatch ): """Regression for text-only Vertex Live sessions answered in audio: the connect-time default setup (AUDIO) used to go out before the client's session.update, and Gemini accepts only one setup per connection, so the requested modality was dropped and billed at the audio rate. The setup now goes out on the client's first message, built from session.update when that - comes first. An audio-only model still gets AUDIO, and the client learns the - downgrade from session.updated.""" + comes first. Under drop_params an audio-only model still gets AUDIO, and the + client learns the downgrade from session.updated.""" from litellm.llms.vertex_ai.realtime.transformation import VertexAIRealtimeConfig monkeypatch.setattr(litellm, "gemini_live_defer_setup", False, raising=False) @@ -2825,7 +2827,7 @@ async def test_async_realtime_vertex_first_session_update_builds_the_only_setup( websocket=client_ws, logging_obj=_realtime_logging_obj(), provider_config=VertexAIRealtimeConfig( - access_token="fake-token", project="fake-project", location="us-central1" + access_token="fake-token", project="fake-project", location="us-central1", drop_params=drop_params ), headers={}, ), diff --git a/tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py b/tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py index ccb824b080e..5cd60b7de4b 100644 --- a/tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py +++ b/tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py @@ -1983,6 +1983,42 @@ def test_gemini_transcribe_live_coerces_audio_modality_to_text(modalities, patch assert setup["generationConfig"]["responseModalities"] == ["TEXT"] +@pytest.mark.parametrize("model", ["gemini-3.5-transcribe-live", "gemini/gemini-3.5-transcribe-live"]) +def test_gemini_transcribe_live_synthetic_session_created_reports_text_output( + model, patch_gemini_transcribe_live_cost_map_entry +): + session = GeminiRealtimeConfig().transform_session_created_event(model, "sess_transcribe")["session"] + + assert session["modalities"] == ["text"] + + +@pytest.mark.parametrize("modalities", [["audio"], ["audio", "text"]]) +@pytest.mark.parametrize("drop_params", [False, True]) +def test_gemini_transcribe_live_audio_output_is_rejected_unless_drop_params( + modalities, drop_params, patch_gemini_transcribe_live_cost_map_entry +): + rejection = GeminiRealtimeConfig(drop_params=drop_params).setup_session_update_rejection( + "gemini-3.5-transcribe-live", {"output_modalities": modalities}, setup_sent=False + ) + + assert (rejection is None) is drop_params + assert rejection is None or rejection.param == "session.output_modalities" + + +@pytest.mark.parametrize("drop_params", [False, True]) +def test_gemini_provider_realtime_config_carries_drop_params(drop_params): + from litellm.types.utils import LlmProviders + from litellm.utils import ProviderConfigManager + + model = "gemini-live-2.5-flash-native-audio" + config = ProviderConfigManager.get_provider_realtime_config( + model=model, provider=LlmProviders.GEMINI, drop_params=drop_params + ) + + rejection = config.setup_session_update_rejection(model, {"modalities": ["text"]}, setup_sent=False) + assert (rejection is None) is drop_params + + def test_gemini_chat_model_with_text_output_modalities_keeps_audio_eager_setup( patch_gemini_transcribe_live_cost_map_entry, ): diff --git a/tests/unit/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py b/tests/unit/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py index 8c4e098417b..714a29c36a2 100644 --- a/tests/unit/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py +++ b/tests/unit/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py @@ -179,6 +179,25 @@ def test_vertex_audio_only_live_model_coerces_text_modality_to_audio( assert setup["generationConfig"]["responseModalities"] == ["AUDIO"] +@pytest.mark.parametrize("drop_params", [False, True]) +def test_vertex_realtime_config_carries_drop_params(drop_params, patch_native_audio_cost_map_entry): + from litellm.llms.vertex_ai.realtime.transformation import vertex_realtime_config + + cfg: Final = vertex_realtime_config( + _NATIVE_AUDIO_MODEL, + access_token="tok", + resolve_access_token=AsyncMock(return_value="tok"), + project="my-proj", + location="us-central1", + drop_params=drop_params, + ) + + rejection: Final = cfg.setup_session_update_rejection( + _NATIVE_AUDIO_MODEL, {"modalities": ["text"]}, setup_sent=False + ) + assert (rejection is None) is drop_params + + def test_vertex_session_update_normalizes_ga_remapped_fields( patch_native_audio_cost_map_entry, ): From c588e734e47dc16a4522c869fcff5f032fd644ac Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 21 Oct 2026 04:59:45 -0700 Subject: [PATCH 07/10] fix(gemini realtime): send the setup before guardrail blocks and idle deadlines, mark push-to-talk turns A pre_call guardrail blocking the first typed message now goes out after the setup, so Gemini and Vertex no longer close the session with 1007. Push-to-talk sessions (turn_detection null or create_response false) wrap each audio turn in empty activityStart and activityEnd messages, which Gemini needs when automatic activity detection is off. A session with no content after 60 seconds sends its setup, since Vertex refuses a setup 300 seconds after connect. Held session.update frames past 10 MB answer session_update_too_large, and deferred mode now errors on a second session.update unless drop_params is set. --- litellm/constants.py | 1 + .../litellm_core_utils/realtime_streaming.py | 72 +++++++-- .../llms/base_llm/realtime/transformation.py | 2 +- .../llms/gemini/realtime/transformation.py | 38 +++-- litellm/types/llms/gemini.py | 16 +- .../test_realtime_streaming.py | 148 +++++++++++++++++- .../test_gemini_realtime_transformation.py | 59 +++++-- 7 files changed, 285 insertions(+), 51 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 6a503463a8c..4e957fbcff7 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -415,6 +415,7 @@ REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS: Final = get_env_float( "REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS", 20.0 ) BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS: Final = get_env_float("BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS", 5.0) +REALTIME_LAZY_SETUP_DEADLINE_SECONDS: Final = 60.0 # RFC 6455 caps the close frame payload at 125 bytes, 2 of which carry the status code WEBSOCKET_CLOSE_REASON_MAX_BYTES: Final = 123 diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index ed75031eb9e..a9aba6d460c 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -1,5 +1,6 @@ import asyncio import json +import time import traceback from collections.abc import Callable, Coroutine, Mapping, MutableMapping, Sequence from dataclasses import dataclass @@ -15,12 +16,17 @@ from litellm._logging import redact_internal_details_from_client_message, verbos from litellm._uuid import uuid from litellm.constants import ( BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY, + REALTIME_LAZY_SETUP_DEADLINE_SECONDS, REALTIME_METERED_USAGE_KEY, REALTIME_SESSION_FAILURE_LOGGED_KEY, REALTIME_SESSION_SUCCESS_LOGGED_KEY, ) from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER -from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeBackend +from litellm.llms.base_llm.realtime.transformation import ( + BaseRealtimeConfig, + RealtimeBackend, + RealtimeSessionUpdateRejection, +) from litellm.types.llms.openai import ( OpenAIRealtimeEvents, OpenAIRealtimeOutputItemDone, @@ -156,6 +162,16 @@ def _session_update_frame(session: Mapping[str, object]) -> str: return json.dumps(frame, default=_SESSION_FIELDS.validate_python) +_HELD_SESSION_UPDATE_TOO_LARGE: Final = RealtimeSessionUpdateRejection( + code="session_update_too_large", + message=( + "The session.update frames sent before the first conversation content add up to more than 10 MB, " + "so this session.update was not applied" + ), + param="session", +) + + def _decode_json_object(payload: str) -> Mapping[str, object]: """Decode a realtime frame into its top-level field mapping.""" return json.loads(payload) @@ -274,8 +290,10 @@ class RealTimeStreaming: event_normalizer: RealtimeEventNormalizer | None = None, usage_meter: RealtimeUsageMeter | None = None, logging_worker: _LoggingWorker = GLOBAL_LOGGING_WORKER, + lazy_setup_deadline_seconds: float = REALTIME_LAZY_SETUP_DEADLINE_SECONDS, ): self.websocket: _ClientWebSocket = websocket + self._lazy_setup_deadline: Final = time.monotonic() + lazy_setup_deadline_seconds self._replay_scope: Final = client_websocket_scope(websocket) self.backend_ws = backend_ws self.logging_obj = logging_obj @@ -332,6 +350,7 @@ class RealTimeStreaming: # content (realtimeInput / clientContent / toolResponse) has been sent. self._content_sent_after_setup: bool = False self._held_session_update: Mapping[str, object] | None = None + self._held_session_update_bytes: int = 0 # Whether this is a transcription-only session (session.type == "transcription", # e.g. gpt-realtime-whisper). Such sessions must not be sent response.create and # their input_audio_transcription.completed usage drives duration-based cost. @@ -649,7 +668,7 @@ class RealTimeStreaming: async def _rejected_setup_session_update(self, message: str) -> bool: provider_config: Final = self.provider_config - if provider_config is None or not self._builds_setup_lazily(): + if provider_config is None or not provider_config.builds_setup_from_session_update(): return False client_event: Final = _decode_json_object(message) session: Final = client_event.get("session") @@ -667,17 +686,23 @@ class RealTimeStreaming: return _session_update_frame(self._held_session_update or _NO_SESSION_FIELDS) async def _hold_session_update(self, message: str) -> None: - session: Final = _decode_json_object(message).get("session") + client_event: Final = _decode_json_object(message) + if self._held_session_update_bytes + len(message.encode("utf-8")) > RealTimeStreaming._MAX_BUFFERED_BYTES: + await self.websocket.send_text( + json.dumps(_HELD_SESSION_UPDATE_TOO_LARGE.error_event(client_event.get("event_id"))) + ) + return + session: Final = client_event.get("session") update: Final = _SESSION_FIELDS.validate_python(session) if isinstance(session, Mapping) else _NO_SESSION_FIELDS self._held_session_update = _merge_session_fields(self._held_session_update or _NO_SESSION_FIELDS, update) - await self._acknowledge_held_session_update() + held_message: Final = self._held_session_update_message() + self._held_session_update_bytes = len(held_message.encode("utf-8")) + await self._acknowledge_held_session_update(held_message) - async def _acknowledge_held_session_update(self) -> None: + async def _acknowledge_held_session_update(self, held_message: str) -> None: if self.provider_config is None: return - preview: Final = self.provider_config.transform_realtime_request( - self._held_session_update_message(), self.model, None - ) + preview: Final = self.provider_config.transform_realtime_request(held_message, self.model, None) setup: Final = next( (frame for frame in preview if isinstance(frame, str) and "setup" in _decode_json_object(frame)), None ) @@ -698,8 +723,26 @@ class RealTimeStreaming: return message: Final = self._held_session_update_message() self._held_session_update = None + self._held_session_update_bytes = 0 await self._send_to_backend(message) + async def _send_setup_before_content(self, msg_type: str | None) -> None: + if self._holds_session_updates_until_first_content(): + await self._send_held_session_update_as_setup() + await self._send_default_setup_before_first_client_message(msg_type) + + async def _receive_client_message_before_setup_deadline(self) -> str | None: + if not self._holds_session_updates_until_first_content(): + return await self._receive_client_message() + receive: Final = asyncio.ensure_future(self._receive_client_message()) + try: + done, _ = await asyncio.wait((receive,), timeout=self._lazy_setup_deadline - time.monotonic()) + if not done: + await self._send_setup_before_content(None) + return await receive + finally: + receive.cancel() + async def _send_default_setup_before_first_client_message(self, msg_type: str | None) -> None: if self.session_configuration_request is not None or self.provider_config is None: return @@ -1591,7 +1634,7 @@ class RealTimeStreaming: client_event: _ClientEventFrame try: while True: - message = await self._receive_client_message() + message = await self._receive_client_message_before_setup_deadline() if message is None: return ClientLoopExit.CLIENT_DISCONNECTED @@ -1606,6 +1649,7 @@ class RealTimeStreaming: msg_type = client_event.get("type") if msg_type == "conversation.item.create": + await self._send_setup_before_content(msg_type) # Check user text messages for prompt injection item = client_event.get("item", {}) # Check function_call_output first so a client cannot @@ -1803,12 +1847,10 @@ class RealTimeStreaming: await self._send_session_gate_once() if msg_type == "session.update" and await self._rejected_setup_session_update(message): continue - if self._holds_session_updates_until_first_content(): - if msg_type == "session.update": - await self._hold_session_update(message) - continue - await self._send_held_session_update_as_setup() - await self._send_default_setup_before_first_client_message(msg_type) + if msg_type == "session.update" and self._holds_session_updates_until_first_content(): + await self._hold_session_update(message) + continue + await self._send_setup_before_content(msg_type) sent = await self._send_to_backend(message) if guardrail_turn_detection_injected and sent: self._guardrail_turn_detection_update_sent = True diff --git a/litellm/llms/base_llm/realtime/transformation.py b/litellm/llms/base_llm/realtime/transformation.py index 47054b66187..2b72bcfff4f 100644 --- a/litellm/llms/base_llm/realtime/transformation.py +++ b/litellm/llms/base_llm/realtime/transformation.py @@ -27,7 +27,7 @@ else: @dataclasses.dataclass(frozen=True, slots=True) class RealtimeSessionUpdateRejection: - code: Literal["invalid_type", "invalid_value", "session_update_not_applied"] + code: Literal["invalid_type", "invalid_value", "session_update_not_applied", "session_update_too_large"] message: str param: str diff --git a/litellm/llms/gemini/realtime/transformation.py b/litellm/llms/gemini/realtime/transformation.py index a8f11ba5484..d131438875c 100644 --- a/litellm/llms/gemini/realtime/transformation.py +++ b/litellm/llms/gemini/realtime/transformation.py @@ -162,6 +162,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): # turns; buffer it here so the next response.done carries the token counts. self._pending_usage_metadata: dict | None = None self._unbilled_input_audio_bytes: int = 0 + self._manual_activity_open: bool = False def is_setup_message(self, msg_obj: dict) -> bool: return "setup" in msg_obj @@ -278,17 +279,26 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): except (json.JSONDecodeError, TypeError, AttributeError): return False + def _start_manual_activity(self, session_configuration_request: str | None) -> tuple[str, ...]: + if self._manual_activity_open or not self._manual_turn_detection_enabled(session_configuration_request): + return () + self._manual_activity_open = True + activity_start: Final[BidiGenerateContentRealtimeInput] = {"activityStart": {}} + return (json.dumps({"realtimeInput": activity_start}),) + + def _end_manual_activity(self) -> tuple[str, ...]: + if not self._manual_activity_open: + return () + self._manual_activity_open = False + activity_end: Final[BidiGenerateContentRealtimeInput] = {"activityEnd": {}} + return (json.dumps({"realtimeInput": activity_end}),) + def _handle_input_audio_buffer_commit_or_end(self, session_configuration_request: str | None) -> list[str]: """Map OpenAI buffer commit/end to Gemini Live turn-boundary signals.""" if self._manual_turn_detection_enabled(session_configuration_request): - realtime_input_dict: BidiGenerateContentRealtimeInput = { - "activityEnd": True, - } - verbose_logger.debug("Gemini Realtime: Sending activityEnd realtimeInput to backend") - else: - realtime_input_dict = {"audioStreamEnd": True} - verbose_logger.debug("Gemini Realtime: Sending audioStreamEnd realtimeInput to backend") - return [json.dumps({"realtimeInput": realtime_input_dict})] + return list(self._end_manual_activity()) + audio_stream_end: Final[BidiGenerateContentRealtimeInput] = {"audioStreamEnd": True} + return [json.dumps({"realtimeInput": audio_stream_end})] def map_automatic_turn_detection(self, value: OpenAIRealtimeTurnDetection) -> AutomaticActivityDetection: """Map OpenAI ``server_vad`` to Gemini ``automaticActivityDetection``. @@ -488,9 +498,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): return RealtimeSessionUpdateRejection( code="session_update_not_applied", message=( - "Gemini Live takes its session configuration once, with the first conversation content, " - "so this session.update was not applied. Send session.update before the first content, " - "or set drop_params to drop it instead" + "Gemini Live accepts one session configuration per connection, and this session already sent " + "its configuration, so this session.update was not applied. Put every setting in session.update " + "before the first conversation content, or set drop_params to drop it instead" ), param="session", ) @@ -708,14 +718,13 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): message_str = str(message) raise ValueError(f"Invalid JSON message: {message_str}") - messages: Final[list[str]] = [] msg_type: Final = json_message.get("type") if msg_type == "session.update": return self._handle_session_update(json_message, model, session_configuration_request) if msg_type == "response.create": - return [] # Gemini responds automatically; nothing to forward + return list(self._end_manual_activity()) if msg_type == "conversation.item.create": return self._handle_conversation_item(json_message) @@ -733,8 +742,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): gemini_msg: Final = json.dumps({"realtimeInput": realtime_input_dict}) verbose_logger.debug("Gemini Realtime: Sending audio realtimeInput to backend") - messages.append(gemini_msg) - return messages + return [*self._start_manual_activity(session_configuration_request), gemini_msg] if msg_type in ("input_audio_buffer.commit", "input_audio_buffer.end"): return self._handle_input_audio_buffer_commit_or_end(session_configuration_request) diff --git a/litellm/types/llms/gemini.py b/litellm/types/llms/gemini.py index c300c26bcd3..e2d17dc8243 100644 --- a/litellm/types/llms/gemini.py +++ b/litellm/types/llms/gemini.py @@ -80,6 +80,14 @@ class BidiGenerateContentServerMessage(TypedDict, total=False): """Output only. The setup complete message.""" +class BidiGenerateContentActivityStart(TypedDict): + pass + + +class BidiGenerateContentActivityEnd(TypedDict): + pass + + class BidiGenerateContentRealtimeInput(TypedDict, total=False): text: str """The text to be sent to the model.""" @@ -93,11 +101,11 @@ class BidiGenerateContentRealtimeInput(TypedDict, total=False): audioStreamEnd: bool """Output only. If true, indicates that the audio stream has ended.""" - activityStart: bool - """Output only. If true, indicates that the activity has started.""" + activityStart: BidiGenerateContentActivityStart + """Marks the start of user activity when automatic activity detection is disabled.""" - activityEnd: bool - """Output only. If true, indicates that the activity has ended.""" + activityEnd: BidiGenerateContentActivityEnd + """Marks the end of user activity when automatic activity detection is disabled.""" StartOfSpeechSensitivityEnum = Literal[ diff --git a/tests/unit/litellm_core_utils/test_realtime_streaming.py b/tests/unit/litellm_core_utils/test_realtime_streaming.py index 1f78823dac7..41a5ded28aa 100644 --- a/tests/unit/litellm_core_utils/test_realtime_streaming.py +++ b/tests/unit/litellm_core_utils/test_realtime_streaming.py @@ -2,7 +2,7 @@ import asyncio import base64 import json import time -from collections.abc import Coroutine, Mapping, Sequence +from collections.abc import Callable, Coroutine, Mapping, Sequence from dataclasses import dataclass from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock, patch @@ -2792,6 +2792,152 @@ async def test_default_mode_folds_transcription_guardrail_vad_disable_into_the_m assert len(injected) == 1 +def _blocking_text_guardrail() -> CustomGuardrail: + class _BlockingTextGuardrail(CustomGuardrail): + async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): + if any("zebrafruit" in text for text in inputs.get("texts", [])): + raise ValueError("Content blocked") + return inputs + + return _BlockingTextGuardrail( + guardrail_name="test_text_guard", event_hook=GuardrailEventHooks.pre_call, default_on=True + ) + + +@pytest.mark.asyncio +async def test_default_mode_sends_the_setup_before_a_guardrail_blocks_the_first_message(monkeypatch): + """Gemini Live closes the socket when content arrives before the setup, so a + guardrail that blocks the first typed message must not reach the backend first.""" + monkeypatch.setattr(litellm, "callbacks", [_blocking_text_guardrail()]) + instructions_update = json.dumps({"type": "session.update", "session": {"instructions": "Be brief."}}) + blocked_item = json.dumps( + { + "type": "conversation.item.create", + "item": {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "zebrafruit"}]}, + } + ) + streaming, backend_ws = _gemini_default_mode_streaming( + monkeypatch, [instructions_update, blocked_item], model="gemini-live-2.5-flash" + ) + + await streaming.client_ack_messages() + + frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] + assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"] + assert frames[0]["setup"]["systemInstruction"]["parts"][0]["text"] == "Be brief." + assert "zebrafruit" not in json.dumps(frames[1]) + errors: Final = [event["error"]["type"] for event in _events_sent_to_client(streaming) if event["type"] == "error"] + assert errors == ["guardrail_violation"] + + +def _client_ws_receiving(*steps: Callable[[], Coroutine[object, object, str]]) -> MagicMock: + remaining: Final = iter(steps) + + async def receive_text() -> str: + return await next(remaining)() + + client_ws: Final = MagicMock() + client_ws.receive_text = receive_text + client_ws.send_text = AsyncMock() + return client_ws + + +@pytest.mark.asyncio +async def test_default_mode_sends_the_held_setup_when_no_content_arrives_before_the_deadline(monkeypatch): + """Vertex AI refuses a setup sent minutes after connecting, so an idle session + sends what it holds at the deadline and answers any later session.update as late.""" + monkeypatch.setattr(litellm, "gemini_live_defer_setup", False, raising=False) + from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig + + setup_sent: Final = asyncio.Event() + + async def instructions_update() -> str: + return json.dumps({"type": "session.update", "session": {"instructions": "Be brief."}}) + + async def late_update_once_the_setup_is_out() -> str: + await setup_sent.wait() + return _LATE_INSTRUCTIONS_UPDATE + + async def disconnect() -> str: + raise ConnectionClosed(None, None) + + backend_ws: Final = MagicMock() + backend_ws.send = AsyncMock(side_effect=lambda frame: setup_sent.set()) + streaming: Final = RealTimeStreaming( + _client_ws_receiving(instructions_update, late_update_once_the_setup_is_out, disconnect), + backend_ws, + MagicMock(), + provider_config=GeminiRealtimeConfig(), + model="gemini-live-2.5-flash", + lazy_setup_deadline_seconds=0.05, + ) + + await asyncio.wait_for(streaming.client_ack_messages(), timeout=5) + + frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] + assert [next(iter(frame)) for frame in frames] == ["setup"] + assert frames[0]["setup"]["systemInstruction"]["parts"][0]["text"] == "Be brief." + events: Final = _events_sent_to_client(streaming) + assert [(event["type"], event.get("error", {}).get("code")) for event in events] == [ + ("session.updated", None), + ("error", "session_update_not_applied"), + ] + + +@pytest.mark.asyncio +async def test_default_mode_refuses_held_session_updates_past_the_size_cap(monkeypatch): + def instructions_update(text: str) -> str: + return json.dumps({"type": "session.update", "event_id": f"evt_{text[0]}", "session": {"instructions": text}}) + + half_the_cap: Final = RealTimeStreaming._MAX_BUFFERED_BYTES // 2 + streaming, backend_ws = _gemini_default_mode_streaming( + monkeypatch, + [instructions_update("a" * half_the_cap), instructions_update("b" * half_the_cap), _TEXT_ITEM_CREATE], + model="gemini-live-2.5-flash", + ) + + await streaming.client_ack_messages() + + events: Final = _events_sent_to_client(streaming) + assert [(event["type"], event.get("error", {}).get("event_id")) for event in events] == [ + ("session.updated", None), + ("error", "evt_b"), + ] + assert events[1]["error"]["code"] == "session_update_too_large" + setup: Final = json.loads(backend_ws.send.await_args_list[0].args[0])["setup"] + assert setup["systemInstruction"]["parts"][0]["text"].startswith("a") + + +@pytest.mark.asyncio +async def test_deferred_setup_answers_a_second_session_update_with_an_error(monkeypatch): + """Gemini Live ignores a second setup, so with deferred setup the second + session.update gets the same not-applied error as in the default mode.""" + monkeypatch.setattr(litellm, "gemini_live_defer_setup", True, raising=False) + from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig + + client_ws: Final = MagicMock() + client_ws.receive_text = AsyncMock( + side_effect=[ + json.dumps({"type": "session.update", "session": {"modalities": ["text"]}}), + _LATE_INSTRUCTIONS_UPDATE, + ConnectionClosed(None, None), + ] + ) + client_ws.send_text = AsyncMock() + backend_ws: Final = MagicMock() + backend_ws.send = AsyncMock() + streaming: Final = RealTimeStreaming( + client_ws, backend_ws, MagicMock(), provider_config=GeminiRealtimeConfig(), model="gemini-live-2.5-flash" + ) + + await streaming.client_ack_messages() + + frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] + assert [next(iter(frame)) for frame in frames] == ["setup"] + errors: Final = [event["error"] for event in _events_sent_to_client(streaming) if event["type"] == "error"] + assert [(error["code"], error["event_id"]) for error in errors] == [("session_update_not_applied", "evt_late")] + + @pytest.mark.asyncio async def test_deferred_setup_buffers_audio_until_backend_setup_complete(monkeypatch): """Pipecat may send audio before session.update when setup is deferred.""" diff --git a/tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py b/tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py index 5cd60b7de4b..141c816bf35 100644 --- a/tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py +++ b/tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py @@ -1,3 +1,4 @@ +import base64 import json from collections.abc import Mapping from typing import Final, cast @@ -1425,22 +1426,50 @@ def test_gemini_input_audio_buffer_clear_is_local_noop(): assert messages == [] -def test_gemini_input_audio_buffer_commit_maps_to_activity_end_when_manual_vad(): +_MANUAL_VAD_SETUP = json.dumps({"setup": {"realtimeInputConfig": {"automaticActivityDetection": {"disabled": True}}}}) +_APPEND = json.dumps({"type": "input_audio_buffer.append", "audio": base64.b64encode(b"\x00\x01" * 80).decode()}) + + +def _realtime_input_keys(config: GeminiRealtimeConfig, client_event: str, setup: str | None) -> list[str]: + frames = config.transform_realtime_request(client_event, "gemini-live-2.5-flash-native-audio", setup) + return [key for frame in frames for key in json.loads(frame)["realtimeInput"]] + + +@pytest.mark.parametrize("turn_end", ["input_audio_buffer.commit", "response.create"]) +def test_gemini_manual_vad_wraps_each_audio_turn_in_activity_start_and_end(turn_end): + """With activity detection off, Gemini Live answers only after activityEnd closes an + activityStart, and it closes the socket on any other shape of those markers.""" config = GeminiRealtimeConfig() - setup = { - "setup": { - "realtimeInputConfig": { - "automaticActivityDetection": {"disabled": True}, - } - } - } - messages = config.transform_realtime_request( - json.dumps({"type": "input_audio_buffer.commit"}), - "gemini-live-2.5-flash-native-audio", - session_configuration_request=json.dumps(setup), - ) - assert len(messages) == 1 - assert json.loads(messages[0]) == {"realtimeInput": {"activityEnd": True}} + end = json.dumps({"type": turn_end}) + + turns = [ + [_realtime_input_keys(config, event, _MANUAL_VAD_SETUP) for event in (_APPEND, _APPEND, end, end)] + for _ in range(2) + ] + + assert turns == [[["activityStart", "audio"], ["audio"], ["activityEnd"], []]] * 2 + + +def test_gemini_manual_vad_activity_markers_are_empty_messages(): + config = GeminiRealtimeConfig() + commit = json.dumps({"type": "input_audio_buffer.commit"}) + + first_append = config.transform_realtime_request(_APPEND, "gemini-live-2.5-flash", _MANUAL_VAD_SETUP) + turn_end = config.transform_realtime_request(commit, "gemini-live-2.5-flash", _MANUAL_VAD_SETUP) + + assert json.loads(first_append[0]) == {"realtimeInput": {"activityStart": {}}} + assert [json.loads(frame) for frame in turn_end] == [{"realtimeInput": {"activityEnd": {}}}] + + +def test_gemini_automatic_vad_sends_no_activity_markers(): + config = GeminiRealtimeConfig() + commit = json.dumps({"type": "input_audio_buffer.commit"}) + response_create = json.dumps({"type": "response.create"}) + + keys = [_realtime_input_keys(config, event, None) for event in (_APPEND, commit)] + + assert keys == [["audio"], ["audioStreamEnd"]] + assert config.transform_realtime_request(response_create, "gemini-live-2.5-flash", None) == [] def test_gemini_tool_call_id_to_name_evicts_oldest_when_capped(): From 58089dfcb742171c8bb08199a805035b30bfda60 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 17:53:15 -0700 Subject: [PATCH 08/10] fix(realtime): ack unchanged Gemini Live session.update resends and honor null turn_detection An update that leaves the applied setup unchanged now gets session.updated instead of session_update_not_applied, so clients that resend their session with the first response keep working. Deferred setup mode merges held updates like the default mode, turn_detection null turns Gemini's activity detection off, drop_params drops a voice outside the documented Live voices, and the lazy setup deadline moves to 240 seconds --- litellm/constants.py | 2 +- .../litellm_core_utils/realtime_streaming.py | 56 ++++++++--- .../llms/gemini/realtime/transformation.py | 98 ++++++++++++++----- .../test_realtime_streaming.py | 50 ++++++++-- .../test_gemini_realtime_transformation.py | 45 ++++++++- 5 files changed, 204 insertions(+), 47 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index a4ec377c707..67ce2c806de 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -416,7 +416,7 @@ REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS: Final = get_env_float( "REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS", 20.0 ) BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS: Final = get_env_float("BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS", 5.0) -REALTIME_LAZY_SETUP_DEADLINE_SECONDS: Final = 60.0 +REALTIME_LAZY_SETUP_DEADLINE_SECONDS: Final = 240.0 # RFC 6455 caps the close frame payload at 125 bytes, 2 of which carry the status code WEBSOCKET_CLOSE_REASON_MAX_BYTES: Final = 123 diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index a9aba6d460c..897bd53c14d 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -350,6 +350,7 @@ class RealTimeStreaming: # content (realtimeInput / clientContent / toolResponse) has been sent. self._content_sent_after_setup: bool = False self._held_session_update: Mapping[str, object] | None = None + self._applied_session_fields: Mapping[str, object] | None = None self._held_session_update_bytes: int = 0 # Whether this is a transcription-only session (session.type == "transcription", # e.g. gpt-realtime-whisper). Such sessions must not be sent response.create and @@ -661,7 +662,7 @@ class RealTimeStreaming: def _builds_setup_lazily(self) -> bool: if self.provider_config is None: return False - return self.provider_config.builds_setup_from_session_update() and not self._uses_deferred_backend_setup() + return self.provider_config.builds_setup_from_session_update() def _holds_session_updates_until_first_content(self) -> bool: return self.session_configuration_request is None and self._builds_setup_lazily() @@ -672,10 +673,12 @@ class RealTimeStreaming: return False client_event: Final = _decode_json_object(message) session: Final = client_event.get("session") + update: Final = _SESSION_FIELDS.validate_python(session) if isinstance(session, Mapping) else _NO_SESSION_FIELDS + if self._leaves_applied_setup_unchanged(update): + await self._acknowledge_session_setup(self._applied_session_fields or _NO_SESSION_FIELDS) + return True rejection: Final = provider_config.setup_session_update_rejection( - self.model, - _SESSION_FIELDS.validate_python(session) if isinstance(session, Mapping) else _NO_SESSION_FIELDS, - setup_sent=self.session_configuration_request is not None, + self.model, update, setup_sent=self.session_configuration_request is not None ) if rejection is None: return False @@ -694,18 +697,37 @@ class RealTimeStreaming: return session: Final = client_event.get("session") update: Final = _SESSION_FIELDS.validate_python(session) if isinstance(session, Mapping) else _NO_SESSION_FIELDS - self._held_session_update = _merge_session_fields(self._held_session_update or _NO_SESSION_FIELDS, update) - held_message: Final = self._held_session_update_message() - self._held_session_update_bytes = len(held_message.encode("utf-8")) - await self._acknowledge_held_session_update(held_message) + held: Final = _merge_session_fields(self._held_session_update or _NO_SESSION_FIELDS, update) + self._held_session_update = held + self._held_session_update_bytes = len(_session_update_frame(held).encode("utf-8")) + await self._acknowledge_session_setup(held) - async def _acknowledge_held_session_update(self, held_message: str) -> None: + def _setup_preview(self, session: Mapping[str, object]) -> str | None: if self.provider_config is None: - return - preview: Final = self.provider_config.transform_realtime_request(held_message, self.model, None) - setup: Final = next( + return None + preview: Final = self.provider_config.transform_realtime_request( + _session_update_frame(session), self.model, None + ) + return next( (frame for frame in preview if isinstance(frame, str) and "setup" in _decode_json_object(frame)), None ) + + def _leaves_applied_setup_unchanged(self, update: Mapping[str, object]) -> bool: + applied: Final = self._applied_session_fields + if self.session_configuration_request is None or applied is None: + return False + applied_setup: Final = self._setup_preview(applied) + merged_setup: Final = self._setup_preview(_merge_session_fields(applied, update)) + return ( + applied_setup is not None + and merged_setup is not None + and _decode_json_object(applied_setup) == _decode_json_object(merged_setup) + ) + + async def _acknowledge_session_setup(self, session: Mapping[str, object]) -> None: + if self.provider_config is None: + return + setup: Final = self._setup_preview(session) if setup is None: return acknowledged: Final = self.provider_config.transform_session_created_event( @@ -719,12 +741,14 @@ class RealTimeStreaming: await self._send_event_to_client(event, event_str) async def _send_held_session_update_as_setup(self) -> None: - if self._held_session_update is None: + held: Final = self._held_session_update + if held is None: return message: Final = self._held_session_update_message() self._held_session_update = None self._held_session_update_bytes = 0 - await self._send_to_backend(message) + if await self._send_to_backend(message): + self._applied_session_fields = held async def _send_setup_before_content(self, msg_type: str | None) -> None: if self._holds_session_updates_until_first_content(): @@ -754,6 +778,7 @@ class RealTimeStreaming: setup: Final = self.maybe_inject_guardrail_auto_response_disable(default_setup) await self.backend_ws.send(setup) self._cache_session_configuration_request(setup) + self._applied_session_fields = _NO_SESSION_FIELDS def _enforce_transcription_session_model(self, message: str) -> str: """Force client transcription session updates to the authorized model. @@ -1817,6 +1842,9 @@ class RealTimeStreaming: # actually forward to the backend. self.store_input(message=message) + if msg_type != "session.update" and self._uses_deferred_backend_setup(): + await self._send_setup_before_content(msg_type) + if self._should_buffer_client_message_until_setup(message): self._buffer_pending_message_until_setup(message) continue diff --git a/litellm/llms/gemini/realtime/transformation.py b/litellm/llms/gemini/realtime/transformation.py index 508cbebdd6d..06965fcc4e4 100644 --- a/litellm/llms/gemini/realtime/transformation.py +++ b/litellm/llms/gemini/realtime/transformation.py @@ -84,21 +84,67 @@ OPENAI_STOCK_REALTIME_VOICES: Final[frozenset[str]] = frozenset( {"alloy", "ash", "ballad", "cedar", "coral", "echo", "marin", "sage", "shimmer", "verse"} ) +# Per https://docs.cloud.google.com/vertex-ai/generative-ai/docs/live-api/configure-language-voice (2026-10-28) +GEMINI_LIVE_DOCUMENTED_VOICES: Final[frozenset[str]] = frozenset( + { + "achernar", + "achird", + "algenib", + "algieba", + "alnilam", + "aoede", + "autonoe", + "callirrhoe", + "charon", + "despina", + "enceladus", + "erinome", + "fenrir", + "gacrux", + "iapetus", + "kore", + "laomedeia", + "leda", + "orus", + "puck", + "pulcherrima", + "rasalgethi", + "sadachbia", + "sadaltager", + "schedar", + "sulafat", + "umbriel", + "vindemiatrix", + "zephyr", + "zubenelgenubi", + } +) -def _gemini_live_speech_config(voice: object) -> Mapping[str, object] | None: + +def _gemini_live_speech_config(voice: object, drop_params: bool) -> Mapping[str, object] | None: """Build the Gemini Live speechConfig for a client-requested voice. - OpenAI stock voice names have no Gemini equivalent and Gemini Live closes - the session on an unknown voice, so they are dropped with a warning and - the model keeps its default voice. Every other name is forwarded verbatim. + Gemini Live closes the session on a voice name it does not know. OpenAI + stock voice names are always dropped, and with drop_params so is any name + outside Google's documented Live voices, keeping the model's default voice. + Every other name is forwarded verbatim for Gemini to accept or refuse. """ - if isinstance(voice, str) and voice.lower() in OPENAI_STOCK_REALTIME_VOICES: + if not isinstance(voice, str): + return VertexGeminiConfig().map_audio_params({"voice": voice}) + if voice.lower() in OPENAI_STOCK_REALTIME_VOICES: verbose_logger.warning( "Gemini Realtime: voice %s is an OpenAI voice with no Gemini equivalent; " "dropping it so the session keeps the model's default voice.", voice, ) return None + if drop_params and voice.lower() not in GEMINI_LIVE_DOCUMENTED_VOICES: + verbose_logger.warning( + "Gemini Realtime: voice %s is not a documented Gemini Live voice; " + "drop_params is set, so it is dropped and the session keeps the model's default voice.", + voice, + ) + return None return VertexGeminiConfig().map_audio_params({"voice": voice}) @@ -373,30 +419,36 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): ) elif key == "input_audio_transcription" and value is not None: optional_params["inputAudioTranscription"] = {} - elif key == "turn_detection" and value is not None: - value_typed = cast(OpenAIRealtimeTurnDetection, value) - if ( - isinstance(value_typed, dict) - and value_typed.get("type") == "semantic_vad" - and "create_response" not in value_typed - ): - # Pipecat/OpenAI GA semantic VAD — skip; Gemini uses its own VAD. - # Only skip when there is no create_response override so that - # a guardrail-injected create_response:false is not dropped. - continue - transformed_audio_activity_config = self.map_automatic_turn_detection(value_typed) - if transformed_audio_activity_config: - optional_params["realtimeInputConfig"] = BidiGenerateContentRealtimeInputConfig( - automaticActivityDetection=transformed_audio_activity_config - ) + elif key == "turn_detection": + realtime_input_config = self._realtime_input_config(cast(object, value)) + if realtime_input_config is not None: + optional_params["realtimeInputConfig"] = realtime_input_config elif key == "voice": - speech_config = _gemini_live_speech_config(value) + speech_config = _gemini_live_speech_config(value, self.drop_params) if speech_config: optional_params["generationConfig"]["speechConfig"] = speech_config if len(optional_params["generationConfig"]) == 0: optional_params.pop("generationConfig") return optional_params + def _realtime_input_config(self, turn_detection: object) -> BidiGenerateContentRealtimeInputConfig | None: + if turn_detection is None: + return BidiGenerateContentRealtimeInputConfig(automaticActivityDetection={"disabled": True}) + value_typed: Final = cast(OpenAIRealtimeTurnDetection, turn_detection) + if ( + isinstance(value_typed, dict) + and value_typed.get("type") == "semantic_vad" + and "create_response" not in value_typed + ): + # Pipecat/OpenAI GA semantic VAD — skip; Gemini uses its own VAD. + # Only skip when there is no create_response override so that + # a guardrail-injected create_response:false is not dropped. + return None + transformed_audio_activity_config: Final = self.map_automatic_turn_detection(value_typed) + if not transformed_audio_activity_config: + return None + return BidiGenerateContentRealtimeInputConfig(automaticActivityDetection=transformed_audio_activity_config) + @staticmethod def _extract_turn_detection(session: dict) -> dict | None: """Extract turn_detection from a session.update payload. @@ -445,6 +497,8 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): if isinstance(input_cfg, dict): if "input_audio_transcription" not in normalized and "transcription" in input_cfg: normalized["input_audio_transcription"] = input_cfg["transcription"] + if "turn_detection" not in normalized and "turn_detection" in input_cfg: + normalized["turn_detection"] = input_cfg["turn_detection"] output_cfg: Final = audio.get("output") if isinstance(output_cfg, dict) and output_cfg.get("voice"): normalized["voice"] = output_cfg["voice"] diff --git a/tests/unit/litellm_core_utils/test_realtime_streaming.py b/tests/unit/litellm_core_utils/test_realtime_streaming.py index 41a5ded28aa..c530df79c89 100644 --- a/tests/unit/litellm_core_utils/test_realtime_streaming.py +++ b/tests/unit/litellm_core_utils/test_realtime_streaming.py @@ -2856,7 +2856,9 @@ async def test_default_mode_sends_the_held_setup_when_no_content_arrives_before_ async def late_update_once_the_setup_is_out() -> str: await setup_sent.wait() - return _LATE_INSTRUCTIONS_UPDATE + return json.dumps( + {"type": "session.update", "event_id": "evt_late", "session": {"instructions": "Answer in French."}} + ) async def disconnect() -> str: raise ConnectionClosed(None, None) @@ -2909,9 +2911,29 @@ async def test_default_mode_refuses_held_session_updates_past_the_size_cap(monke @pytest.mark.asyncio -async def test_deferred_setup_answers_a_second_session_update_with_an_error(monkeypatch): - """Gemini Live ignores a second setup, so with deferred setup the second - session.update gets the same not-applied error as in the default mode.""" +async def test_default_mode_acknowledges_an_unchanged_session_update_after_content(monkeypatch): + """Pipecat resends its unchanged session.update right after its first + conversation items and treats any error event as fatal.""" + session_update: Final = json.dumps( + {"type": "session.update", "session": {"modalities": ["text"], "instructions": "Be brief."}} + ) + streaming, backend_ws = _gemini_default_mode_streaming( + monkeypatch, [session_update, _TEXT_ITEM_CREATE, session_update], model="gemini-live-2.5-flash" + ) + + await streaming.client_ack_messages() + + frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] + assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"] + events: Final = _events_sent_to_client(streaming) + assert [event["type"] for event in events] == ["session.updated", "session.updated"] + assert events[0]["session"] == events[1]["session"] + + +@pytest.mark.asyncio +async def test_deferred_setup_merges_session_updates_until_the_first_content(monkeypatch): + """Deferred setup merges every session.update sent before the first content, + the same as the default mode, and answers a changed one after it as late.""" monkeypatch.setattr(litellm, "gemini_live_defer_setup", True, raising=False) from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig @@ -2919,7 +2941,11 @@ async def test_deferred_setup_answers_a_second_session_update_with_an_error(monk client_ws.receive_text = AsyncMock( side_effect=[ json.dumps({"type": "session.update", "session": {"modalities": ["text"]}}), - _LATE_INSTRUCTIONS_UPDATE, + json.dumps({"type": "session.update", "session": {"instructions": "Be brief."}}), + _TEXT_ITEM_CREATE, + json.dumps( + {"type": "session.update", "event_id": "evt_late", "session": {"instructions": "Answer in French."}} + ), ConnectionClosed(None, None), ] ) @@ -2933,7 +2959,9 @@ async def test_deferred_setup_answers_a_second_session_update_with_an_error(monk await streaming.client_ack_messages() frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] - assert [next(iter(frame)) for frame in frames] == ["setup"] + assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"] + assert frames[0]["setup"]["generationConfig"]["responseModalities"] == ["TEXT"] + assert frames[0]["setup"]["systemInstruction"]["parts"][0]["text"] == "Be brief." errors: Final = [event["error"] for event in _events_sent_to_client(streaming) if event["type"] == "error"] assert [(error["code"], error["event_id"]) for error in errors] == [("session_update_not_applied", "evt_late")] @@ -2973,14 +3001,18 @@ async def test_deferred_setup_buffers_audio_until_backend_setup_complete(monkeyp @pytest.mark.asyncio -async def test_deferred_setup_sends_session_update_before_buffered_audio(monkeypatch): +async def test_deferred_setup_sends_the_held_session_update_when_more_audio_arrives(monkeypatch): monkeypatch.setattr(litellm, "gemini_live_defer_setup", True, raising=False) from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig client_ws = MagicMock() + client_ws.send_text = AsyncMock() audio_msg = json.dumps({"type": "input_audio_buffer.append", "audio": "AA=="}) session_update = json.dumps({"type": "session.update", "session": {"modalities": ["audio"]}}) - client_ws.receive_text = AsyncMock(side_effect=[audio_msg, session_update, ConnectionClosed(None, None)]) + later_audio_msg = json.dumps({"type": "input_audio_buffer.append", "audio": "AQ=="}) + client_ws.receive_text = AsyncMock( + side_effect=[audio_msg, session_update, later_audio_msg, ConnectionClosed(None, None)] + ) backend_ws = MagicMock() backend_ws.send = AsyncMock() logging_obj: Final = _mock_logging() @@ -3000,7 +3032,7 @@ async def test_deferred_setup_sends_session_update_before_buffered_audio(monkeyp sent_payload = json.loads(backend_ws.send.await_args_list[0].args[0]) assert "setup" in sent_payload assert "realtimeInput" not in sent_payload - assert streaming._pending_messages_until_setup == [audio_msg] + assert streaming._pending_messages_until_setup == [audio_msg, later_audio_msg] @pytest.mark.asyncio diff --git a/tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py b/tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py index 141c816bf35..9e35e07fd54 100644 --- a/tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py +++ b/tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py @@ -8,7 +8,7 @@ import pytest import litellm -from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig +from litellm.llms.gemini.realtime.transformation import GEMINI_LIVE_DOCUMENTED_VOICES, GeminiRealtimeConfig from litellm.types.llms.gemini import BidiGenerateContentServerMessage @@ -1461,6 +1461,27 @@ def test_gemini_manual_vad_activity_markers_are_empty_messages(): assert [json.loads(frame) for frame in turn_end] == [{"realtimeInput": {"activityEnd": {}}}] +@pytest.mark.parametrize( + ("session", "expected_keys"), + [ + ({"turn_detection": None}, [["activityStart", "audio"], ["activityEnd"]]), + ({"type": "realtime", "audio": {"input": {"turn_detection": None}}}, [["activityStart", "audio"], ["activityEnd"]]), + ({"type": "realtime", "audio": {"input": {}}}, [["audio"], ["audioStreamEnd"]]), + ], +) +def test_gemini_null_turn_detection_makes_the_client_mark_each_turn(session, expected_keys): + """OpenAI's turn_detection null turns server VAD off, so the client commits each turn itself.""" + config = GeminiRealtimeConfig() + setup = config.transform_realtime_request( + json.dumps({"type": "session.update", "session": session}), "gemini-live-2.5-flash-native-audio", None + ) + commit = json.dumps({"type": "input_audio_buffer.commit"}) + + keys = [_realtime_input_keys(config, event, setup[0]) for event in (_APPEND, commit)] + + assert keys == expected_keys + + def test_gemini_automatic_vad_sends_no_activity_markers(): config = GeminiRealtimeConfig() commit = json.dumps({"type": "input_audio_buffer.commit"}) @@ -1948,6 +1969,28 @@ def test_map_openai_params_drops_stock_voice_case_insensitively(): assert passthrough["generationConfig"]["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Kore" +_DOCUMENTED_VOICE: Final = sorted(GEMINI_LIVE_DOCUMENTED_VOICES)[0].capitalize() + + +@pytest.mark.parametrize( + ("voice", "drop_params", "forwarded"), + [ + ("Notavoice", True, None), + ("Notavoice", False, "Notavoice"), + (_DOCUMENTED_VOICE, True, _DOCUMENTED_VOICE), + ], +) +def test_map_openai_params_drops_an_undocumented_voice_only_with_drop_params(voice, drop_params, forwarded): + """Gemini Live closes the session on a voice it does not know, and AI Studio knows + names Vertex AI refuses, so only drop_params narrows to the documented voices.""" + mapped = GeminiRealtimeConfig(drop_params=drop_params).map_openai_params( + optional_params={}, non_default_params={"voice": voice} + ) + + speech_config = mapped.get("generationConfig", {}).get("speechConfig") + assert (speech_config["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] if speech_config else None) == forwarded + + @pytest.fixture(autouse=False) def patch_gemini_transcribe_live_cost_map_entry(monkeypatch): """Inject the gemini-3.5-transcribe-live registry entry locally. From ebad57065f713b47adfae64e0fb205e432cd7232 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 18:22:32 -0700 Subject: [PATCH 09/10] fix(realtime): keep Vertex activity detection off for null turn_detection and set up deferred sessions before text content --- .../litellm_core_utils/realtime_streaming.py | 5 +++- .../llms/vertex_ai/realtime/transformation.py | 7 +++-- .../test_realtime_streaming.py | 30 +++++++++++++++++++ .../test_vertex_ai_realtime_transformation.py | 20 +++++++++++++ 4 files changed, 58 insertions(+), 4 deletions(-) diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index 897bd53c14d..9186a92a6c9 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -770,7 +770,10 @@ class RealTimeStreaming: async def _send_default_setup_before_first_client_message(self, msg_type: str | None) -> None: if self.session_configuration_request is not None or self.provider_config is None: return - if msg_type == "session.update" or self._uses_deferred_backend_setup(): + if msg_type == "session.update" or ( + self._uses_deferred_backend_setup() + and (msg_type in RealTimeStreaming._CLIENT_AUDIO_BUFFER_TYPES or self._pending_messages_until_setup) + ): return default_setup: Final = self.provider_config.session_configuration_request(self.model) if not default_setup: diff --git a/litellm/llms/vertex_ai/realtime/transformation.py b/litellm/llms/vertex_ai/realtime/transformation.py index fa3992120a6..d7a79dd32e6 100644 --- a/litellm/llms/vertex_ai/realtime/transformation.py +++ b/litellm/llms/vertex_ai/realtime/transformation.py @@ -177,15 +177,16 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig): # ``silence_duration_ms``). ``map_automatic_turn_detection`` sets # ``disabled=True`` whenever ``create_response`` is absent or # ``False``. Force ``disabled=False`` only when the client did - # not explicitly request ``create_response: False`` — that path + # not explicitly request ``create_response: False`` or + # ``turn_detection: null`` — the first of those paths # is how transcription guardrails suppress automatic responses, # and overriding it here would silently bypass the guardrail. # Vertex Live has no "VAD on, no auto-response" mode, so callers # that need that behaviour must accept that VAD is off. client_turn_detection: Final = session_params.get("turn_detection") client_disabled_auto_response: Final = ( - isinstance(client_turn_detection, dict) and client_turn_detection.get("create_response") is False - ) + "turn_detection" in session_params and client_turn_detection is None + ) or (isinstance(client_turn_detection, dict) and client_turn_detection.get("create_response") is False) realtime_input_config: Final = setup_config.setdefault("realtimeInputConfig", {}) automatic_detection: Final = realtime_input_config.setdefault("automaticActivityDetection", {}) if not client_disabled_auto_response: diff --git a/tests/unit/litellm_core_utils/test_realtime_streaming.py b/tests/unit/litellm_core_utils/test_realtime_streaming.py index c530df79c89..97b0511b953 100644 --- a/tests/unit/litellm_core_utils/test_realtime_streaming.py +++ b/tests/unit/litellm_core_utils/test_realtime_streaming.py @@ -2966,6 +2966,36 @@ async def test_deferred_setup_merges_session_updates_until_the_first_content(mon assert [(error["code"], error["event_id"]) for error in errors] == [("session_update_not_applied", "evt_late")] +@pytest.mark.asyncio +async def test_deferred_setup_sends_the_default_setup_before_text_content_that_comes_first(monkeypatch): + """Gemini Live closes the socket on content sent before any setup, so text that arrives + before a session.update in deferred mode gets the model's default setup first.""" + monkeypatch.setattr(litellm, "gemini_live_defer_setup", True, raising=False) + from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig + + client_ws: Final = MagicMock() + client_ws.receive_text = AsyncMock( + side_effect=[ + _TEXT_ITEM_CREATE, + json.dumps({"type": "session.update", "event_id": "evt_late", "session": {"modalities": ["text"]}}), + ConnectionClosed(None, None), + ] + ) + client_ws.send_text = AsyncMock() + backend_ws: Final = MagicMock() + backend_ws.send = AsyncMock() + streaming: Final = RealTimeStreaming( + client_ws, backend_ws, MagicMock(), provider_config=GeminiRealtimeConfig(), model="gemini-live-2.5-flash" + ) + + await streaming.client_ack_messages() + + frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] + assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"] + errors: Final = [event["error"] for event in _events_sent_to_client(streaming) if event["type"] == "error"] + assert [(error["code"], error["event_id"]) for error in errors] == [("session_update_not_applied", "evt_late")] + + @pytest.mark.asyncio async def test_deferred_setup_buffers_audio_until_backend_setup_complete(monkeypatch): """Pipecat may send audio before session.update when setup is deferred.""" diff --git a/tests/unit/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py b/tests/unit/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py index 714a29c36a2..0839718d890 100644 --- a/tests/unit/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py +++ b/tests/unit/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py @@ -272,6 +272,26 @@ def test_vertex_keeps_turn_detection_tuning_when_auto_response_is_off(): } + +@pytest.mark.parametrize( + ("session", "disabled"), + [ + ({"turn_detection": None}, True), + ({"type": "realtime", "audio": {"input": {"turn_detection": None}}}, True), + ({"type": "realtime", "audio": {"input": {}}}, False), + ], +) +def test_vertex_turns_activity_detection_off_for_null_turn_detection(session, disabled): + """OpenAI's turn_detection null is push-to-talk, so Vertex must wait for the client's activity markers.""" + cfg: Final = VertexAIRealtimeConfig(access_token="tok", project="my-proj", location="us-central1") + messages: Final = cfg.transform_realtime_request( + json.dumps({"type": "session.update", "session": session}), + _NATIVE_AUDIO_MODEL, + session_configuration_request=None, + ) + setup: Final = json.loads(messages[0])["setup"] + assert setup["realtimeInputConfig"]["automaticActivityDetection"]["disabled"] is disabled + # --------------------------------------------------------------------------- # Round-trip test: text-in / text-out via RealTimeStreaming # --------------------------------------------------------------------------- From ef52e611b3b7d69e6f6f19f7a572ee87a43caa7e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 20:59:27 -0700 Subject: [PATCH 10/10] fix(realtime): refuse unscreenable Gemini audio, send held setup after buffered audio, reject unknown Vertex voices A realtime_input_transcription guardrail cannot screen Gemini Live audio before the answer starts, so audio input under one is refused with an audio_input_not_screenable error instead of reaching the model. In deferred setup mode a session.update that follows buffered audio is sent as setup at once. Only conversation content triggers the setup, so control events no longer lock in a default one. Vertex rejects an undocumented voice on the session.update, or drops it with drop_params, while Google AI Studio forwards any name. --- .../litellm_core_utils/realtime_streaming.py | 53 +++++++++++++- .../llms/gemini/realtime/transformation.py | 45 ++++++++++-- .../llms/vertex_ai/realtime/transformation.py | 3 + .../test_realtime_streaming.py | 69 ++++++++++++++++++- .../test_gemini_realtime_transformation.py | 24 +++---- .../test_vertex_ai_realtime_transformation.py | 34 +++++++++ 6 files changed, 202 insertions(+), 26 deletions(-) diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index 9186a92a6c9..2c8c0b8b6d5 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -172,6 +172,23 @@ _HELD_SESSION_UPDATE_TOO_LARGE: Final = RealtimeSessionUpdateRejection( ) +def _unscreened_audio_error_event(client_event_id: object) -> dict[str, object]: + return { + "type": "error", + "event_id": f"event_{uuid.uuid4()}", + "error": { + "type": "invalid_request_error", + "code": "audio_input_not_screenable", + "message": ( + "A realtime_input_transcription guardrail is on, and Gemini Live only transcribes a turn after it " + "starts answering it, so audio input was not sent. Send text with conversation.item.create instead" + ), + "param": None, + "event_id": client_event_id if isinstance(client_event_id, str) else None, + }, + } + + def _decode_json_object(payload: str) -> Mapping[str, object]: """Decode a realtime frame into its top-level field mapping.""" return json.loads(payload) @@ -352,6 +369,7 @@ class RealTimeStreaming: self._held_session_update: Mapping[str, object] | None = None self._applied_session_fields: Mapping[str, object] | None = None self._held_session_update_bytes: int = 0 + self._unscreened_audio_refused: bool = False # Whether this is a transcription-only session (session.type == "transcription", # e.g. gpt-realtime-whisper). Such sessions must not be sent response.create and # their input_audio_transcription.completed usage drives duration-based cost. @@ -389,6 +407,15 @@ class RealTimeStreaming: ] ) _CLIENT_AUDIO_BUFFER_COMMIT_TYPES = frozenset(["input_audio_buffer.commit", "input_audio_buffer.end"]) + _CLIENT_CONTENT_TYPES = frozenset( + [ + "conversation.item.create", + "response.create", + "input_audio_buffer.append", + "input_audio_buffer.commit", + "input_audio_buffer.end", + ] + ) _AUDIO_FORMAT_MAP: dict[str, dict[str, str | int]] = { "pcm16": {"type": "audio/pcm", "rate": 24000}, "g711_ulaw": {"type": "audio/G711-ulaw", "rate": 8000}, @@ -1134,6 +1161,21 @@ class RealTimeStreaming: """Transcription-only sessions have no assistant turn to gate and reject realtime session updates.""" return not self._is_transcription_session and self._has_audio_transcription_guardrails() + def _refuses_unscreened_audio(self, msg_type: str | None) -> bool: + return ( + msg_type in RealTimeStreaming._CLIENT_AUDIO_BUFFER_TYPES + and self._builds_setup_lazily() + and self._should_disable_vad_auto_response() + ) + + async def _refuse_unscreened_audio(self, msg_type: str | None, message: str) -> None: + if self._unscreened_audio_refused and msg_type not in RealTimeStreaming._CLIENT_AUDIO_BUFFER_COMMIT_TYPES: + return + self._unscreened_audio_refused = True + await self.websocket.send_text( + json.dumps(_unscreened_audio_error_event(_decode_json_object(message).get("event_id"))) + ) + async def run_realtime_guardrails( self, transcript: str, @@ -1839,13 +1881,17 @@ class RealTimeStreaming: except (json.JSONDecodeError, AttributeError): pass + if self._refuses_unscreened_audio(msg_type): + await self._refuse_unscreened_audio(msg_type, message) + continue + ## LOGGING # Log after any in-place modifications (GA remap, guardrail # turn_detection injection) so audit logs reflect what we # actually forward to the backend. self.store_input(message=message) - if msg_type != "session.update" and self._uses_deferred_backend_setup(): + if msg_type in RealTimeStreaming._CLIENT_CONTENT_TYPES and self._uses_deferred_backend_setup(): await self._send_setup_before_content(msg_type) if self._should_buffer_client_message_until_setup(message): @@ -1880,8 +1926,11 @@ class RealTimeStreaming: continue if msg_type == "session.update" and self._holds_session_updates_until_first_content(): await self._hold_session_update(message) + if self._pending_messages_until_setup: + await self._send_held_session_update_as_setup() continue - await self._send_setup_before_content(msg_type) + if msg_type in RealTimeStreaming._CLIENT_CONTENT_TYPES: + await self._send_setup_before_content(msg_type) sent = await self._send_to_backend(message) if guardrail_turn_detection_injected and sent: self._guardrail_turn_detection_update_sent = True diff --git a/litellm/llms/gemini/realtime/transformation.py b/litellm/llms/gemini/realtime/transformation.py index 06965fcc4e4..19e87be36c5 100644 --- a/litellm/llms/gemini/realtime/transformation.py +++ b/litellm/llms/gemini/realtime/transformation.py @@ -121,13 +121,21 @@ GEMINI_LIVE_DOCUMENTED_VOICES: Final[frozenset[str]] = frozenset( ) -def _gemini_live_speech_config(voice: object, drop_params: bool) -> Mapping[str, object] | None: +def _is_undocumented_live_voice(voice: object) -> bool: + return ( + isinstance(voice, str) + and voice.lower() not in OPENAI_STOCK_REALTIME_VOICES + and voice.lower() not in GEMINI_LIVE_DOCUMENTED_VOICES + ) + + +def _gemini_live_speech_config(voice: object, drop_undocumented: bool) -> Mapping[str, object] | None: """Build the Gemini Live speechConfig for a client-requested voice. Gemini Live closes the session on a voice name it does not know. OpenAI - stock voice names are always dropped, and with drop_params so is any name - outside Google's documented Live voices, keeping the model's default voice. - Every other name is forwarded verbatim for Gemini to accept or refuse. + stock voice names are always dropped, and so is any name outside Google's + documented Live voices when ``drop_undocumented`` is set, keeping the + model's default voice. Every other name is forwarded verbatim. """ if not isinstance(voice, str): return VertexGeminiConfig().map_audio_params({"voice": voice}) @@ -138,7 +146,7 @@ def _gemini_live_speech_config(voice: object, drop_params: bool) -> Mapping[str, voice, ) return None - if drop_params and voice.lower() not in GEMINI_LIVE_DOCUMENTED_VOICES: + if drop_undocumented and _is_undocumented_live_voice(voice): verbose_logger.warning( "Gemini Realtime: voice %s is not a documented Gemini Live voice; " "drop_params is set, so it is dropped and the session keeps the model's default voice.", @@ -226,6 +234,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): """Google AI Studio closes the socket (1007) on padding or silence tuning next to ``disabled: true``.""" return True + def _rejects_undocumented_voices(self) -> bool: + return False + @staticmethod def _usage_detail_alias(details: object, defaults: dict[str, int]) -> dict[str, object]: detail_mapping: Final = as_str_mapping(details) @@ -424,7 +435,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): if realtime_input_config is not None: optional_params["realtimeInputConfig"] = realtime_input_config elif key == "voice": - speech_config = _gemini_live_speech_config(value, self.drop_params) + speech_config = _gemini_live_speech_config( + value, self.drop_params and self._rejects_undocumented_voices() + ) if speech_config: optional_params["generationConfig"]["speechConfig"] = speech_config if len(optional_params["generationConfig"]) == 0: @@ -564,6 +577,10 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): ), param="session", ) + return self._modality_rejection(model, session) or self._voice_rejection(model, session) + + @staticmethod + def _modality_rejection(model: str, session: Mapping[str, object]) -> RealtimeSessionUpdateRejection | None: param: Final = "output_modalities" if "output_modalities" in session else "modalities" unsupported: Final = GeminiRealtimeConfig._unsupported_response_modalities( model, GeminiRealtimeConfig._requested_response_modalities(session.get(param)) @@ -579,6 +596,22 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): param=f"session.{param}", ) + def _voice_rejection(self, model: str, session: Mapping[str, object]) -> RealtimeSessionUpdateRejection | None: + audio_output: Final = as_str_mapping((as_str_mapping(session.get("audio")) or {}).get("output")) or {} + output_voice: Final = audio_output.get("voice") + voice: Final = output_voice or session.get("voice") + if not self._rejects_undocumented_voices() or not _is_undocumented_live_voice(voice): + return None + param: Final = "session.audio.output.voice" if output_voice else "session.voice" + return RealtimeSessionUpdateRejection( + code="invalid_value", + message=( + f"Invalid value for '{param}': {voice} is not a Gemini Live voice {model} offers. " + "Set drop_params to drop it instead" + ), + param=param, + ) + @staticmethod def _coerce_response_modalities(model: str, modalities: Sequence[object]) -> tuple[str, ...]: """Drop modalities Gemini Live cannot produce at all, swap the ones this diff --git a/litellm/llms/vertex_ai/realtime/transformation.py b/litellm/llms/vertex_ai/realtime/transformation.py index d7a79dd32e6..c8c5a51be74 100644 --- a/litellm/llms/vertex_ai/realtime/transformation.py +++ b/litellm/llms/vertex_ai/realtime/transformation.py @@ -51,6 +51,9 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig): def _rejects_vad_tuning_with_detection_off(self) -> bool: return False + def _rejects_undocumented_voices(self) -> bool: + return True + # ------------------------------------------------------------------ # URL # ------------------------------------------------------------------ diff --git a/tests/unit/litellm_core_utils/test_realtime_streaming.py b/tests/unit/litellm_core_utils/test_realtime_streaming.py index 97b0511b953..7f07848fa9c 100644 --- a/tests/unit/litellm_core_utils/test_realtime_streaming.py +++ b/tests/unit/litellm_core_utils/test_realtime_streaming.py @@ -2509,9 +2509,9 @@ async def test_follow_up_setup_updates_cached_session_configuration_request(): def _gemini_default_mode_streaming( - monkeypatch, client_messages, model="gemini-live-2.5-flash-native-audio", drop_params=False + monkeypatch, client_messages, model="gemini-live-2.5-flash-native-audio", drop_params=False, defer_setup=False ): - monkeypatch.setattr(litellm, "gemini_live_defer_setup", False, raising=False) + monkeypatch.setattr(litellm, "gemini_live_defer_setup", defer_setup, raising=False) from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig client_ws = MagicMock() @@ -2996,6 +2996,71 @@ async def test_deferred_setup_sends_the_default_setup_before_text_content_that_c assert [(error["code"], error["event_id"]) for error in errors] == [("session_update_not_applied", "evt_late")] +@pytest.mark.asyncio +async def test_deferred_setup_sends_a_session_update_that_follows_buffered_audio_at_once(monkeypatch): + append: Final = json.dumps({"type": "input_audio_buffer.append", "audio": "AAAA"}) + commit: Final = json.dumps({"type": "input_audio_buffer.commit"}) + session_update: Final = json.dumps({"type": "session.update", "session": {"instructions": "Be a pirate."}}) + streaming, backend_ws = _gemini_default_mode_streaming( + monkeypatch, [append, commit, session_update], model="gemini-live-2.5-flash", defer_setup=True + ) + + await streaming.client_ack_messages() + + frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] + assert [next(iter(frame)) for frame in frames] == ["setup"] + assert "Be a pirate." in json.dumps(frames[0]["setup"]["systemInstruction"]) + + +@pytest.mark.parametrize("defer_setup", [False, True]) +@pytest.mark.asyncio +async def test_a_transcription_guardrail_refuses_gemini_audio_it_cannot_screen(monkeypatch, defer_setup): + monkeypatch.setattr(litellm, "callbacks", [_transcription_guardrail()]) + session_update: Final = json.dumps( + {"type": "session.update", "session": {"turn_detection": {"type": "server_vad", "create_response": False}}} + ) + append: Final = json.dumps({"type": "input_audio_buffer.append", "audio": "AAAA"}) + commit: Final = json.dumps({"type": "input_audio_buffer.commit", "event_id": "commit_1"}) + streaming, backend_ws = _gemini_default_mode_streaming( + monkeypatch, + [session_update, append, append, commit, json.dumps({"type": "response.create"}), _TEXT_ITEM_CREATE], + model="gemini-live-2.5-flash", + defer_setup=defer_setup, + ) + + await streaming.client_ack_messages() + + frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] + assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"] + refusals: Final = [ + event["error"]["event_id"] + for event in _events_sent_to_client(streaming) + if event["type"] == "error" and event["error"]["code"] == "audio_input_not_screenable" + ] + assert refusals == [None, "commit_1"] + + +@pytest.mark.parametrize("defer_setup", [False, True]) +@pytest.mark.parametrize("control_type", ["response.cancel", "conversation.item.delete", "input_audio_buffer.clear"]) +@pytest.mark.asyncio +async def test_a_control_event_before_session_update_leaves_the_setup_open(monkeypatch, defer_setup, control_type): + control_event: Final = json.dumps({"type": control_type, "item_id": "item_x"}) + session_update: Final = json.dumps({"type": "session.update", "session": {"modalities": ["text"]}}) + streaming, backend_ws = _gemini_default_mode_streaming( + monkeypatch, + [control_event, session_update, _TEXT_ITEM_CREATE], + model="gemini-live-2.5-flash", + defer_setup=defer_setup, + ) + + await streaming.client_ack_messages() + + frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] + assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"] + assert frames[0]["setup"]["generationConfig"]["responseModalities"] == ["TEXT"] + assert [event["type"] for event in _events_sent_to_client(streaming) if event["type"] == "error"] == [] + + @pytest.mark.asyncio async def test_deferred_setup_buffers_audio_until_backend_setup_complete(monkeypatch): """Pipecat may send audio before session.update when setup is deferred.""" diff --git a/tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py b/tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py index 9e35e07fd54..e9757e47ef3 100644 --- a/tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py +++ b/tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py @@ -1972,23 +1972,15 @@ def test_map_openai_params_drops_stock_voice_case_insensitively(): _DOCUMENTED_VOICE: Final = sorted(GEMINI_LIVE_DOCUMENTED_VOICES)[0].capitalize() -@pytest.mark.parametrize( - ("voice", "drop_params", "forwarded"), - [ - ("Notavoice", True, None), - ("Notavoice", False, "Notavoice"), - (_DOCUMENTED_VOICE, True, _DOCUMENTED_VOICE), - ], -) -def test_map_openai_params_drops_an_undocumented_voice_only_with_drop_params(voice, drop_params, forwarded): - """Gemini Live closes the session on a voice it does not know, and AI Studio knows - names Vertex AI refuses, so only drop_params narrows to the documented voices.""" - mapped = GeminiRealtimeConfig(drop_params=drop_params).map_openai_params( - optional_params={}, non_default_params={"voice": voice} - ) +@pytest.mark.parametrize("drop_params", [False, True]) +@pytest.mark.parametrize("voice", ["Nova", _DOCUMENTED_VOICE]) +def test_google_ai_studio_forwards_any_gemini_voice_name(voice, drop_params): + """AI Studio Live speaks voices outside Google's documented list, so the name is forwarded even with drop_params.""" + config = GeminiRealtimeConfig(drop_params=drop_params) + mapped = config.map_openai_params(optional_params={}, non_default_params={"voice": voice}) - speech_config = mapped.get("generationConfig", {}).get("speechConfig") - assert (speech_config["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] if speech_config else None) == forwarded + assert mapped["generationConfig"]["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == voice + assert config.setup_session_update_rejection("gemini-3.8-live", {"voice": voice}, setup_sent=False) is None @pytest.fixture(autouse=False) diff --git a/tests/unit/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py b/tests/unit/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py index 0839718d890..9d7fba8b11e 100644 --- a/tests/unit/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py +++ b/tests/unit/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py @@ -665,3 +665,37 @@ def test_vertex_session_update_two_output_modalities_collapse_to_audio(): assert len(messages) == 1 assert json.loads(messages[0])["setup"]["generationConfig"]["responseModalities"] == ["AUDIO"] + + +@pytest.mark.parametrize( + ("session", "param"), + [ + ({"voice": "Nova"}, "session.voice"), + ({"type": "realtime", "audio": {"output": {"voice": "Nova"}}}, "session.audio.output.voice"), + ], +) +def test_vertex_rejects_an_undocumented_voice_on_the_session_update(session, param): + cfg: Final = VertexAIRealtimeConfig(access_token="tok", project="my-proj", location="us-central1") + + rejection: Final = cfg.setup_session_update_rejection("gemini-live-2.5-flash", session, setup_sent=False) + + assert rejection is not None + assert (rejection.code, rejection.param) == ("invalid_value", param) + + +@pytest.mark.parametrize("voice", ["Kore", "alloy"]) +def test_vertex_accepts_documented_and_openai_stock_voices_on_the_session_update(voice): + cfg: Final = VertexAIRealtimeConfig(access_token="tok", project="my-proj", location="us-central1") + + assert cfg.setup_session_update_rejection("gemini-live-2.5-flash", {"voice": voice}, setup_sent=False) is None + + +def test_vertex_drops_an_undocumented_voice_with_drop_params(): + cfg: Final = VertexAIRealtimeConfig(access_token="tok", project="my-proj", location="us-central1", drop_params=True) + session_update: Final = {"type": "session.update", "session": {"voice": "Nova", "instructions": "Be brief."}} + + assert cfg.setup_session_update_rejection("gemini-live-2.5-flash", session_update["session"], setup_sent=False) is None + messages: Final = cfg.transform_realtime_request( + json.dumps(session_update), "gemini-live-2.5-flash", session_configuration_request=None + ) + assert "speechConfig" not in json.loads(messages[0])["setup"].get("generationConfig", {})