fix(realtime): ack unchanged Gemini Live session.update resends and honor null turn_detection

An update that leaves the applied setup unchanged now gets session.updated
instead of session_update_not_applied, so clients that resend their session
with the first response keep working. Deferred setup mode merges held updates
like the default mode, turn_detection null turns Gemini's activity detection
off, drop_params drops a voice outside the documented Live voices, and the
lazy setup deadline moves to 240 seconds
This commit is contained in:
mateo-berri 2026-10-28 17:53:15 -07:00
parent d8994c152a
commit 58089dfcb7
5 changed files with 204 additions and 47 deletions

View file

@ -416,7 +416,7 @@ REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS: Final = get_env_float(
"REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS", 20.0
)
BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS: Final = get_env_float("BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS", 5.0)
REALTIME_LAZY_SETUP_DEADLINE_SECONDS: Final = 60.0
REALTIME_LAZY_SETUP_DEADLINE_SECONDS: Final = 240.0
# RFC 6455 caps the close frame payload at 125 bytes, 2 of which carry the status code
WEBSOCKET_CLOSE_REASON_MAX_BYTES: Final = 123

View file

@ -350,6 +350,7 @@ class RealTimeStreaming:
# content (realtimeInput / clientContent / toolResponse) has been sent.
self._content_sent_after_setup: bool = False
self._held_session_update: Mapping[str, object] | None = None
self._applied_session_fields: Mapping[str, object] | None = None
self._held_session_update_bytes: int = 0
# Whether this is a transcription-only session (session.type == "transcription",
# e.g. gpt-realtime-whisper). Such sessions must not be sent response.create and
@ -661,7 +662,7 @@ class RealTimeStreaming:
def _builds_setup_lazily(self) -> bool:
if self.provider_config is None:
return False
return self.provider_config.builds_setup_from_session_update() and not self._uses_deferred_backend_setup()
return self.provider_config.builds_setup_from_session_update()
def _holds_session_updates_until_first_content(self) -> bool:
return self.session_configuration_request is None and self._builds_setup_lazily()
@ -672,10 +673,12 @@ class RealTimeStreaming:
return False
client_event: Final = _decode_json_object(message)
session: Final = client_event.get("session")
update: Final = _SESSION_FIELDS.validate_python(session) if isinstance(session, Mapping) else _NO_SESSION_FIELDS
if self._leaves_applied_setup_unchanged(update):
await self._acknowledge_session_setup(self._applied_session_fields or _NO_SESSION_FIELDS)
return True
rejection: Final = provider_config.setup_session_update_rejection(
self.model,
_SESSION_FIELDS.validate_python(session) if isinstance(session, Mapping) else _NO_SESSION_FIELDS,
setup_sent=self.session_configuration_request is not None,
self.model, update, setup_sent=self.session_configuration_request is not None
)
if rejection is None:
return False
@ -694,18 +697,37 @@ class RealTimeStreaming:
return
session: Final = client_event.get("session")
update: Final = _SESSION_FIELDS.validate_python(session) if isinstance(session, Mapping) else _NO_SESSION_FIELDS
self._held_session_update = _merge_session_fields(self._held_session_update or _NO_SESSION_FIELDS, update)
held_message: Final = self._held_session_update_message()
self._held_session_update_bytes = len(held_message.encode("utf-8"))
await self._acknowledge_held_session_update(held_message)
held: Final = _merge_session_fields(self._held_session_update or _NO_SESSION_FIELDS, update)
self._held_session_update = held
self._held_session_update_bytes = len(_session_update_frame(held).encode("utf-8"))
await self._acknowledge_session_setup(held)
async def _acknowledge_held_session_update(self, held_message: str) -> None:
def _setup_preview(self, session: Mapping[str, object]) -> str | None:
if self.provider_config is None:
return
preview: Final = self.provider_config.transform_realtime_request(held_message, self.model, None)
setup: Final = next(
return None
preview: Final = self.provider_config.transform_realtime_request(
_session_update_frame(session), self.model, None
)
return next(
(frame for frame in preview if isinstance(frame, str) and "setup" in _decode_json_object(frame)), None
)
def _leaves_applied_setup_unchanged(self, update: Mapping[str, object]) -> bool:
applied: Final = self._applied_session_fields
if self.session_configuration_request is None or applied is None:
return False
applied_setup: Final = self._setup_preview(applied)
merged_setup: Final = self._setup_preview(_merge_session_fields(applied, update))
return (
applied_setup is not None
and merged_setup is not None
and _decode_json_object(applied_setup) == _decode_json_object(merged_setup)
)
async def _acknowledge_session_setup(self, session: Mapping[str, object]) -> None:
if self.provider_config is None:
return
setup: Final = self._setup_preview(session)
if setup is None:
return
acknowledged: Final = self.provider_config.transform_session_created_event(
@ -719,12 +741,14 @@ class RealTimeStreaming:
await self._send_event_to_client(event, event_str)
async def _send_held_session_update_as_setup(self) -> None:
if self._held_session_update is None:
held: Final = self._held_session_update
if held is None:
return
message: Final = self._held_session_update_message()
self._held_session_update = None
self._held_session_update_bytes = 0
await self._send_to_backend(message)
if await self._send_to_backend(message):
self._applied_session_fields = held
async def _send_setup_before_content(self, msg_type: str | None) -> None:
if self._holds_session_updates_until_first_content():
@ -754,6 +778,7 @@ class RealTimeStreaming:
setup: Final = self.maybe_inject_guardrail_auto_response_disable(default_setup)
await self.backend_ws.send(setup)
self._cache_session_configuration_request(setup)
self._applied_session_fields = _NO_SESSION_FIELDS
def _enforce_transcription_session_model(self, message: str) -> str:
"""Force client transcription session updates to the authorized model.
@ -1817,6 +1842,9 @@ class RealTimeStreaming:
# actually forward to the backend.
self.store_input(message=message)
if msg_type != "session.update" and self._uses_deferred_backend_setup():
await self._send_setup_before_content(msg_type)
if self._should_buffer_client_message_until_setup(message):
self._buffer_pending_message_until_setup(message)
continue

View file

@ -84,21 +84,67 @@ OPENAI_STOCK_REALTIME_VOICES: Final[frozenset[str]] = frozenset(
{"alloy", "ash", "ballad", "cedar", "coral", "echo", "marin", "sage", "shimmer", "verse"}
)
# Per https://docs.cloud.google.com/vertex-ai/generative-ai/docs/live-api/configure-language-voice (2026-10-28)
GEMINI_LIVE_DOCUMENTED_VOICES: Final[frozenset[str]] = frozenset(
{
"achernar",
"achird",
"algenib",
"algieba",
"alnilam",
"aoede",
"autonoe",
"callirrhoe",
"charon",
"despina",
"enceladus",
"erinome",
"fenrir",
"gacrux",
"iapetus",
"kore",
"laomedeia",
"leda",
"orus",
"puck",
"pulcherrima",
"rasalgethi",
"sadachbia",
"sadaltager",
"schedar",
"sulafat",
"umbriel",
"vindemiatrix",
"zephyr",
"zubenelgenubi",
}
)
def _gemini_live_speech_config(voice: object) -> Mapping[str, object] | None:
def _gemini_live_speech_config(voice: object, drop_params: bool) -> Mapping[str, object] | None:
"""Build the Gemini Live speechConfig for a client-requested voice.
OpenAI stock voice names have no Gemini equivalent and Gemini Live closes
the session on an unknown voice, so they are dropped with a warning and
the model keeps its default voice. Every other name is forwarded verbatim.
Gemini Live closes the session on a voice name it does not know. OpenAI
stock voice names are always dropped, and with drop_params so is any name
outside Google's documented Live voices, keeping the model's default voice.
Every other name is forwarded verbatim for Gemini to accept or refuse.
"""
if isinstance(voice, str) and voice.lower() in OPENAI_STOCK_REALTIME_VOICES:
if not isinstance(voice, str):
return VertexGeminiConfig().map_audio_params({"voice": voice})
if voice.lower() in OPENAI_STOCK_REALTIME_VOICES:
verbose_logger.warning(
"Gemini Realtime: voice %s is an OpenAI voice with no Gemini equivalent; "
"dropping it so the session keeps the model's default voice.",
voice,
)
return None
if drop_params and voice.lower() not in GEMINI_LIVE_DOCUMENTED_VOICES:
verbose_logger.warning(
"Gemini Realtime: voice %s is not a documented Gemini Live voice; "
"drop_params is set, so it is dropped and the session keeps the model's default voice.",
voice,
)
return None
return VertexGeminiConfig().map_audio_params({"voice": voice})
@ -373,30 +419,36 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
)
elif key == "input_audio_transcription" and value is not None:
optional_params["inputAudioTranscription"] = {}
elif key == "turn_detection" and value is not None:
value_typed = cast(OpenAIRealtimeTurnDetection, value)
if (
isinstance(value_typed, dict)
and value_typed.get("type") == "semantic_vad"
and "create_response" not in value_typed
):
# Pipecat/OpenAI GA semantic VAD — skip; Gemini uses its own VAD.
# Only skip when there is no create_response override so that
# a guardrail-injected create_response:false is not dropped.
continue
transformed_audio_activity_config = self.map_automatic_turn_detection(value_typed)
if transformed_audio_activity_config:
optional_params["realtimeInputConfig"] = BidiGenerateContentRealtimeInputConfig(
automaticActivityDetection=transformed_audio_activity_config
)
elif key == "turn_detection":
realtime_input_config = self._realtime_input_config(cast(object, value))
if realtime_input_config is not None:
optional_params["realtimeInputConfig"] = realtime_input_config
elif key == "voice":
speech_config = _gemini_live_speech_config(value)
speech_config = _gemini_live_speech_config(value, self.drop_params)
if speech_config:
optional_params["generationConfig"]["speechConfig"] = speech_config
if len(optional_params["generationConfig"]) == 0:
optional_params.pop("generationConfig")
return optional_params
def _realtime_input_config(self, turn_detection: object) -> BidiGenerateContentRealtimeInputConfig | None:
if turn_detection is None:
return BidiGenerateContentRealtimeInputConfig(automaticActivityDetection={"disabled": True})
value_typed: Final = cast(OpenAIRealtimeTurnDetection, turn_detection)
if (
isinstance(value_typed, dict)
and value_typed.get("type") == "semantic_vad"
and "create_response" not in value_typed
):
# Pipecat/OpenAI GA semantic VAD — skip; Gemini uses its own VAD.
# Only skip when there is no create_response override so that
# a guardrail-injected create_response:false is not dropped.
return None
transformed_audio_activity_config: Final = self.map_automatic_turn_detection(value_typed)
if not transformed_audio_activity_config:
return None
return BidiGenerateContentRealtimeInputConfig(automaticActivityDetection=transformed_audio_activity_config)
@staticmethod
def _extract_turn_detection(session: dict) -> dict | None:
"""Extract turn_detection from a session.update payload.
@ -445,6 +497,8 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
if isinstance(input_cfg, dict):
if "input_audio_transcription" not in normalized and "transcription" in input_cfg:
normalized["input_audio_transcription"] = input_cfg["transcription"]
if "turn_detection" not in normalized and "turn_detection" in input_cfg:
normalized["turn_detection"] = input_cfg["turn_detection"]
output_cfg: Final = audio.get("output")
if isinstance(output_cfg, dict) and output_cfg.get("voice"):
normalized["voice"] = output_cfg["voice"]

View file

@ -2856,7 +2856,9 @@ async def test_default_mode_sends_the_held_setup_when_no_content_arrives_before_
async def late_update_once_the_setup_is_out() -> str:
await setup_sent.wait()
return _LATE_INSTRUCTIONS_UPDATE
return json.dumps(
{"type": "session.update", "event_id": "evt_late", "session": {"instructions": "Answer in French."}}
)
async def disconnect() -> str:
raise ConnectionClosed(None, None)
@ -2909,9 +2911,29 @@ async def test_default_mode_refuses_held_session_updates_past_the_size_cap(monke
@pytest.mark.asyncio
async def test_deferred_setup_answers_a_second_session_update_with_an_error(monkeypatch):
"""Gemini Live ignores a second setup, so with deferred setup the second
session.update gets the same not-applied error as in the default mode."""
async def test_default_mode_acknowledges_an_unchanged_session_update_after_content(monkeypatch):
"""Pipecat resends its unchanged session.update right after its first
conversation items and treats any error event as fatal."""
session_update: Final = json.dumps(
{"type": "session.update", "session": {"modalities": ["text"], "instructions": "Be brief."}}
)
streaming, backend_ws = _gemini_default_mode_streaming(
monkeypatch, [session_update, _TEXT_ITEM_CREATE, session_update], model="gemini-live-2.5-flash"
)
await streaming.client_ack_messages()
frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list]
assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"]
events: Final = _events_sent_to_client(streaming)
assert [event["type"] for event in events] == ["session.updated", "session.updated"]
assert events[0]["session"] == events[1]["session"]
@pytest.mark.asyncio
async def test_deferred_setup_merges_session_updates_until_the_first_content(monkeypatch):
"""Deferred setup merges every session.update sent before the first content,
the same as the default mode, and answers a changed one after it as late."""
monkeypatch.setattr(litellm, "gemini_live_defer_setup", True, raising=False)
from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig
@ -2919,7 +2941,11 @@ async def test_deferred_setup_answers_a_second_session_update_with_an_error(monk
client_ws.receive_text = AsyncMock(
side_effect=[
json.dumps({"type": "session.update", "session": {"modalities": ["text"]}}),
_LATE_INSTRUCTIONS_UPDATE,
json.dumps({"type": "session.update", "session": {"instructions": "Be brief."}}),
_TEXT_ITEM_CREATE,
json.dumps(
{"type": "session.update", "event_id": "evt_late", "session": {"instructions": "Answer in French."}}
),
ConnectionClosed(None, None),
]
)
@ -2933,7 +2959,9 @@ async def test_deferred_setup_answers_a_second_session_update_with_an_error(monk
await streaming.client_ack_messages()
frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list]
assert [next(iter(frame)) for frame in frames] == ["setup"]
assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"]
assert frames[0]["setup"]["generationConfig"]["responseModalities"] == ["TEXT"]
assert frames[0]["setup"]["systemInstruction"]["parts"][0]["text"] == "Be brief."
errors: Final = [event["error"] for event in _events_sent_to_client(streaming) if event["type"] == "error"]
assert [(error["code"], error["event_id"]) for error in errors] == [("session_update_not_applied", "evt_late")]
@ -2973,14 +3001,18 @@ async def test_deferred_setup_buffers_audio_until_backend_setup_complete(monkeyp
@pytest.mark.asyncio
async def test_deferred_setup_sends_session_update_before_buffered_audio(monkeypatch):
async def test_deferred_setup_sends_the_held_session_update_when_more_audio_arrives(monkeypatch):
monkeypatch.setattr(litellm, "gemini_live_defer_setup", True, raising=False)
from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig
client_ws = MagicMock()
client_ws.send_text = AsyncMock()
audio_msg = json.dumps({"type": "input_audio_buffer.append", "audio": "AA=="})
session_update = json.dumps({"type": "session.update", "session": {"modalities": ["audio"]}})
client_ws.receive_text = AsyncMock(side_effect=[audio_msg, session_update, ConnectionClosed(None, None)])
later_audio_msg = json.dumps({"type": "input_audio_buffer.append", "audio": "AQ=="})
client_ws.receive_text = AsyncMock(
side_effect=[audio_msg, session_update, later_audio_msg, ConnectionClosed(None, None)]
)
backend_ws = MagicMock()
backend_ws.send = AsyncMock()
logging_obj: Final = _mock_logging()
@ -3000,7 +3032,7 @@ async def test_deferred_setup_sends_session_update_before_buffered_audio(monkeyp
sent_payload = json.loads(backend_ws.send.await_args_list[0].args[0])
assert "setup" in sent_payload
assert "realtimeInput" not in sent_payload
assert streaming._pending_messages_until_setup == [audio_msg]
assert streaming._pending_messages_until_setup == [audio_msg, later_audio_msg]
@pytest.mark.asyncio

View file

@ -8,7 +8,7 @@ import pytest
import litellm
from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig
from litellm.llms.gemini.realtime.transformation import GEMINI_LIVE_DOCUMENTED_VOICES, GeminiRealtimeConfig
from litellm.types.llms.gemini import BidiGenerateContentServerMessage
@ -1461,6 +1461,27 @@ def test_gemini_manual_vad_activity_markers_are_empty_messages():
assert [json.loads(frame) for frame in turn_end] == [{"realtimeInput": {"activityEnd": {}}}]
@pytest.mark.parametrize(
("session", "expected_keys"),
[
({"turn_detection": None}, [["activityStart", "audio"], ["activityEnd"]]),
({"type": "realtime", "audio": {"input": {"turn_detection": None}}}, [["activityStart", "audio"], ["activityEnd"]]),
({"type": "realtime", "audio": {"input": {}}}, [["audio"], ["audioStreamEnd"]]),
],
)
def test_gemini_null_turn_detection_makes_the_client_mark_each_turn(session, expected_keys):
"""OpenAI's turn_detection null turns server VAD off, so the client commits each turn itself."""
config = GeminiRealtimeConfig()
setup = config.transform_realtime_request(
json.dumps({"type": "session.update", "session": session}), "gemini-live-2.5-flash-native-audio", None
)
commit = json.dumps({"type": "input_audio_buffer.commit"})
keys = [_realtime_input_keys(config, event, setup[0]) for event in (_APPEND, commit)]
assert keys == expected_keys
def test_gemini_automatic_vad_sends_no_activity_markers():
config = GeminiRealtimeConfig()
commit = json.dumps({"type": "input_audio_buffer.commit"})
@ -1948,6 +1969,28 @@ def test_map_openai_params_drops_stock_voice_case_insensitively():
assert passthrough["generationConfig"]["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Kore"
_DOCUMENTED_VOICE: Final = sorted(GEMINI_LIVE_DOCUMENTED_VOICES)[0].capitalize()
@pytest.mark.parametrize(
("voice", "drop_params", "forwarded"),
[
("Notavoice", True, None),
("Notavoice", False, "Notavoice"),
(_DOCUMENTED_VOICE, True, _DOCUMENTED_VOICE),
],
)
def test_map_openai_params_drops_an_undocumented_voice_only_with_drop_params(voice, drop_params, forwarded):
"""Gemini Live closes the session on a voice it does not know, and AI Studio knows
names Vertex AI refuses, so only drop_params narrows to the documented voices."""
mapped = GeminiRealtimeConfig(drop_params=drop_params).map_openai_params(
optional_params={}, non_default_params={"voice": voice}
)
speech_config = mapped.get("generationConfig", {}).get("speechConfig")
assert (speech_config["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] if speech_config else None) == forwarded
@pytest.fixture(autouse=False)
def patch_gemini_transcribe_live_cost_map_entry(monkeypatch):
"""Inject the gemini-3.5-transcribe-live registry entry locally.