From aabbc3204b51e9ea7f35e2b4fc0ec3027b167a40 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 26 Aug 2026 15:06:23 -0700 Subject: [PATCH] fix(gemini-realtime): drop the native-audio speechConfig strip on Google AI Studio too Live probes against every gemini_native_audio model on both providers show setup accepts a valid prebuilt voice and 1007s only unknown voice names, so the strip predicate rested on a false premise and silently discarded the client's voice on AI Studio native-audio sessions --- litellm/llms/gemini/realtime/transformation.py | 17 ++--------------- .../llms/vertex_ai/realtime/transformation.py | 3 --- .../test_gemini_realtime_transformation.py | 17 +---------------- .../test_vertex_ai_realtime_transformation.py | 6 +++--- 4 files changed, 6 insertions(+), 37 deletions(-) diff --git a/litellm/llms/gemini/realtime/transformation.py b/litellm/llms/gemini/realtime/transformation.py index f00cd2c713d..51801e91356 100644 --- a/litellm/llms/gemini/realtime/transformation.py +++ b/litellm/llms/gemini/realtime/transformation.py @@ -115,13 +115,6 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): """Google AI Studio Gemini 3.5+ accepts ``id`` on functionResponses; Vertex AI rejects it.""" return True - def _strip_native_audio_speech_config(self) -> bool: - """Google AI Studio native-audio Live was never verified to accept ``speechConfig`` on setup. - - Vertex AI accepts it and reads the requested voice. - """ - return True - @staticmethod def _usage_detail_alias(details: Any, defaults: dict[str, int]) -> dict[str, Any]: if not isinstance(details, dict): @@ -390,10 +383,6 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): entry: Final = GeminiRealtimeConfig._model_cost_entry(model) return bool(entry.get("gemini_native_audio") or entry.get("gemini_audio_only_live")) - @staticmethod - def _is_native_audio_model(model: str) -> bool: - return bool(GeminiRealtimeConfig._model_cost_entry(model).get("gemini_native_audio")) - @staticmethod def _coerce_response_modalities(model: str, modalities: list[Any]) -> list[str]: """Map unsupported TEXT responseModalities to AUDIO for audio-only Live models.""" @@ -407,8 +396,8 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): without_text: Final = [modality for modality in normalized if modality != "TEXT"] return without_text if without_text else ["AUDIO"] - def _finalize_gemini_live_setup(self, model: str, setup: dict[str, Any]) -> dict[str, Any]: - """Drop fields Gemini Live native-audio rejects on ``setup``.""" + @staticmethod + def _finalize_gemini_live_setup(model: str, setup: dict[str, Any]) -> dict[str, Any]: generation_config: Final = setup.get("generationConfig") if isinstance(generation_config, dict): modalities: Final = generation_config.get("responseModalities") @@ -416,8 +405,6 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): generation_config["responseModalities"] = GeminiRealtimeConfig._coerce_response_modalities( model, modalities ) - if self._strip_native_audio_speech_config() and GeminiRealtimeConfig._is_native_audio_model(model): - generation_config.pop("speechConfig", None) return setup def _handle_session_update( diff --git a/litellm/llms/vertex_ai/realtime/transformation.py b/litellm/llms/vertex_ai/realtime/transformation.py index 0a59dca519a..fe59034c27b 100644 --- a/litellm/llms/vertex_ai/realtime/transformation.py +++ b/litellm/llms/vertex_ai/realtime/transformation.py @@ -35,9 +35,6 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig): def _include_function_response_id(self) -> bool: return False - def _strip_native_audio_speech_config(self) -> bool: - return False - # ------------------------------------------------------------------ # URL # ------------------------------------------------------------------ diff --git a/tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py b/tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py index e80724f3347..42e330925a0 100644 --- a/tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py +++ b/tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py @@ -1298,8 +1298,7 @@ def test_gemini_realtime_pipecat_ga_session_voice_and_tools(patch_gemini_audio_c assert len(messages) == 1 setup = json.loads(messages[0])["setup"] assert setup["generationConfig"]["responseModalities"] == ["AUDIO"] - # Native-audio Live rejects speechConfig on setup (see _finalize_gemini_live_setup). - assert "speechConfig" not in setup.get("generationConfig", {}) + assert setup["generationConfig"]["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Kore" assert setup["tools"][0]["function_declarations"][0]["name"] == "terminate_call" assert setup["realtimeInputConfig"]["automaticActivityDetection"]["disabled"] is False @@ -1843,20 +1842,6 @@ def test_is_audio_only_live_model_uses_cost_map(model, expected, patch_gemini_au assert GeminiRealtimeConfig._is_audio_only_live_model(model) == expected -@pytest.mark.parametrize( - "model,expected", - [ - ("gemini-2.5-flash-native-audio-latest", True), - ("gemini/gemini-2.5-flash-native-audio-latest", True), - ("gemini-3.1-flash-live-preview", False), - ("gemini/gemini-3.1-flash-live-preview", False), - ("gemini-2.0-flash", False), - ], -) -def test_is_native_audio_model_uses_cost_map(model, expected, patch_gemini_audio_cost_map_entries): - assert GeminiRealtimeConfig._is_native_audio_model(model) == expected - - def test_is_setup_message_and_is_content_message(): config = GeminiRealtimeConfig() assert config.is_setup_message({"setup": {}}) is True diff --git a/tests/test_litellm/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py b/tests/test_litellm/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py index 5a45a762e0b..d4cf58bc0b4 100644 --- a/tests/test_litellm/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py @@ -491,8 +491,8 @@ def test_vertex_native_audio_keeps_requested_voice(patch_native_audio_cost_map_e assert generation_config["responseModalities"] == ["AUDIO"] -def test_google_ai_studio_native_audio_still_strips_voice(patch_native_audio_cost_map_entry): - """Google AI Studio's own native-audio Live is untouched: only Vertex was verified to accept speechConfig.""" +def test_google_ai_studio_native_audio_keeps_requested_voice(patch_native_audio_cost_map_entry): + """Regression: AI Studio native-audio Live accepts speechConfig too, so the voice survives on both providers.""" from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig messages = GeminiRealtimeConfig().transform_realtime_request( @@ -510,7 +510,7 @@ def test_google_ai_studio_native_audio_still_strips_voice(patch_native_audio_cos ) generation_config = json.loads(messages[0])["setup"]["generationConfig"] - assert "speechConfig" not in generation_config + assert generation_config["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Aoede" def test_vertex_native_audio_drops_openai_stock_voice(patch_native_audio_cost_map_entry):