diff --git a/litellm/llms/gemini/realtime/transformation.py b/litellm/llms/gemini/realtime/transformation.py index ea576750cf3..f83a8993aeb 100644 --- a/litellm/llms/gemini/realtime/transformation.py +++ b/litellm/llms/gemini/realtime/transformation.py @@ -92,6 +92,10 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): """Google AI Studio Gemini 3.5+ accepts ``id`` on functionResponses; Vertex AI rejects it.""" return True + def _strip_native_audio_speech_config(self) -> bool: + """Google AI Studio native-audio Live rejects ``speechConfig`` on setup; Vertex AI accepts it.""" + return True + @staticmethod def _usage_detail_alias(details: Any, defaults: dict[str, int]) -> dict[str, Any]: if not isinstance(details, dict): @@ -382,8 +386,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): without_text: Final = [modality for modality in normalized if modality != "TEXT"] return without_text if without_text else ["AUDIO"] - @staticmethod - def _finalize_gemini_live_setup(model: str, setup: dict[str, Any]) -> dict[str, Any]: + def _finalize_gemini_live_setup(self, model: str, setup: dict[str, Any]) -> dict[str, Any]: """Drop fields Gemini Live native-audio rejects on ``setup``.""" generation_config: Final = setup.get("generationConfig") if isinstance(generation_config, dict): @@ -392,7 +395,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): generation_config["responseModalities"] = GeminiRealtimeConfig._coerce_response_modalities( model, modalities ) - if GeminiRealtimeConfig._is_native_audio_model(model): + if self._strip_native_audio_speech_config() and GeminiRealtimeConfig._is_native_audio_model(model): generation_config.pop("speechConfig", None) return setup diff --git a/litellm/llms/vertex_ai/realtime/transformation.py b/litellm/llms/vertex_ai/realtime/transformation.py index fe59034c27b..0a59dca519a 100644 --- a/litellm/llms/vertex_ai/realtime/transformation.py +++ b/litellm/llms/vertex_ai/realtime/transformation.py @@ -35,6 +35,9 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig): def _include_function_response_id(self) -> bool: return False + def _strip_native_audio_speech_config(self) -> bool: + return False + # ------------------------------------------------------------------ # URL # ------------------------------------------------------------------ diff --git a/tests/test_litellm/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py b/tests/test_litellm/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py index 720c629cbf7..02c2b7baee8 100644 --- a/tests/test_litellm/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py @@ -462,3 +462,53 @@ def test_vertex_function_call_output_omits_id(): assert "id" not in function_response assert function_response["name"] == "terminate_call" assert function_response["response"] == {"status": "ok"} + + +def test_vertex_native_audio_keeps_requested_voice(patch_native_audio_cost_map_entry): + """Regression: Vertex Live accepts speechConfig on native audio, so the client's voice must survive. + + Stripping it silently dropped voice selection for every Vertex native-audio + session. TEXT is still coerced away, which Vertex does reject. + """ + cfg = VertexAIRealtimeConfig( + access_token="tok", project="my-proj", location="us-central1" + ) + session_update = { + "type": "session.update", + "session": { + "output_modalities": ["text"], + "audio": {"output": {"voice": "Aoede"}}, + }, + } + + messages = cfg.transform_realtime_request( + json.dumps(session_update), + _NATIVE_AUDIO_MODEL, + session_configuration_request=None, + ) + + generation_config = json.loads(messages[0])["setup"]["generationConfig"] + assert generation_config["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Aoede" + assert generation_config["responseModalities"] == ["AUDIO"] + + +def test_google_ai_studio_native_audio_still_strips_voice(patch_native_audio_cost_map_entry): + """Google AI Studio's own native-audio Live is untouched: only Vertex was verified to accept speechConfig.""" + from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig + + messages = GeminiRealtimeConfig().transform_realtime_request( + json.dumps( + { + "type": "session.update", + "session": { + "output_modalities": ["audio"], + "audio": {"output": {"voice": "Aoede"}}, + }, + } + ), + _NATIVE_AUDIO_MODEL, + session_configuration_request=None, + ) + + generation_config = json.loads(messages[0])["setup"]["generationConfig"] + assert "speechConfig" not in generation_config