fix(gemini-realtime): drop the native-audio speechConfig strip on Google AI Studio too

Live probes against every gemini_native_audio model on both providers show
setup accepts a valid prebuilt voice and 1007s only unknown voice names, so
the strip predicate rested on a false premise and silently discarded the
client's voice on AI Studio native-audio sessions
This commit is contained in:
mateo-berri 2026-08-26 15:06:23 -07:00
parent 27207659f8
commit aabbc3204b
4 changed files with 6 additions and 37 deletions

View file

@ -115,13 +115,6 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
"""Google AI Studio Gemini 3.5+ accepts ``id`` on functionResponses; Vertex AI rejects it."""
return True
def _strip_native_audio_speech_config(self) -> bool:
"""Google AI Studio native-audio Live was never verified to accept ``speechConfig`` on setup.
Vertex AI accepts it and reads the requested voice.
"""
return True
@staticmethod
def _usage_detail_alias(details: Any, defaults: dict[str, int]) -> dict[str, Any]:
if not isinstance(details, dict):
@ -390,10 +383,6 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
entry: Final = GeminiRealtimeConfig._model_cost_entry(model)
return bool(entry.get("gemini_native_audio") or entry.get("gemini_audio_only_live"))
@staticmethod
def _is_native_audio_model(model: str) -> bool:
return bool(GeminiRealtimeConfig._model_cost_entry(model).get("gemini_native_audio"))
@staticmethod
def _coerce_response_modalities(model: str, modalities: list[Any]) -> list[str]:
"""Map unsupported TEXT responseModalities to AUDIO for audio-only Live models."""
@ -407,8 +396,8 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
without_text: Final = [modality for modality in normalized if modality != "TEXT"]
return without_text if without_text else ["AUDIO"]
def _finalize_gemini_live_setup(self, model: str, setup: dict[str, Any]) -> dict[str, Any]:
"""Drop fields Gemini Live native-audio rejects on ``setup``."""
@staticmethod
def _finalize_gemini_live_setup(model: str, setup: dict[str, Any]) -> dict[str, Any]:
generation_config: Final = setup.get("generationConfig")
if isinstance(generation_config, dict):
modalities: Final = generation_config.get("responseModalities")
@ -416,8 +405,6 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
generation_config["responseModalities"] = GeminiRealtimeConfig._coerce_response_modalities(
model, modalities
)
if self._strip_native_audio_speech_config() and GeminiRealtimeConfig._is_native_audio_model(model):
generation_config.pop("speechConfig", None)
return setup
def _handle_session_update(

View file

@ -35,9 +35,6 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig):
def _include_function_response_id(self) -> bool:
return False
def _strip_native_audio_speech_config(self) -> bool:
return False
# ------------------------------------------------------------------
# URL
# ------------------------------------------------------------------

View file

@ -1298,8 +1298,7 @@ def test_gemini_realtime_pipecat_ga_session_voice_and_tools(patch_gemini_audio_c
assert len(messages) == 1
setup = json.loads(messages[0])["setup"]
assert setup["generationConfig"]["responseModalities"] == ["AUDIO"]
# Native-audio Live rejects speechConfig on setup (see _finalize_gemini_live_setup).
assert "speechConfig" not in setup.get("generationConfig", {})
assert setup["generationConfig"]["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Kore"
assert setup["tools"][0]["function_declarations"][0]["name"] == "terminate_call"
assert setup["realtimeInputConfig"]["automaticActivityDetection"]["disabled"] is False
@ -1843,20 +1842,6 @@ def test_is_audio_only_live_model_uses_cost_map(model, expected, patch_gemini_au
assert GeminiRealtimeConfig._is_audio_only_live_model(model) == expected
@pytest.mark.parametrize(
"model,expected",
[
("gemini-2.5-flash-native-audio-latest", True),
("gemini/gemini-2.5-flash-native-audio-latest", True),
("gemini-3.1-flash-live-preview", False),
("gemini/gemini-3.1-flash-live-preview", False),
("gemini-2.0-flash", False),
],
)
def test_is_native_audio_model_uses_cost_map(model, expected, patch_gemini_audio_cost_map_entries):
assert GeminiRealtimeConfig._is_native_audio_model(model) == expected
def test_is_setup_message_and_is_content_message():
config = GeminiRealtimeConfig()
assert config.is_setup_message({"setup": {}}) is True

View file

@ -491,8 +491,8 @@ def test_vertex_native_audio_keeps_requested_voice(patch_native_audio_cost_map_e
assert generation_config["responseModalities"] == ["AUDIO"]
def test_google_ai_studio_native_audio_still_strips_voice(patch_native_audio_cost_map_entry):
"""Google AI Studio's own native-audio Live is untouched: only Vertex was verified to accept speechConfig."""
def test_google_ai_studio_native_audio_keeps_requested_voice(patch_native_audio_cost_map_entry):
"""Regression: AI Studio native-audio Live accepts speechConfig too, so the voice survives on both providers."""
from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig
messages = GeminiRealtimeConfig().transform_realtime_request(
@ -510,7 +510,7 @@ def test_google_ai_studio_native_audio_still_strips_voice(patch_native_audio_cos
)
generation_config = json.loads(messages[0])["setup"]["generationConfig"]
assert "speechConfig" not in generation_config
assert generation_config["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Aoede"
def test_vertex_native_audio_drops_openai_stock_voice(patch_native_audio_cost_map_entry):