mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-07 02:59:05 +00:00
fix(gemini-realtime): drop the native-audio speechConfig strip on Google AI Studio too
Live probes against every gemini_native_audio model on both providers show setup accepts a valid prebuilt voice and 1007s only unknown voice names, so the strip predicate rested on a false premise and silently discarded the client's voice on AI Studio native-audio sessions
This commit is contained in:
parent
27207659f8
commit
aabbc3204b
4 changed files with 6 additions and 37 deletions
|
|
@ -115,13 +115,6 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
"""Google AI Studio Gemini 3.5+ accepts ``id`` on functionResponses; Vertex AI rejects it."""
|
||||
return True
|
||||
|
||||
def _strip_native_audio_speech_config(self) -> bool:
|
||||
"""Google AI Studio native-audio Live was never verified to accept ``speechConfig`` on setup.
|
||||
|
||||
Vertex AI accepts it and reads the requested voice.
|
||||
"""
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _usage_detail_alias(details: Any, defaults: dict[str, int]) -> dict[str, Any]:
|
||||
if not isinstance(details, dict):
|
||||
|
|
@ -390,10 +383,6 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
entry: Final = GeminiRealtimeConfig._model_cost_entry(model)
|
||||
return bool(entry.get("gemini_native_audio") or entry.get("gemini_audio_only_live"))
|
||||
|
||||
@staticmethod
|
||||
def _is_native_audio_model(model: str) -> bool:
|
||||
return bool(GeminiRealtimeConfig._model_cost_entry(model).get("gemini_native_audio"))
|
||||
|
||||
@staticmethod
|
||||
def _coerce_response_modalities(model: str, modalities: list[Any]) -> list[str]:
|
||||
"""Map unsupported TEXT responseModalities to AUDIO for audio-only Live models."""
|
||||
|
|
@ -407,8 +396,8 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
without_text: Final = [modality for modality in normalized if modality != "TEXT"]
|
||||
return without_text if without_text else ["AUDIO"]
|
||||
|
||||
def _finalize_gemini_live_setup(self, model: str, setup: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Drop fields Gemini Live native-audio rejects on ``setup``."""
|
||||
@staticmethod
|
||||
def _finalize_gemini_live_setup(model: str, setup: dict[str, Any]) -> dict[str, Any]:
|
||||
generation_config: Final = setup.get("generationConfig")
|
||||
if isinstance(generation_config, dict):
|
||||
modalities: Final = generation_config.get("responseModalities")
|
||||
|
|
@ -416,8 +405,6 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
generation_config["responseModalities"] = GeminiRealtimeConfig._coerce_response_modalities(
|
||||
model, modalities
|
||||
)
|
||||
if self._strip_native_audio_speech_config() and GeminiRealtimeConfig._is_native_audio_model(model):
|
||||
generation_config.pop("speechConfig", None)
|
||||
return setup
|
||||
|
||||
def _handle_session_update(
|
||||
|
|
|
|||
|
|
@ -35,9 +35,6 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig):
|
|||
def _include_function_response_id(self) -> bool:
|
||||
return False
|
||||
|
||||
def _strip_native_audio_speech_config(self) -> bool:
|
||||
return False
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# URL
|
||||
# ------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -1298,8 +1298,7 @@ def test_gemini_realtime_pipecat_ga_session_voice_and_tools(patch_gemini_audio_c
|
|||
assert len(messages) == 1
|
||||
setup = json.loads(messages[0])["setup"]
|
||||
assert setup["generationConfig"]["responseModalities"] == ["AUDIO"]
|
||||
# Native-audio Live rejects speechConfig on setup (see _finalize_gemini_live_setup).
|
||||
assert "speechConfig" not in setup.get("generationConfig", {})
|
||||
assert setup["generationConfig"]["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Kore"
|
||||
assert setup["tools"][0]["function_declarations"][0]["name"] == "terminate_call"
|
||||
assert setup["realtimeInputConfig"]["automaticActivityDetection"]["disabled"] is False
|
||||
|
||||
|
|
@ -1843,20 +1842,6 @@ def test_is_audio_only_live_model_uses_cost_map(model, expected, patch_gemini_au
|
|||
assert GeminiRealtimeConfig._is_audio_only_live_model(model) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model,expected",
|
||||
[
|
||||
("gemini-2.5-flash-native-audio-latest", True),
|
||||
("gemini/gemini-2.5-flash-native-audio-latest", True),
|
||||
("gemini-3.1-flash-live-preview", False),
|
||||
("gemini/gemini-3.1-flash-live-preview", False),
|
||||
("gemini-2.0-flash", False),
|
||||
],
|
||||
)
|
||||
def test_is_native_audio_model_uses_cost_map(model, expected, patch_gemini_audio_cost_map_entries):
|
||||
assert GeminiRealtimeConfig._is_native_audio_model(model) == expected
|
||||
|
||||
|
||||
def test_is_setup_message_and_is_content_message():
|
||||
config = GeminiRealtimeConfig()
|
||||
assert config.is_setup_message({"setup": {}}) is True
|
||||
|
|
|
|||
|
|
@ -491,8 +491,8 @@ def test_vertex_native_audio_keeps_requested_voice(patch_native_audio_cost_map_e
|
|||
assert generation_config["responseModalities"] == ["AUDIO"]
|
||||
|
||||
|
||||
def test_google_ai_studio_native_audio_still_strips_voice(patch_native_audio_cost_map_entry):
|
||||
"""Google AI Studio's own native-audio Live is untouched: only Vertex was verified to accept speechConfig."""
|
||||
def test_google_ai_studio_native_audio_keeps_requested_voice(patch_native_audio_cost_map_entry):
|
||||
"""Regression: AI Studio native-audio Live accepts speechConfig too, so the voice survives on both providers."""
|
||||
from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig
|
||||
|
||||
messages = GeminiRealtimeConfig().transform_realtime_request(
|
||||
|
|
@ -510,7 +510,7 @@ def test_google_ai_studio_native_audio_still_strips_voice(patch_native_audio_cos
|
|||
)
|
||||
|
||||
generation_config = json.loads(messages[0])["setup"]["generationConfig"]
|
||||
assert "speechConfig" not in generation_config
|
||||
assert generation_config["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Aoede"
|
||||
|
||||
|
||||
def test_vertex_native_audio_drops_openai_stock_voice(patch_native_audio_cost_map_entry):
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue