From f4e8f03d270f9f309f89fcb60d4b815a350c3385 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Tue, 15 Sep 2026 12:12:58 -0500 Subject: [PATCH 01/20] feat(vertex-ai): support gemini 3.1 flash tts --- ci_cd/generate_model_prices_schema.py | 1 + .../transformation.py | 22 +- .../gemini/google_genai/transformation.py | 10 +- .../vertex_and_google_ai_studio_gemini.py | 121 ++++++- .../vertex_ai/google_genai/transformation.py | 31 +- .../text_to_speech/transformation.py | 223 +++++++++++-- litellm/main.py | 11 +- ...odel_prices_and_context_window_backup.json | 68 +++- litellm/proxy/proxy_server.py | 37 ++- litellm/types/llms/vertex_ai.py | 12 + .../types/llms/vertex_ai_text_to_speech.py | 17 +- litellm/utils.py | 30 ++ model_prices_and_context_window.json | 68 +++- model_prices_and_context_window.schema.json | 3 + .../test_google_genai_transformation.py | 102 ++++-- .../llms/gemini/test_gemini_tts.py | 230 +++++++++---- .../proxy/proxy_server/test_routes_audio.py | 90 ++++- .../test_gemini_3_1_flash_tts_metadata.py | 75 +++++ tests/test_litellm/test_utils.py | 1 + .../text_to_speech/test_transformation.py | 307 +++++++++++++++++- 20 files changed, 1271 insertions(+), 188 deletions(-) create mode 100644 tests/test_litellm/test_gemini_3_1_flash_tts_metadata.py diff --git a/ci_cd/generate_model_prices_schema.py b/ci_cd/generate_model_prices_schema.py index 8eec07dadda..76d66e002ed 100644 --- a/ci_cd/generate_model_prices_schema.py +++ b/ci_cd/generate_model_prices_schema.py @@ -288,6 +288,7 @@ def string_key_schemas(modes: tuple) -> dict[str, JsonSchema]: "type": "string", "enum": ["lyria_predict", "lyria_interactions"], }, + "health_check_voice": STRING, } diff --git a/litellm/endpoints/speech/speech_to_completion_bridge/transformation.py b/litellm/endpoints/speech/speech_to_completion_bridge/transformation.py index 2ed140c0208..41a46832fea 100644 --- a/litellm/endpoints/speech/speech_to_completion_bridge/transformation.py +++ b/litellm/endpoints/speech/speech_to_completion_bridge/transformation.py @@ -30,6 +30,11 @@ class ChatAudioParam(TypedDict): format: ReadOnly[NotRequired[str]] +class ChatSpeechConfigAudioParam(TypedDict): + speech_config: ReadOnly[Mapping[str, object]] + format: ReadOnly[str] + + class SpeechToCompletionBridgeTransformationHandler: def _validate_response_format( self, model: str, custom_llm_provider: str, optional_params: Mapping[str, object] @@ -68,7 +73,17 @@ class SpeechToCompletionBridgeTransformationHandler: def _chat_audio_param( self, model: str, voice: str | Mapping[str, object] | None, optional_params: Mapping[str, object] - ) -> ChatAudioParam | None: + ) -> ChatAudioParam | ChatSpeechConfigAudioParam | None: + if isinstance(voice, Mapping) and self._is_gemini_tts_model(model): + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + normalize_gemini_speech_config, + ) + + speech_audio: Final[ChatSpeechConfigAudioParam] = { + "speech_config": normalize_gemini_speech_config(voice), + "format": GEMINI_TTS_CHAT_AUDIO_FORMAT, + } + return speech_audio if not isinstance(voice, str): return None audio_format: Final = self._chat_audio_format(model, optional_params) @@ -145,8 +160,9 @@ class SpeechToCompletionBridgeTransformationHandler: return wav_header + pcm_data def _is_gemini_tts_model(self, model: str) -> bool: - """Check if the model is a Gemini TTS model that returns PCM16 data.""" - return "gemini" in model.lower() and ("tts" in model.lower() or "preview-tts" in model.lower()) + from litellm.utils import is_gemini_tts_model + + return is_gemini_tts_model(model) def _gemini_tts_response_body(self, decoded_audio: bytes, response_format: str | None) -> tuple[bytes, str]: if response_format == GEMINI_TTS_RAW_RESPONSE_FORMAT: diff --git a/litellm/llms/gemini/google_genai/transformation.py b/litellm/llms/gemini/google_genai/transformation.py index 1189af2d6a3..01c9665fe45 100644 --- a/litellm/llms/gemini/google_genai/transformation.py +++ b/litellm/llms/gemini/google_genai/transformation.py @@ -16,7 +16,10 @@ from litellm.llms.vertex_ai.common_utils import ( _build_vertex_schema, supports_response_json_schema, ) -from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM +from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexLLM, + normalize_gemini_speech_config, +) from litellm.types.router import GenericLiteLLMParams if TYPE_CHECKING: @@ -140,7 +143,10 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM): if is_supported: # Always output in camelCase for Google GenAI API output_key = param_camel if param != param_camel else param - _generate_content_config_dict[output_key] = value + if param_snake == "speech_config" and isinstance(value, dict): + _generate_content_config_dict[output_key] = normalize_gemini_speech_config(value) + else: + _generate_content_config_dict[output_key] = value return _generate_content_config_dict def validate_environment( diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index a50665c71df..ccb9ae0a632 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -7,6 +7,7 @@ import time from collections.abc import Callable, Mapping, Sequence from copy import deepcopy from functools import partial +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Union, cast, get_args import httpx @@ -133,6 +134,73 @@ def _served_model_name(model_version: object) -> str | None: return model_version.split("@", 1)[0] +_GEMINI_SPEECH_CONFIG_KEY_MAP: Final[Mapping[str, str]] = MappingProxyType( + { + "multi_speaker_voice_config": "multiSpeakerVoiceConfig", + "speaker_voice_configs": "speakerVoiceConfigs", + "voice_config": "voiceConfig", + "prebuilt_voice_config": "prebuiltVoiceConfig", + "voice_name": "voiceName", + "language_code": "languageCode", + } +) + + +def _normalize_gemini_speech_config_item(value: object) -> object: + if isinstance(value, Mapping): + normalized: ( # mutable-ok: the iterative walk fills exactly one normalized provider payload + dict[str, object] | list[object] + ) = {} # mutable-ok: the iterative walk fills the normalized tree; rebind-ok: each shape branch initializes the shared result + elif isinstance(value, list): + normalized = [] # mutable-ok: the iterative walk fills the normalized tree; rebind-ok: each shape branch initializes the shared result + else: + return value + + pending: Final[ # mutable-ok: the explicit worklist must grow while avoiding recursive traversal + list[tuple[dict[str, object] | list[object], Mapping[object, object] | list[object]]] + ] = [(normalized, value)] + while pending: + target, source = pending.pop() + items = source.items() if isinstance(source, Mapping) else enumerate(source) + for key, item in items: + if isinstance(source, Mapping): + if not isinstance(key, str) or not isinstance(target, dict): + continue + normalized_key: str | int = _GEMINI_SPEECH_CONFIG_KEY_MAP.get(key, key) + else: + if not isinstance(target, list): + continue + normalized_key = len(target) + + child: object + if isinstance(item, Mapping): + child = {} # mutable-ok: nested mappings are filled by the worklist + elif isinstance(item, list): + child = [] # mutable-ok: nested lists are filled by the worklist + else: + child = item + + if isinstance(target, dict) and isinstance(normalized_key, str): + target[normalized_key] = child + elif isinstance(target, list): + target.append(child) + + if isinstance(child, (dict, list)) and isinstance(item, (Mapping, list)): + pending.append((child, item)) + return normalized + + +def normalize_gemini_speech_config( + value: Mapping[str, object], +) -> dict[str, object]: # mutable-ok: provider request serialization requires a concrete dict + normalized_value: Final = _normalize_gemini_speech_config_item(value) + if isinstance(normalized_value, dict): + return cast( # cast-ok: the runtime dict check establishes the normalizer's concrete result shape + dict[str, object], normalized_value + ) + return {} # mutable-ok: provider request serialization requires a concrete empty dict + + class VertexAIBaseConfig: def get_mapped_special_auth_params(self) -> dict: """ @@ -1035,12 +1103,6 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): languageCode: "en-US", } """ - from litellm.types.llms.vertex_ai import ( - PrebuiltVoiceConfig, - SpeechConfig, - VoiceConfig, - ) - # Validate audio format - Gemini TTS only supports pcm16 audio_format: Final = value.get("format") if audio_format is not None and audio_format != "pcm16": @@ -1050,18 +1112,42 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): f"Please set audio format to 'pcm16'." ) - # Map OpenAI audio parameter to Gemini speech config - speech_config: Final[SpeechConfig] = {} + camel_speech_config: Final = value.get("speechConfig") + snake_speech_config: Final = value.get("speech_config") + speech_config_value: Final = ( + camel_speech_config if isinstance(camel_speech_config, Mapping) else snake_speech_config + ) + language_code: Final = value.get("language_code", value.get("languageCode")) + if isinstance(speech_config_value, Mapping): + speech_config: Final = normalize_gemini_speech_config(speech_config_value) + if language_code is not None and "languageCode" not in speech_config: + return { # mutable-ok: provider request serialization requires a concrete dict + **speech_config, + "languageCode": language_code, + } + return speech_config - if "voice" in value: - prebuilt_voice_config: Final[PrebuiltVoiceConfig] = {"voiceName": value["voice"]} - voice_config: Final[VoiceConfig] = {"prebuiltVoiceConfig": prebuilt_voice_config} - speech_config["voiceConfig"] = voice_config + speech_config_without_language: Final[ # mutable-ok: provider request serialization requires a concrete dict + dict[str, object] # mutable-ok: provider request serialization requires a concrete dict + ] = ( + { # mutable-ok: provider request serialization requires a concrete dict + "voiceConfig": { # mutable-ok: nested provider payload is serialized as a dict + "prebuiltVoiceConfig": { # mutable-ok: nested provider payload is serialized as a dict + "voiceName": value["voice"], + } + } + } + if "voice" in value + else {} # mutable-ok: provider request serialization requires a concrete empty dict + ) - if "language_code" in value: - speech_config["languageCode"] = value["language_code"] + if language_code is not None: + return { # mutable-ok: provider request serialization requires a concrete dict + **speech_config_without_language, + "languageCode": language_code, + } - return cast(dict, speech_config) + return speech_config_without_language @staticmethod def _apply_include_server_side_tool_invocations( @@ -1228,7 +1314,10 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): optional_params["responseModalities"].append("AUDIO") # Set default temperature to 1.0 for Gemini 3 models if not specified - if VertexGeminiConfig._is_gemini_3_or_newer(model): + if VertexGeminiConfig._is_gemini_3_or_newer(model) and not litellm.utils.is_gemini_tts_model( + model, + custom_llm_provider="vertex_ai", + ): if "temperature" not in optional_params: optional_params["temperature"] = 1.0 diff --git a/litellm/llms/vertex_ai/google_genai/transformation.py b/litellm/llms/vertex_ai/google_genai/transformation.py index 8c22ae06af0..a856d86df9a 100644 --- a/litellm/llms/vertex_ai/google_genai/transformation.py +++ b/litellm/llms/vertex_ai/google_genai/transformation.py @@ -38,34 +38,15 @@ class VertexAIGoogleGenAIConfig(GoogleGenAIConfig): return default_headers - def _camel_to_snake(self, camel_str: str) -> str: - """Convert camelCase to snake_case""" - import re - - return re.sub(r"(? dict[str, Any]: # mutable-ok: inherited provider interface returns a request dict + return super().map_generate_content_optional_params( + generate_content_config_dict=generate_content_config_dict, + model=model, + ) def transform_generate_content_request( self, diff --git a/litellm/llms/vertex_ai/text_to_speech/transformation.py b/litellm/llms/vertex_ai/text_to_speech/transformation.py index d382f43495f..48598411670 100644 --- a/litellm/llms/vertex_ai/text_to_speech/transformation.py +++ b/litellm/llms/vertex_ai/text_to_speech/transformation.py @@ -6,9 +6,9 @@ Reference: https://cloud.google.com/text-to-speech/docs/reference/rest/v1/text/s """ import base64 -from collections.abc import Coroutine +from collections.abc import Coroutine, Mapping from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, TypeAlias, Union +from typing import TYPE_CHECKING, Any, ClassVar, Final, TypeAlias, Union import httpx @@ -78,14 +78,27 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): } # Response format mappings from OpenAI to Google Cloud audio encoding - FORMAT_MAPPINGS = { - "mp3": "MP3", - "opus": "OGG_OPUS", - "aac": "MP3", # Google doesn't have AAC, use MP3 - "flac": "FLAC", - "wav": "LINEAR16", - "pcm": "LINEAR16", - } + FORMAT_MAPPINGS: ClassVar[Mapping[str, str]] = MappingProxyType( + { + "mp3": "MP3", + "opus": "OGG_OPUS", + "aac": "MP3", # Google doesn't have AAC, use MP3 + "flac": "FLAC", + "wav": "LINEAR16", + "pcm": "LINEAR16", + } + ) + GEMINI_FORMAT_MAPPINGS: ClassVar[Mapping[str, str]] = MappingProxyType( + { + **FORMAT_MAPPINGS, + "alaw": "ALAW", + "mulaw": "MULAW", + "ogg_opus": "OGG_OPUS", + "pcm": "LINEAR16", + "pcm16": "LINEAR16", + "linear16": "LINEAR16", + } + ) def __init__(self) -> None: BaseTextToSpeechConfig.__init__(self) @@ -94,6 +107,7 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): def _map_voice_to_vertex_format( self, voice: str | dict | None, + model: str | None = None, ) -> tuple[str | None, dict | None]: """ Map voice to Vertex AI format. @@ -111,6 +125,9 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): if voice is None: return None, None + if model is not None and self._is_gemini_tts_model(model): + return self._map_gemini_tts_voice_to_vertex_format(model=model, voice=voice) + if isinstance(voice, dict): # Already in Vertex AI format return None, voice @@ -139,6 +156,156 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): return voice_str, voice_dict + @staticmethod + def _is_gemini_tts_model(model: str) -> bool: + from litellm.utils import is_gemini_tts_model + + return is_gemini_tts_model(model, custom_llm_provider="vertex_ai") + + @staticmethod + def _get_str_value( + source: dict, # mutable-ok: provider voice payloads arrive as concrete dictionaries + *keys: str, + ) -> str | None: + for key in keys: + value = source.get(key) + if isinstance(value, str): + return value + return None + + @staticmethod + def _get_dict_value( + source: dict, # mutable-ok: provider voice payloads arrive as concrete dictionaries + *keys: str, + ) -> dict | None: # mutable-ok: nested provider payloads remain concrete dictionaries + for key in keys: + value = source.get(key) + if isinstance(value, dict): + return value + return None + + def _extract_gemini_tts_speaker_configs( + self, + voice: dict, # mutable-ok: provider voice payload arrives as a concrete dictionary + ) -> list[dict[str, str]]: # mutable-ok: provider request serialization requires a concrete list + speech_config: Final = self._get_dict_value(voice, "speechConfig", "speech_config") or voice + multi_speaker_config: Final = self._get_dict_value( + speech_config, + "multiSpeakerVoiceConfig", + "multi_speaker_voice_config", + ) + if multi_speaker_config is None: + return [] # mutable-ok: provider request serialization requires a concrete empty list + raw_speaker_configs: Final = multi_speaker_config.get( + "speakerVoiceConfigs", + multi_speaker_config.get( + "speaker_voice_configs", + [], # mutable-ok: missing speaker configuration uses a concrete empty-list sentinel + ), + ) + if not isinstance(raw_speaker_configs, list): + return [] # mutable-ok: malformed speaker configuration produces a concrete empty list + speaker_configs: Final[ # mutable-ok: validated payloads are accumulated for serialization + list[dict[str, str]] + ] = [] + for raw_config in raw_speaker_configs: + if not isinstance(raw_config, dict): + continue + speaker_alias = self._get_str_value( + raw_config, + "speakerAlias", + "speaker_alias", + "speaker", + ) + speaker_id = self._get_str_value(raw_config, "speakerId", "speaker_id") + if speaker_id is None: + voice_config = self._get_dict_value(raw_config, "voiceConfig", "voice_config") + if voice_config is not None: + prebuilt_voice_config = self._get_dict_value( + voice_config, + "prebuiltVoiceConfig", + "prebuilt_voice_config", + ) + if prebuilt_voice_config is not None: + speaker_id = self._get_str_value( + prebuilt_voice_config, + "voiceName", + "voice_name", + ) + if speaker_alias is not None and speaker_id is not None: + speaker_configs.append( + { # mutable-ok: provider request serialization requires a concrete dict + "speakerAlias": speaker_alias, + "speakerId": speaker_id, + } + ) + return speaker_configs + + def _extract_gemini_tts_voice_name( + self, + voice: dict, # mutable-ok: provider voice payload arrives as a concrete dictionary + ) -> str | None: + voice_name: Final = self._get_str_value(voice, "name", "voiceName", "voice_name") + if voice_name is not None: + return voice_name + speech_config: Final = self._get_dict_value(voice, "speechConfig", "speech_config") or voice + voice_config: Final = self._get_dict_value(speech_config, "voiceConfig", "voice_config") + if voice_config is None: + return None + prebuilt_voice_config: Final = self._get_dict_value( + voice_config, + "prebuiltVoiceConfig", + "prebuilt_voice_config", + ) + if prebuilt_voice_config is None: + return None + return self._get_str_value(prebuilt_voice_config, "voiceName", "voice_name") + + def _map_gemini_tts_voice_to_vertex_format( + self, + model: str, + voice: str | dict, # mutable-ok: provider voice payload arrives as a concrete dictionary + ) -> tuple[str | None, dict]: # mutable-ok: provider request serialization requires a concrete dict + if isinstance(voice, str): + return voice, { # mutable-ok: provider request serialization requires a concrete dict + "languageCode": self.DEFAULT_LANGUAGE_CODE, + "modelName": model, + "name": voice, + } + + language_code: Final = self._get_str_value(voice, "languageCode", "language_code") or self.DEFAULT_LANGUAGE_CODE + model_name: Final = model + speaker_configs: Final = self._extract_gemini_tts_speaker_configs(voice) + if speaker_configs: + return None, { # mutable-ok: provider request serialization requires a concrete dict + "languageCode": language_code, + "modelName": model_name, + "multiSpeakerVoiceConfig": { # mutable-ok: nested provider payload is serialized as a dict + "speakerVoiceConfigs": speaker_configs, + }, + } + voice_name: Final = self._extract_gemini_tts_voice_name(voice) + if voice_name is not None: + return None, { # mutable-ok: provider request serialization requires a concrete dict + "languageCode": language_code, + "modelName": model_name, + "name": voice_name, + } + return None, { # mutable-ok: provider request serialization requires a concrete dict + **voice, + "languageCode": language_code, + "modelName": model_name, + } + + @staticmethod + def _dispatch_voice_name(voice: str | Mapping[str, object] | None) -> str | None: + if isinstance(voice, str): + return voice + if not isinstance(voice, Mapping): + return None + name: Final = voice.get("name") + return name if isinstance(name, str) else None + def dispatch_text_to_speech( self, model: str, @@ -172,14 +339,16 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): vertex_project: Final = self.safe_get_vertex_ai_project(litellm_params_dict) vertex_location: Final = self.safe_get_vertex_ai_location(litellm_params_dict) - # Convert voice to string if it's a dict (extract name) - # Actual voice mapping happens in map_openai_params - voice_str: str | None = None - if isinstance(voice, str): - voice_str = voice - elif isinstance(voice, dict): - # Extract voice name from dict if needed - voice_str = voice.get("name") if voice else None + mapped_voice, mapped_params = ( + (self._dispatch_voice_name(voice), optional_params) + if "audioEncoding" in optional_params + else self.map_openai_params( + model=model, + voice=voice, + optional_params=optional_params, + kwargs=kwargs, + ) + ) # Store credentials in litellm_params for use in transform methods litellm_params_dict.update( @@ -195,9 +364,9 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): response: Final = base_llm_http_handler.text_to_speech_handler( model=model, input=input, - voice=voice_str, + voice=mapped_voice, text_to_speech_provider_config=self, - text_to_speech_optional_params=optional_params, + text_to_speech_optional_params=mapped_params, custom_llm_provider="vertex_ai", litellm_params=litellm_params_dict, logging_obj=logging_obj, @@ -245,15 +414,21 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): ########################################################## # Map voice using helper ########################################################## - mapped_voice_str, voice_dict = self._map_voice_to_vertex_format(voice) + mapped_voice_str, voice_dict = self._map_voice_to_vertex_format( + voice=voice, + model=model, + ) if voice_dict is not None: mapped_params["vertex_voice_dict"] = voice_dict # Map response format if "response_format" in optional_params: format_name: Final = optional_params["response_format"] - if format_name in self.FORMAT_MAPPINGS: - mapped_params["audioEncoding"] = self.FORMAT_MAPPINGS[format_name] + format_mappings: Final = ( + self.GEMINI_FORMAT_MAPPINGS if self._is_gemini_tts_model(model) else self.FORMAT_MAPPINGS + ) + if format_name in format_mappings: + mapped_params["audioEncoding"] = format_mappings[format_name] else: # Try to use it directly as Google Cloud format mapped_params["audioEncoding"] = format_name @@ -410,7 +585,7 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): voice_dict: Final = litellm_params.get("vertex_voice_dict") or optional_params.get("vertex_voice_dict") if voice_dict is not None and isinstance(voice_dict, dict): vertex_voice = VertexTextToSpeechVoice(**voice_dict) - elif voice is not None and isinstance(voice, str): + elif voice is not None: # Handle string voice (shouldn't normally happen if dispatch was called) parts: Final = voice.split("-") if len(parts) >= 2: diff --git a/litellm/main.py b/litellm/main.py index 7f4b34d28a0..761496d2d3d 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -8198,6 +8198,7 @@ def speech( kwargs.pop("tags", []) optional_params = {} + original_voice: Final = voice if response_format is not None: optional_params["response_format"] = response_format if speed is not None: @@ -8450,7 +8451,13 @@ def speech( generic_optional_params: Final = GenericLiteLLMParams(**kwargs) # Handle Gemini models separately (they use speech_to_completion_bridge) - if "gemini" in model: + if ( + litellm.utils.is_gemini_tts_model( + model, + custom_llm_provider=custom_llm_provider, + ) + and response_format is None + ): from .endpoints.speech.speech_to_completion_bridge.handler import ( speech_to_completion_bridge_handler, ) @@ -8458,7 +8465,7 @@ def speech( return speech_to_completion_bridge_handler.speech( model=model, input=input, - voice=voice, + voice=original_voice, optional_params=optional_params, litellm_params=litellm_params_dict, headers=headers or {}, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index ae02744b8e1..c34f6997384 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -48029,6 +48029,34 @@ "web_search_billing_unit": "per_query", "google_maps_grounding_cost_per_query": 0.014 }, + "vertex_ai/gemini-3.1-flash-tts-preview": { + "input_cost_per_token": 1e-06, + "input_cost_per_token_batches": 5e-07, + "litellm_provider": "vertex_ai-language-models", + "mode": "audio_speech", + "output_cost_per_audio_token": 2e-05, + "output_cost_per_token": 2e-05, + "source": "https://cloud.google.com/text-to-speech/pricing", + "health_check_voice": "Kore", + "max_input_tokens": 8192, + "max_output_tokens": 16384, + "max_tokens": 16384, + "supported_endpoints": [ + "/v1/audio/speech" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "audio" + ], + "supports_audio_input": false, + "supports_audio_output": true, + "supports_function_calling": false, + "supports_prompt_caching": false, + "tpm": 4000000, + "rpm": 10 + }, "vertex_ai/gemini-3.1-flash-lite": { "cache_read_input_audio_token_cost": 5e-08, "deprecation_date": "2027-05-07", @@ -54830,6 +54858,33 @@ "supports_response_schema": false, "supports_web_search": false }, + "gemini-3.1-flash-tts-preview": { + "health_check_voice": "Kore", + "input_cost_per_token": 1e-06, + "litellm_provider": "gemini", + "max_input_tokens": 8192, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "audio_speech", + "output_cost_per_audio_token": 2e-05, + "output_cost_per_token": 2e-05, + "source": "https://cloud.google.com/text-to-speech/pricing", + "supported_endpoints": [ + "/v1/audio/speech" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "audio" + ], + "supports_audio_input": false, + "supports_audio_output": true, + "supports_function_calling": false, + "supports_prompt_caching": false, + "tpm": 4000000, + "rpm": 10 + }, "gemini-flash-latest": { "cache_read_input_token_cost": 7.5e-08, "input_cost_per_token": 7.5e-07, @@ -66189,13 +66244,14 @@ "output_cost_per_token": 1e-05, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, - "vertex_ai/gemini-3.1-flash-tts-preview": { - "input_cost_per_token": 1e-06, - "input_cost_per_token_batches": 5e-07, + "vertex_ai/gemini-3.1-flash-live-preview": { + "input_cost_per_audio_token": 3e-06, + "input_cost_per_second": 8.33333333333e-05, + "input_cost_per_token": 7.5e-07, "litellm_provider": "vertex_ai", - "mode": "audio_speech", - "output_cost_per_audio_token": 2e-05, - "output_cost_per_token": 2e-05, + "mode": "realtime", + "output_cost_per_audio_token": 1.2e-05, + "output_cost_per_token": 4.5e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/gemini-3.5-transcribe": { diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 92a75bf953a..ab66191ae25 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -12133,12 +12133,40 @@ async def audio_speech( custom_headers.update(callback_headers) requested_format: Final = data.get("response_format") + requested_format_str: Final = requested_format if isinstance(requested_format, str) else None + extra_response_format_media_types: Final = MappingProxyType( + { + "pcm16": "audio/wav", + "linear16": "audio/wav", + "alaw": "audio/x-alaw-basic", + "mulaw": "audio/basic", + } + ) + extra_media_type: Final = ( + extra_response_format_media_types.get(requested_format_str.lower()) + if requested_format_str is not None + else None + ) upstream_content_type: Final = ( response.response.headers.get("content-type") if isinstance(response, HttpxBinaryResponseContent) else None ) - media_type: Final = resolve_speech_media_type( + resolved_media_type: Final = resolve_speech_media_type( upstream_content_type=upstream_content_type, - response_format=requested_format if isinstance(requested_format, str) else None, + response_format=requested_format_str, + ) + request_model: Final = data.get("model", "") + media_type: Final = ( + extra_media_type + if extra_media_type is not None + else ( + "audio/wav" + if ( + requested_format_str is None + and isinstance(request_model, str) + and litellm.utils.is_gemini_tts_model(request_model) + ) + else resolved_media_type + ) ) return StreamingResponse( @@ -12162,13 +12190,16 @@ async def audio_speech( detail=e.detail, headers=headers_with_litellm_call_id(e.headers, litellm_call_id), ) + raw_status_code: Final = getattr(e, "status_code", None) + if not isinstance(raw_status_code, int): + raise raise ProxyException( message=getattr(e, "message", f"{e}"), type=getattr(e, "type", "None"), param=getattr(e, "param", "None"), headers=litellm_call_id_headers(litellm_call_id), openai_code=getattr(e, "code", None), - code=getattr(e, "status_code", 500), + code=raw_status_code, ) diff --git a/litellm/types/llms/vertex_ai.py b/litellm/types/llms/vertex_ai.py index ce51e46ef15..8a9b81bf890 100644 --- a/litellm/types/llms/vertex_ai.py +++ b/litellm/types/llms/vertex_ai.py @@ -1,7 +1,9 @@ +from collections.abc import Sequence from enum import Enum from typing import Any, Final, Literal, Protocol from typing_extensions import ( + ReadOnly, Required, TypedDict, ) @@ -226,8 +228,18 @@ class VoiceConfig(TypedDict): prebuiltVoiceConfig: PrebuiltVoiceConfig +class SpeakerVoiceConfig(TypedDict): + speaker: ReadOnly[str] + voiceConfig: ReadOnly[VoiceConfig] + + +class MultiSpeakerVoiceConfig(TypedDict): + speakerVoiceConfigs: ReadOnly[Sequence[SpeakerVoiceConfig]] + + class SpeechConfig(TypedDict, total=False): voiceConfig: VoiceConfig + multiSpeakerVoiceConfig: ReadOnly[MultiSpeakerVoiceConfig] languageCode: str diff --git a/litellm/types/llms/vertex_ai_text_to_speech.py b/litellm/types/llms/vertex_ai_text_to_speech.py index e8f9fbe4a24..86b131dce59 100644 --- a/litellm/types/llms/vertex_ai_text_to_speech.py +++ b/litellm/types/llms/vertex_ai_text_to_speech.py @@ -4,7 +4,9 @@ Type definitions for Vertex AI Text-to-Speech API Reference: https://cloud.google.com/text-to-speech/docs/reference/rest/v1/text/synthesize """ -from typing_extensions import TypedDict +from collections.abc import Sequence + +from typing_extensions import ReadOnly, TypedDict class VertexTextToSpeechInput(TypedDict, total=False): @@ -16,6 +18,16 @@ class VertexTextToSpeechInput(TypedDict, total=False): text: str | None ssml: str | None + prompt: ReadOnly[str | None] + + +class VertexTextToSpeechSpeakerVoiceConfig(TypedDict): + speakerAlias: ReadOnly[str] + speakerId: ReadOnly[str] + + +class VertexTextToSpeechMultiSpeakerVoiceConfig(TypedDict): + speakerVoiceConfigs: ReadOnly[Sequence[VertexTextToSpeechSpeakerVoiceConfig]] class VertexTextToSpeechVoice(TypedDict, total=False): @@ -29,6 +41,8 @@ class VertexTextToSpeechVoice(TypedDict, total=False): languageCode: str name: str + modelName: ReadOnly[str] + multiSpeakerVoiceConfig: ReadOnly[VertexTextToSpeechMultiSpeakerVoiceConfig] class VertexTextToSpeechAudioConfig(TypedDict, total=False): @@ -42,6 +56,7 @@ class VertexTextToSpeechAudioConfig(TypedDict, total=False): audioEncoding: str speakingRate: str + sampleRateHertz: ReadOnly[int] class VertexTextToSpeechRequest(TypedDict, total=False): diff --git a/litellm/utils.py b/litellm/utils.py index dd35c17809f..9a763476cf2 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -724,6 +724,36 @@ def _is_gemini_model(model: str | None, custom_llm_provider: str | None) -> bool return model is not None and "gemini" in model.lower() +def is_gemini_tts_model(model: str, custom_llm_provider: str | None = None) -> bool: + try: + runtime_model_info: Final = get_model_info( + model=model, + custom_llm_provider=custom_llm_provider, + ) + except Exception: # noqa: BLE001 # model lookup failures intentionally fall back to the bundled registry + provider_model: Final = ( + f"{custom_llm_provider}/{model}" + if custom_llm_provider is not None and not model.startswith(f"{custom_llm_provider}/") + else model + ) + bundled_model_cost: Final = _get_bundled_model_cost_map() + bundled_model_info: Final = bundled_model_cost.get(provider_model) or bundled_model_cost.get(model) + return ( + bundled_model_info is not None + and bundled_model_info.get("mode") == "audio_speech" + and bundled_model_info.get("litellm_provider") + in ( + "gemini", + "vertex_ai-language-models", + ) + ) + + return runtime_model_info.get("mode") == "audio_speech" and runtime_model_info.get("litellm_provider") in ( + "gemini", + "vertex_ai-language-models", + ) + + def _remove_thought_signature_from_id(tool_call_id: str, separator: str) -> str: """ Remove thought signature from a tool call ID. diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index ae02744b8e1..c34f6997384 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -48029,6 +48029,34 @@ "web_search_billing_unit": "per_query", "google_maps_grounding_cost_per_query": 0.014 }, + "vertex_ai/gemini-3.1-flash-tts-preview": { + "input_cost_per_token": 1e-06, + "input_cost_per_token_batches": 5e-07, + "litellm_provider": "vertex_ai-language-models", + "mode": "audio_speech", + "output_cost_per_audio_token": 2e-05, + "output_cost_per_token": 2e-05, + "source": "https://cloud.google.com/text-to-speech/pricing", + "health_check_voice": "Kore", + "max_input_tokens": 8192, + "max_output_tokens": 16384, + "max_tokens": 16384, + "supported_endpoints": [ + "/v1/audio/speech" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "audio" + ], + "supports_audio_input": false, + "supports_audio_output": true, + "supports_function_calling": false, + "supports_prompt_caching": false, + "tpm": 4000000, + "rpm": 10 + }, "vertex_ai/gemini-3.1-flash-lite": { "cache_read_input_audio_token_cost": 5e-08, "deprecation_date": "2027-05-07", @@ -54830,6 +54858,33 @@ "supports_response_schema": false, "supports_web_search": false }, + "gemini-3.1-flash-tts-preview": { + "health_check_voice": "Kore", + "input_cost_per_token": 1e-06, + "litellm_provider": "gemini", + "max_input_tokens": 8192, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "audio_speech", + "output_cost_per_audio_token": 2e-05, + "output_cost_per_token": 2e-05, + "source": "https://cloud.google.com/text-to-speech/pricing", + "supported_endpoints": [ + "/v1/audio/speech" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "audio" + ], + "supports_audio_input": false, + "supports_audio_output": true, + "supports_function_calling": false, + "supports_prompt_caching": false, + "tpm": 4000000, + "rpm": 10 + }, "gemini-flash-latest": { "cache_read_input_token_cost": 7.5e-08, "input_cost_per_token": 7.5e-07, @@ -66189,13 +66244,14 @@ "output_cost_per_token": 1e-05, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, - "vertex_ai/gemini-3.1-flash-tts-preview": { - "input_cost_per_token": 1e-06, - "input_cost_per_token_batches": 5e-07, + "vertex_ai/gemini-3.1-flash-live-preview": { + "input_cost_per_audio_token": 3e-06, + "input_cost_per_second": 8.33333333333e-05, + "input_cost_per_token": 7.5e-07, "litellm_provider": "vertex_ai", - "mode": "audio_speech", - "output_cost_per_audio_token": 2e-05, - "output_cost_per_token": 2e-05, + "mode": "realtime", + "output_cost_per_audio_token": 1.2e-05, + "output_cost_per_token": 4.5e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, "vertex_ai/gemini-3.5-transcribe": { diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index 737a9b7fa60..0cfd83f388f 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -262,6 +262,9 @@ "minimum": 0 } }, + "health_check_voice": { + "type": "string" + }, "input_cost_per_audio_per_second": { "type": "number", "minimum": 0 diff --git a/tests/test_litellm/google_genai/test_google_genai_transformation.py b/tests/test_litellm/google_genai/test_google_genai_transformation.py index f0d0fc6126d..e68c3a47369 100644 --- a/tests/test_litellm/google_genai/test_google_genai_transformation.py +++ b/tests/test_litellm/google_genai/test_google_genai_transformation.py @@ -3,10 +3,10 @@ Test to verify the Google GenAI transformation logic for generateContent parameters """ - import pytest from litellm.llms.gemini.google_genai.transformation import GoogleGenAIConfig +from litellm.llms.vertex_ai.google_genai.transformation import VertexAIGoogleGenAIConfig from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, ) @@ -31,10 +31,7 @@ def test_map_generate_content_optional_params_response_json_schema_camelcase(): # responseJsonSchema should be in the result (camelCase format for Google GenAI API) assert "responseJsonSchema" in result - assert ( - result["responseJsonSchema"] - == generate_content_config_dict["responseJsonSchema"] - ) + assert result["responseJsonSchema"] == generate_content_config_dict["responseJsonSchema"] assert "temperature" in result assert result["temperature"] == 1.0 @@ -58,10 +55,7 @@ def test_map_generate_content_optional_params_response_schema_snakecase(): # response_schema should be converted to responseJsonSchema (camelCase) assert "responseJsonSchema" in result - assert ( - result["responseJsonSchema"] - == generate_content_config_dict["response_json_schema"] - ) + assert result["responseJsonSchema"] == generate_content_config_dict["response_json_schema"] assert "temperature" in result @@ -157,6 +151,68 @@ def test_map_generate_content_optional_params_response_mime_type(): assert "responseJsonSchema" in result +@pytest.mark.parametrize("config_class", [GoogleGenAIConfig, VertexAIGoogleGenAIConfig]) +def test_map_generate_content_optional_params_speech_config(config_class): + config = config_class() + + result = config.map_generate_content_optional_params( + generate_content_config_dict={ + "response_modalities": ["AUDIO"], + "speech_config": { + "multi_speaker_voice_config": { + "speaker_voice_configs": [ + { + "speaker": "Ryan", + "voice_config": { + "prebuilt_voice_config": { + "voice_name": "Umbriel", + }, + }, + }, + { + "speaker": "Katie", + "voice_config": { + "prebuilt_voice_config": { + "voice_name": "Leda", + }, + }, + }, + ], + }, + "language_code": "en-US", + }, + }, + model="gemini-3.1-flash-tts-preview", + ) + + assert result["responseModalities"] == ["AUDIO"] + assert "response_modalities" not in result + assert "speech_config" not in result + assert result["speechConfig"] == { + "multiSpeakerVoiceConfig": { + "speakerVoiceConfigs": [ + { + "speaker": "Ryan", + "voiceConfig": { + "prebuiltVoiceConfig": { + "voiceName": "Umbriel", + }, + }, + }, + { + "speaker": "Katie", + "voiceConfig": { + "prebuiltVoiceConfig": { + "voiceName": "Leda", + }, + }, + }, + ], + }, + "languageCode": "en-US", + } + + def test_responses_api_reasoning_dict_format(): """Test that reasoning parameter with dict format is mapped to reasoning_effort""" from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams @@ -254,9 +310,7 @@ def test_transform_generate_content_request_with_system_instruction(): # Verify that systemInstruction is in the request assert "systemInstruction" in result, "systemInstruction should be in request body" - assert ( - result["systemInstruction"] == system_instruction - ), "systemInstruction should match input" + assert result["systemInstruction"] == system_instruction, "systemInstruction should match input" assert result["model"] == "gemini-3-flash-preview" assert result["contents"] == contents @@ -279,9 +333,7 @@ def test_transform_generate_content_request_without_system_instruction(): ) # Verify that systemInstruction is NOT in the request when not provided - assert ( - "systemInstruction" not in result - ), "systemInstruction should not be in request when None" + assert "systemInstruction" not in result, "systemInstruction should not be in request when None" assert result["model"] == "gemini-3-flash-preview" assert result["contents"] == contents @@ -290,9 +342,7 @@ def test_transform_generate_content_request_system_instruction_with_tools(): """Test that systemInstruction works correctly alongside tools""" config = GoogleGenAIConfig() - system_instruction = { - "parts": [{"text": "You are a helpful assistant that uses tools"}] - } + system_instruction = {"parts": [{"text": "You are a helpful assistant that uses tools"}]} contents = [{"role": "user", "parts": [{"text": "What's the weather?"}]}] @@ -374,9 +424,7 @@ def test_transform_generate_content_request_normalizes_response_schema_2_5(): assert "responseJsonSchema" in gen_config normalized = gen_config["responseJsonSchema"] assert "$defs" in normalized - assert normalized["properties"]["highlights"]["items"] == { - "$ref": "#/$defs/Highlight" - } + assert normalized["properties"]["highlights"]["items"] == {"$ref": "#/$defs/Highlight"} def test_transform_generate_content_request_flattens_response_schema_1_5(): @@ -581,12 +629,8 @@ def test_validate_environment_with_dict_api_key(): # The dict should be merged into headers, not set as a value assert "x-goog-api-key" in result, "x-goog-api-key should be in headers" - assert ( - result["x-goog-api-key"] == "sk-test-key-123" - ), "API key should be the string value, not a dict" - assert isinstance( - result["x-goog-api-key"], str - ), "Header value should be a string, not a dict" + assert result["x-goog-api-key"] == "sk-test-key-123", "API key should be the string value, not a dict" + assert isinstance(result["x-goog-api-key"], str), "Header value should be a string, not a dict" assert "Content-Type" in result, "Content-Type should be in headers" assert result["Content-Type"] == "application/json" @@ -632,9 +676,7 @@ def test_validate_environment_with_extra_headers(): # Both the auth dict and extra headers should be merged assert "x-goog-api-key" in result, "x-goog-api-key should be in headers" - assert ( - result["x-goog-api-key"] == "sk-test-key-789" - ), "API key should be correctly set" + assert result["x-goog-api-key"] == "sk-test-key-789", "API key should be correctly set" assert isinstance(result["x-goog-api-key"], str), "Header value should be a string" assert "X-Custom-Header" in result, "Extra headers should be merged" assert result["X-Custom-Header"] == "custom-value" diff --git a/tests/test_litellm/llms/gemini/test_gemini_tts.py b/tests/test_litellm/llms/gemini/test_gemini_tts.py index 4893825373a..b22372b2f8c 100644 --- a/tests/test_litellm/llms/gemini/test_gemini_tts.py +++ b/tests/test_litellm/llms/gemini/test_gemini_tts.py @@ -5,11 +5,63 @@ Test Gemini TTS (Text-to-Speech) functionality import pytest from unittest.mock import patch, MagicMock - import litellm +from litellm.endpoints.speech.speech_to_completion_bridge.transformation import ( + SpeechToCompletionBridgeTransformationHandler, +) from litellm.llms.gemini.chat.transformation import GoogleAIStudioGeminiConfig from litellm.utils import get_supported_openai_params +GEMINI_3_1_FLASH_TTS_MODEL = "gemini-3.1-flash-tts-preview" + +MULTI_SPEAKER_SPEECH_CONFIG = { + "multi_speaker_voice_config": { + "speaker_voice_configs": [ + { + "speaker": "Ryan", + "voice_config": { + "prebuilt_voice_config": { + "voice_name": "Umbriel", + }, + }, + }, + { + "speaker": "Katie", + "voice_config": { + "prebuilt_voice_config": { + "voice_name": "Leda", + }, + }, + }, + ], + }, + "language_code": "en-US", +} + +NORMALIZED_MULTI_SPEAKER_SPEECH_CONFIG = { + "multiSpeakerVoiceConfig": { + "speakerVoiceConfigs": [ + { + "speaker": "Ryan", + "voiceConfig": { + "prebuiltVoiceConfig": { + "voiceName": "Umbriel", + }, + }, + }, + { + "speaker": "Katie", + "voiceConfig": { + "prebuiltVoiceConfig": { + "voiceName": "Leda", + }, + }, + }, + ], + }, + "languageCode": "en-US", +} + class TestGeminiTTSTransformation: """Test Gemini TTS transformation functionality""" @@ -19,17 +71,16 @@ class TestGeminiTTSTransformation: config = GoogleAIStudioGeminiConfig() # Test TTS models (both preview and non-preview versions) - assert ( - config.is_model_gemini_audio_model("gemini-2.5-flash-preview-tts") == True - ) - assert config.is_model_gemini_audio_model("gemini-2.5-pro-preview-tts") == True - assert config.is_model_gemini_audio_model("gemini-2.5-flash-tts") == True - assert config.is_model_gemini_audio_model("gemini-2.5-pro-tts") == True + assert config.is_model_gemini_audio_model("gemini-2.5-flash-preview-tts") + assert config.is_model_gemini_audio_model("gemini-2.5-pro-preview-tts") + assert config.is_model_gemini_audio_model("gemini-2.5-flash-tts") + assert config.is_model_gemini_audio_model("gemini-2.5-pro-tts") + assert config.is_model_gemini_audio_model(GEMINI_3_1_FLASH_TTS_MODEL) # Test non-TTS models - assert config.is_model_gemini_audio_model("gemini-2.5-flash") == False - assert config.is_model_gemini_audio_model("gemini-2.5-pro") == False - assert config.is_model_gemini_audio_model("gpt-4o-audio-preview") == False + assert not config.is_model_gemini_audio_model("gemini-2.5-flash") + assert not config.is_model_gemini_audio_model("gemini-2.5-pro") + assert not config.is_model_gemini_audio_model("gpt-4o-audio-preview") def test_gemini_tts_supported_params(self): """Test that audio parameter is included for TTS models""" @@ -66,10 +117,7 @@ class TestGeminiTTSTransformation: assert "speechConfig" in result assert "voiceConfig" in result["speechConfig"] assert "prebuiltVoiceConfig" in result["speechConfig"]["voiceConfig"] - assert ( - result["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] - == "Kore" - ) + assert result["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Kore" # Check response modalities assert "responseModalities" in result @@ -78,9 +126,7 @@ class TestGeminiTTSTransformation: def test_gemini_tts_audio_parameter_mapping_with_language_code(self): config = GoogleAIStudioGeminiConfig() - non_default_params = { - "audio": {"voice": "Kore", "format": "pcm16", "language_code": "en-US"} - } + non_default_params = {"audio": {"voice": "Kore", "format": "pcm16", "language_code": "en-US"}} optional_params = {} result = config.map_openai_params( @@ -92,17 +138,12 @@ class TestGeminiTTSTransformation: assert "speechConfig" in result assert result["speechConfig"]["languageCode"] == "en-US" - assert ( - result["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] - == "Kore" - ) + assert result["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Kore" def test_map_audio_params_language_code(self): config = GoogleAIStudioGeminiConfig() - result = config._map_audio_params( - {"voice": "Kore", "format": "pcm16", "language_code": "de-DE"} - ) + result = config._map_audio_params({"voice": "Kore", "format": "pcm16", "language_code": "de-DE"}) assert result["languageCode"] == "de-DE" assert result["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Kore" @@ -115,6 +156,29 @@ class TestGeminiTTSTransformation: assert "languageCode" not in result assert result["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Kore" + def test_gemini_tts_multi_speaker_audio_parameter_mapping(self): + """Test multi-speaker audio parameter mapping for Gemini 3.1 TTS models""" + config = GoogleAIStudioGeminiConfig() + + non_default_params = { + "audio": { + "speech_config": MULTI_SPEAKER_SPEECH_CONFIG, + "format": "pcm16", + } + } + optional_params = {} + + result = config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model=GEMINI_3_1_FLASH_TTS_MODEL, + drop_params=False, + ) + + assert result["speechConfig"] == NORMALIZED_MULTI_SPEAKER_SPEECH_CONFIG + assert result["responseModalities"] == ["AUDIO"] + assert "temperature" not in result + def test_gemini_tts_audio_parameter_with_existing_modalities(self): """Test audio parameter mapping when modalities already exist""" config = GoogleAIStudioGeminiConfig() @@ -198,9 +262,7 @@ class TestGeminiTTSTransformation: } optional_params = {} - with pytest.raises( - ValueError, match="Unsupported audio format for Gemini TTS models" - ): + with pytest.raises(ValueError, match="Unsupported audio format for Gemini TTS models"): config.map_openai_params( non_default_params=non_default_params, optional_params=optional_params, @@ -239,6 +301,28 @@ def test_gemini_tts_completion_mock(): assert response.choices[0].message.content is not None +def test_gemini_tts_speech_bridge_accepts_multi_speaker_voice_dict(): + handler = SpeechToCompletionBridgeTransformationHandler() + + result = handler.transform_request( + model=f"vertex_ai/{GEMINI_3_1_FLASH_TTS_MODEL}", + input="Ryan: How are you doing today Katie?\nKatie: Not too bad.", + voice=MULTI_SPEAKER_SPEECH_CONFIG, + optional_params={"response_format": "pcm"}, + litellm_params={}, + headers={}, + litellm_logging_obj=MagicMock(), + custom_llm_provider="vertex_ai", + ) + + assert result["modalities"] == ["audio"] + assert result["audio"] == { + "speech_config": NORMALIZED_MULTI_SPEAKER_SPEECH_CONFIG, + "format": "pcm16", + } + assert "response_format" not in result + + class TestGeminiTTSSpeechConfigInRequestBody: """Test that speechConfig is properly included in the final request body. @@ -255,11 +339,11 @@ class TestGeminiTTSSpeechConfigInRequestBody: ("gemini-2.5-flash-preview-tts", "vertex_ai"), ("gemini-2.5-flash-preview-tts", "gemini"), ("gemini-2.5-pro-tts", "vertex_ai"), + (GEMINI_3_1_FLASH_TTS_MODEL, "vertex_ai"), + (GEMINI_3_1_FLASH_TTS_MODEL, "gemini"), ], ) - def test_speechconfig_in_generation_config_transform_request_body( - self, model, custom_llm_provider - ): + def test_speechconfig_in_generation_config_transform_request_body(self, model, custom_llm_provider): """Test that speechConfig is included in generationConfig after _transform_request_body()""" from litellm.llms.vertex_ai.gemini.transformation import ( _transform_request_body, @@ -267,9 +351,7 @@ class TestGeminiTTSSpeechConfigInRequestBody: # Simulate optional_params after map_openai_params() has run optional_params = { - "speechConfig": { - "voiceConfig": {"prebuiltVoiceConfig": {"voiceName": "Kore"}} - }, + "speechConfig": {"voiceConfig": {"prebuiltVoiceConfig": {"voiceName": "Kore"}}}, "responseModalities": ["AUDIO"], } @@ -292,12 +374,7 @@ class TestGeminiTTSSpeechConfigInRequestBody: f"speechConfig was filtered out of generationConfig for model={model}, provider={custom_llm_provider}. " "Ensure speechConfig is in the GenerationConfig TypedDict." ) - assert ( - generation_config["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"][ - "voiceName" - ] - == "Kore" - ) + assert generation_config["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Kore" @pytest.mark.parametrize( "model,custom_llm_provider", @@ -305,16 +382,18 @@ class TestGeminiTTSSpeechConfigInRequestBody: ("gemini-2.5-flash-tts", "vertex_ai"), ("gemini-2.5-flash-tts", "gemini"), ("gemini-2.5-flash-preview-tts", "vertex_ai"), + (GEMINI_3_1_FLASH_TTS_MODEL, "vertex_ai"), + (GEMINI_3_1_FLASH_TTS_MODEL, "gemini"), ], ) def test_speechconfig_end_to_end_mapping(self, model, custom_llm_provider): """Test full pipeline: audio param -> map_openai_params -> _transform_request_body""" - from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( - VertexGeminiConfig, - ) from litellm.llms.vertex_ai.gemini.transformation import ( _transform_request_body, ) + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexGeminiConfig, + ) config = VertexGeminiConfig() @@ -351,17 +430,55 @@ class TestGeminiTTSSpeechConfigInRequestBody: f"speechConfig was filtered out during _transform_request_body() for model={model}, provider={custom_llm_provider}. " "This breaks Gemini TTS - speechConfig must be in GenerationConfig TypedDict." ) - assert ( - generation_config["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"][ - "voiceName" - ] - == "Puck" - ) + assert generation_config["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Puck" # Also verify responseModalities is present assert "responseModalities" in generation_config assert "AUDIO" in generation_config["responseModalities"] + @pytest.mark.parametrize( + "custom_llm_provider", + [ + "vertex_ai", + "gemini", + ], + ) + def test_multi_speaker_speechconfig_end_to_end_mapping(self, custom_llm_provider): + """Test full pipeline for Gemini 3.1 multi-speaker TTS speechConfig""" + from litellm.llms.vertex_ai.gemini.transformation import ( + _transform_request_body, + ) + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexGeminiConfig, + ) + + config = VertexGeminiConfig() + + mapped_params = config.map_openai_params( + non_default_params={ + "audio": { + "speech_config": MULTI_SPEAKER_SPEECH_CONFIG, + "format": "pcm16", + } + }, + optional_params={}, + model=GEMINI_3_1_FLASH_TTS_MODEL, + drop_params=False, + ) + + request_body = _transform_request_body( + messages=[{"role": "user", "content": "Ryan: Hi.\nKatie: Hello."}], + model=GEMINI_3_1_FLASH_TTS_MODEL, + optional_params=mapped_params, + custom_llm_provider=custom_llm_provider, + litellm_params={}, + cached_content=None, + ) + + generation_config = request_body["generationConfig"] + assert generation_config["speechConfig"] == NORMALIZED_MULTI_SPEAKER_SPEECH_CONFIG + assert generation_config["responseModalities"] == ["AUDIO"] + assert "temperature" not in generation_config @pytest.mark.parametrize( "model,custom_llm_provider", @@ -372,18 +489,16 @@ class TestGeminiTTSSpeechConfigInRequestBody: ], ) def test_language_code_end_to_end_mapping(self, model, custom_llm_provider): - from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( - VertexGeminiConfig, - ) from litellm.llms.vertex_ai.gemini.transformation import ( _transform_request_body, ) + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexGeminiConfig, + ) config = VertexGeminiConfig() - non_default_params = { - "audio": {"voice": "Puck", "format": "pcm16", "language_code": "pt-BR"} - } + non_default_params = {"audio": {"voice": "Puck", "format": "pcm16", "language_code": "pt-BR"}} optional_params = {} mapped_params = config.map_openai_params( @@ -406,12 +521,7 @@ class TestGeminiTTSSpeechConfigInRequestBody: generation_config = request_body["generationConfig"] assert generation_config["speechConfig"]["languageCode"] == "pt-BR" - assert ( - generation_config["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"][ - "voiceName" - ] - == "Puck" - ) + assert generation_config["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Puck" assert "AUDIO" in generation_config["responseModalities"] diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_audio.py b/tests/test_litellm/proxy/proxy_server/test_routes_audio.py index de76c7257cf..9216cb09927 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_audio.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_audio.py @@ -14,6 +14,7 @@ from unittest.mock import AsyncMock, MagicMock import httpx import pytest +from fastapi import HTTPException from litellm.proxy import proxy_server from litellm.types.llms.openai import HttpxBinaryResponseContent @@ -85,7 +86,7 @@ def patched_speech_error(monkeypatch): def patched_speech_provider_rejection(monkeypatch, patched_speech_error): import litellm - async def _raise(*args, **kwargs): + async def _raise_rejection(*args, **kwargs): raise litellm.BadRequestError( message=( "Gemini TTS only produces raw PCM16 audio, so response_format='mp3' is not supported." @@ -95,6 +96,29 @@ def patched_speech_provider_rejection(monkeypatch, patched_speech_error): llm_provider="gemini", ) + monkeypatch.setattr(proxy_server, "route_request", _raise_rejection) + yield + + +@pytest.fixture +def patched_speech_proxy_error(monkeypatch, patched_speech): + async def _raise(*args, **kwargs): + raise proxy_server.ProxyException( + message="Unsupported audio encoding.", + type="invalid_request_error", + param=None, + code=400, + ) + + monkeypatch.setattr(proxy_server, "route_request", _raise) + yield + + +@pytest.fixture +def patched_speech_http_error(monkeypatch, patched_speech): + async def _raise(*args, **kwargs): + raise HTTPException(status_code=422, detail="Provider rejected the speech request.") + monkeypatch.setattr(proxy_server, "route_request", _raise) yield @@ -119,9 +143,7 @@ def patched_transcription(monkeypatch): return data monkeypatch.setattr(proxy_server, "add_litellm_data_to_request", _add_data) - monkeypatch.setattr( - proxy_server, "check_file_size_under_limit", lambda **kwargs: True - ) + monkeypatch.setattr(proxy_server, "check_file_size_under_limit", lambda **kwargs: True) async def _form_data(request): from starlette.datastructures import FormData, UploadFile @@ -201,6 +223,35 @@ def test_audio_speech_content_type_matches_audio_format( assert response.headers.get("content-type", "").split(";")[0] == expected_content_type +@pytest.mark.parametrize("path", ["/v1/audio/speech", "/audio/speech"]) +@pytest.mark.parametrize( + "response_format,expected_media_type", + [ + ("pcm16", "audio/wav"), + ("alaw", "audio/x-alaw-basic"), + ("mulaw", "audio/basic"), + ], +) +def test_audio_speech_uses_requested_media_type( + client, + auth_as, + patched_speech, + path, + response_format, + expected_media_type, +): + payload = { + "model": "gemini-3.1-flash-tts-preview", + "input": "Hi", + "voice": "Umbriel", + "response_format": response_format, + } + with auth_as(): + response = client.post(path, json=payload) + assert response.status_code == 200 + assert response.headers.get("content-type", "") == expected_media_type + + @pytest.mark.parametrize("path", ["/v1/audio/speech", "/audio/speech"]) def test_audio_speech_error(client, auth_as, patched_speech_error, path): """Pins ``POST /v1/audio/speech`` and ``POST /audio/speech`` (error).""" @@ -208,7 +259,13 @@ def test_audio_speech_error(client, auth_as, patched_speech_error, path): with auth_as(): response = client.post(path, json=payload) assert response.status_code == 500 - assert len(response.content) > 0 + assert response.json() == { + "error": { + "message": "Internal server error", + "type": "internal_server_error", + } + } + assert "speech boom" not in response.text def test_audio_speech_bad_request_maps_to_400(client, auth_as, patched_speech_provider_rejection): @@ -223,6 +280,29 @@ def test_audio_speech_bad_request_maps_to_400(client, auth_as, patched_speech_pr assert "wav" in error["message"] +@pytest.mark.parametrize("path", ["/v1/audio/speech", "/audio/speech"]) +def test_audio_speech_proxy_error_preserves_status_and_message( + client, + auth_as, + patched_speech_proxy_error, + path, +): + payload = {"model": "tts-1", "input": "Hi", "voice": "alloy"} + with auth_as(): + response = client.post(path, json=payload) + assert response.status_code == 400 + assert response.json()["error"]["message"] == "Unsupported audio encoding." + + +@pytest.mark.parametrize("path", ["/v1/audio/speech", "/audio/speech"]) +def test_audio_speech_http_error_preserves_detail(client, auth_as, patched_speech_http_error, path): + payload = {"model": "tts-1", "input": "Hi", "voice": "alloy"} + with auth_as(): + response = client.post(path, json=payload) + assert response.status_code == 422 + assert response.json()["error"]["message"] == "Provider rejected the speech request." + + @pytest.mark.parametrize("path", ["/v1/audio/transcriptions", "/audio/transcriptions"]) def test_audio_transcription_happy_path(client, auth_as, patched_transcription, path): """Pins ``POST /v1/audio/transcriptions`` / ``POST /audio/transcriptions`` (happy).""" diff --git a/tests/test_litellm/test_gemini_3_1_flash_tts_metadata.py b/tests/test_litellm/test_gemini_3_1_flash_tts_metadata.py new file mode 100644 index 00000000000..37d5a1dd9f2 --- /dev/null +++ b/tests/test_litellm/test_gemini_3_1_flash_tts_metadata.py @@ -0,0 +1,75 @@ +import json +from pathlib import Path + +import pytest + +import litellm +from litellm.utils import _invalidate_model_cost_lowercase_map, is_gemini_tts_model + +EXPECTED_MODELS = { + "gemini-3.1-flash-tts-preview": "gemini", + "gemini/gemini-3.1-flash-tts-preview": "gemini", + "vertex_ai/gemini-3.1-flash-tts-preview": "vertex_ai-language-models", +} + + +def _load_model_cost_map(path: Path) -> dict: + with open(path, encoding="utf-8") as model_cost_file: + return json.load(model_cost_file) + + +@pytest.mark.parametrize("model,provider", EXPECTED_MODELS.items()) +def test_gemini_3_1_flash_tts_model_metadata(model, provider): + model_cost = _load_model_cost_map(Path(__file__).parents[2] / "model_prices_and_context_window.json") + + info = model_cost.get(model) + assert info is not None, f"{model} not found in model cost map" + + assert info["litellm_provider"] == provider + assert info["mode"] == "audio_speech" + assert info["input_cost_per_token"] == 1e-06 + assert info["output_cost_per_token"] == 2e-05 + assert info["max_input_tokens"] == 8192 + assert info["max_output_tokens"] == 16384 + assert info["max_tokens"] == 16384 + assert info["supported_endpoints"] == ["/v1/audio/speech"] + + if model == "gemini/gemini-3.1-flash-tts-preview": + assert info["rpm"] == 10 + assert info["tpm"] == 4000000 + else: + assert info["output_cost_per_audio_token"] == 2e-05 + assert info["supported_modalities"] == ["text"] + assert info["supported_output_modalities"] == ["audio"] + assert info["supports_audio_input"] is False + assert info["supports_audio_output"] is True + assert info["supports_function_calling"] is False + assert info["supports_prompt_caching"] is False + assert info["health_check_voice"] == "Kore" + + +def test_gemini_3_1_flash_tts_backup_matches_main(): + repo_root = Path(__file__).parents[2] + main_cost = _load_model_cost_map(repo_root / "model_prices_and_context_window.json") + backup_cost = _load_model_cost_map(repo_root / "litellm" / "model_prices_and_context_window_backup.json") + + for model in EXPECTED_MODELS: + assert backup_cost.get(model) == main_cost.get(model), ( + f"{model} differs between main and backup model cost maps" + ) + + +def test_gemini_tts_detection_uses_model_metadata(monkeypatch): + model: str = "gemini/future-generative-speech" + monkeypatch.setitem( + litellm.model_cost, + model, + { + "litellm_provider": "gemini", + "mode": "audio_speech", + }, + ) + litellm.get_model_info.cache_clear() + _invalidate_model_cost_lowercase_map() + + assert is_gemini_tts_model(model) diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index ce280cc3513..2724b255e74 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -835,6 +835,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "ocr_cost_per_page_batches": {"type": "number"}, "ocr_cost_per_credit": {"type": "number"}, "code_interpreter_cost_per_session": {"type": "number"}, + "health_check_voice": {"type": "string"}, "inference_geo": {"type": "string"}, "litellm_provider": {"type": "string"}, "max_input_tokens": {"type": "number"}, diff --git a/tests/unit/llms/vertex_ai/text_to_speech/test_transformation.py b/tests/unit/llms/vertex_ai/text_to_speech/test_transformation.py index b5eec42b569..e85c81807bb 100644 --- a/tests/unit/llms/vertex_ai/text_to_speech/test_transformation.py +++ b/tests/unit/llms/vertex_ai/text_to_speech/test_transformation.py @@ -42,8 +42,12 @@ class TestVertexAITextToSpeechConfig: assert url == custom_url - @patch.object(VertexAITextToSpeechConfig, "_ensure_access_token") - @patch.object(VertexAITextToSpeechConfig, "_get_token_and_url") + @patch.object( # test-quality-ok: isolates provider credentials while testing request serialization + VertexAITextToSpeechConfig, "_ensure_access_token" + ) + @patch.object( # test-quality-ok: fixes the provider URL at the authentication boundary + VertexAITextToSpeechConfig, "_get_token_and_url" + ) def test_transform_text_to_speech_request_body(self, mock_get_token, mock_ensure_token): """Test that transform_text_to_speech_request generates correct request body""" # Mock authentication @@ -125,6 +129,222 @@ class TestVertexAITextToSpeechConfig: assert voice_str is None assert voice_dict == voice_input + def test_gemini_tts_multi_speaker_voice_mapping(self): + config = VertexAITextToSpeechConfig() + + voice = { + "multi_speaker_voice_config": { + "speaker_voice_configs": [ + { + "speaker": "Ryan", + "voice_config": { + "prebuilt_voice_config": { + "voice_name": "Umbriel", + }, + }, + }, + { + "speaker": "Katie", + "voice_config": { + "prebuilt_voice_config": { + "voice_name": "Leda", + }, + }, + }, + ], + }, + } + + expected_voice = { + "languageCode": "en-US", + "modelName": "gemini-3.1-flash-tts-preview", + "multiSpeakerVoiceConfig": { + "speakerVoiceConfigs": [ + { + "speakerAlias": "Ryan", + "speakerId": "Umbriel", + }, + { + "speakerAlias": "Katie", + "speakerId": "Leda", + }, + ], + }, + } + + voice_str, optional_params = config.map_openai_params( + model="gemini-3.1-flash-tts-preview", + optional_params={"response_format": "mp3"}, + voice=voice, + ) + assert voice_str is None + assert optional_params["audioEncoding"] == "MP3" + assert optional_params["vertex_voice_dict"] == expected_voice + + voice_str, optional_params = config.map_openai_params( + model="gemini-3.1-flash-tts-preview", + optional_params={"response_format": "pcm16"}, + voice=voice, + ) + assert voice_str is None + assert optional_params["audioEncoding"] == "LINEAR16" + assert optional_params["vertex_voice_dict"] == expected_voice + + @pytest.mark.parametrize( + "voice", + [ + {"name": "Umbriel", "modelName": "gemini-2.5-flash-tts"}, + {"name": "Umbriel", "model_name": "gemini-2.5-flash-tts"}, + {"modelName": "gemini-2.5-flash-tts", "model_name": "chirp-3"}, + { + "modelName": "gemini-2.5-flash-tts", + "model_name": "chirp-3", + "multi_speaker_voice_config": { + "speaker_voice_configs": [ + { + "speaker": "Ryan", + "voice_config": { + "prebuilt_voice_config": { + "voice_name": "Umbriel", + }, + }, + }, + ], + }, + }, + ], + ) + def test_gemini_tts_ignores_voice_model_name_override(self, voice): + config = VertexAITextToSpeechConfig() + routed_model = "gemini-3.1-flash-tts-preview" + + _, optional_params = config.map_openai_params( + model=routed_model, + optional_params={"response_format": "mp3"}, + voice=voice, + ) + assert optional_params["vertex_voice_dict"]["modelName"] == routed_model + + def test_dispatch_maps_gemini_cloud_tts_params_when_provider_config_skipped(self): + config = VertexAITextToSpeechConfig() + handler = MagicMock() + handler.text_to_speech_handler.return_value = "ok" + + result = config.dispatch_text_to_speech( + model="gemini-3.1-flash-tts-preview", + input="Hi", + voice={ + "name": "Umbriel", + "modelName": "gemini-2.5-flash-tts", + }, + optional_params={"response_format": "mp3"}, + litellm_params_dict={}, + logging_obj=MagicMock(), + timeout=10, + extra_headers=None, + base_llm_http_handler=handler, + aspeech=False, + api_base=None, + api_key=None, + ) + + assert result == "ok" + mapped_params = handler.text_to_speech_handler.call_args.kwargs["text_to_speech_optional_params"] + assert mapped_params["audioEncoding"] == "MP3" + assert mapped_params["vertex_voice_dict"]["name"] == "Umbriel" + assert mapped_params["vertex_voice_dict"]["modelName"] == "gemini-3.1-flash-tts-preview" + + def test_dispatch_keeps_pre_mapped_cloud_tts_params(self): + config = VertexAITextToSpeechConfig() + handler = MagicMock() + handler.text_to_speech_handler.return_value = "ok" + optional_params = { + "audioEncoding": "OGG_OPUS", + "vertex_voice_dict": {"languageCode": "en-US", "name": "en-US-Chirp3-HD-Charon"}, + } + + config.dispatch_text_to_speech( + model="chirp", + input="Hi", + voice="en-US-Chirp3-HD-Charon", + optional_params=optional_params, + litellm_params_dict={}, + logging_obj=MagicMock(), + timeout=10, + extra_headers=None, + base_llm_http_handler=handler, + aspeech=False, + api_base=None, + api_key=None, + ) + + call_kwargs = handler.text_to_speech_handler.call_args.kwargs + assert call_kwargs["voice"] == "en-US-Chirp3-HD-Charon" + assert call_kwargs["text_to_speech_optional_params"] is optional_params + assert call_kwargs["text_to_speech_optional_params"]["audioEncoding"] == "OGG_OPUS" + + @patch.object( # test-quality-ok: isolates provider credentials while testing Gemini request serialization + VertexAITextToSpeechConfig, "_ensure_access_token" + ) + @patch.object( # test-quality-ok: fixes the Gemini provider URL at the authentication boundary + VertexAITextToSpeechConfig, "_get_token_and_url" + ) + def test_gemini_tts_mp3_request_body(self, mock_get_token, mock_ensure_token): + mock_ensure_token.return_value = ("mock-token", "test-project") + mock_get_token.return_value = ("mock-token", "mock-url") + config = VertexAITextToSpeechConfig() + + result = config.transform_text_to_speech_request( + model="gemini-3.1-flash-tts-preview", + input="Ryan: Hi.\nKatie: Hello.", + voice=None, + optional_params={ + "audioEncoding": "MP3", + "vertex_voice_dict": { + "languageCode": "en-US", + "modelName": "gemini-3.1-flash-tts-preview", + "multiSpeakerVoiceConfig": { + "speakerVoiceConfigs": [ + { + "speakerAlias": "Ryan", + "speakerId": "Umbriel", + }, + { + "speakerAlias": "Katie", + "speakerId": "Leda", + }, + ], + }, + }, + }, + litellm_params={ + "vertex_credentials": None, + "vertex_project": "test-project", + "vertex_location": "global", + }, + headers={}, + ) + + request_body = result["dict_body"] + assert request_body["input"] == {"text": "Ryan: Hi.\nKatie: Hello."} + assert request_body["voice"] == { + "languageCode": "en-US", + "modelName": "gemini-3.1-flash-tts-preview", + "multiSpeakerVoiceConfig": { + "speakerVoiceConfigs": [ + { + "speakerAlias": "Ryan", + "speakerId": "Umbriel", + }, + { + "speakerAlias": "Katie", + "speakerId": "Leda", + }, + ], + }, + } + assert request_body["audioConfig"]["audioEncoding"] == "MP3" + @pytest.mark.parametrize( ("audio", "expected_content_type"), @@ -553,9 +773,15 @@ class TestVertexAILyriaTextToSpeechConfig: assert mock_post.call_args.kwargs["json"] == expected_body -@patch("litellm.llms.custom_httpx.llm_http_handler.HTTPHandler.post") -@patch.object(VertexAITextToSpeechConfig, "_ensure_access_token") -@patch.object(VertexAITextToSpeechConfig, "_get_token_and_url") +@patch( # test-quality-ok: exercises public speech dispatch up to the outbound HTTP boundary + "litellm.llms.custom_httpx.llm_http_handler.HTTPHandler.post" +) +@patch.object( # test-quality-ok: isolates provider credentials in the public API test + VertexAITextToSpeechConfig, "_ensure_access_token" +) +@patch.object( # test-quality-ok: fixes the provider URL for deterministic dispatch assertions + VertexAITextToSpeechConfig, "_get_token_and_url" +) def test_litellm_speech_vertex_ai_chirp(mock_get_token, mock_ensure_token, mock_post): """ Test that litellm.speech(model="vertex_ai/chirp") sends the correct URL and request body @@ -607,3 +833,74 @@ def test_litellm_speech_vertex_ai_chirp(mock_get_token, mock_ensure_token, mock_ assert "headers" in call_kwargs assert "Authorization" in call_kwargs["headers"] assert call_kwargs["headers"]["Authorization"] == "Bearer mock-token" + + +@patch( # test-quality-ok: exercises public Gemini speech dispatch up to the outbound HTTP boundary + "litellm.llms.custom_httpx.llm_http_handler.HTTPHandler.post" +) +@patch.object( # test-quality-ok: isolates provider credentials in the public Gemini API test + VertexAITextToSpeechConfig, "_ensure_access_token" +) +@patch.object( # test-quality-ok: fixes the Gemini provider URL for deterministic dispatch assertions + VertexAITextToSpeechConfig, "_get_token_and_url" +) +def test_litellm_speech_vertex_ai_gemini_tts_mp3_uses_cloud_tts(mock_get_token, mock_ensure_token, mock_post): + mock_ensure_token.return_value = ("mock-token", "test-project") + mock_get_token.return_value = ("mock-token", "mock-url") + mock_response = Mock(spec=httpx.Response) + mock_response.status_code = 200 + mock_response.headers = {"content-type": "application/json"} + mock_response.json.return_value = {"audioContent": "SGVsbG8gV29ybGQ="} + mock_post.return_value = mock_response + + litellm.speech( + model="vertex_ai/gemini-3.1-flash-tts-preview", + input="Ryan: Hi.\nKatie: Hello.", + voice={ + "multi_speaker_voice_config": { + "speaker_voice_configs": [ + { + "speaker": "Ryan", + "voice_config": { + "prebuilt_voice_config": { + "voice_name": "Umbriel", + }, + }, + }, + { + "speaker": "Katie", + "voice_config": { + "prebuilt_voice_config": { + "voice_name": "Leda", + }, + }, + }, + ], + }, + }, + response_format="mp3", + vertex_project="test-project", + vertex_location="global", + ) + + mock_post.assert_called_once() + call_kwargs = mock_post.call_args.kwargs + assert call_kwargs["url"] == "https://texttospeech.googleapis.com/v1/text:synthesize" + request_body = call_kwargs["json"] + assert request_body["audioConfig"]["audioEncoding"] == "MP3" + assert request_body["voice"] == { + "languageCode": "en-US", + "modelName": "gemini-3.1-flash-tts-preview", + "multiSpeakerVoiceConfig": { + "speakerVoiceConfigs": [ + { + "speakerAlias": "Ryan", + "speakerId": "Umbriel", + }, + { + "speakerAlias": "Katie", + "speakerId": "Leda", + }, + ], + }, + } From 34ffd955850d7bdbd414b7221f69034aaeb6412c Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Tue, 15 Sep 2026 12:35:26 -0500 Subject: [PATCH 02/20] fix(vertex-ai): keep TTS metadata compatible with the cost map guard --- ci_cd/generate_model_prices_schema.py | 5 +++-- litellm/model_prices_and_context_window_backup.json | 2 -- model_prices_and_context_window.json | 2 -- model_prices_and_context_window.schema.json | 3 --- tests/test_litellm/test_gemini_3_1_flash_tts_metadata.py | 1 - tests/test_litellm/test_utils.py | 1 - 6 files changed, 3 insertions(+), 11 deletions(-) diff --git a/ci_cd/generate_model_prices_schema.py b/ci_cd/generate_model_prices_schema.py index 76d66e002ed..9531cb80cfe 100644 --- a/ci_cd/generate_model_prices_schema.py +++ b/ci_cd/generate_model_prices_schema.py @@ -288,7 +288,6 @@ def string_key_schemas(modes: tuple) -> dict[str, JsonSchema]: "type": "string", "enum": ["lyria_predict", "lyria_interactions"], }, - "health_check_voice": STRING, } @@ -376,7 +375,9 @@ def render(schema: JsonSchema) -> str: def validation_errors(prices: dict, schema: JsonSchema) -> tuple: - validator = jsonschema.Draft202012Validator(schema, format_checker=jsonschema.Draft202012Validator.FORMAT_CHECKER) + validator = jsonschema.Draft202012Validator( + schema, format_checker=jsonschema.Draft202012Validator.FORMAT_CHECKER + ) return tuple( f"{'.'.join(str(part) for part in error.absolute_path)}: {error.message}" for error in validator.iter_errors(prices) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index c34f6997384..d36d20fb57b 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -48037,7 +48037,6 @@ "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2e-05, "source": "https://cloud.google.com/text-to-speech/pricing", - "health_check_voice": "Kore", "max_input_tokens": 8192, "max_output_tokens": 16384, "max_tokens": 16384, @@ -54859,7 +54858,6 @@ "supports_web_search": false }, "gemini-3.1-flash-tts-preview": { - "health_check_voice": "Kore", "input_cost_per_token": 1e-06, "litellm_provider": "gemini", "max_input_tokens": 8192, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index c34f6997384..d36d20fb57b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -48037,7 +48037,6 @@ "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2e-05, "source": "https://cloud.google.com/text-to-speech/pricing", - "health_check_voice": "Kore", "max_input_tokens": 8192, "max_output_tokens": 16384, "max_tokens": 16384, @@ -54859,7 +54858,6 @@ "supports_web_search": false }, "gemini-3.1-flash-tts-preview": { - "health_check_voice": "Kore", "input_cost_per_token": 1e-06, "litellm_provider": "gemini", "max_input_tokens": 8192, diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index 0cfd83f388f..737a9b7fa60 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -262,9 +262,6 @@ "minimum": 0 } }, - "health_check_voice": { - "type": "string" - }, "input_cost_per_audio_per_second": { "type": "number", "minimum": 0 diff --git a/tests/test_litellm/test_gemini_3_1_flash_tts_metadata.py b/tests/test_litellm/test_gemini_3_1_flash_tts_metadata.py index 37d5a1dd9f2..9a95bb750b4 100644 --- a/tests/test_litellm/test_gemini_3_1_flash_tts_metadata.py +++ b/tests/test_litellm/test_gemini_3_1_flash_tts_metadata.py @@ -45,7 +45,6 @@ def test_gemini_3_1_flash_tts_model_metadata(model, provider): assert info["supports_audio_output"] is True assert info["supports_function_calling"] is False assert info["supports_prompt_caching"] is False - assert info["health_check_voice"] == "Kore" def test_gemini_3_1_flash_tts_backup_matches_main(): diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 2724b255e74..ce280cc3513 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -835,7 +835,6 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "ocr_cost_per_page_batches": {"type": "number"}, "ocr_cost_per_credit": {"type": "number"}, "code_interpreter_cost_per_session": {"type": "number"}, - "health_check_voice": {"type": "string"}, "inference_geo": {"type": "string"}, "litellm_provider": {"type": "string"}, "max_input_tokens": {"type": "number"}, From f67af139f35eeb6e9f4dce7de8d2f85bd606299c Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Tue, 15 Sep 2026 13:00:42 -0500 Subject: [PATCH 03/20] fix(vertex-ai): preserve existing Gemini TTS models and voice mappings --- .../vertex_and_google_ai_studio_gemini.py | 24 +++++++++--- ...odel_prices_and_context_window_backup.json | 38 ++++++++++++++++++- model_prices_and_context_window.json | 38 ++++++++++++++++++- .../llms/gemini/test_gemini_tts.py | 25 ++++++++++++ .../test_gemini_3_1_flash_tts_metadata.py | 11 ++++++ 5 files changed, 128 insertions(+), 8 deletions(-) diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index ccb9ae0a632..bddd239b65b 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -193,12 +193,24 @@ def _normalize_gemini_speech_config_item(value: object) -> object: def normalize_gemini_speech_config( value: Mapping[str, object], ) -> dict[str, object]: # mutable-ok: provider request serialization requires a concrete dict - normalized_value: Final = _normalize_gemini_speech_config_item(value) - if isinstance(normalized_value, dict): - return cast( # cast-ok: the runtime dict check establishes the normalizer's concrete result shape - dict[str, object], normalized_value - ) - return {} # mutable-ok: provider request serialization requires a concrete empty dict + nested_config: Final = value.get("speechConfig", value.get("speech_config")) + config: Final = nested_config if isinstance(nested_config, Mapping) else value + normalized_value: Final = _normalize_gemini_speech_config_item(config) + if not isinstance(normalized_value, dict): + return {} # mutable-ok: provider request serialization requires a concrete empty dict + normalized: Final = cast( # cast-ok: runtime dict check establishes the normalizer's concrete result shape + dict[str, object], normalized_value + ) + voice_name: Final = next( + (normalized[key] for key in ("name", "voiceName", "voice") if isinstance(normalized.get(key), str)), + None, + ) + if voice_name is None or "voiceConfig" in normalized or "multiSpeakerVoiceConfig" in normalized: + return normalized + return { # mutable-ok: provider request serialization requires a concrete dict + **{key: item for key, item in normalized.items() if key not in ("name", "voiceName", "voice", "modelName")}, + "voiceConfig": {"prebuiltVoiceConfig": {"voiceName": voice_name}}, + } class VertexAIBaseConfig: diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index d36d20fb57b..16fd1706f71 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -66233,10 +66233,46 @@ "output_cost_per_token": 2e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, + "vertex_ai/gemini-2.5-flash-tts": { + "input_cost_per_token": 5e-07, + "litellm_provider": "vertex_ai-language-models", + "mode": "audio_speech", + "output_cost_per_audio_token": 9.999999999999999e-06, + "output_cost_per_token": 9.999999999999999e-06, + "source": "https://cloud.google.com/text-to-speech/pricing", + "supported_endpoints": [ + "/v1/audio/speech" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "audio" + ], + "supports_audio_output": true + }, + "vertex_ai/gemini-2.5-pro-tts": { + "input_cost_per_token": 1e-06, + "litellm_provider": "vertex_ai-language-models", + "mode": "audio_speech", + "output_cost_per_audio_token": 1.9999999999999998e-05, + "output_cost_per_token": 1.9999999999999998e-05, + "source": "https://cloud.google.com/text-to-speech/pricing", + "supported_endpoints": [ + "/v1/audio/speech" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "audio" + ], + "supports_audio_output": true + }, "vertex_ai/gemini-2.5-flash-preview-tts": { "input_cost_per_token": 5e-07, "input_cost_per_token_batches": 2.5e-07, - "litellm_provider": "vertex_ai", + "litellm_provider": "vertex_ai-language-models", "mode": "audio_speech", "output_cost_per_audio_token": 1e-05, "output_cost_per_token": 1e-05, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index d36d20fb57b..16fd1706f71 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -66233,10 +66233,46 @@ "output_cost_per_token": 2e-06, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing" }, + "vertex_ai/gemini-2.5-flash-tts": { + "input_cost_per_token": 5e-07, + "litellm_provider": "vertex_ai-language-models", + "mode": "audio_speech", + "output_cost_per_audio_token": 9.999999999999999e-06, + "output_cost_per_token": 9.999999999999999e-06, + "source": "https://cloud.google.com/text-to-speech/pricing", + "supported_endpoints": [ + "/v1/audio/speech" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "audio" + ], + "supports_audio_output": true + }, + "vertex_ai/gemini-2.5-pro-tts": { + "input_cost_per_token": 1e-06, + "litellm_provider": "vertex_ai-language-models", + "mode": "audio_speech", + "output_cost_per_audio_token": 1.9999999999999998e-05, + "output_cost_per_token": 1.9999999999999998e-05, + "source": "https://cloud.google.com/text-to-speech/pricing", + "supported_endpoints": [ + "/v1/audio/speech" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "audio" + ], + "supports_audio_output": true + }, "vertex_ai/gemini-2.5-flash-preview-tts": { "input_cost_per_token": 5e-07, "input_cost_per_token_batches": 2.5e-07, - "litellm_provider": "vertex_ai", + "litellm_provider": "vertex_ai-language-models", "mode": "audio_speech", "output_cost_per_audio_token": 1e-05, "output_cost_per_token": 1e-05, diff --git a/tests/test_litellm/llms/gemini/test_gemini_tts.py b/tests/test_litellm/llms/gemini/test_gemini_tts.py index b22372b2f8c..ae6f7a3abd8 100644 --- a/tests/test_litellm/llms/gemini/test_gemini_tts.py +++ b/tests/test_litellm/llms/gemini/test_gemini_tts.py @@ -527,3 +527,28 @@ class TestGeminiTTSSpeechConfigInRequestBody: if __name__ == "__main__": pytest.main([__file__]) + + +@pytest.mark.parametrize("voice_key", ["name", "voiceName", "voice_name", "voice"]) +@pytest.mark.parametrize("wrapper", [None, "speechConfig", "speech_config"]) +def test_speech_bridge_preserves_single_speaker_voice_mapping(voice_key: str, wrapper: str | None): + handler = SpeechToCompletionBridgeTransformationHandler() + config = {voice_key: "Umbriel", "language_code": "en-US"} + voice = {wrapper: config} if wrapper else config + result = handler.transform_request( + model=f"vertex_ai/{GEMINI_3_1_FLASH_TTS_MODEL}", + input="Hello", + voice=voice, + optional_params={}, + litellm_params={}, + headers={}, + litellm_logging_obj=MagicMock(), + custom_llm_provider="vertex_ai", + ) + assert result["audio"] == { + "speech_config": { + "voiceConfig": {"prebuiltVoiceConfig": {"voiceName": "Umbriel"}}, + "languageCode": "en-US", + }, + "format": "pcm16", + } diff --git a/tests/test_litellm/test_gemini_3_1_flash_tts_metadata.py b/tests/test_litellm/test_gemini_3_1_flash_tts_metadata.py index 9a95bb750b4..676b116790f 100644 --- a/tests/test_litellm/test_gemini_3_1_flash_tts_metadata.py +++ b/tests/test_litellm/test_gemini_3_1_flash_tts_metadata.py @@ -72,3 +72,14 @@ def test_gemini_tts_detection_uses_model_metadata(monkeypatch): _invalidate_model_cost_lowercase_map() assert is_gemini_tts_model(model) + + +@pytest.mark.parametrize("model", ["gemini-2.5-flash-preview-tts", "gemini-2.5-flash-tts", "gemini-2.5-pro-tts"]) +@pytest.mark.parametrize("prefixed", [False, True]) +def test_existing_vertex_tts_models_keep_gemini_dispatch(model: str, prefixed: bool, monkeypatch): + model_cost = _load_model_cost_map(Path(__file__).parents[2] / "model_prices_and_context_window.json") + monkeypatch.setattr(litellm, "model_cost", model_cost) + litellm.get_model_info.cache_clear() + _invalidate_model_cost_lowercase_map() + requested_model = f"vertex_ai/{model}" if prefixed else model + assert is_gemini_tts_model(requested_model, custom_llm_provider="vertex_ai") From 349b668fed4dd463298532bc2b02cb9276bb9454 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Tue, 15 Sep 2026 13:13:20 -0500 Subject: [PATCH 04/20] fix(vertex-ai): recognize provider-stripped TTS model names --- litellm/utils.py | 6 +++++- tests/test_litellm/llms/gemini/test_gemini_tts.py | 5 +++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index 9a763476cf2..6f06563438b 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -737,7 +737,11 @@ def is_gemini_tts_model(model: str, custom_llm_provider: str | None = None) -> b else model ) bundled_model_cost: Final = _get_bundled_model_cost_map() - bundled_model_info: Final = bundled_model_cost.get(provider_model) or bundled_model_cost.get(model) + bundled_model_info: Final = ( + bundled_model_cost.get(provider_model) + or bundled_model_cost.get(model) + or (bundled_model_cost.get(f"vertex_ai/{model}") if custom_llm_provider is None else None) + ) return ( bundled_model_info is not None and bundled_model_info.get("mode") == "audio_speech" diff --git a/tests/test_litellm/llms/gemini/test_gemini_tts.py b/tests/test_litellm/llms/gemini/test_gemini_tts.py index ae6f7a3abd8..0710590cb97 100644 --- a/tests/test_litellm/llms/gemini/test_gemini_tts.py +++ b/tests/test_litellm/llms/gemini/test_gemini_tts.py @@ -531,12 +531,13 @@ if __name__ == "__main__": @pytest.mark.parametrize("voice_key", ["name", "voiceName", "voice_name", "voice"]) @pytest.mark.parametrize("wrapper", [None, "speechConfig", "speech_config"]) -def test_speech_bridge_preserves_single_speaker_voice_mapping(voice_key: str, wrapper: str | None): +@pytest.mark.parametrize("model", ["gemini-2.5-flash-tts", "gemini-2.5-pro-tts", GEMINI_3_1_FLASH_TTS_MODEL]) +def test_speech_bridge_preserves_single_speaker_voice_mapping(voice_key: str, wrapper: str | None, model: str): handler = SpeechToCompletionBridgeTransformationHandler() config = {voice_key: "Umbriel", "language_code": "en-US"} voice = {wrapper: config} if wrapper else config result = handler.transform_request( - model=f"vertex_ai/{GEMINI_3_1_FLASH_TTS_MODEL}", + model=model, input="Hello", voice=voice, optional_params={}, From 5fb51d0507719741dd258ec781066c829f2763f6 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Tue, 22 Sep 2026 17:57:17 -0500 Subject: [PATCH 05/20] test(vertex-ai): match current speech HTTP error response --- tests/test_litellm/proxy/proxy_server/test_routes_audio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_audio.py b/tests/test_litellm/proxy/proxy_server/test_routes_audio.py index 9216cb09927..f4230fe33aa 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_audio.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_audio.py @@ -300,7 +300,7 @@ def test_audio_speech_http_error_preserves_detail(client, auth_as, patched_speec with auth_as(): response = client.post(path, json=payload) assert response.status_code == 422 - assert response.json()["error"]["message"] == "Provider rejected the speech request." + assert response.json()["detail"] == "Provider rejected the speech request." @pytest.mark.parametrize("path", ["/v1/audio/transcriptions", "/audio/transcriptions"]) From c37b63ebbdb1012abedb5df9918f5064a896ff6e Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Tue, 22 Sep 2026 19:16:11 -0500 Subject: [PATCH 06/20] fix(vertex-ai): preserve nested speech locale and voice --- .../text_to_speech/transformation.py | 12 +++++++++-- .../text_to_speech/test_transformation.py | 21 ++++++++++++++++--- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/litellm/llms/vertex_ai/text_to_speech/transformation.py b/litellm/llms/vertex_ai/text_to_speech/transformation.py index 48598411670..2b799145422 100644 --- a/litellm/llms/vertex_ai/text_to_speech/transformation.py +++ b/litellm/llms/vertex_ai/text_to_speech/transformation.py @@ -245,10 +245,13 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): self, voice: dict, # mutable-ok: provider voice payload arrives as a concrete dictionary ) -> str | None: - voice_name: Final = self._get_str_value(voice, "name", "voiceName", "voice_name") + voice_name: Final = self._get_str_value(voice, "name", "voiceName", "voice_name", "voice") if voice_name is not None: return voice_name speech_config: Final = self._get_dict_value(voice, "speechConfig", "speech_config") or voice + nested_voice_name: Final = self._get_str_value(speech_config, "name", "voiceName", "voice_name", "voice") + if nested_voice_name is not None: + return nested_voice_name voice_config: Final = self._get_dict_value(speech_config, "voiceConfig", "voice_config") if voice_config is None: return None @@ -273,7 +276,12 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): "name": voice, } - language_code: Final = self._get_str_value(voice, "languageCode", "language_code") or self.DEFAULT_LANGUAGE_CODE + speech_config: Final = self._get_dict_value(voice, "speechConfig", "speech_config") or voice + language_code: Final = ( + self._get_str_value(voice, "languageCode", "language_code") + or self._get_str_value(speech_config, "languageCode", "language_code") + or self.DEFAULT_LANGUAGE_CODE + ) model_name: Final = model speaker_configs: Final = self._extract_gemini_tts_speaker_configs(voice) if speaker_configs: diff --git a/tests/unit/llms/vertex_ai/text_to_speech/test_transformation.py b/tests/unit/llms/vertex_ai/text_to_speech/test_transformation.py index e85c81807bb..9c31a8a67d2 100644 --- a/tests/unit/llms/vertex_ai/text_to_speech/test_transformation.py +++ b/tests/unit/llms/vertex_ai/text_to_speech/test_transformation.py @@ -190,6 +190,23 @@ class TestVertexAITextToSpeechConfig: assert optional_params["audioEncoding"] == "LINEAR16" assert optional_params["vertex_voice_dict"] == expected_voice + def test_gemini_tts_nested_language_and_voice_alias_reach_cloud_tts(self): + config: Final = VertexAITextToSpeechConfig() + voice: Final = {"speech_config": {"language_code": "fr-FR"}, "voice": "Kore"} + + voice_name, optional_params = config.map_openai_params( + model="gemini-3.1-flash-tts-preview", + optional_params={"response_format": "mp3"}, + voice=voice, + ) + + assert voice_name is None + assert optional_params["vertex_voice_dict"] == { + "languageCode": "fr-FR", + "modelName": "gemini-3.1-flash-tts-preview", + "name": "Kore", + } + @pytest.mark.parametrize( "voice", [ @@ -481,9 +498,7 @@ class TestVertexAILyriaTextToSpeechConfig: ) def test_get_complete_url_encodes_injected_predict_path_segments(self, monkeypatch: pytest.MonkeyPatch) -> None: - injected: Final = ( - "victim-project/locations/us-central1/publishers/google/models/other-model:predict?ignored=" - ) + injected: Final = "victim-project/locations/us-central1/publishers/google/models/other-model:predict?ignored=" encoded: Final = ( "victim-project%2Flocations%2Fus-central1%2Fpublishers%2Fgoogle" "%2Fmodels%2Fother-model%3Apredict%3Fignored%3D" From a72002a84ee139db2bf82bd23a0d9ed6be7aaada Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Wed, 23 Sep 2026 04:40:54 -0500 Subject: [PATCH 07/20] fix(vertex-ai): use provider speech media type in proxy --- litellm/proxy/proxy_server.py | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index ab66191ae25..6f67845f537 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -12150,24 +12150,10 @@ async def audio_speech( upstream_content_type: Final = ( response.response.headers.get("content-type") if isinstance(response, HttpxBinaryResponseContent) else None ) - resolved_media_type: Final = resolve_speech_media_type( + media_type: Final = extra_media_type or resolve_speech_media_type( upstream_content_type=upstream_content_type, response_format=requested_format_str, ) - request_model: Final = data.get("model", "") - media_type: Final = ( - extra_media_type - if extra_media_type is not None - else ( - "audio/wav" - if ( - requested_format_str is None - and isinstance(request_model, str) - and litellm.utils.is_gemini_tts_model(request_model) - ) - else resolved_media_type - ) - ) return StreamingResponse( _audio_speech_chunk_generator(response), From f5c6223bf1f407c43692a08d549a1474f8b561df Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Wed, 23 Sep 2026 05:08:59 -0500 Subject: [PATCH 08/20] fix(gemini): preserve TTS dispatch and nested speech settings --- .../speech_to_completion_bridge/handler.py | 1 + .../transformation.py | 3 +- .../vertex_and_google_ai_studio_gemini.py | 21 ++++++----- ...odel_prices_and_context_window_backup.json | 6 ++++ litellm/utils.py | 15 +++++--- model_prices_and_context_window.json | 6 ++++ .../llms/gemini/test_gemini_tts.py | 10 ++++++ .../test_gemini_3_1_flash_tts_metadata.py | 13 ++++++- .../test_transformation.py | 36 +++++++++++++++++-- 9 files changed, 92 insertions(+), 19 deletions(-) diff --git a/litellm/endpoints/speech/speech_to_completion_bridge/handler.py b/litellm/endpoints/speech/speech_to_completion_bridge/handler.py index 6c33621ec89..6dc044fa4f0 100644 --- a/litellm/endpoints/speech/speech_to_completion_bridge/handler.py +++ b/litellm/endpoints/speech/speech_to_completion_bridge/handler.py @@ -119,6 +119,7 @@ class SpeechToCompletionBridgeHandler: if isinstance(result, ModelResponse): return self.transformation_handler.transform_response( model_response=result, + model=model, response_format=requested_response_format if isinstance(requested_response_format, str) else None, ) else: diff --git a/litellm/endpoints/speech/speech_to_completion_bridge/transformation.py b/litellm/endpoints/speech/speech_to_completion_bridge/transformation.py index 41a46832fea..ea26f2e7244 100644 --- a/litellm/endpoints/speech/speech_to_completion_bridge/transformation.py +++ b/litellm/endpoints/speech/speech_to_completion_bridge/transformation.py @@ -170,7 +170,7 @@ class SpeechToCompletionBridgeTransformationHandler: return self._convert_pcm16_to_wav(decoded_audio), "audio/wav" def transform_response( - self, model_response: "ModelResponse", response_format: str | None + self, model_response: "ModelResponse", model: str, response_format: str | None ) -> "HttpxBinaryResponseContent": import base64 @@ -184,7 +184,6 @@ class SpeechToCompletionBridgeTransformationHandler: raise ValueError("No audio part found in the response") decoded_audio: Final = base64.b64decode(audio_part.data) - model: Final = getattr(model_response, "model", "") content, content_type = ( self._gemini_tts_response_body(decoded_audio, response_format) if self._is_gemini_tts_model(model) diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index bddd239b65b..71072f3aa8b 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -194,7 +194,14 @@ def normalize_gemini_speech_config( value: Mapping[str, object], ) -> dict[str, object]: # mutable-ok: provider request serialization requires a concrete dict nested_config: Final = value.get("speechConfig", value.get("speech_config")) - config: Final = nested_config if isinstance(nested_config, Mapping) else value + config: Final = ( + { + **{key: item for key, item in value.items() if key not in ("speechConfig", "speech_config", "format")}, + **nested_config, + } + if isinstance(nested_config, Mapping) + else value + ) normalized_value: Final = _normalize_gemini_speech_config_item(config) if not isinstance(normalized_value, dict): return {} # mutable-ok: provider request serialization requires a concrete empty dict @@ -205,7 +212,9 @@ def normalize_gemini_speech_config( (normalized[key] for key in ("name", "voiceName", "voice") if isinstance(normalized.get(key), str)), None, ) - if voice_name is None or "voiceConfig" in normalized or "multiSpeakerVoiceConfig" in normalized: + if "voiceConfig" in normalized or "multiSpeakerVoiceConfig" in normalized: + return {key: item for key, item in normalized.items() if key not in ("name", "voiceName", "voice")} + if voice_name is None: return normalized return { # mutable-ok: provider request serialization requires a concrete dict **{key: item for key, item in normalized.items() if key not in ("name", "voiceName", "voice", "modelName")}, @@ -1131,13 +1140,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): ) language_code: Final = value.get("language_code", value.get("languageCode")) if isinstance(speech_config_value, Mapping): - speech_config: Final = normalize_gemini_speech_config(speech_config_value) - if language_code is not None and "languageCode" not in speech_config: - return { # mutable-ok: provider request serialization requires a concrete dict - **speech_config, - "languageCode": language_code, - } - return speech_config + return normalize_gemini_speech_config(value) speech_config_without_language: Final[ # mutable-ok: provider request serialization requires a concrete dict dict[str, object] # mutable-ok: provider request serialization requires a concrete dict diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 16fd1706f71..fd37b666fd1 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -26821,6 +26821,9 @@ "mode": "chat", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2e-05, + "provider_specific_entry": { + "gemini_tts": 1 + }, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_modalities": [ "text" @@ -28625,6 +28628,9 @@ "mode": "chat", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2e-05, + "provider_specific_entry": { + "gemini_tts": 1 + }, "rpm": 10000, "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_modalities": [ diff --git a/litellm/utils.py b/litellm/utils.py index 6f06563438b..1e86b87092b 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -742,19 +742,26 @@ def is_gemini_tts_model(model: str, custom_llm_provider: str | None = None) -> b or bundled_model_cost.get(model) or (bundled_model_cost.get(f"vertex_ai/{model}") if custom_llm_provider is None else None) ) + bundled_entry: Final = ( + bundled_model_info.get("provider_specific_entry") if bundled_model_info is not None else None + ) return ( bundled_model_info is not None - and bundled_model_info.get("mode") == "audio_speech" and bundled_model_info.get("litellm_provider") in ( "gemini", "vertex_ai-language-models", ) + and ( + bundled_model_info.get("mode") == "audio_speech" + or (isinstance(bundled_entry, dict) and bundled_entry.get("gemini_tts") == 1) + ) ) - return runtime_model_info.get("mode") == "audio_speech" and runtime_model_info.get("litellm_provider") in ( - "gemini", - "vertex_ai-language-models", + provider_entry: Final = runtime_model_info.get("provider_specific_entry") + return runtime_model_info.get("litellm_provider") in ("gemini", "vertex_ai-language-models") and ( + runtime_model_info.get("mode") == "audio_speech" + or (isinstance(provider_entry, dict) and provider_entry.get("gemini_tts") == 1) ) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 16fd1706f71..fd37b666fd1 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -26821,6 +26821,9 @@ "mode": "chat", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2e-05, + "provider_specific_entry": { + "gemini_tts": 1 + }, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", "supported_modalities": [ "text" @@ -28625,6 +28628,9 @@ "mode": "chat", "output_cost_per_audio_token": 2e-05, "output_cost_per_token": 2e-05, + "provider_specific_entry": { + "gemini_tts": 1 + }, "rpm": 10000, "source": "https://ai.google.dev/gemini-api/docs/pricing", "supported_modalities": [ diff --git a/tests/test_litellm/llms/gemini/test_gemini_tts.py b/tests/test_litellm/llms/gemini/test_gemini_tts.py index 0710590cb97..2a6c0367e99 100644 --- a/tests/test_litellm/llms/gemini/test_gemini_tts.py +++ b/tests/test_litellm/llms/gemini/test_gemini_tts.py @@ -148,6 +148,16 @@ class TestGeminiTTSTransformation: assert result["languageCode"] == "de-DE" assert result["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Kore" + def test_map_audio_params_keeps_voice_beside_nested_speech_config(self): + config = GoogleAIStudioGeminiConfig() + + result = config._map_audio_params( + {"speech_config": {"language_code": "fr-FR"}, "voice": "Kore", "format": "pcm16"} + ) + + assert result["languageCode"] == "fr-FR" + assert result["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Kore" + def test_map_audio_params_no_language_code(self): config = GoogleAIStudioGeminiConfig() diff --git a/tests/test_litellm/test_gemini_3_1_flash_tts_metadata.py b/tests/test_litellm/test_gemini_3_1_flash_tts_metadata.py index 676b116790f..1ebe727ad5d 100644 --- a/tests/test_litellm/test_gemini_3_1_flash_tts_metadata.py +++ b/tests/test_litellm/test_gemini_3_1_flash_tts_metadata.py @@ -74,7 +74,10 @@ def test_gemini_tts_detection_uses_model_metadata(monkeypatch): assert is_gemini_tts_model(model) -@pytest.mark.parametrize("model", ["gemini-2.5-flash-preview-tts", "gemini-2.5-flash-tts", "gemini-2.5-pro-tts"]) +@pytest.mark.parametrize( + "model", + ["gemini-2.5-flash-preview-tts", "gemini-2.5-flash-tts", "gemini-2.5-pro-tts", "gemini-2.5-pro-preview-tts"], +) @pytest.mark.parametrize("prefixed", [False, True]) def test_existing_vertex_tts_models_keep_gemini_dispatch(model: str, prefixed: bool, monkeypatch): model_cost = _load_model_cost_map(Path(__file__).parents[2] / "model_prices_and_context_window.json") @@ -83,3 +86,11 @@ def test_existing_vertex_tts_models_keep_gemini_dispatch(model: str, prefixed: b _invalidate_model_cost_lowercase_map() requested_model = f"vertex_ai/{model}" if prefixed else model assert is_gemini_tts_model(requested_model, custom_llm_provider="vertex_ai") + + +def test_existing_gemini_pro_preview_tts_keeps_speech_dispatch() -> None: + assert is_gemini_tts_model("gemini/gemini-2.5-pro-preview-tts", custom_llm_provider="gemini") + + +def test_gemini_music_model_does_not_use_speech_dispatch() -> None: + assert not is_gemini_tts_model("gemini/lyria-3-clip-preview", custom_llm_provider="gemini") diff --git a/tests/unit/endpoints/speech/speech_to_completion_bridge/test_transformation.py b/tests/unit/endpoints/speech/speech_to_completion_bridge/test_transformation.py index 953f028af3c..f33c8868266 100644 --- a/tests/unit/endpoints/speech/speech_to_completion_bridge/test_transformation.py +++ b/tests/unit/endpoints/speech/speech_to_completion_bridge/test_transformation.py @@ -16,9 +16,7 @@ PCM_BYTES: Final = b"\x01\x02\x03\x04" * 6 def _model_response(model: str, pcm: bytes) -> ModelResponse: - audio: Final = ChatCompletionAudioResponse( - data=base64.b64encode(pcm).decode(), expires_at=0, transcript="hello" - ) + audio: Final = ChatCompletionAudioResponse(data=base64.b64encode(pcm).decode(), expires_at=0, transcript="hello") return ModelResponse(model=model, choices=[Choices(message=Message(content=None, audio=audio))]) @@ -72,6 +70,24 @@ def test_non_gemini_request_forwards_speech_response_format_as_audio_format() -> assert request["audio"] == {"voice": "alloy", "format": "wav"} +def test_gemini_request_keeps_voice_beside_nested_speech_config() -> None: + request: Final = SpeechToCompletionBridgeTransformationHandler().transform_request( + model=GEMINI_TTS_MODEL, + input="Bonjour", + voice={"speech_config": {"language_code": "fr-FR"}, "voice": "Kore"}, + optional_params={}, + litellm_params={}, + headers={}, + litellm_logging_obj=MagicMock(), + custom_llm_provider="gemini", + ) + + assert request["audio"]["speech_config"] == { + "languageCode": "fr-FR", + "voiceConfig": {"prebuiltVoiceConfig": {"voiceName": "Kore"}}, + } + + @pytest.mark.parametrize("response_format", ["mp3", "flac", "opus", "aac"]) def test_gemini_tts_request_rejects_formats_gemini_cannot_produce(response_format: str) -> None: with pytest.raises(litellm.BadRequestError) as excinfo: @@ -86,6 +102,7 @@ def test_gemini_tts_request_rejects_formats_gemini_cannot_produce(response_forma def test_gemini_tts_pcm_response_returns_raw_pcm_bytes() -> None: response: Final = SpeechToCompletionBridgeTransformationHandler().transform_response( model_response=_model_response(GEMINI_TTS_MODEL, PCM_BYTES), + model=GEMINI_TTS_MODEL, response_format="pcm", ) @@ -97,6 +114,7 @@ def test_gemini_tts_pcm_response_returns_raw_pcm_bytes() -> None: def test_gemini_tts_wav_and_default_responses_wrap_pcm_in_wav(response_format: str | None) -> None: response: Final = SpeechToCompletionBridgeTransformationHandler().transform_response( model_response=_model_response(GEMINI_TTS_MODEL, PCM_BYTES), + model=GEMINI_TTS_MODEL, response_format=response_format, ) @@ -110,8 +128,20 @@ def test_gemini_tts_wav_and_default_responses_wrap_pcm_in_wav(response_format: s def test_non_gemini_response_keeps_original_bytes_and_mpeg_content_type() -> None: response: Final = SpeechToCompletionBridgeTransformationHandler().transform_response( model_response=_model_response("gpt-4o-audio-preview", PCM_BYTES), + model="gpt-4o-audio-preview", response_format="mp3", ) assert response.response.content == PCM_BYTES assert response.response.headers["content-type"] == "audio/mpeg" + + +def test_gemini_response_uses_requested_model_when_served_name_differs() -> None: + response: Final = SpeechToCompletionBridgeTransformationHandler().transform_response( + model_response=_model_response("served-speech-alias", PCM_BYTES), + model=GEMINI_TTS_MODEL, + response_format=None, + ) + + assert response.response.content[:4] == b"RIFF" + assert response.response.headers["content-type"] == "audio/wav" From 0883607cb347dc94dcdaff6363e9d6e4a644a1b5 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Wed, 23 Sep 2026 05:34:20 -0500 Subject: [PATCH 09/20] test(gemini): run TTS regressions in CI shards --- .../test_transformation.py | 2 ++ .../llms/gemini/test_gemini_tts.py | 4 ++-- .../text_to_speech/test_transformation.py | 2 ++ .../test_gemini_3_1_flash_tts_metadata.py | 21 +++---------------- .../speech_to_completion_bridge/__init__.py | 0 .../llms/vertex_ai/text_to_speech/__init__.py | 0 6 files changed, 9 insertions(+), 20 deletions(-) rename tests/{unit => test_litellm}/endpoints/speech/speech_to_completion_bridge/test_transformation.py (98%) rename tests/{unit => test_litellm}/llms/vertex_ai/text_to_speech/test_transformation.py (99%) delete mode 100644 tests/unit/endpoints/speech/speech_to_completion_bridge/__init__.py delete mode 100644 tests/unit/llms/vertex_ai/text_to_speech/__init__.py diff --git a/tests/unit/endpoints/speech/speech_to_completion_bridge/test_transformation.py b/tests/test_litellm/endpoints/speech/speech_to_completion_bridge/test_transformation.py similarity index 98% rename from tests/unit/endpoints/speech/speech_to_completion_bridge/test_transformation.py rename to tests/test_litellm/endpoints/speech/speech_to_completion_bridge/test_transformation.py index f33c8868266..6b01df20024 100644 --- a/tests/unit/endpoints/speech/speech_to_completion_bridge/test_transformation.py +++ b/tests/test_litellm/endpoints/speech/speech_to_completion_bridge/test_transformation.py @@ -11,6 +11,8 @@ from litellm.endpoints.speech.speech_to_completion_bridge.transformation import ) from litellm.types.utils import ChatCompletionAudioResponse, Choices, Message, ModelResponse +pytestmark = pytest.mark.usefixtures("local_model_cost_map") + GEMINI_TTS_MODEL: Final = "gemini-3.1-flash-tts-preview" PCM_BYTES: Final = b"\x01\x02\x03\x04" * 6 diff --git a/tests/test_litellm/llms/gemini/test_gemini_tts.py b/tests/test_litellm/llms/gemini/test_gemini_tts.py index 2a6c0367e99..b625d6dfe29 100644 --- a/tests/test_litellm/llms/gemini/test_gemini_tts.py +++ b/tests/test_litellm/llms/gemini/test_gemini_tts.py @@ -12,6 +12,8 @@ from litellm.endpoints.speech.speech_to_completion_bridge.transformation import from litellm.llms.gemini.chat.transformation import GoogleAIStudioGeminiConfig from litellm.utils import get_supported_openai_params +pytestmark = pytest.mark.usefixtures("local_model_cost_map") + GEMINI_3_1_FLASH_TTS_MODEL = "gemini-3.1-flash-tts-preview" MULTI_SPEAKER_SPEECH_CONFIG = { @@ -187,7 +189,6 @@ class TestGeminiTTSTransformation: assert result["speechConfig"] == NORMALIZED_MULTI_SPEAKER_SPEECH_CONFIG assert result["responseModalities"] == ["AUDIO"] - assert "temperature" not in result def test_gemini_tts_audio_parameter_with_existing_modalities(self): """Test audio parameter mapping when modalities already exist""" @@ -488,7 +489,6 @@ class TestGeminiTTSSpeechConfigInRequestBody: generation_config = request_body["generationConfig"] assert generation_config["speechConfig"] == NORMALIZED_MULTI_SPEAKER_SPEECH_CONFIG assert generation_config["responseModalities"] == ["AUDIO"] - assert "temperature" not in generation_config @pytest.mark.parametrize( "model,custom_llm_provider", diff --git a/tests/unit/llms/vertex_ai/text_to_speech/test_transformation.py b/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py similarity index 99% rename from tests/unit/llms/vertex_ai/text_to_speech/test_transformation.py rename to tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py index 9c31a8a67d2..50a73e39c12 100644 --- a/tests/unit/llms/vertex_ai/text_to_speech/test_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py @@ -13,6 +13,8 @@ from litellm.llms.vertex_ai.text_to_speech.transformation import ( from litellm.types.utils import LlmProviders from litellm.utils import ProviderConfigManager +pytestmark = pytest.mark.usefixtures("local_model_cost_map") + class TestVertexAITextToSpeechConfig: """Tests for VertexAITextToSpeechConfig transformation""" diff --git a/tests/test_litellm/test_gemini_3_1_flash_tts_metadata.py b/tests/test_litellm/test_gemini_3_1_flash_tts_metadata.py index 1ebe727ad5d..6420b5b9dc3 100644 --- a/tests/test_litellm/test_gemini_3_1_flash_tts_metadata.py +++ b/tests/test_litellm/test_gemini_3_1_flash_tts_metadata.py @@ -6,6 +6,8 @@ import pytest import litellm from litellm.utils import _invalidate_model_cost_lowercase_map, is_gemini_tts_model +pytestmark = pytest.mark.usefixtures("local_model_cost_map") + EXPECTED_MODELS = { "gemini-3.1-flash-tts-preview": "gemini", "gemini/gemini-3.1-flash-tts-preview": "gemini", @@ -27,24 +29,7 @@ def test_gemini_3_1_flash_tts_model_metadata(model, provider): assert info["litellm_provider"] == provider assert info["mode"] == "audio_speech" - assert info["input_cost_per_token"] == 1e-06 - assert info["output_cost_per_token"] == 2e-05 - assert info["max_input_tokens"] == 8192 - assert info["max_output_tokens"] == 16384 - assert info["max_tokens"] == 16384 - assert info["supported_endpoints"] == ["/v1/audio/speech"] - - if model == "gemini/gemini-3.1-flash-tts-preview": - assert info["rpm"] == 10 - assert info["tpm"] == 4000000 - else: - assert info["output_cost_per_audio_token"] == 2e-05 - assert info["supported_modalities"] == ["text"] - assert info["supported_output_modalities"] == ["audio"] - assert info["supports_audio_input"] is False - assert info["supports_audio_output"] is True - assert info["supports_function_calling"] is False - assert info["supports_prompt_caching"] is False + assert "/v1/audio/speech" in info["supported_endpoints"] def test_gemini_3_1_flash_tts_backup_matches_main(): diff --git a/tests/unit/endpoints/speech/speech_to_completion_bridge/__init__.py b/tests/unit/endpoints/speech/speech_to_completion_bridge/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/unit/llms/vertex_ai/text_to_speech/__init__.py b/tests/unit/llms/vertex_ai/text_to_speech/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 From db15c8ec4c40f8211aa3202a0f21732f12b8bb0f Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Wed, 23 Sep 2026 06:31:31 -0500 Subject: [PATCH 10/20] fix(vertex-ai): account for Gemini Cloud TTS audio cost --- litellm/cost_calculator.py | 12 ++- .../text_to_speech/transformation.py | 32 +++++++- litellm/types/llms/openai.py | 7 +- pyproject.toml | 2 +- .../text_to_speech/test_transformation.py | 76 ++++++++++++++++++- uv.lock | 4 +- 6 files changed, 122 insertions(+), 11 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 7990832dc48..2e90c073780 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -972,7 +972,11 @@ def get_usage_object( ( completion_response.get("usage") if isinstance(completion_response, dict) - else getattr(completion_response, "get", lambda x: None)("usage") + else ( + completion_response.usage + if isinstance(completion_response, HttpxBinaryResponseContent) + else getattr(completion_response, "get", lambda x: None)("usage") + ) ), ) @@ -1415,8 +1419,10 @@ def completion_cost( verbose_logger.debug("selected model name for cost calculation: %s", model) if completion_response is not None and ( - isinstance(completion_response, BaseModel) or isinstance(completion_response, dict) - ): # tts returns a custom class + isinstance(completion_response, (BaseModel, dict)) + or isinstance(completion_response, HttpxBinaryResponseContent) + and completion_response.usage is not None + ): if isinstance(completion_response, dict): usage_obj: dict | Usage | None = completion_response.get("usage", {}) else: diff --git a/litellm/llms/vertex_ai/text_to_speech/transformation.py b/litellm/llms/vertex_ai/text_to_speech/transformation.py index 2b799145422..2fcb68b0f44 100644 --- a/litellm/llms/vertex_ai/text_to_speech/transformation.py +++ b/litellm/llms/vertex_ai/text_to_speech/transformation.py @@ -6,6 +6,7 @@ Reference: https://cloud.google.com/text-to-speech/docs/reference/rest/v1/text/s """ import base64 +import math from collections.abc import Coroutine, Mapping from types import MappingProxyType from typing import TYPE_CHECKING, Any, ClassVar, Final, TypeAlias, Union @@ -16,6 +17,7 @@ import litellm from litellm.exceptions import UnsupportedParamsError from litellm.litellm_core_utils.audio_utils.utils import ( DEFAULT_SPEECH_MEDIA_TYPE, + calculate_request_duration, speech_media_type_from_audio_bytes, ) from litellm.litellm_core_utils.url_utils import encode_url_path_segment @@ -665,8 +667,34 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): content=binary_data, ) - # Initialize the HttpxBinaryResponseContent instance - return HttpxBinaryResponseContent(response) + binary_response: Final = HttpxBinaryResponseContent(response) + if self._is_gemini_tts_model(model): + from litellm.types.utils import CompletionTokensDetailsWrapper, Usage + + request_body: Final = logging_obj.model_call_details["additional_args"]["complete_input_dict"]["dict_body"] + input_data: Final = request_body["input"] + audio_config: Final = request_body["audioConfig"] + container_duration: Final = calculate_request_duration(binary_data) + sample_rate: Final = audio_config.get("sampleRateHertz") or 24000 + duration: Final = ( + len(binary_data) / (2 * sample_rate) + if container_duration is None and audio_config["audioEncoding"] == "PCM" + else container_duration + ) + if duration is None: + raise ValueError("Cannot determine Gemini TTS output duration for cost calculation") + input_text: Final = " ".join( + value for value in (input_data.get("text"), input_data.get("ssml"), input_data.get("prompt")) if value + ) + prompt_tokens: Final = litellm.token_counter(model=model, text=input_text) + audio_tokens: Final = math.ceil(duration * 25) + binary_response.usage = Usage( + prompt_tokens=prompt_tokens, + completion_tokens=audio_tokens, + total_tokens=prompt_tokens + audio_tokens, + completion_tokens_details=CompletionTokensDetailsWrapper(audio_tokens=audio_tokens), + ) + return binary_response class VertexAILyriaTextToSpeechConfig(VertexAITextToSpeechConfig): diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 599b1a76249..b7a086d151d 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1,7 +1,7 @@ from collections.abc import Iterable, Mapping from enum import Enum from os import PathLike -from typing import IO, Any, Final, Literal, Optional, TypeAlias, Union +from typing import IO, TYPE_CHECKING, Any, Final, Literal, Optional, TypeAlias, Union import httpx from openai import Omit @@ -93,6 +93,9 @@ from litellm.types.responses.main import ( from .base import CachedTokensDetails +if TYPE_CHECKING: + from litellm.types.utils import Usage + FileContent = IO[bytes] | bytes | PathLike FileTypes = ( @@ -120,10 +123,12 @@ class BinaryResponseSummary(TypedDict): class HttpxBinaryResponseContent(_HttpxBinaryResponseContent): _hidden_params: dict + usage: "Usage | None" def __init__(self, response: httpx.Response) -> None: super().__init__(response) self._hidden_params = {} # mutable-ok: mutable-dict contract shared with ModelResponse logging consumers + self.usage = None def logging_summary(self) -> BinaryResponseSummary: return { diff --git a/pyproject.toml b/pyproject.toml index ba72378989a..55e68a59524 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ dependencies = [ "pydantic-settings>=2.14.1,<3.0", "jsonschema>=4.0.0,<5.0", "boto3>=1.43.1,<2.0", + "soundfile>=0.12.1,<1.0", ] [project.urls] @@ -82,7 +83,6 @@ proxy = [ "InquirerPy>=0.3.4,<1.0", "tomlkit>=0.13.3,<1.0", "polars>=1.38.1,<2.0", - "soundfile>=0.12.1,<1.0", "pyroscope-io>=0.8.16,<1.0; sys_platform != 'win32'", "expression>=5.6.0,<6.0", ] diff --git a/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py b/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py index 50a73e39c12..62916841ea5 100644 --- a/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py @@ -1,4 +1,8 @@ import base64 +import io +import math +import wave +from pathlib import Path from typing import Final from unittest.mock import MagicMock, Mock, patch @@ -10,6 +14,8 @@ from litellm.llms.vertex_ai.text_to_speech.transformation import ( VertexAILyriaTextToSpeechConfig, VertexAITextToSpeechConfig, ) +from litellm.cost_calculator import response_cost_calculator +from litellm.litellm_core_utils.audio_utils.utils import calculate_request_duration from litellm.types.utils import LlmProviders from litellm.utils import ProviderConfigManager @@ -407,6 +413,66 @@ def test_transform_text_to_speech_response_leaves_unknown_bytes_unlabeled(): assert result.response.content == raw_pcm +@pytest.mark.parametrize("encoding", ["LINEAR16", "PCM", "MP3"]) +def test_gemini_cloud_tts_response_bills_text_and_audio(encoding: str): + model: Final = "gemini-3.1-flash-tts-preview" + input_text: Final = "Hello from Gemini text to speech" + pcm_audio: Final = b"\x00\x00" * 24000 + wav_buffer: Final = io.BytesIO() + with wave.open(wav_buffer, "wb") as wav_file: + wav_file.setnchannels(1) + wav_file.setsampwidth(2) + wav_file.setframerate(24000) + wav_file.writeframes(pcm_audio) + audio_by_encoding: Final = { + "PCM": pcm_audio, + "LINEAR16": wav_buffer.getvalue(), + "MP3": (Path(__file__).resolve().parents[4] / "audio_tests/speech_vertex.mp3").read_bytes(), + } + audio_bytes: Final = audio_by_encoding[encoding] + logger: Final = MagicMock() + logger.model_call_details = { + "additional_args": { + "complete_input_dict": { + "dict_body": { + "input": {"text": input_text}, + "audioConfig": {"audioEncoding": encoding, "sampleRateHertz": 24000}, + } + } + } + } + raw_response: Final = httpx.Response(200, json={"audioContent": base64.b64encode(audio_bytes).decode()}) + + result: Final = VertexAITextToSpeechConfig().transform_text_to_speech_response(model, raw_response, logger) + usage: Final = result.usage + assert usage is not None + duration: Final = len(audio_bytes) / 48000 if encoding == "PCM" else calculate_request_duration(audio_bytes) + assert duration is not None + assert usage.completion_tokens == math.ceil(duration * 25), ( + "Google Cloud TTS pricing, 2026-09-23: https://cloud.google.com/text-to-speech/pricing" + ) + assert usage.prompt_tokens == litellm.token_counter(model=model, text=input_text) + assert usage.total_tokens == usage.prompt_tokens + usage.completion_tokens + assert usage.completion_tokens_details is not None + assert usage.completion_tokens_details.audio_tokens == usage.completion_tokens + + model_info: Final = litellm.get_model_info(model, custom_llm_provider="vertex_ai") + cost: Final = response_cost_calculator( + response_object=result, + model=model, + custom_llm_provider="vertex_ai", + call_type="speech", + optional_params={}, + prompt=input_text, + ) + expected_cost: Final = ( + usage.prompt_tokens * model_info["input_cost_per_token"] + + usage.completion_tokens * model_info["output_cost_per_audio_token"] + ) + assert cost == pytest.approx(expected_cost) + assert cost > usage.prompt_tokens * model_info["input_cost_per_token"] + + class TestVertexAILyriaTextToSpeechConfig: @pytest.mark.parametrize( "model", @@ -867,10 +933,11 @@ def test_litellm_speech_vertex_ai_gemini_tts_mp3_uses_cloud_tts(mock_get_token, mock_response = Mock(spec=httpx.Response) mock_response.status_code = 200 mock_response.headers = {"content-type": "application/json"} - mock_response.json.return_value = {"audioContent": "SGVsbG8gV29ybGQ="} + audio_bytes: Final = (Path(__file__).resolve().parents[4] / "audio_tests/speech_vertex.mp3").read_bytes() + mock_response.json.return_value = {"audioContent": base64.b64encode(audio_bytes).decode()} mock_post.return_value = mock_response - litellm.speech( + result: Final = litellm.speech( model="vertex_ai/gemini-3.1-flash-tts-preview", input="Ryan: Hi.\nKatie: Hello.", voice={ @@ -900,6 +967,11 @@ def test_litellm_speech_vertex_ai_gemini_tts_mp3_uses_cloud_tts(mock_get_token, vertex_location="global", ) + assert result.response.content == audio_bytes + assert result.usage is not None + assert result.usage.prompt_tokens > 0 + assert result.usage.completion_tokens > 0 + mock_post.assert_called_once() call_kwargs = mock_post.call_args.kwargs assert call_kwargs["url"] == "https://texttospeech.googleapis.com/v1/text:synthesize" diff --git a/uv.lock b/uv.lock index 85e2b6d4e52..590812613de 100644 --- a/uv.lock +++ b/uv.lock @@ -4518,6 +4518,7 @@ dependencies = [ { name = "pydantic-settings" }, { name = "python-dotenv" }, { name = "pyyaml" }, + { name = "soundfile" }, { name = "tiktoken" }, { name = "tokenizers" }, ] @@ -4592,7 +4593,6 @@ proxy = [ { name = "restrictedpython" }, { name = "rich" }, { name = "rq" }, - { name = "soundfile" }, { name = "starlette" }, { name = "tomlkit" }, { name = "uvicorn" }, @@ -4841,7 +4841,7 @@ requires-dist = [ { name = "rq", marker = "extra == 'proxy'", specifier = ">=2.7.0,<3.0" }, { name = "semantic-router", marker = "python_full_version < '3.14' and extra == 'semantic-router'", specifier = ">=0.1.15,<1.0" }, { name = "sentry-sdk", marker = "extra == 'proxy-runtime'", specifier = ">=2.21.0,<3.0" }, - { name = "soundfile", marker = "extra == 'proxy'", specifier = ">=0.12.1,<1.0" }, + { name = "soundfile", specifier = ">=0.12.1,<1.0" }, { name = "soundfile", marker = "extra == 'stt-nvidia-riva'", specifier = ">=0.12.1" }, { name = "starlette", marker = "extra == 'proxy'", specifier = ">=1.0.1,<2.0" }, { name = "tiktoken", marker = "python_full_version < '3.14'", specifier = ">=0.8.0,<1.0" }, From d77bed6d78185a808370b04a860b162f0921d7fa Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Wed, 23 Sep 2026 06:50:10 -0500 Subject: [PATCH 11/20] fix(vertex-ai): bill Gemini TTS without optional audio decoder --- .../text_to_speech/transformation.py | 57 +++++++++++---- pyproject.toml | 2 +- tests/audio_tests/gemini_tts_speech.mp3 | Bin 0 -> 4992 bytes tests/audio_tests/gemini_tts_speech.ogg | Bin 0 -> 4179 bytes .../text_to_speech/test_transformation.py | 68 +++++++++++++++++- uv.lock | 4 +- 6 files changed, 113 insertions(+), 18 deletions(-) create mode 100644 tests/audio_tests/gemini_tts_speech.mp3 create mode 100644 tests/audio_tests/gemini_tts_speech.ogg diff --git a/litellm/llms/vertex_ai/text_to_speech/transformation.py b/litellm/llms/vertex_ai/text_to_speech/transformation.py index 2fcb68b0f44..15c296950d6 100644 --- a/litellm/llms/vertex_ai/text_to_speech/transformation.py +++ b/litellm/llms/vertex_ai/text_to_speech/transformation.py @@ -34,6 +34,7 @@ from litellm.types.llms.vertex_ai import VERTEX_CREDENTIALS_TYPES from litellm.types.llms.vertex_ai_text_to_speech import ( VertexTextToSpeechAudioConfig, VertexTextToSpeechInput, + VertexTextToSpeechSpeakerVoiceConfig, VertexTextToSpeechVoice, ) @@ -50,6 +51,36 @@ _LyriaVoice: TypeAlias = ( ) # mutable-ok: inherited interface supports structured provider voice dictionaries +def _fallback_gemini_tts_audio_duration(audio: bytes, encoding: str, sample_rate: int) -> float | None: + if encoding == "PCM": + return len(audio) / (2 * sample_rate) if sample_rate > 0 else None + if encoding in ("ALAW", "MULAW") and not (audio[:4] == b"RIFF" and audio[8:12] == b"WAVE"): + return len(audio) / sample_rate if sample_rate > 0 else None + + if encoding in ("LINEAR16", "ALAW", "MULAW") and audio[:4] == b"RIFF" and audio[8:12] == b"WAVE": + format_offset: Final = audio.find(b"fmt ", 12) + data_offset: Final = audio.find(b"data", 12) + if format_offset >= 0 and data_offset >= 0 and format_offset + 20 <= len(audio): + byte_rate: Final = int.from_bytes(audio[format_offset + 16 : format_offset + 20], "little") + data_size: Final = int.from_bytes(audio[data_offset + 4 : data_offset + 8], "little") + if byte_rate > 0 and data_size <= len(audio) - data_offset - 8: + return data_size / byte_rate + + if encoding == "MP3" and speech_media_type_from_audio_bytes(audio) == "audio/mpeg": + return len(audio) / 4000 + + if encoding == "OGG_OPUS" and audio[:4] == b"OggS": + opus_header: Final = audio.find(b"OpusHead") + last_page: Final = audio.rfind(b"OggS") + if opus_header >= 0 and opus_header + 12 <= len(audio) and last_page + 14 <= len(audio): + pre_skip: Final = int.from_bytes(audio[opus_header + 10 : opus_header + 12], "little") + granule: Final = int.from_bytes(audio[last_page + 6 : last_page + 14], "little") + if pre_skip <= granule < 2**64 - 1: + return (granule - pre_skip) / 48000 + + return None + + class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): """ Configuration for Google Cloud/Vertex AI Text-to-Speech @@ -166,7 +197,7 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): @staticmethod def _get_str_value( - source: dict, # mutable-ok: provider voice payloads arrive as concrete dictionaries + source: Mapping[str, object], *keys: str, ) -> str | None: for key in keys: @@ -177,9 +208,9 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): @staticmethod def _get_dict_value( - source: dict, # mutable-ok: provider voice payloads arrive as concrete dictionaries + source: Mapping[str, object], *keys: str, - ) -> dict | None: # mutable-ok: nested provider payloads remain concrete dictionaries + ) -> dict[str, object] | None: # mutable-ok: nested provider payloads remain concrete dictionaries for key in keys: value = source.get(key) if isinstance(value, dict): @@ -188,8 +219,10 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): def _extract_gemini_tts_speaker_configs( self, - voice: dict, # mutable-ok: provider voice payload arrives as a concrete dictionary - ) -> list[dict[str, str]]: # mutable-ok: provider request serialization requires a concrete list + voice: Mapping[str, object], + ) -> list[ + VertexTextToSpeechSpeakerVoiceConfig + ]: # mutable-ok: provider request serialization requires a concrete list speech_config: Final = self._get_dict_value(voice, "speechConfig", "speech_config") or voice multi_speaker_config: Final = self._get_dict_value( speech_config, @@ -208,7 +241,7 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): if not isinstance(raw_speaker_configs, list): return [] # mutable-ok: malformed speaker configuration produces a concrete empty list speaker_configs: Final[ # mutable-ok: validated payloads are accumulated for serialization - list[dict[str, str]] + list[VertexTextToSpeechSpeakerVoiceConfig] ] = [] for raw_config in raw_speaker_configs: if not isinstance(raw_config, dict): @@ -245,7 +278,7 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): def _extract_gemini_tts_voice_name( self, - voice: dict, # mutable-ok: provider voice payload arrives as a concrete dictionary + voice: Mapping[str, object], ) -> str | None: voice_name: Final = self._get_str_value(voice, "name", "voiceName", "voice_name", "voice") if voice_name is not None: @@ -269,8 +302,8 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): def _map_gemini_tts_voice_to_vertex_format( self, model: str, - voice: str | dict, # mutable-ok: provider voice payload arrives as a concrete dictionary - ) -> tuple[str | None, dict]: # mutable-ok: provider request serialization requires a concrete dict + voice: str | Mapping[str, object], + ) -> tuple[str | None, dict[str, object]]: # mutable-ok: provider request serialization requires a concrete dict if isinstance(voice, str): return voice, { # mutable-ok: provider request serialization requires a concrete dict "languageCode": self.DEFAULT_LANGUAGE_CODE, @@ -677,9 +710,9 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): container_duration: Final = calculate_request_duration(binary_data) sample_rate: Final = audio_config.get("sampleRateHertz") or 24000 duration: Final = ( - len(binary_data) / (2 * sample_rate) - if container_duration is None and audio_config["audioEncoding"] == "PCM" - else container_duration + container_duration + if container_duration is not None + else _fallback_gemini_tts_audio_duration(binary_data, audio_config["audioEncoding"], sample_rate) ) if duration is None: raise ValueError("Cannot determine Gemini TTS output duration for cost calculation") diff --git a/pyproject.toml b/pyproject.toml index 55e68a59524..ba72378989a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,6 @@ dependencies = [ "pydantic-settings>=2.14.1,<3.0", "jsonschema>=4.0.0,<5.0", "boto3>=1.43.1,<2.0", - "soundfile>=0.12.1,<1.0", ] [project.urls] @@ -83,6 +82,7 @@ proxy = [ "InquirerPy>=0.3.4,<1.0", "tomlkit>=0.13.3,<1.0", "polars>=1.38.1,<2.0", + "soundfile>=0.12.1,<1.0", "pyroscope-io>=0.8.16,<1.0; sys_platform != 'win32'", "expression>=5.6.0,<6.0", ] diff --git a/tests/audio_tests/gemini_tts_speech.mp3 b/tests/audio_tests/gemini_tts_speech.mp3 new file mode 100644 index 0000000000000000000000000000000000000000..71a916ad6c0803a578e56162e8d97996d999b228 GIT binary patch literal 4992 zcmeI#cTiK?x&ZLK6TlE!0s$fdhN6b1p$P~G0U}K$Vi2&=5wL|S7ETbUL8T+oq>B+1 z1Tlh05fB0CHUt4hsRtB6kRw?y@ywhvZ|?i+zI*<=-^`xbv-a%Y`sQ2Tti2cbX3-2` zrozHua}2zV0b;Xi8?;SfoPMQ2Ph|GD@s|Mn7)DulJAx&{{9yzZ|7CepQAy#aHW+_e zIEq)g2LM=+6N^qnYCru6bsC-fX6AYpIbyvbcfn)bYkx1m^~EIp><`27L{yRG9phay1Y`NEPBEGIaQ2#u1rsCn%xv5{`QpN46tG03i! z`c2h$W-N?$(|}q3xRjettV&5bE$igU@_}7}vBbEMf*7jd%<-!!}WthPrLz?uWIbs2F<<>GvG* z-PcHujIWM>19PMhjn~KR&fPd~qjO!o@Mueu?sxD500yZb>W~)=*$l6eM_Cz0L|1o+ z=ZoTc5IXQBJd9kHg|xrfSm%+T2G}IgN?i+1d=}1Fp#{LHiHXKr8uiMV%_Wi)H*W@Eiwv&@`CVp)N*ii}G2!T!* z*4&lAVBWG!^+pN%^Q+Kd*uJyzhPRgB_DI~X{3UOrE#L?uoR-qqLp@te8Nm~`&YNd? ztWk!1vYIK>mhW$TvMOvWf+bbbQq~5>TI~&!dTQ=J@*Y`Byh^Y}b+&&LN|V;SH4Yd% zX1A^ys7ggXZ-V_c%V1S_J{o7wBEd2GP@Hme&tVAry^PFdBOD7aqjf?Uq zaFaa)zLXCI2@1L=k;?W15EUP(WL8)#D4OBO?NLYF!+wR{;&Fn?z;F7^m_^-aL~h_^ zAwQlkgl!wNf+H{x+deuaXUqowF!xN34ZaNhG5##6r?jJc#D}f5r^|qmAC9m!@$Zj5vQi;F~_NR%N0Ey51uX@k|4okw2A|^242wDGOyH_{mLKL$g+Z?@leQ(Tf$Cy_^<=5 zC9e5c+?wvso^YJlpq}G*1%fx{o6FJDjte@n1cFeSlOTzN5P! z6iU&P&>UcV{d#in!L&MYBU95R5U6&)ddA>p>U@BhLP_&yhC{A-?NsEsixHy^CrEAn zu2PCQaHQ^p+IGFq$&Q!@t$LE_Pg00Yje0abp|?kPz9yz2`2u_x5BVuCP$Co^IQPyh za!Ifvu*sR++m86O!vgnWV2?Q--rq7-@v2sd9~xGfiFLh7G(ria%#|8Zjgva!U%s`J zF!Agir=#p*cE%ja=qB*BRt@revTT!L6@1+Zx{|ftZe%fD%%FeUF}==rs~<8dCJf4J zRKIN$KJGoj02B6wFOrQk00Ui)a4OHx{XL=Mq5VMr+0`fIgiYqo2P@JoFrr4uqI_@v zSZq5uS9^B`=J^U(!R=M>TOx!{DjGPQRe5F#d>?XtlaE!#2d_X9{)~z$^+N!x7G20} zRqa6QyK*#2?a0toq(;!)l3v}fN?Jyu%|AW{$ZtHTrnEu5FvsNG?K|B4ZuV{aL7u-2 ztHLaUIwMoBznxE^NOciBOiTzz%qQvrm_p@JhJBBx(kYMoYWB-$a_+$>Xr0VWlGsXa zY+SxPa@IcDGxYxRL2vJ_$lj$OW9~aZyD%+D!wg-)7-CKGd}DFn)-q@yGSvek;{slT zwY|RQCF~!(3H45C-F!yW`}jts+om3sOWMD;Zppc}7eOcof5Z1yI^F#uf)tTD?j!)Yvro4J&+~U--V_u;U4-2isrwcJ`soJ;XBz_8^V=R~ zHJ-m4&F=di$2@K_bo$2kSyRQ?Oz$nV>1}BK#?J~paEEmS-XnVF{$a(04(^9E_~6y> z*P_SO&l+1lMCulT%f% zS|H1)oc#~#>dFJ}_itl8tW#L+77!bFEt9?T+mth<8|qEB7pVszr`QDmIfjsz;B&k@ zpTZx>Jl{(^-=+=nu}?i@_)^E(W5CBhe4lu9@RLIo*s?&C5u*?5BQ?Id6%ntZa!-s^ z5yl#hFgY=FXR*$vPA@OZ)xvXOa|-gcpd`dWA$7Nayia0zXkcomZC+n?U@Xr+iG{U# zAbl#G-m|wuhQ%)5*W(RH69AkXKJw)$_v zPMtsM+%&7@M6szCjD{D;HAwO z(jxL;i2=Z}-@-^i$A&alMQT@jcj`WQCHyI~am)wNsBS3-KYC7Ux&yKLW>?$q{_ZiB zat;AmX>xLEwXZu_ZOZD3SORO%1aO1vGsXQ?tDycqIsl$tj0IgVY&$Q`(KXgw5foX- z?ke7(V8@^A=!>aCph_hA>~vFFlUdGB&eT8t$OAYnk81u7vp+0KS(` zPl6sQX{w+*sL@Lx(Ze_XsRq$8P zakwLU2WZ9dVaNzaw*wI ztoU1e^U&cdz35RjmSSjb;(KpR#Dxv{Jii9}s34J-|9&+4l$ch0$4rRl9$K;wZ9aIu zc37Ibuice+q7ik)7|+`1g-SB}h)itrOW1KY)amSlP@m|#v7Jd9WH(_V`B+)9JJz4K z0V`4Wa%(?dOS-oAD9>+|ZfY%pY?|3=vxh74Pd(ppd9X8?kJu~i z9^f65pxVX!erC1)cL3G`y}p?nP-%FAJ8|#o=%#RMR1Dj7&&k`1OLk>JSaf6ymDTQE zxbiE1Sv7Piq(*(@dQqTSygUL0lt7F{HUkC2wz-wz1>PouJP*h{DMOLEi3AM-_or2hT6_NKO3u%$%ce1 zQp18W4_8VlES!?kZc*8J8jU;cvYfEgs|UcMW{GBTzlN8FB$JAFAJB^m12uBt53xIvA$y2@WCDic)!QkC^Ph&-=u4Usi~ zo5f_RlMDm=ELBBca#9sSn!emk-b2yes!1X_I5_-y&+|VjZcx>SFO#4@A8?=jCxX#- zB3#s*=ik9jYLVb32=at>X$8h+Jx1z4d)~t}5Guc=L63LXV+*JgAg1hLBt0BK#LFUs zAgpP_8{r;^BC7s06WS!IKGZADt_+NI7@C*3I{}use*;lRK&itB&o6@@q!q*s3+{kL z9^dHc0xBO}lTLuGS~oTb>{Ul(o0f|Wg}cHg7la`U{YKaqa+837#PUeA018EP-3Gpt zAYc*(HR|x%1;oixliZ`F^cy^09|``oKmX-dzb6Pgg=>eyF%VJ!V;oFr-x~Lfc4j+|FJXBQnPGzW?c2*f5hYQ3V+ zSlp8VH-8{gyWR>W<6-b)S=Jpn{+x*t;&)%#AD(~Y>F9Q=4DV-joW4j)TT?iR9i%k< zVX`TIOu;pw?e2MEkjJ`cj$NlBVV_bKShJ!Tq(*T|qmmkBe%(q5{860F^CxgJ)5^T_ z#bmQ%Z}U}si-;!kW($f%8pyw*Vrv$jcA3xD97dD!6hqJm(F%aXp=gjyLtW;PDv#aj?#xjB>8O-H_3UW>VGvF z>)(#Q`*TK8d|C&NBJ%q4-{$`1#lQV23H>D;+=Tz9{J;9=AIkr~{=Y0Pci`{!{b%a) T=bZm*|M|bt_t2m^|QgoM(m zln4j{FP`&wp7Y_k-gVu3U-#a>|9{Xx3iVgh`EVtj%E0>359 z*O$#>q>Hu96X|P+yoSPp*HDPf$==H4+WB|pptft>b^Ol!yDC+*21W=6$4pH@T0lVP z-d!m^5fNTJOJk(5HUeR2W}%~}g+Rdd;6{)1byd}MG!^AUc{$nGX09%;7eKP9oSI5z zmXRP2JJCH*ygkViZtp8Mt0Sgo{uPxqd8WtBUBj}?(pdGvdsFh|6+guQbnsgV{OYeB z{8h*Qa|42qWRLK2ipLcrC?2c z$K*k~wy`ZmfWL8;#8kUxkak}^&^kiHhBiP$4hV90AN(NYprEptMQDmeF8Yc`aq9M* z&*jTbqCxiyWBI1}8I1I4>0HzHVLhDe?9+T2(3s*-jE3`pc(^-{jjdi%Et3QT(}R*AF{hxudg40IT)G zd-PVx`Z0IMcB!#7lV zzhGO5gOtML;MN1plP$$AG{6jVAn-h3D(Y9b(TR(B;JEU&=567!5D36}@rozy{PlE# zz_EsU$;k;~(8i0$_{!UTK2S^hW=i0)$~>(SsS-P?N<`oJC_Z{8*{%ie_F6KguM0^{Mw+VCZ2ojr4*etO4Amr;EgGd>Wu`)jfKtfEY z%|@D+weJpZW%FVcwycF>?&u5Z)&%x5>2c>Jjdvl5AAZik>wmQbf&ie$wEah!X+|!g z?4K6$nc!UlS*h(LBlFqVM_xXG)nGH*Wtbe9ER7WN;woR`cm~xZ830;ge`mrVd16hi zId*H{#SLWS%hbZ$#tWGxMZx`uJJD)+kHa?@RI4Lk(Cmpy9LhTI3_!3D!+hxYn0+kW z5&VE7(`T8GeeEV~3Mxij*3M!AQD}^S9J^IM$->aF!cuR*CSv7@w9L6fHuAu*o~tun zD1_@kmMfVMTOba|MO@Z36xAY=N}s^dMQ#kWd~B!^S;N7a za~*H`v7cV1(J0!yhY8)*jj1`El)--0*8VFD|ILU9*w(}-NY@G|SZs3Odje*UT69ig zdRBJ|NG6ej;QzopKxT59Gtfw^;?>(mK=lHW1t`ayN2GC2h*0-wfS~+69mL zs+$MA>2CA8t?cZ@Dm{nl?+NeVZASlOxg5x*w7p{}O{5plexUdr$Qsb^4SQ(tb&V`V zV*Z59wj)}j+#~pm>xUqR&GZlCbSX?6H^lmxJm+lDj`+^zw8@OU##NfqR|Ko0q`^kh zF6Xa(maSP_sw7!XYTungcKu7^*puKOR7Us=e<_J!lDj(4n7!oDH)n{QGDo8wtx~S; z1IMb~D z^x*4+pi#LHS9mtKs4iFT&67N!`kmwf{_dd4(Kq2L{xPUDJH&5s+PwMfMKBHYGX;L+ z`5kknS(>g+8~8!ET}kVR^{|$c8`;!r9M(>Z0XWX8cV9!BIr}yp0Sx@)nf?|~uI&8- z=9?e?HN1Y85H0KA)GU6}(PtQoZ)ulxf>WzwX4iyGp1XC!k!nx%|CA4|Ue+(2L~{ganH$ ziR=*<7xrICiVq2Y#sxfkUR{__PKY3+Nh82voC#@swo}Cop&&$%v>b3I?8^(H4oD#Ge=X{3qzo%68 zM89QKyVy37BUX69~XDGfKFlC8{aYr?s5=xgr1;s4Hi-~BOC>_sg>!+&q6CCK@F?; zf^N#`dwc!eFRa%WiuuUoS?0z)qEha%YWYcEOpgS%OSR}?ECetL(yetsgUzG4M|X6B zpZ2I2QXQ1<^#vok;TXpQ(|e?`w8kFR$gPFa@1O+$D5#nzG{nSs_>#pBIFhL0JzB&I zx2fQF&!C2guelEUjP4$~(L-UuaQNb4tLLOeXM=-ira6$h60gV4_yD|g7@nX!xWusG z8Ld3A>bKW}tq!3hw^`syhvD?{;cJ#)ch7eORvzM~MN_^JhZ)acUMSE`c#cHcdWqmc zQYL4>k*&JeD^5U~=!$=};*+raC^}H3$CUBT_PHsEXXVE?OwPE7Vl+}LHLdK?VECm= z4?-Fgn0gBF>T!uGI(6GktkZUCH2;=>u9DhRBeFZ^tP!|89c6sA1Egh1&;JsjpYQkn z9N{HIlrZ#?bK`NT>Pz!Q^qh??g$H169a4h3yF{`m&{<~mc}z`^krk@n zHW+WMtQRM_z<_GaR^6ny$~0L=2I6%5bbsRY-WJkrBvFDGt)M~^dlF{R>uVn81v%(9 z0xHaKomKemK7A_~TefU&5fi4~F45(D7`f%>J+^4f%oEd31_6JU{xD}vEgb#JBwybT zE}s1*r#u&HPy>n7+8-0-%N;Kt%TB@}M=~Z%I9UGDUB)Mf+86~&bT2viQcsyl(4((* zDO!*TVP?(na*nEE8aEUJNt>x%nv~)ppUXvx0!c;JHj2vM)({`pt{{Q2W4lXEYWaTg zAz9eHot`LsQZH44fZwL9L$h4|Dc0n zhU!Y<@zSgdhh6#P%yvtbT^Jd|Ns8 zb$tJ&;^jw^{Hfp-RQRyRb zeq9=t80j|jkc=qZ@S;$6&d?c1&YTQrb|ox1>O=bwEyJZn_Z{Y45Nq@JZ&}2nXEK|3 zCbl~mGm{qe_jCkC$3@?5TI9|2#POe8!f8KYnyp=|4ywZ)M8a6U58dDp6fw7i-nGV{ux0JgIH)Fk|V zAw7AWYua+zWf=rWlst0Bt1R_C_p=ts=YBe593X7QiaLyw#qmBA;r$wn-xhAQ$C#-o zQlfY&!8>;{!x19)O=JHfiHWf?6NbE4R+eI0`QZa0Um=@)C)gJR8 ze0$I;>%ux@M|k&r8)?I?I-q!~Ea&HDK3g}o@N)n+(qc8;O=qj3o~b{eUFW5X;7Ig7 zndKO@gek$WZTh(x>hdjOY(1PB#Fw37z6+UtO-h zX#)G86P^ou2|Xds{%W~(MwpOuAi2gGpg-}r+Er4Kl+C+QQ@*?vMaoB7EVo40Ub-z!GjeWfB@q&h%xHUr@9$5_1)bak^a#>==o9@|AtbcVo@Lc zKb#a2^PQQpIs21ui=485n%K|$Il4%Y)?XtXuEfsYL)7n)4(EF7PY#bJy~WGV&-Nz( zg87Kl_JWzrP@cFwAzXVJWZW0ApH_;!~fAN3&|8OG5JA;4ntNhDf@AF@O3j5Mu=L7$P6CwY%zvJcK{@?lk_$~eq@QA+< literal 0 HcmV?d00001 diff --git a/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py b/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py index 62916841ea5..018863c555f 100644 --- a/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py @@ -1,6 +1,7 @@ import base64 import io import math +import struct import wave from pathlib import Path from typing import Final @@ -13,6 +14,7 @@ import litellm from litellm.llms.vertex_ai.text_to_speech.transformation import ( VertexAILyriaTextToSpeechConfig, VertexAITextToSpeechConfig, + _fallback_gemini_tts_audio_duration, ) from litellm.cost_calculator import response_cost_calculator from litellm.litellm_core_utils.audio_utils.utils import calculate_request_duration @@ -413,7 +415,7 @@ def test_transform_text_to_speech_response_leaves_unknown_bytes_unlabeled(): assert result.response.content == raw_pcm -@pytest.mark.parametrize("encoding", ["LINEAR16", "PCM", "MP3"]) +@pytest.mark.parametrize("encoding", ["LINEAR16", "PCM", "MP3", "OGG_OPUS", "ALAW", "MULAW"]) def test_gemini_cloud_tts_response_bills_text_and_audio(encoding: str): model: Final = "gemini-3.1-flash-tts-preview" input_text: Final = "Hello from Gemini text to speech" @@ -424,10 +426,33 @@ def test_gemini_cloud_tts_response_bills_text_and_audio(encoding: str): wav_file.setsampwidth(2) wav_file.setframerate(24000) wav_file.writeframes(pcm_audio) + g711_audio: Final = ( + struct.pack( + "<4sI4s4sIHHIIHH4sI", + b"RIFF", + 36 + 24000, + b"WAVE", + b"fmt ", + 16, + 6 if encoding == "ALAW" else 7, + 1, + 24000, + 24000, + 1, + 8, + b"data", + 24000, + ) + + b"\x00" * 24000 + ) + fixture_dir: Final = Path(__file__).resolve().parents[4] / "audio_tests" audio_by_encoding: Final = { "PCM": pcm_audio, "LINEAR16": wav_buffer.getvalue(), - "MP3": (Path(__file__).resolve().parents[4] / "audio_tests/speech_vertex.mp3").read_bytes(), + "MP3": (fixture_dir / "gemini_tts_speech.mp3").read_bytes(), + "OGG_OPUS": (fixture_dir / "gemini_tts_speech.ogg").read_bytes(), + "ALAW": g711_audio, + "MULAW": g711_audio, } audio_bytes: Final = audio_by_encoding[encoding] logger: Final = MagicMock() @@ -448,6 +473,11 @@ def test_gemini_cloud_tts_response_bills_text_and_audio(encoding: str): assert usage is not None duration: Final = len(audio_bytes) / 48000 if encoding == "PCM" else calculate_request_duration(audio_bytes) assert duration is not None + assert _fallback_gemini_tts_audio_duration(audio_bytes, encoding, 24000) == pytest.approx(duration), ( + "Google Cloud AudioEncoding and RFC 7845, checked 2026-09-23: " + "https://cloud.google.com/text-to-speech/docs/reference/rest/v1/AudioEncoding " + "https://www.rfc-editor.org/rfc/rfc7845.html" + ) assert usage.completion_tokens == math.ceil(duration * 25), ( "Google Cloud TTS pricing, 2026-09-23: https://cloud.google.com/text-to-speech/pricing" ) @@ -455,6 +485,9 @@ def test_gemini_cloud_tts_response_bills_text_and_audio(encoding: str): assert usage.total_tokens == usage.prompt_tokens + usage.completion_tokens assert usage.completion_tokens_details is not None assert usage.completion_tokens_details.audio_tokens == usage.completion_tokens + with patch("litellm.llms.vertex_ai.text_to_speech.transformation.calculate_request_duration", return_value=None): + sdk_result: Final = VertexAITextToSpeechConfig().transform_text_to_speech_response(model, raw_response, logger) + assert sdk_result.usage == usage model_info: Final = litellm.get_model_info(model, custom_llm_provider="vertex_ai") cost: Final = response_cost_calculator( @@ -473,6 +506,35 @@ def test_gemini_cloud_tts_response_bills_text_and_audio(encoding: str): assert cost > usage.prompt_tokens * model_info["input_cost_per_token"] +@pytest.mark.parametrize("encoding", ["ALAW", "MULAW"]) +@pytest.mark.parametrize("audio", [b"\x12" * 24000, b"RIFF" + b"\x12" * 23996]) +def test_gemini_cloud_tts_returns_raw_g711_audio_when_duration_decoder_is_unavailable(encoding: str, audio: bytes): + raw_response: Final = httpx.Response(200, json={"audioContent": base64.b64encode(audio).decode()}) + logger: Final = MagicMock() + logger.model_call_details = { + "additional_args": { + "complete_input_dict": { + "dict_body": { + "input": {"text": "Hello"}, + "audioConfig": {"audioEncoding": encoding, "sampleRateHertz": 24000}, + } + } + } + } + + with patch("litellm.llms.vertex_ai.text_to_speech.transformation.calculate_request_duration", return_value=None): + result: Final = VertexAITextToSpeechConfig().transform_text_to_speech_response( + "gemini-3.1-flash-tts-preview", raw_response, logger + ) + + assert result.response.content == audio + assert result.usage is not None + assert result.usage.prompt_tokens > 0 + assert result.usage.completion_tokens == 25, ( + "Google Cloud TTS pricing, 2026-09-23: https://cloud.google.com/text-to-speech/pricing" + ) + + class TestVertexAILyriaTextToSpeechConfig: @pytest.mark.parametrize( "model", @@ -933,7 +995,7 @@ def test_litellm_speech_vertex_ai_gemini_tts_mp3_uses_cloud_tts(mock_get_token, mock_response = Mock(spec=httpx.Response) mock_response.status_code = 200 mock_response.headers = {"content-type": "application/json"} - audio_bytes: Final = (Path(__file__).resolve().parents[4] / "audio_tests/speech_vertex.mp3").read_bytes() + audio_bytes: Final = (Path(__file__).resolve().parents[4] / "audio_tests/gemini_tts_speech.mp3").read_bytes() mock_response.json.return_value = {"audioContent": base64.b64encode(audio_bytes).decode()} mock_post.return_value = mock_response diff --git a/uv.lock b/uv.lock index 590812613de..85e2b6d4e52 100644 --- a/uv.lock +++ b/uv.lock @@ -4518,7 +4518,6 @@ dependencies = [ { name = "pydantic-settings" }, { name = "python-dotenv" }, { name = "pyyaml" }, - { name = "soundfile" }, { name = "tiktoken" }, { name = "tokenizers" }, ] @@ -4593,6 +4592,7 @@ proxy = [ { name = "restrictedpython" }, { name = "rich" }, { name = "rq" }, + { name = "soundfile" }, { name = "starlette" }, { name = "tomlkit" }, { name = "uvicorn" }, @@ -4841,7 +4841,7 @@ requires-dist = [ { name = "rq", marker = "extra == 'proxy'", specifier = ">=2.7.0,<3.0" }, { name = "semantic-router", marker = "python_full_version < '3.14' and extra == 'semantic-router'", specifier = ">=0.1.15,<1.0" }, { name = "sentry-sdk", marker = "extra == 'proxy-runtime'", specifier = ">=2.21.0,<3.0" }, - { name = "soundfile", specifier = ">=0.12.1,<1.0" }, + { name = "soundfile", marker = "extra == 'proxy'", specifier = ">=0.12.1,<1.0" }, { name = "soundfile", marker = "extra == 'stt-nvidia-riva'", specifier = ">=0.12.1" }, { name = "starlette", marker = "extra == 'proxy'", specifier = ">=1.0.1,<2.0" }, { name = "tiktoken", marker = "python_full_version < '3.14'", specifier = ">=0.8.0,<1.0" }, From ac1455170c9f3a925563205190cb1be0fa5d0c65 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Wed, 23 Sep 2026 07:26:06 -0500 Subject: [PATCH 12/20] test(gemini): cover TTS mapping and billing edge cases --- .../vertex_and_google_ai_studio_gemini.py | 29 ++--- .../llms/gemini/test_gemini_tts.py | 35 ++++- .../text_to_speech/test_transformation.py | 120 +++++++++++++++++- 3 files changed, 160 insertions(+), 24 deletions(-) diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 71072f3aa8b..d466c586753 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -146,15 +146,10 @@ _GEMINI_SPEECH_CONFIG_KEY_MAP: Final[Mapping[str, str]] = MappingProxyType( ) -def _normalize_gemini_speech_config_item(value: object) -> object: - if isinstance(value, Mapping): - normalized: ( # mutable-ok: the iterative walk fills exactly one normalized provider payload - dict[str, object] | list[object] - ) = {} # mutable-ok: the iterative walk fills the normalized tree; rebind-ok: each shape branch initializes the shared result - elif isinstance(value, list): - normalized = [] # mutable-ok: the iterative walk fills the normalized tree; rebind-ok: each shape branch initializes the shared result - else: - return value +def _normalize_gemini_speech_config_item(value: Mapping[str, object]) -> dict[str, object]: + normalized: Final[ + dict[str, object] + ] = {} # mutable-ok: iterative traversal preserves deeply nested provider payloads pending: Final[ # mutable-ok: the explicit worklist must grow while avoiding recursive traversal list[tuple[dict[str, object] | list[object], Mapping[object, object] | list[object]]] @@ -164,12 +159,10 @@ def _normalize_gemini_speech_config_item(value: object) -> object: items = source.items() if isinstance(source, Mapping) else enumerate(source) for key, item in items: if isinstance(source, Mapping): - if not isinstance(key, str) or not isinstance(target, dict): + if not isinstance(key, str): continue normalized_key: str | int = _GEMINI_SPEECH_CONFIG_KEY_MAP.get(key, key) else: - if not isinstance(target, list): - continue normalized_key = len(target) child: object @@ -180,9 +173,10 @@ def _normalize_gemini_speech_config_item(value: object) -> object: else: child = item - if isinstance(target, dict) and isinstance(normalized_key, str): + if isinstance(target, dict): + assert isinstance(normalized_key, str) target[normalized_key] = child - elif isinstance(target, list): + else: target.append(child) if isinstance(child, (dict, list)) and isinstance(item, (Mapping, list)): @@ -202,12 +196,7 @@ def normalize_gemini_speech_config( if isinstance(nested_config, Mapping) else value ) - normalized_value: Final = _normalize_gemini_speech_config_item(config) - if not isinstance(normalized_value, dict): - return {} # mutable-ok: provider request serialization requires a concrete empty dict - normalized: Final = cast( # cast-ok: runtime dict check establishes the normalizer's concrete result shape - dict[str, object], normalized_value - ) + normalized: Final = _normalize_gemini_speech_config_item(config) voice_name: Final = next( (normalized[key] for key in ("name", "voiceName", "voice") if isinstance(normalized.get(key), str)), None, diff --git a/tests/test_litellm/llms/gemini/test_gemini_tts.py b/tests/test_litellm/llms/gemini/test_gemini_tts.py index b625d6dfe29..cb7dd3ee11a 100644 --- a/tests/test_litellm/llms/gemini/test_gemini_tts.py +++ b/tests/test_litellm/llms/gemini/test_gemini_tts.py @@ -2,9 +2,11 @@ Test Gemini TTS (Text-to-Speech) functionality """ -import pytest +from typing import Final from unittest.mock import patch, MagicMock +import pytest + import litellm from litellm.endpoints.speech.speech_to_completion_bridge.transformation import ( SpeechToCompletionBridgeTransformationHandler, @@ -168,6 +170,37 @@ class TestGeminiTTSTransformation: assert "languageCode" not in result assert result["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Kore" + def test_map_audio_params_preserves_nested_list_and_values_without_voice(self): + config: Final = GoogleAIStudioGeminiConfig() + + result: Final = config._map_audio_params( + { + "speech_config": { + "language_code": "fr-FR", + "multi_speaker_voice_config": { + "speaker_voice_configs": [ + {"speaker": "Ryan", "voice_config": {"prebuilt_voice_config": {"voice_name": "Umbriel"}}} + ] + }, + "metadata": {1: "discard", "source": ["studio", 2]}, + }, + "format": "pcm16", + } + ) + + assert result == { + "languageCode": "fr-FR", + "multiSpeakerVoiceConfig": { + "speakerVoiceConfigs": [ + {"speaker": "Ryan", "voiceConfig": {"prebuiltVoiceConfig": {"voiceName": "Umbriel"}}} + ] + }, + "metadata": {"source": ["studio", 2]}, + } + assert config._map_audio_params({"speech_config": {"language_code": "fr-FR"}, "format": "pcm16"}) == { + "languageCode": "fr-FR" + } + def test_gemini_tts_multi_speaker_audio_parameter_mapping(self): """Test multi-speaker audio parameter mapping for Gemini 3.1 TTS models""" config = GoogleAIStudioGeminiConfig() diff --git a/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py b/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py index 018863c555f..a83ce60f385 100644 --- a/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py @@ -102,6 +102,20 @@ class TestVertexAITextToSpeechConfig: assert "headers" in result assert "Authorization" in result["headers"] + raw_voice_request: Final = config.transform_text_to_speech_request( + model="vertex_ai/chirp", + input="Hello", + voice="en-US-Chirp3-HD-Charon", + optional_params={}, + litellm_params={ + "vertex_credentials": None, + "vertex_project": "test-project", + "vertex_location": "us-central1", + }, + headers={}, + ) + assert raw_voice_request["dict_body"]["voice"]["name"] == "en-US-Chirp3-HD-Charon" + def test_voice_mapping_openai_to_vertex(self): """Test that OpenAI voice names are correctly mapped to Vertex AI voices""" config = VertexAITextToSpeechConfig() @@ -217,6 +231,75 @@ class TestVertexAITextToSpeechConfig: "name": "Kore", } + def test_gemini_tts_string_voice_reaches_cloud_tts(self): + config: Final = VertexAITextToSpeechConfig() + + voice_name, optional_params = config.map_openai_params( + model="gemini-3.1-flash-tts-preview", + optional_params={"response_format": "mp3"}, + voice="Kore", + ) + + assert voice_name == "Kore" + assert optional_params["vertex_voice_dict"] == { + "languageCode": "en-US", + "modelName": "gemini-3.1-flash-tts-preview", + "name": "Kore", + } + + def test_gemini_tts_ignores_incomplete_speaker_entries(self): + config: Final = VertexAITextToSpeechConfig() + voice: Final = { + "multiSpeakerVoiceConfig": { + "speakerVoiceConfigs": [ + None, + {"speakerAlias": "Ryan"}, + {"speakerAlias": "Katie", "speakerId": "Leda"}, + ] + } + } + + _, optional_params = config.map_openai_params( + model="gemini-3.1-flash-tts-preview", + optional_params={"response_format": "mp3"}, + voice=voice, + ) + + assert optional_params["vertex_voice_dict"]["multiSpeakerVoiceConfig"] == { + "speakerVoiceConfigs": [{"speakerAlias": "Katie", "speakerId": "Leda"}] + } + + def test_gemini_tts_ignores_non_list_speaker_entries(self): + config: Final = VertexAITextToSpeechConfig() + + assert config._extract_gemini_tts_speaker_configs( + {"multiSpeakerVoiceConfig": {"speakerVoiceConfigs": "invalid"}} + ) == [] + + @pytest.mark.parametrize( + ("voice", "expected_name"), + [ + ({"speechConfig": {"voice": "Kore"}}, "Kore"), + ({"speechConfig": {"voiceConfig": {"prebuiltVoiceConfig": {"voiceName": "Kore"}}}}, "Kore"), + ({"speechConfig": {"voiceConfig": {}}}, None), + ], + ) + def test_gemini_tts_nested_voice_mapping(self, voice: dict[str, object], expected_name: str | None): + config: Final = VertexAITextToSpeechConfig() + + _, optional_params = config.map_openai_params( + model="gemini-3.1-flash-tts-preview", + optional_params={"response_format": "mp3"}, + voice=voice, + ) + + mapped_voice: Final = optional_params["vertex_voice_dict"] + if expected_name is None: + assert mapped_voice["speechConfig"]["voiceConfig"] == {} + assert "name" not in mapped_voice + else: + assert mapped_voice["name"] == expected_name + @pytest.mark.parametrize( "voice", [ @@ -281,7 +364,17 @@ class TestVertexAITextToSpeechConfig: assert mapped_params["vertex_voice_dict"]["name"] == "Umbriel" assert mapped_params["vertex_voice_dict"]["modelName"] == "gemini-3.1-flash-tts-preview" - def test_dispatch_keeps_pre_mapped_cloud_tts_params(self): + @pytest.mark.parametrize( + ("voice", "expected_name"), + [ + ("en-US-Chirp3-HD-Charon", "en-US-Chirp3-HD-Charon"), + ({"name": "en-US-Chirp3-HD-Charon"}, "en-US-Chirp3-HD-Charon"), + (None, None), + ], + ) + def test_dispatch_keeps_pre_mapped_cloud_tts_params( + self, voice: str | dict[str, str] | None, expected_name: str | None + ): config = VertexAITextToSpeechConfig() handler = MagicMock() handler.text_to_speech_handler.return_value = "ok" @@ -293,7 +386,7 @@ class TestVertexAITextToSpeechConfig: config.dispatch_text_to_speech( model="chirp", input="Hi", - voice="en-US-Chirp3-HD-Charon", + voice=voice, optional_params=optional_params, litellm_params_dict={}, logging_obj=MagicMock(), @@ -306,7 +399,7 @@ class TestVertexAITextToSpeechConfig: ) call_kwargs = handler.text_to_speech_handler.call_args.kwargs - assert call_kwargs["voice"] == "en-US-Chirp3-HD-Charon" + assert call_kwargs["voice"] == expected_name assert call_kwargs["text_to_speech_optional_params"] is optional_params assert call_kwargs["text_to_speech_optional_params"]["audioEncoding"] == "OGG_OPUS" @@ -535,6 +628,27 @@ def test_gemini_cloud_tts_returns_raw_g711_audio_when_duration_decoder_is_unavai ) +def test_gemini_cloud_tts_rejects_audio_without_measurable_duration(): + raw_response: Final = httpx.Response(200, json={"audioContent": base64.b64encode(b"invalid").decode()}) + logger: Final = MagicMock() + logger.model_call_details = { + "additional_args": { + "complete_input_dict": { + "dict_body": { + "input": {"text": "Hello"}, + "audioConfig": {"audioEncoding": "MP3", "sampleRateHertz": 24000}, + } + } + } + } + + with patch("litellm.llms.vertex_ai.text_to_speech.transformation.calculate_request_duration", return_value=None): + with pytest.raises(ValueError, match="Cannot determine Gemini TTS output duration"): + VertexAITextToSpeechConfig().transform_text_to_speech_response( + "gemini-3.1-flash-tts-preview", raw_response, logger + ) + + class TestVertexAILyriaTextToSpeechConfig: @pytest.mark.parametrize( "model", From c6b07c8f6e636954fbeca5f44d5edebe7b1e8826 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Wed, 23 Sep 2026 07:43:34 -0500 Subject: [PATCH 13/20] test(vertex-ai): cover Gemini voice normalization in provider shard --- .../test_litellm/llms/gemini/test_gemini_tts.py | 7 ++----- .../test_vertex_and_google_ai_studio_gemini.py | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/tests/test_litellm/llms/gemini/test_gemini_tts.py b/tests/test_litellm/llms/gemini/test_gemini_tts.py index cb7dd3ee11a..c1c8492bd52 100644 --- a/tests/test_litellm/llms/gemini/test_gemini_tts.py +++ b/tests/test_litellm/llms/gemini/test_gemini_tts.py @@ -182,7 +182,7 @@ class TestGeminiTTSTransformation: {"speaker": "Ryan", "voice_config": {"prebuilt_voice_config": {"voice_name": "Umbriel"}}} ] }, - "metadata": {1: "discard", "source": ["studio", 2]}, + "metadata": ["studio", 2], }, "format": "pcm16", } @@ -195,10 +195,7 @@ class TestGeminiTTSTransformation: {"speaker": "Ryan", "voiceConfig": {"prebuiltVoiceConfig": {"voiceName": "Umbriel"}}} ] }, - "metadata": {"source": ["studio", 2]}, - } - assert config._map_audio_params({"speech_config": {"language_code": "fr-FR"}, "format": "pcm16"}) == { - "languageCode": "fr-FR" + "metadata": ["studio", 2], } def test_gemini_tts_multi_speaker_audio_parameter_mapping(self): diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 88ba7fc37d9..d4b2c96e3b4 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -17,12 +17,28 @@ from litellm.llms.gemini.chat.transformation import GoogleAIStudioGeminiConfig from litellm.llms.vertex_ai.common_utils import VertexAIError from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( VertexGeminiConfig, + normalize_gemini_speech_config, ) from litellm.types.llms.vertex_ai import GeminiFinishReason, UsageMetadata from litellm.types.utils import ChoiceLogprobs, Usage from litellm.utils import CustomStreamWrapper +def test_gemini_speech_config_drops_non_string_keys_without_adding_voice(): + speech_config: Final = { + "speech_config": { + "language_code": "fr-FR", + "metadata": {1: "discard", "source": ["studio", 2]}, + }, + "format": "pcm16", + } + + assert normalize_gemini_speech_config(speech_config) == { + "languageCode": "fr-FR", + "metadata": {"source": ["studio", 2]}, + } + + def test_top_logprobs(): non_default_params = { "top_logprobs": 2, From 66a99f8961e7fd063ea9853636e387b99862f0b8 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Wed, 23 Sep 2026 13:16:54 -0500 Subject: [PATCH 14/20] fix(vertex-ai): keep Gemini TTS routing with generic provider metadata --- litellm/utils.py | 18 +++++++++++++++++- .../test_gemini_3_1_flash_tts_metadata.py | 15 +++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/litellm/utils.py b/litellm/utils.py index 1e86b87092b..db04a3e7bb3 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -759,9 +759,25 @@ def is_gemini_tts_model(model: str, custom_llm_provider: str | None = None) -> b ) provider_entry: Final = runtime_model_info.get("provider_specific_entry") - return runtime_model_info.get("litellm_provider") in ("gemini", "vertex_ai-language-models") and ( + if runtime_model_info.get("litellm_provider") in ("gemini", "vertex_ai-language-models") and ( runtime_model_info.get("mode") == "audio_speech" or (isinstance(provider_entry, dict) and provider_entry.get("gemini_tts") == 1) + ): + return True + if runtime_model_info.get("litellm_provider") != "vertex_ai" or runtime_model_info.get("mode") != "audio_speech": + return False + bundled_model_name: Final = model if model.startswith("vertex_ai/") else f"vertex_ai/{model}" + vertex_model_info: Final = _get_bundled_model_cost_map().get(bundled_model_name) + return ( + vertex_model_info is not None + and vertex_model_info.get("litellm_provider") == "vertex_ai-language-models" + and ( + vertex_model_info.get("mode") == "audio_speech" + or ( + isinstance(vertex_model_info.get("provider_specific_entry"), dict) + and vertex_model_info["provider_specific_entry"].get("gemini_tts") == 1 + ) + ) ) diff --git a/tests/test_litellm/test_gemini_3_1_flash_tts_metadata.py b/tests/test_litellm/test_gemini_3_1_flash_tts_metadata.py index 6420b5b9dc3..421c9538a41 100644 --- a/tests/test_litellm/test_gemini_3_1_flash_tts_metadata.py +++ b/tests/test_litellm/test_gemini_3_1_flash_tts_metadata.py @@ -1,5 +1,6 @@ import json from pathlib import Path +from typing import Final import pytest @@ -59,6 +60,20 @@ def test_gemini_tts_detection_uses_model_metadata(monkeypatch): assert is_gemini_tts_model(model) +def test_gemini_tts_detection_uses_bundled_metadata_when_runtime_provider_is_generic(monkeypatch): + model: Final = "gemini-3.1-flash-tts-preview" + monkeypatch.setitem( + litellm.model_cost, + f"vertex_ai/{model}", + {"litellm_provider": "vertex_ai", "mode": "audio_speech"}, + ) + litellm.get_model_info.cache_clear() + _invalidate_model_cost_lowercase_map() + + assert is_gemini_tts_model(model, custom_llm_provider="vertex_ai") + assert not is_gemini_tts_model("lyria-3-clip-preview", custom_llm_provider="vertex_ai") + + @pytest.mark.parametrize( "model", ["gemini-2.5-flash-preview-tts", "gemini-2.5-flash-tts", "gemini-2.5-pro-tts", "gemini-2.5-pro-preview-tts"], From 3e65e9ef0ef0048880140e1f38cecfd082065147 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Sat, 26 Sep 2026 05:15:27 -0500 Subject: [PATCH 15/20] chore(vertex-ai): remove obsolete lint suppressions --- .../vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py | 2 +- litellm/llms/vertex_ai/text_to_speech/transformation.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 0383ed53337..fa98f8f190d 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -1132,7 +1132,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): return normalize_gemini_speech_config(value) speech_config_without_language: Final[ # mutable-ok: provider request serialization requires a concrete dict - dict[str, object] # mutable-ok: provider request serialization requires a concrete dict + dict[str, object] ] = ( { # mutable-ok: provider request serialization requires a concrete dict "voiceConfig": { # mutable-ok: nested provider payload is serialized as a dict diff --git a/litellm/llms/vertex_ai/text_to_speech/transformation.py b/litellm/llms/vertex_ai/text_to_speech/transformation.py index f92e67792e0..dce7347d69b 100644 --- a/litellm/llms/vertex_ai/text_to_speech/transformation.py +++ b/litellm/llms/vertex_ai/text_to_speech/transformation.py @@ -218,9 +218,7 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): def _extract_gemini_tts_speaker_configs( self, voice: Mapping[str, object], - ) -> list[ - VertexTextToSpeechSpeakerVoiceConfig - ]: # mutable-ok: provider request serialization requires a concrete list + ) -> list[VertexTextToSpeechSpeakerVoiceConfig]: speech_config: Final = self._get_dict_value(voice, "speechConfig", "speech_config") or voice multi_speaker_config: Final = self._get_dict_value( speech_config, From 2fdb818327f070cfa96e175231df7ade46f01c1a Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Sat, 26 Sep 2026 08:52:05 -0500 Subject: [PATCH 16/20] fix(gemini): strip audio format from speech configuration --- .../vertex_and_google_ai_studio_gemini.py | 4 +++- .../test_transformation.py | 20 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index fa98f8f190d..ea0a14c3d1e 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -196,7 +196,9 @@ def normalize_gemini_speech_config( if isinstance(nested_config, Mapping) else value ) - normalized: Final = _normalize_gemini_speech_config_item(config) + normalized: Final = _normalize_gemini_speech_config_item( + {key: item for key, item in config.items() if key != "format"} + ) voice_name: Final = next( (normalized[key] for key in ("name", "voiceName", "voice") if isinstance(normalized.get(key), str)), None, diff --git a/tests/unit/endpoints/speech/speech_to_completion_bridge/test_transformation.py b/tests/unit/endpoints/speech/speech_to_completion_bridge/test_transformation.py index 6b01df20024..e65a82980e0 100644 --- a/tests/unit/endpoints/speech/speech_to_completion_bridge/test_transformation.py +++ b/tests/unit/endpoints/speech/speech_to_completion_bridge/test_transformation.py @@ -90,6 +90,26 @@ def test_gemini_request_keeps_voice_beside_nested_speech_config() -> None: } +@pytest.mark.parametrize("wrapper", (None, "speech_config", "speechConfig")) +def test_gemini_request_keeps_voice_and_language_without_audio_format(wrapper: str | None) -> None: + voice: Final = {"language_code": "fr-FR", "voice": "Kore", "format": "pcm16"} + request: Final = SpeechToCompletionBridgeTransformationHandler().transform_request( + model=GEMINI_TTS_MODEL, + input="Bonjour", + voice=voice if wrapper is None else {wrapper: voice}, + optional_params={}, + litellm_params={}, + headers={}, + litellm_logging_obj=MagicMock(), + custom_llm_provider="gemini", + ) + + assert request["audio"]["speech_config"] == { + "languageCode": "fr-FR", + "voiceConfig": {"prebuiltVoiceConfig": {"voiceName": "Kore"}}, + } + + @pytest.mark.parametrize("response_format", ["mp3", "flac", "opus", "aac"]) def test_gemini_tts_request_rejects_formats_gemini_cannot_produce(response_format: str) -> None: with pytest.raises(litellm.BadRequestError) as excinfo: From 0281ce43efef52408260605fb5614cbefb8662e1 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Tue, 6 Oct 2026 08:41:11 -0500 Subject: [PATCH 17/20] chore(vertex-ai): remove obsolete mutation suppressions --- .../vertex_and_google_ai_studio_gemini.py | 20 +++++++++---------- .../text_to_speech/transformation.py | 18 ++++++++--------- 2 files changed, 18 insertions(+), 20 deletions(-) diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 4cf6b33cd10..8c3513e064e 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -147,9 +147,7 @@ _GEMINI_SPEECH_CONFIG_KEY_MAP: Final[Mapping[str, str]] = MappingProxyType( def _normalize_gemini_speech_config_item(value: Mapping[str, object]) -> dict[str, object]: - normalized: Final[ - dict[str, object] - ] = {} # mutable-ok: iterative traversal preserves deeply nested provider payloads + normalized: Final[dict[str, object]] = {} pending: Final[ # mutable-ok: the explicit worklist must grow while avoiding recursive traversal list[tuple[dict[str, object] | list[object], Mapping[object, object] | list[object]]] @@ -167,9 +165,9 @@ def _normalize_gemini_speech_config_item(value: Mapping[str, object]) -> dict[st child: object if isinstance(item, Mapping): - child = {} # mutable-ok: nested mappings are filled by the worklist + child = {} elif isinstance(item, list): - child = [] # mutable-ok: nested lists are filled by the worklist + child = [] else: child = item @@ -207,7 +205,7 @@ def normalize_gemini_speech_config( return {key: item for key, item in normalized.items() if key not in ("name", "voiceName", "voice")} if voice_name is None: return normalized - return { # mutable-ok: provider request serialization requires a concrete dict + return { **{key: item for key, item in normalized.items() if key not in ("name", "voiceName", "voice", "modelName")}, "voiceConfig": {"prebuiltVoiceConfig": {"voiceName": voice_name}}, } @@ -1136,19 +1134,19 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): speech_config_without_language: Final[ # mutable-ok: provider request serialization requires a concrete dict dict[str, object] ] = ( - { # mutable-ok: provider request serialization requires a concrete dict - "voiceConfig": { # mutable-ok: nested provider payload is serialized as a dict - "prebuiltVoiceConfig": { # mutable-ok: nested provider payload is serialized as a dict + { + "voiceConfig": { + "prebuiltVoiceConfig": { "voiceName": value["voice"], } } } if "voice" in value - else {} # mutable-ok: provider request serialization requires a concrete empty dict + else {} ) if language_code is not None: - return { # mutable-ok: provider request serialization requires a concrete dict + return { **speech_config_without_language, "languageCode": language_code, } diff --git a/litellm/llms/vertex_ai/text_to_speech/transformation.py b/litellm/llms/vertex_ai/text_to_speech/transformation.py index 3fcbcb898c0..261c5f09768 100644 --- a/litellm/llms/vertex_ai/text_to_speech/transformation.py +++ b/litellm/llms/vertex_ai/text_to_speech/transformation.py @@ -230,16 +230,16 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): "multi_speaker_voice_config", ) if multi_speaker_config is None: - return [] # mutable-ok: provider request serialization requires a concrete empty list + return [] raw_speaker_configs: Final = multi_speaker_config.get( "speakerVoiceConfigs", multi_speaker_config.get( "speaker_voice_configs", - [], # mutable-ok: missing speaker configuration uses a concrete empty-list sentinel + [], ), ) if not isinstance(raw_speaker_configs, list): - return [] # mutable-ok: malformed speaker configuration produces a concrete empty list + return [] speaker_configs: Final[ # mutable-ok: validated payloads are accumulated for serialization list[VertexTextToSpeechSpeakerVoiceConfig] ] = [] @@ -269,7 +269,7 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): ) if speaker_alias is not None and speaker_id is not None: speaker_configs.append( - { # mutable-ok: provider request serialization requires a concrete dict + { "speakerAlias": speaker_alias, "speakerId": speaker_id, } @@ -305,7 +305,7 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): voice: str | Mapping[str, object], ) -> tuple[str | None, dict[str, object]]: # mutable-ok: provider request serialization requires a concrete dict if isinstance(voice, str): - return voice, { # mutable-ok: provider request serialization requires a concrete dict + return voice, { "languageCode": self.DEFAULT_LANGUAGE_CODE, "modelName": model, "name": voice, @@ -320,21 +320,21 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): model_name: Final = model speaker_configs: Final = self._extract_gemini_tts_speaker_configs(voice) if speaker_configs: - return None, { # mutable-ok: provider request serialization requires a concrete dict + return None, { "languageCode": language_code, "modelName": model_name, - "multiSpeakerVoiceConfig": { # mutable-ok: nested provider payload is serialized as a dict + "multiSpeakerVoiceConfig": { "speakerVoiceConfigs": speaker_configs, }, } voice_name: Final = self._extract_gemini_tts_voice_name(voice) if voice_name is not None: - return None, { # mutable-ok: provider request serialization requires a concrete dict + return None, { "languageCode": language_code, "modelName": model_name, "name": voice_name, } - return None, { # mutable-ok: provider request serialization requires a concrete dict + return None, { **voice, "languageCode": language_code, "modelName": model_name, From 9c54ed898b64b0a2b99c20708080de3ebbe4053a Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Tue, 6 Oct 2026 09:05:57 -0500 Subject: [PATCH 18/20] fix(vertex-ai): preserve FLAC billing and wrapped audio headers --- .../llms/vertex_ai/text_to_speech/transformation.py | 8 ++++++++ litellm/proxy/proxy_server.py | 8 ++++++-- .../vertex_ai/text_to_speech/test_transformation.py | 13 ++++++++++--- tests/unit/proxy/proxy_server/test_routes_audio.py | 4 ++++ 4 files changed, 28 insertions(+), 5 deletions(-) diff --git a/litellm/llms/vertex_ai/text_to_speech/transformation.py b/litellm/llms/vertex_ai/text_to_speech/transformation.py index 261c5f09768..9ce4d39524f 100644 --- a/litellm/llms/vertex_ai/text_to_speech/transformation.py +++ b/litellm/llms/vertex_ai/text_to_speech/transformation.py @@ -68,6 +68,14 @@ def _fallback_gemini_tts_audio_duration(audio: bytes, encoding: str, sample_rate if byte_rate > 0 and data_size <= len(audio) - data_offset - 8: return data_size / byte_rate + if encoding == "FLAC" and audio[:4] == b"fLaC" and len(audio) >= 42: + if audio[4] & 0x7F != 0 or int.from_bytes(audio[5:8], "big") != 34: + return None + stream_info: Final = int.from_bytes(audio[18:26], "big") + flac_sample_rate: Final = stream_info >> 44 + total_samples: Final = stream_info & ((1 << 36) - 1) + return total_samples / flac_sample_rate if flac_sample_rate > 0 and total_samples > 0 else None + if encoding == "MP3" and speech_media_type_from_audio_bytes(audio) == "audio/mpeg": return len(audio) / 4000 diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 0ec86edc4a7..39e3fa2f2e9 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -12733,8 +12733,12 @@ async def audio_speech( upstream_content_type: Final = ( response.response.headers.get("content-type") if isinstance(response, HttpxBinaryResponseContent) else None ) - media_type: Final = extra_media_type or resolve_speech_media_type( - upstream_content_type=upstream_content_type, + media_type: Final = resolve_speech_media_type( + upstream_content_type=( + upstream_content_type + if upstream_content_type is not None and upstream_content_type.lower().startswith("audio/") + else extra_media_type or upstream_content_type + ), response_format=requested_format_str, ) diff --git a/tests/unit/llms/vertex_ai/text_to_speech/test_transformation.py b/tests/unit/llms/vertex_ai/text_to_speech/test_transformation.py index f4026d08df0..4a7fb7f0fad 100644 --- a/tests/unit/llms/vertex_ai/text_to_speech/test_transformation.py +++ b/tests/unit/llms/vertex_ai/text_to_speech/test_transformation.py @@ -2,6 +2,7 @@ import base64 import io import math import struct +import sys import wave from pathlib import Path from typing import Final @@ -9,6 +10,7 @@ from unittest.mock import MagicMock, Mock, patch import httpx import pytest +import soundfile from pydantic import ValidationError import litellm @@ -509,8 +511,8 @@ def test_transform_text_to_speech_response_leaves_unknown_bytes_unlabeled(): assert result.response.content == raw_pcm -@pytest.mark.parametrize("encoding", ["LINEAR16", "PCM", "MP3", "OGG_OPUS", "ALAW", "MULAW"]) -def test_gemini_cloud_tts_response_bills_text_and_audio(encoding: str): +@pytest.mark.parametrize("encoding", ["LINEAR16", "PCM", "MP3", "OGG_OPUS", "ALAW", "MULAW", "FLAC"]) +def test_gemini_cloud_tts_response_bills_text_and_audio(encoding: str, monkeypatch: pytest.MonkeyPatch) -> None: model: Final = "gemini-3.1-flash-tts-preview" input_text: Final = "Hello from Gemini text to speech" pcm_audio: Final = b"\x00\x00" * 24000 @@ -520,6 +522,8 @@ def test_gemini_cloud_tts_response_bills_text_and_audio(encoding: str): wav_file.setsampwidth(2) wav_file.setframerate(24000) wav_file.writeframes(pcm_audio) + flac_buffer: Final = io.BytesIO() + soundfile.write(flac_buffer, (0.0,) * 24000, 48000, format="FLAC", subtype="PCM_16") g711_audio: Final = ( struct.pack( "<4sI4s4sIHHIIHH4sI", @@ -547,6 +551,7 @@ def test_gemini_cloud_tts_response_bills_text_and_audio(encoding: str): "OGG_OPUS": (fixture_dir / "gemini_tts_speech.ogg").read_bytes(), "ALAW": g711_audio, "MULAW": g711_audio, + "FLAC": flac_buffer.getvalue(), } audio_bytes: Final = audio_by_encoding[encoding] logger: Final = MagicMock() @@ -571,6 +576,7 @@ def test_gemini_cloud_tts_response_bills_text_and_audio(encoding: str): "Google Cloud AudioEncoding and RFC 7845, checked 2026-09-23: " "https://cloud.google.com/text-to-speech/docs/reference/rest/v1/AudioEncoding " "https://www.rfc-editor.org/rfc/rfc7845.html" + " https://www.rfc-editor.org/rfc/rfc9639.html#section-8.2, checked 2026-10-06" ) assert usage.completion_tokens == math.ceil(duration * 25), ( "Google Cloud TTS pricing, 2026-09-23: https://cloud.google.com/text-to-speech/pricing" @@ -579,7 +585,8 @@ def test_gemini_cloud_tts_response_bills_text_and_audio(encoding: str): assert usage.total_tokens == usage.prompt_tokens + usage.completion_tokens assert usage.completion_tokens_details is not None assert usage.completion_tokens_details.audio_tokens == usage.completion_tokens - with patch("litellm.llms.vertex_ai.text_to_speech.transformation.calculate_request_duration", return_value=None): + with monkeypatch.context() as dependencies: + dependencies.setitem(sys.modules, "soundfile", None) sdk_result: Final = VertexAITextToSpeechConfig().transform_text_to_speech_response(model, raw_response, logger) assert sdk_result.usage == usage diff --git a/tests/unit/proxy/proxy_server/test_routes_audio.py b/tests/unit/proxy/proxy_server/test_routes_audio.py index f4230fe33aa..fc98266f139 100644 --- a/tests/unit/proxy/proxy_server/test_routes_audio.py +++ b/tests/unit/proxy/proxy_server/test_routes_audio.py @@ -201,6 +201,9 @@ def test_audio_speech_happy_path(client, auth_as, patched_speech, path): ("audio/flac", "flac", "audio/flac"), ("audio/pcm", "pcm", "audio/pcm"), ("audio/wav", "mp3", "audio/wav"), + ("audio/wav", "alaw", "audio/wav"), + ("audio/wav", "mulaw", "audio/wav"), + ("audio/wav", "pcm16", "audio/wav"), ("application/json", "flac", "audio/flac"), (None, "wav", "audio/wav"), (None, None, "audio/mpeg"), @@ -224,6 +227,7 @@ def test_audio_speech_content_type_matches_audio_format( @pytest.mark.parametrize("path", ["/v1/audio/speech", "/audio/speech"]) +@pytest.mark.parametrize("patched_speech", (None, "application/json"), indirect=True) @pytest.mark.parametrize( "response_format,expected_media_type", [ From 95d6f4563b6dea26d8d9833abfaa5314a721e5eb Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Tue, 6 Oct 2026 09:39:21 -0500 Subject: [PATCH 19/20] fix(vertex-ai): reject unmeasurable FLAC before billing --- .../text_to_speech/transformation.py | 4 ++- .../text_to_speech/test_transformation.py | 35 +++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/litellm/llms/vertex_ai/text_to_speech/transformation.py b/litellm/llms/vertex_ai/text_to_speech/transformation.py index 9ce4d39524f..3e8484447c0 100644 --- a/litellm/llms/vertex_ai/text_to_speech/transformation.py +++ b/litellm/llms/vertex_ai/text_to_speech/transformation.py @@ -718,7 +718,9 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): container_duration: Final = calculate_request_duration(binary_data) sample_rate: Final = audio_config.get("sampleRateHertz") or 24000 duration: Final = ( - container_duration + _fallback_gemini_tts_audio_duration(binary_data, "FLAC", sample_rate) + if audio_config["audioEncoding"] == "FLAC" + else container_duration if container_duration is not None else _fallback_gemini_tts_audio_duration(binary_data, audio_config["audioEncoding"], sample_rate) ) diff --git a/tests/unit/llms/vertex_ai/text_to_speech/test_transformation.py b/tests/unit/llms/vertex_ai/text_to_speech/test_transformation.py index 4a7fb7f0fad..839705dc540 100644 --- a/tests/unit/llms/vertex_ai/text_to_speech/test_transformation.py +++ b/tests/unit/llms/vertex_ai/text_to_speech/test_transformation.py @@ -607,6 +607,41 @@ def test_gemini_cloud_tts_response_bills_text_and_audio(encoding: str, monkeypat assert cost > usage.prompt_tokens * model_info["input_cost_per_token"] +@pytest.mark.parametrize("decoder_available", (False, True)) +@pytest.mark.parametrize("invalid_streaminfo", (False, True)) +def test_gemini_cloud_tts_rejects_unbillable_flac( + monkeypatch: pytest.MonkeyPatch, decoder_available: bool, invalid_streaminfo: bool +) -> None: + output: Final = io.BytesIO() + soundfile.write(output, (0.0,) * 24000, 48000, format="FLAC", subtype="PCM_16") + audio: Final = output.getvalue() + stream_info: Final = int.from_bytes(audio[18:26], "big") & ~((1 << 36) - 1) + metadata_header: Final = b"\x01\x00\x00\x22" if invalid_streaminfo else audio[4:8] + unknown_duration_audio: Final = audio[:4] + metadata_header + audio[8:18] + stream_info.to_bytes(8, "big") + audio[26:] + raw_response: Final = httpx.Response( + 200, json={"audioContent": base64.b64encode(unknown_duration_audio).decode()} + ) + logger: Final = MagicMock( + model_call_details={ + "additional_args": { + "complete_input_dict": { + "dict_body": { + "input": {"text": "Hello"}, + "audioConfig": {"audioEncoding": "FLAC", "sampleRateHertz": 48000}, + } + } + } + } + ) + if not decoder_available: + monkeypatch.setitem(sys.modules, "soundfile", None) + + with pytest.raises(ValueError, match="Cannot determine Gemini TTS output duration"): + VertexAITextToSpeechConfig().transform_text_to_speech_response( + "gemini-3.1-flash-tts-preview", raw_response, logger + ) + + @pytest.mark.parametrize("encoding", ["ALAW", "MULAW"]) @pytest.mark.parametrize("audio", [b"\x12" * 24000, b"RIFF" + b"\x12" * 23996]) def test_gemini_cloud_tts_returns_raw_g711_audio_when_duration_decoder_is_unavailable(encoding: str, audio: bytes): From d10acc96ec85910d81fcf9dcccfcefd9dd2a60d8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:19:12 +0000 Subject: [PATCH 20/20] chore(deps): bump sharp from 0.35.4 to 0.35.5 in /ui/litellm-dashboard Bumps [sharp](https://github.com/lovell/sharp) from 0.35.4 to 0.35.5. - [Release notes](https://github.com/lovell/sharp/releases) - [Commits](https://github.com/lovell/sharp/compare/v0.35.4...v0.35.5) --- updated-dependencies: - dependency-name: sharp dependency-version: 0.35.5 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- ui/litellm-dashboard/package-lock.json | 236 ++++++++++++------------- ui/litellm-dashboard/package.json | 2 +- 2 files changed, 119 insertions(+), 119 deletions(-) diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json index 6e1211ba038..3c501f3bcbb 100644 --- a/ui/litellm-dashboard/package-lock.json +++ b/ui/litellm-dashboard/package-lock.json @@ -1499,9 +1499,9 @@ } }, "node_modules/@img/sharp-darwin-arm64": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.35.4.tgz", - "integrity": "sha512-Uhfl4V4lhP2nbUVF9+hyH1+luj86f1gUFeo8ALYxFoULoU+G87D43BfeMP8XHsk9boxAnCY/bf2EHwhA7MuGsA==", + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.35.5.tgz", + "integrity": "sha512-QRUlFQ0WxvdWyqqG/WtI3iupfD5rBzmCHXSdPsY91sAtVtTo7Q4cb6zOccZ3gqEqkr0f1As1ehLqmEpDsRf+lg==", "cpu": [ "arm64" ], @@ -1517,13 +1517,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-darwin-arm64": "1.3.3" + "@img/sharp-libvips-darwin-arm64": "1.3.4" } }, "node_modules/@img/sharp-darwin-x64": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.35.4.tgz", - "integrity": "sha512-hWniXY3bG5qKpkKrAwPe4y+VTPmf086YQAnkxWh7uA1YrlRouWGa0M0Mxj3ZjnXFkv7/TD1bTy9lGUK26vRvWw==", + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.35.5.tgz", + "integrity": "sha512-+BR255RhDlpygUpOc/Jdt1nT6DQ3XG/ERo5wbcdOf5Q320dKtPCKPLR1LJs9VGXRaMa8l1uUa0tkCNOXiAxZUw==", "cpu": [ "x64" ], @@ -1539,20 +1539,20 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-darwin-x64": "1.3.3" + "@img/sharp-libvips-darwin-x64": "1.3.4" } }, "node_modules/@img/sharp-freebsd-wasm32": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-freebsd-wasm32/-/sharp-freebsd-wasm32-0.35.4.tgz", - "integrity": "sha512-lIsKw/BU+kjB4eZjxrYrZmwOJYi3Ajrv66iAlBmUPyKc3HpnloevB1g3wxGD9P/5BbQ1brBGl65VRRrCvQDEqA==", + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-freebsd-wasm32/-/sharp-freebsd-wasm32-0.35.5.tgz", + "integrity": "sha512-Y/z91nEZ4uIBX5X3nfTovjU9lHNKFYbL2lpHCLVNmXQK03VIZvXBBt0KxbPGp2SdGSF+2mQU4e+hQaWOt86iAw==", "license": "Apache-2.0", "optional": true, "os": [ "freebsd" ], "dependencies": { - "@img/sharp-wasm32": "0.35.4" + "@img/sharp-wasm32": "0.35.5" }, "engines": { "node": ">=20.9.0" @@ -1562,9 +1562,9 @@ } }, "node_modules/@img/sharp-libvips-darwin-arm64": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.3.3.tgz", - "integrity": "sha512-suTBPTDGrI9WodccaDdwZItTSaBYASlBk1NSfElSHrUfzu3szG6lvIF58+WiFvnfzuK8ZBFS5zE00PxqxnRiPg==", + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.3.4.tgz", + "integrity": "sha512-5R89nBYiRdUlSWJxPhO+GVtaXzXSxKnRu/xqMn3KTA3L9EB9Oy/P+Nn2f2vlhPuUdy/Zusb2DarbyTpGCfEDuw==", "cpu": [ "arm64" ], @@ -1578,9 +1578,9 @@ } }, "node_modules/@img/sharp-libvips-darwin-x64": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.3.3.tgz", - "integrity": "sha512-FVJZ5mITMobmXIz/hPDTw0EintTW5H3WfrxwLqEqjiIihlu+hVRyGrFQ60xl0Lxn7Bt3zdpevPaQi0HEzqz9fw==", + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.3.4.tgz", + "integrity": "sha512-iR2OKH80yi0U+dUplyh3/xdpFvps6YkCwsXenIJxqxR1v9o+xtKTGbS9H7cps+2Vxjc8B1j96p75NmTGjIhtpQ==", "cpu": [ "x64" ], @@ -1594,9 +1594,9 @@ } }, "node_modules/@img/sharp-libvips-linux-arm": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.3.3.tgz", - "integrity": "sha512-3rbU4vqXXc3hY/OiXdl52xZvT0F1yEngWfvqudtPJg/KkyiaQw2DRsFrNzpmLvfavbwOq3qXn36GP8obHRULQA==", + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.3.4.tgz", + "integrity": "sha512-LmRtTsOHuvM2+wlO2Db37dx5MiZhB0FvSunciw48YjdOkZz9KAiRbm8ujeMOA1INqmei5NapFxYEK1D1ZSidmw==", "cpu": [ "arm" ], @@ -1613,9 +1613,9 @@ } }, "node_modules/@img/sharp-libvips-linux-arm64": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.3.3.tgz", - "integrity": "sha512-0DaL0A6Xu6sQSQFwe4iVCrKWU2cCTItnRsYsCdxAMm9NF6twAA9BKnoqy4hqz4+azQ0JHuA26qiUKsf1XJ/v5A==", + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.3.4.tgz", + "integrity": "sha512-Y3dgX/6lE2QhQb+Gxy0WZxfg9MEm/JBjamZpS2IklP7xIQoKN4hzAm7KcMVGtaVDt3neE9OKBC7vAfonA/Lr1A==", "cpu": [ "arm64" ], @@ -1632,9 +1632,9 @@ } }, "node_modules/@img/sharp-libvips-linux-ppc64": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.3.3.tgz", - "integrity": "sha512-cdn1OvUBwsXhbC0zSzJnNzf5MZ/mTrobawDvNXBTxe8VtqKAm0sRuEY2Evzovb/w9JMk4TvRxqt1mekSuJz64w==", + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.3.4.tgz", + "integrity": "sha512-Le6boB8Tai0Nis+gIxIpKx68UDVVIqdR8Tin5Yf1z2LJJQLDJvCDRqRu+jC2qCoD+eIomonmOwB4smBRxfVpYQ==", "cpu": [ "ppc64" ], @@ -1651,9 +1651,9 @@ } }, "node_modules/@img/sharp-libvips-linux-riscv64": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.3.3.tgz", - "integrity": "sha512-HjPVx7yKz+0lqdhDlTw1tt90wamBoxhiXpvl1XZpJLiHH4RCJ5yDTqH+VlYPv2fwFs89JFw4c1IexYOcQUi4IQ==", + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.3.4.tgz", + "integrity": "sha512-aHkkIEHPRdQEegJN20MLmGtxYD9R2wQr3Cwpddnu5+YKMt6Uzax7S9h5gpZTo8wyrGuZSlfQ63OevL5mTyOC7Q==", "cpu": [ "riscv64" ], @@ -1670,9 +1670,9 @@ } }, "node_modules/@img/sharp-libvips-linux-s390x": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.3.3.tgz", - "integrity": "sha512-neWLh+3yCNThxnfy3c4BbVBeGgt9aftno+XbT56iK28RgeDs3UOFWviLWlUu0bArYVYJaFDK+RRohbicUNCm8Q==", + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.3.4.tgz", + "integrity": "sha512-ra/mB6MikESDUO7Yg+Mi95bFBb9GsObURuhnOv3OqknjGe9sZrG8tCe9q0xSIGrtLgvgw0gKnFWcK4blSgQOuQ==", "cpu": [ "s390x" ], @@ -1689,9 +1689,9 @@ } }, "node_modules/@img/sharp-libvips-linux-x64": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.3.3.tgz", - "integrity": "sha512-4vKmvAst9nrowcqquKFAyZJUDolUaIp8uRiN0mWFguJ1IplC9/pitXtlnnlU4aa/eJw3J7i67V+pwUL+wZGdsA==", + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.3.4.tgz", + "integrity": "sha512-GJ//SSXbnwSDes02umB3nDJLFcQzw8a18V8fyhqr6tV515tOEMdImjjxj1AoafMRz56F3PHgftnj1QEKSU1zkw==", "cpu": [ "x64" ], @@ -1708,9 +1708,9 @@ } }, "node_modules/@img/sharp-libvips-linuxmusl-arm64": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.3.3.tgz", - "integrity": "sha512-Y9kQaLMuNoB0bPYOOdcZMaseNrFpPodIWWMrx+CZyydf2xn68j9WYc6sWWRrDwNkzCQjKYfc68L7jKjGlHMibw==", + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.3.4.tgz", + "integrity": "sha512-hvulFwtjUcagsis6BBxHwGFwWoNZjgYmULGVrZcyfNbjA8hKILbRxGg15/7w5HDyXHXUos/j6baAWqnCyQ2DWA==", "cpu": [ "arm64" ], @@ -1727,9 +1727,9 @@ } }, "node_modules/@img/sharp-libvips-linuxmusl-x64": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.3.3.tgz", - "integrity": "sha512-fj8Mv0HHfD1Rr+4I68+3agJynxDWtBFgicTbSOb9Bke6pIwzGcJ+RX/yHjmiEGFMCavY/dxvem7MyNaJF+wDiw==", + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.3.4.tgz", + "integrity": "sha512-6zXKeE/p39I1AmA3cJG35eyBGNqNddLnUXjhwBnsGjFPWqf5VKkDBEqaEkPDoTEtkxwi2vv8Tcr2mDyP4So7Fg==", "cpu": [ "x64" ], @@ -1746,9 +1746,9 @@ } }, "node_modules/@img/sharp-linux-arm": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.35.4.tgz", - "integrity": "sha512-7OAS8gI0EReKGVN2HssHlM6umJgxF5VI3xN0p9FA91p/YO+ou5hiNghLdZ5BEHztwaaK5+bLKRf8x/o2L2nk9A==", + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.35.5.tgz", + "integrity": "sha512-LEaXK2WdXVK5ykcw0buWyPMsmLLL2vpHLD6yrNSW+JGEL3BZPA4tpKN6iaMc4AxTTAoaX/sU1rOL51lcIz48ZQ==", "cpu": [ "arm" ], @@ -1767,13 +1767,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-arm": "1.3.3" + "@img/sharp-libvips-linux-arm": "1.3.4" } }, "node_modules/@img/sharp-linux-arm64": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.35.4.tgz", - "integrity": "sha512-De4jpEnAU8Hd5oT0j1G3uL4ZvTuipVMn7YC6vPaJhy6/7EwEae0SVAoBrUMYQbkLGDm85taVWwuPc1a44LTzCQ==", + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.35.5.tgz", + "integrity": "sha512-LYVx5JTsOM2CBzmxreh+nl64/3H6Xb09iSLknqH47z2T2DFFxDeFLP5y4dJwe6H7uGQlHPyEEtIqyo3DYsRwdQ==", "cpu": [ "arm64" ], @@ -1792,13 +1792,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-arm64": "1.3.3" + "@img/sharp-libvips-linux-arm64": "1.3.4" } }, "node_modules/@img/sharp-linux-ppc64": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.35.4.tgz", - "integrity": "sha512-2oYZJeIl4kCcMGk4ouZVjnkCtFrpQFlNEtJ6GbxzhHQchwH0NH/qEb9ykmOl29dqwMq+JhFdZn+1ak2FKhI9fQ==", + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.35.5.tgz", + "integrity": "sha512-QVxAAq8evVRI9ia2vqgwrmWucn5Dfv+JdWzj75pD8omHLPSP7f8p20O8jxzjCcuCEQEOtYOZUmX1hkiZ0kdevA==", "cpu": [ "ppc64" ], @@ -1817,13 +1817,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-ppc64": "1.3.3" + "@img/sharp-libvips-linux-ppc64": "1.3.4" } }, "node_modules/@img/sharp-linux-riscv64": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.35.4.tgz", - "integrity": "sha512-cPbNChoRURAWdebDIHSenxRpgEdy7JkPydSnUxRm9VvKD7m0/xVaR/8Fzlu81pk5nHEvHH87UZUA7cTtwnbJSA==", + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.35.5.tgz", + "integrity": "sha512-LtdreXguaavKODPIfzJ4kffx7UNt1omwtK0rch4EBbbSTXPnxWmYSayXdLJw0fJzQ97kHt1gL/yh4tvU+nCyRQ==", "cpu": [ "riscv64" ], @@ -1842,13 +1842,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-riscv64": "1.3.3" + "@img/sharp-libvips-linux-riscv64": "1.3.4" } }, "node_modules/@img/sharp-linux-s390x": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.35.4.tgz", - "integrity": "sha512-RY0JFY8Fd6RonCBtHz+DvadaPkXDSI1AUn6yWL9TipqkZ1vY8w8evqdgyDFnkm4/K1ve1TvZiaePP5oSd4+WVQ==", + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.35.5.tgz", + "integrity": "sha512-UZasTOFiYzotTsGOCu42BfUzP6Tu6Do/947iRm1RsLKvlllxwGcn4RN27LibGWceix4Y+Pmw3jsnTcCQIgWjqA==", "cpu": [ "s390x" ], @@ -1867,13 +1867,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-s390x": "1.3.3" + "@img/sharp-libvips-linux-s390x": "1.3.4" } }, "node_modules/@img/sharp-linux-x64": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.35.4.tgz", - "integrity": "sha512-9qvvEAuk8k89TfWUoX2htWjbAMX8p+NxCppjpcg5k6xMsjhBQPTsoIh36h9Qde4WRuGpJeYnOjdosDn/cnv+OA==", + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.35.5.tgz", + "integrity": "sha512-SxFtLTeJInhAA9Q836kux2vZNeOBQEx658qvbboZScr0wIARym3IcGmW7KpVD5sbVg0Ojy+udFQdayYIZyoNog==", "cpu": [ "x64" ], @@ -1892,13 +1892,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-x64": "1.3.3" + "@img/sharp-libvips-linux-x64": "1.3.4" } }, "node_modules/@img/sharp-linuxmusl-arm64": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.35.4.tgz", - "integrity": "sha512-KB5jxpfWQTr0nc3xdHtWChdbifHrBGsd2SM62Eyxrl8afikm+f5qGBU75SJIZBT/S1MC8XyacdlXBMSWq6OURA==", + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.35.5.tgz", + "integrity": "sha512-9HbMclmI1zlNkFRs3z9/eBtDjfD0sGlrX1z6b1qwmiFY5ElDLh4BC0LPBdVp7z1DXFiKlIcznf+ZlsuZzLxQqg==", "cpu": [ "arm64" ], @@ -1917,13 +1917,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-arm64": "1.3.3" + "@img/sharp-libvips-linuxmusl-arm64": "1.3.4" } }, "node_modules/@img/sharp-linuxmusl-x64": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.35.4.tgz", - "integrity": "sha512-f+eZJZIQNEEd26RPSW+76chwOf1XtA2Y/O+5ocVyLliHkeih3e+jhLVBdNTd2rS3IbNXK8+ug93Vf5ZXtF5Lxg==", + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.35.5.tgz", + "integrity": "sha512-4KOphqB035HrVdqLZfCgMzzERrQkkzOwRhl4OAkRO1YCldbaFjySXMaK534Mo0V+LndnlJk+sbUyLeU0ULyD1A==", "cpu": [ "x64" ], @@ -1942,13 +1942,13 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-x64": "1.3.3" + "@img/sharp-libvips-linuxmusl-x64": "1.3.4" } }, "node_modules/@img/sharp-wasm32": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.35.4.tgz", - "integrity": "sha512-zQnl4Kwp7Q6NHsENtU2T/00Zi+w3AQNwz3+UaTyVBy2FpXrzXzGjndpK61onhZjRtRpQXxCTeqw19bVyXOh7jA==", + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.35.5.tgz", + "integrity": "sha512-Ptsga1su4tQx+LLF1ECS9U6nz5kmrXKo6XVbtR48Ke3ZRxxgaWBu7IDtEe1quo8hiupwm6WFqxVlXaSf7IINGQ==", "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", "optional": true, "dependencies": { @@ -1962,16 +1962,16 @@ } }, "node_modules/@img/sharp-webcontainers-wasm32": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-webcontainers-wasm32/-/sharp-webcontainers-wasm32-0.35.4.tgz", - "integrity": "sha512-ESfNkywmCfPNyaZjxooddJQiQ+l/nTpGEOGthxiLnIHXC/CmcBixnfwUleX9mCz9ovrUUvKMap/pm8RYbzfwaA==", + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-webcontainers-wasm32/-/sharp-webcontainers-wasm32-0.35.5.tgz", + "integrity": "sha512-hfhF/FmoQyTUkA0bIKFOtw536BQSeBMe6BF6QyWlrPxT754+TFLaZ7sKKTfvvM0yJgKgaYTwnFCIZ/GuDw5SUA==", "cpu": [ "wasm32" ], "license": "Apache-2.0", "optional": true, "dependencies": { - "@img/sharp-wasm32": "0.35.4" + "@img/sharp-wasm32": "0.35.5" }, "engines": { "node": ">=20.9.0" @@ -1981,9 +1981,9 @@ } }, "node_modules/@img/sharp-win32-arm64": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.35.4.tgz", - "integrity": "sha512-iNdlBX9gLVvqe2I3uIJSIKTq6wckP/DYxZtcqxm09x5Gi24DnFBmPAWZmr60ZyYMG0xlzo6goG3670ar+RXvRw==", + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.35.5.tgz", + "integrity": "sha512-X4t7g+7ZA5DKblCBEXGjUqqemj4vczING/5viFwAL8h4N3qYeyjwdCvRLHi4EdOUI+2Z7UFlp1VM+p/AuEtm6Q==", "cpu": [ "arm64" ], @@ -2000,9 +2000,9 @@ } }, "node_modules/@img/sharp-win32-ia32": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.35.4.tgz", - "integrity": "sha512-kqRsbaa5CS6KHlpxnN7WhE6vAAugXyZButpRdvDWetlv6Qv4N9WTcrWzF7tXfB9T7MsoadqdI8hmwLq6UlLvtw==", + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.35.5.tgz", + "integrity": "sha512-5Zm82LoBc43nhwNybZlG7Y1KO//Zhsn306fQl29ZOuStHLGTo3BWL83q3cznX0poxSAMuYL1On/BHBxkBeKr6A==", "cpu": [ "ia32" ], @@ -2019,9 +2019,9 @@ } }, "node_modules/@img/sharp-win32-x64": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.35.4.tgz", - "integrity": "sha512-XtmnYhBcrORsJ4XJngyzr/EWP0hRZLAZRFaApdKuviyqF78+ylxh2y06ZmtULAMOnObJ3ucpN0AcwSWnMowTRg==", + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.35.5.tgz", + "integrity": "sha512-x76eH0vEiHlcMQu8Y8IenntaACtddpT6W0wmXtWrnKcnKI7ME5DdgqhAD6SEWOEl1v2zDvkZDhFA9KnURwpfqg==", "cpu": [ "x64" ], @@ -11241,9 +11241,9 @@ } }, "node_modules/sharp": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.35.4.tgz", - "integrity": "sha512-n++8XWcj+jCOr2IOl7h8LbKnGBDY4aPbmprMONBNFdn0ImXqpGVv5zliDs0V9HbmbCQLpbuo2ej9rAoOQTvMDA==", + "version": "0.35.5", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.35.5.tgz", + "integrity": "sha512-Ywn4OnzGukp7CDMrp08RQ50YKmuwG47brZgIVPTvBaaAfQlRlygrRqSrxdCiL9M+LlzLBiJ68IR1QqvzHyjC7g==", "license": "Apache-2.0", "optional": true, "dependencies": { @@ -11258,31 +11258,31 @@ "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-darwin-arm64": "0.35.4", - "@img/sharp-darwin-x64": "0.35.4", - "@img/sharp-freebsd-wasm32": "0.35.4", - "@img/sharp-libvips-darwin-arm64": "1.3.3", - "@img/sharp-libvips-darwin-x64": "1.3.3", - "@img/sharp-libvips-linux-arm": "1.3.3", - "@img/sharp-libvips-linux-arm64": "1.3.3", - "@img/sharp-libvips-linux-ppc64": "1.3.3", - "@img/sharp-libvips-linux-riscv64": "1.3.3", - "@img/sharp-libvips-linux-s390x": "1.3.3", - "@img/sharp-libvips-linux-x64": "1.3.3", - "@img/sharp-libvips-linuxmusl-arm64": "1.3.3", - "@img/sharp-libvips-linuxmusl-x64": "1.3.3", - "@img/sharp-linux-arm": "0.35.4", - "@img/sharp-linux-arm64": "0.35.4", - "@img/sharp-linux-ppc64": "0.35.4", - "@img/sharp-linux-riscv64": "0.35.4", - "@img/sharp-linux-s390x": "0.35.4", - "@img/sharp-linux-x64": "0.35.4", - "@img/sharp-linuxmusl-arm64": "0.35.4", - "@img/sharp-linuxmusl-x64": "0.35.4", - "@img/sharp-webcontainers-wasm32": "0.35.4", - "@img/sharp-win32-arm64": "0.35.4", - "@img/sharp-win32-ia32": "0.35.4", - "@img/sharp-win32-x64": "0.35.4" + "@img/sharp-darwin-arm64": "0.35.5", + "@img/sharp-darwin-x64": "0.35.5", + "@img/sharp-freebsd-wasm32": "0.35.5", + "@img/sharp-libvips-darwin-arm64": "1.3.4", + "@img/sharp-libvips-darwin-x64": "1.3.4", + "@img/sharp-libvips-linux-arm": "1.3.4", + "@img/sharp-libvips-linux-arm64": "1.3.4", + "@img/sharp-libvips-linux-ppc64": "1.3.4", + "@img/sharp-libvips-linux-riscv64": "1.3.4", + "@img/sharp-libvips-linux-s390x": "1.3.4", + "@img/sharp-libvips-linux-x64": "1.3.4", + "@img/sharp-libvips-linuxmusl-arm64": "1.3.4", + "@img/sharp-libvips-linuxmusl-x64": "1.3.4", + "@img/sharp-linux-arm": "0.35.5", + "@img/sharp-linux-arm64": "0.35.5", + "@img/sharp-linux-ppc64": "0.35.5", + "@img/sharp-linux-riscv64": "0.35.5", + "@img/sharp-linux-s390x": "0.35.5", + "@img/sharp-linux-x64": "0.35.5", + "@img/sharp-linuxmusl-arm64": "0.35.5", + "@img/sharp-linuxmusl-x64": "0.35.5", + "@img/sharp-webcontainers-wasm32": "0.35.5", + "@img/sharp-win32-arm64": "0.35.5", + "@img/sharp-win32-ia32": "0.35.5", + "@img/sharp-win32-x64": "0.35.5" }, "peerDependenciesMeta": { "@types/node": { diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index a6ec2a9a65e..5e989fd608a 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -119,7 +119,7 @@ "axios": "1.13.6", "postcss": "8.5.23", "esbuild": "0.28.1", - "sharp": "^0.35.4" + "sharp": "^0.35.5" }, "engines": { "node": ">=24.14.1",