From b96844dd0c23a084108191df7ff37423a40f862f Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Wed, 15 Jul 2026 19:56:53 -0500 Subject: [PATCH] feat(vertex): expose Lyria through audio speech --- litellm/cost_calculator.py | 12 +- .../text_to_speech/transformation.py | 160 ++++++++++ litellm/main.py | 6 +- ...odel_prices_and_context_window_backup.json | 15 +- .../vertex_passthrough_logging_handler.py | 12 +- litellm/proxy/proxy_server.py | 11 +- litellm/types/utils.py | 4 + litellm/utils.py | 6 + model_prices_and_context_window.json | 15 +- ...test_vertex_passthrough_logging_handler.py | 33 ++ .../text_to_speech/test_transformation.py | 298 +++++++++++++++++- tests/test_litellm/test_cost_calculator.py | 18 ++ tests/test_litellm/test_utils.py | 15 +- 13 files changed, 565 insertions(+), 40 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index b83e9b395a8..2bac5e234bc 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -496,9 +496,19 @@ def cost_per_token( # see this https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models if call_type == "speech" or call_type == "aspeech": speech_model_info = litellm.get_model_info(model=model_without_prefix, custom_llm_provider=custom_llm_provider) - cost_metric: Final = select_cost_metric_for_model(speech_model_info) prompt_cost: float = 0.0 completion_cost: float = 0.0 + if not speech_model_info.get("input_cost_per_character") and not speech_model_info.get( + "input_cost_per_token" + ): + output_cost_per_generation: Final = speech_model_info.get("output_cost_per_image") + output_cost_per_second: Final = speech_model_info.get("output_cost_per_second") + audio_seconds_per_prediction: Final = speech_model_info.get("audio_seconds_per_prediction") + if output_cost_per_generation is not None: + return prompt_cost, float(output_cost_per_generation) + if output_cost_per_second is not None and audio_seconds_per_prediction is not None: + return prompt_cost, float(output_cost_per_second) * float(audio_seconds_per_prediction) + cost_metric: Final = select_cost_metric_for_model(speech_model_info) if cost_metric == "cost_per_character": if prompt_characters is None: raise ValueError( diff --git a/litellm/llms/vertex_ai/text_to_speech/transformation.py b/litellm/llms/vertex_ai/text_to_speech/transformation.py index 332f892ae6b..ad39aa35f8c 100644 --- a/litellm/llms/vertex_ai/text_to_speech/transformation.py +++ b/litellm/llms/vertex_ai/text_to_speech/transformation.py @@ -12,6 +12,8 @@ from typing import TYPE_CHECKING, Any, Final, Union import httpx +import litellm +from litellm.exceptions import UnsupportedParamsError from litellm.litellm_core_utils.audio_utils.utils import ( speech_media_type_from_audio_bytes, ) @@ -471,3 +473,161 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): # Initialize the HttpxBinaryResponseContent instance return HttpxBinaryResponseContent(response) + + +class VertexAILyriaTextToSpeechConfig(VertexAITextToSpeechConfig): + LYRIA_MODELS = { + "lyria-002", + "lyria-3-clip-preview", + "lyria-3-pro-preview", + } + + @classmethod + def is_lyria_model(cls, model: str) -> bool: + return model.removeprefix("vertex_ai/") in cls.LYRIA_MODELS + + def get_supported_openai_params(self, model: str) -> list: + return ["response_format"] + + def map_openai_params( + self, + model: str, + optional_params: dict, + voice: str | dict | None = None, + drop_params: bool = False, + kwargs: dict = {}, + ) -> tuple[str | None, dict]: + mapped_params = dict(optional_params) + base_model = model.removeprefix("vertex_ai/") + unsupported_params = [param for param in ("speed", "instructions") if mapped_params.get(param) is not None] + if unsupported_params: + if drop_params or litellm.drop_params: + for param in unsupported_params: + mapped_params.pop(param, None) + else: + raise UnsupportedParamsError( + status_code=400, + message=( + f"Vertex AI {base_model} does not support the OpenAI parameters: " + f"{', '.join(unsupported_params)}. To drop unsupported openai params " + "from the call, set `litellm.drop_params = True`" + ), + ) + response_format = mapped_params.get("response_format") + supported_formats = ( + {"wav"} if base_model == "lyria-002" else {"mp3", "wav"} if base_model == "lyria-3-pro-preview" else {"mp3"} + ) + if response_format is not None and response_format not in supported_formats: + if drop_params or litellm.drop_params: + mapped_params.pop("response_format", None) + else: + raise UnsupportedParamsError( + status_code=400, + message=( + f"Vertex AI {base_model} does not support response_format={response_format!r}. " + f"Supported values: {', '.join(sorted(supported_formats))}. " + "To drop unsupported openai params from the call, set `litellm.drop_params = True`" + ), + ) + return voice if isinstance(voice, str) else None, mapped_params + + def get_complete_url( + self, + model: str, + api_base: str | None, + litellm_params: dict, + ) -> str: + base_model = model.removeprefix("vertex_ai/") + project = self.safe_get_vertex_ai_project(litellm_params) + if project is None: + _, project = self._ensure_access_token( + credentials=self.safe_get_vertex_ai_credentials(litellm_params), + project_id=None, + custom_llm_provider="vertex_ai", + ) + if base_model.startswith("lyria-3-"): + from litellm.llms.vertex_ai.interactions.transformation import ( + VertexAIInteractionsConfig, + ) + + return VertexAIInteractionsConfig().get_complete_url( + api_base=api_base, + model=base_model, + litellm_params={**litellm_params, "vertex_project": project}, + ) + location = self.safe_get_vertex_ai_location(litellm_params) or self.get_default_vertex_location() + base_url = self.get_api_base(api_base=api_base, vertex_location=location).rstrip("/") + return f"{base_url}/v1/projects/{project}/locations/{location}/publishers/google/models/{base_model}:predict" + + def transform_text_to_speech_request( + self, + model: str, + input: str, + voice: str | None, + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> TextToSpeechRequestData: + access_token, project = self._ensure_access_token( + credentials=self.safe_get_vertex_ai_credentials(litellm_params), + project_id=self.safe_get_vertex_ai_project(litellm_params), + custom_llm_provider="vertex_ai", + ) + headers.update( + { + "Authorization": f"Bearer {access_token}", + "x-goog-user-project": project, + "Content-Type": "application/json", + } + ) + base_model = model.removeprefix("vertex_ai/") + if base_model == "lyria-002": + request_body = { + "instances": [{"prompt": input}], + "parameters": {"sample_count": 1}, + } + else: + request_body = {"model": base_model, "input": input} + if optional_params.get("response_format") == "wav": + request_body["response_format"] = { + "type": "audio", + "mime_type": "audio/wav", + } + return TextToSpeechRequestData(dict_body=request_body, headers=headers) + + def transform_text_to_speech_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: "LiteLLMLoggingObj", + ) -> "HttpxBinaryResponseContent": + from litellm.types.llms.openai import HttpxBinaryResponseContent + + response_json = raw_response.json() + base_model = model.removeprefix("vertex_ai/") + audio_data: str | None = None + mime_type: str | None = None + if base_model == "lyria-002": + predictions = response_json.get("predictions") or [] + if predictions: + audio_data = predictions[0].get("audioContent") or predictions[0].get("bytesBase64Encoded") + mime_type = predictions[0].get("mimeType") + else: + for step in response_json.get("steps") or response_json.get("outputs") or []: + content_items = step.get("content") or [] if step.get("type") == "model_output" else [step] + for content in content_items: + if content.get("type") == "audio" and content.get("data"): + audio_data = content["data"] + mime_type = content.get("mime_type") + if audio_data is None: + raise ValueError(f"No generated audio found in Vertex AI {base_model} response") + mime_type = mime_type or ("audio/wav" if base_model == "lyria-002" else "audio/mpeg") + response = HttpxBinaryResponseContent( + httpx.Response( + status_code=raw_response.status_code, + content=base64.b64decode(audio_data), + headers={"content-type": mime_type}, + ) + ) + response._hidden_params = {"audio_mime_type": mime_type} + return response diff --git a/litellm/main.py b/litellm/main.py index 0128e4defe5..55db92d44b3 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -8235,6 +8235,7 @@ def speech( ) elif custom_llm_provider == "vertex_ai" or custom_llm_provider == "vertex_ai_beta": from litellm.llms.vertex_ai.text_to_speech.transformation import ( + VertexAILyriaTextToSpeechConfig, VertexAITextToSpeechConfig, ) @@ -8259,7 +8260,10 @@ def speech( # Vertex AI Text-to-Speech (Google Cloud TTS) if text_to_speech_provider_config is None: - text_to_speech_provider_config = VertexAITextToSpeechConfig() + if VertexAILyriaTextToSpeechConfig.is_lyria_model(model): + text_to_speech_provider_config = VertexAILyriaTextToSpeechConfig() + else: + text_to_speech_provider_config = VertexAITextToSpeechConfig() # Cast to specific Vertex AI config type to access dispatch method vertex_config: Final = cast(VertexAITextToSpeechConfig, text_to_speech_provider_config) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 1a1d92805e2..2955a0ffa4e 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -45516,9 +45516,12 @@ "litellm_provider": "vertex_ai", "max_audio_length_hours": 0.009111111111111111, "max_audio_per_prompt": 4, - "mode": "chat", + "mode": "audio_speech", "output_cost_per_second": 0.002, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria", + "supported_endpoints": [ + "/v1/audio/speech" + ], "supported_modalities": [ "text" ], @@ -45533,12 +45536,13 @@ "max_input_tokens": 131072, "max_output_tokens": 8192, "max_tokens": 8192, - "mode": "chat", + "mode": "audio_speech", "output_cost_per_image": 0.04, "output_cost_per_token": 0, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria", "supported_endpoints": [ - "/v1beta/interactions" + "/v1beta/interactions", + "/v1/audio/speech" ], "supported_modalities": [ "text", @@ -45566,12 +45570,13 @@ "max_input_tokens": 131072, "max_output_tokens": 8192, "max_tokens": 8192, - "mode": "chat", + "mode": "audio_speech", "output_cost_per_image": 0.08, "output_cost_per_token": 0, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria", "supported_endpoints": [ - "/v1beta/interactions" + "/v1beta/interactions", + "/v1/audio/speech" ], "supported_modalities": [ "text", diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py index 6b2d7763fa0..ab3b24f470f 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py @@ -368,14 +368,8 @@ class VertexPassthroughLoggingHandler: @staticmethod def _is_audio_predict_response(model: str, json_response: dict) -> bool: return ( - VertexPassthroughLoggingHandler._get_audio_prediction_count( - json_response=json_response - ) - > 0 - and VertexPassthroughLoggingHandler._get_audio_prediction_unit_cost( - model=model - ) - is not None + VertexPassthroughLoggingHandler._get_audio_prediction_count(json_response=json_response) > 0 + and VertexPassthroughLoggingHandler._get_audio_prediction_unit_cost(model=model) is not None ) @staticmethod @@ -397,7 +391,7 @@ class VertexPassthroughLoggingHandler: return sum( 1 for prediction in predictions - if isinstance(prediction, dict) and prediction.get("audioContent") + if isinstance(prediction, dict) and (prediction.get("audioContent") or prediction.get("bytesBase64Encoded")) ) @staticmethod diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 27132c90e05..f11cbc92224 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -11176,9 +11176,14 @@ async def audio_speech( upstream_content_type: Final = ( response.response.headers.get("content-type") if isinstance(response, HttpxBinaryResponseContent) else None ) - media_type: Final = resolve_speech_media_type( - upstream_content_type=upstream_content_type, - response_format=requested_format if isinstance(requested_format, str) else None, + hidden_audio_mime_type: Final = hidden_params.get("audio_mime_type") + media_type: Final = ( + hidden_audio_mime_type + if isinstance(hidden_audio_mime_type, str) + else resolve_speech_media_type( + upstream_content_type=upstream_content_type, + response_format=requested_format if isinstance(requested_format, str) else None, + ) ) return StreamingResponse( diff --git a/litellm/types/utils.py b/litellm/types/utils.py index ee6f09e05dc..d2a09639362 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -310,6 +310,9 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): output_cost_per_video_per_second: float | None # only for vertex ai models output_cost_per_audio_per_second: float | None # only for vertex ai models output_cost_per_second: float | None # for OpenAI Speech models + audio_seconds_per_prediction: float | None + max_audio_length_hours: float | None + max_audio_per_prompt: int | None output_cost_per_second_1080p: ( float | None ) # video_generation tier: key output_cost_per_second_ (e.g. 1080p, 720p) @@ -333,6 +336,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): "image_generation", "chat", "audio_transcription", + "audio_speech", "responses", "ocr", "realtime", diff --git a/litellm/utils.py b/litellm/utils.py index ba456fc353b..7c8974906ed 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5880,6 +5880,9 @@ def _get_model_info_helper( "output_cost_per_token_above_512k_tokens", None ), output_cost_per_second=_model_info.get("output_cost_per_second", None), + audio_seconds_per_prediction=_model_info.get("audio_seconds_per_prediction", None), + max_audio_length_hours=_model_info.get("max_audio_length_hours", None), + max_audio_per_prompt=_model_info.get("max_audio_per_prompt", None), output_cost_per_second_1080p=_model_info.get("output_cost_per_second_1080p", None), output_cost_per_second_480p=_model_info.get("output_cost_per_second_480p", None), output_cost_per_second_4k=_model_info.get("output_cost_per_second_4k", None), @@ -9415,9 +9418,12 @@ class ProviderConfigManager: # mapping would drop response_format before the bridge sees it (LIT-6501) return None from litellm.llms.vertex_ai.text_to_speech.transformation import ( + VertexAILyriaTextToSpeechConfig, VertexAITextToSpeechConfig, ) + if VertexAILyriaTextToSpeechConfig.is_lyria_model(model): + return VertexAILyriaTextToSpeechConfig() return VertexAITextToSpeechConfig() elif litellm.LlmProviders.MINIMAX == provider: from litellm.llms.minimax.text_to_speech.transformation import ( diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 1a1d92805e2..2955a0ffa4e 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -45516,9 +45516,12 @@ "litellm_provider": "vertex_ai", "max_audio_length_hours": 0.009111111111111111, "max_audio_per_prompt": 4, - "mode": "chat", + "mode": "audio_speech", "output_cost_per_second": 0.002, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria", + "supported_endpoints": [ + "/v1/audio/speech" + ], "supported_modalities": [ "text" ], @@ -45533,12 +45536,13 @@ "max_input_tokens": 131072, "max_output_tokens": 8192, "max_tokens": 8192, - "mode": "chat", + "mode": "audio_speech", "output_cost_per_image": 0.04, "output_cost_per_token": 0, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria", "supported_endpoints": [ - "/v1beta/interactions" + "/v1beta/interactions", + "/v1/audio/speech" ], "supported_modalities": [ "text", @@ -45566,12 +45570,13 @@ "max_input_tokens": 131072, "max_output_tokens": 8192, "max_tokens": 8192, - "mode": "chat", + "mode": "audio_speech", "output_cost_per_image": 0.08, "output_cost_per_token": 0, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria", "supported_endpoints": [ - "/v1beta/interactions" + "/v1beta/interactions", + "/v1/audio/speech" ], "supported_modalities": [ "text", diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py b/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py index 9f65fa09b1b..ccce19f1634 100644 --- a/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py @@ -110,3 +110,36 @@ def test_audio_predict_response_uses_model_map_metadata( assert result["kwargs"]["model"] == "music-audio-preview" assert result["kwargs"]["response_cost"] == pytest.approx(6.0) assert logging_obj.model_call_details["response_cost"] == pytest.approx(6.0) + + +def test_audio_predict_response_supports_bytes_base64_encoded( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setitem( + litellm.model_cost, + "vertex_ai/lyria-002", + { + "audio_seconds_per_prediction": 30, + "output_cost_per_second": 0.002, + }, + ) + logging_obj = MagicMock() + logging_obj.model_call_details = {} + response = httpx.Response( + status_code=200, + json={"predictions": [{"bytesBase64Encoded": "clip"}]}, + ) + + result = VertexPassthroughLoggingHandler.vertex_passthrough_handler( + httpx_response=response, + logging_obj=logging_obj, + url_route="/v1/projects/test/locations/us-central1/publishers/google/models/lyria-002:predict", + result=response.text, + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + request_body={"instances": [{"prompt": "ambient piano"}]}, + ) + + assert result["kwargs"]["response_cost"] == pytest.approx(0.06) + assert logging_obj.model_call_details["response_cost"] == pytest.approx(0.06) diff --git a/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py b/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py index fba337b5f2c..910bc977a79 100644 --- a/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py @@ -4,11 +4,13 @@ from unittest.mock import MagicMock, Mock, patch import httpx import pytest - import litellm from litellm.llms.vertex_ai.text_to_speech.transformation import ( + VertexAILyriaTextToSpeechConfig, VertexAITextToSpeechConfig, ) +from litellm.types.utils import LlmProviders +from litellm.utils import ProviderConfigManager class TestVertexAITextToSpeechConfig: @@ -41,9 +43,7 @@ class TestVertexAITextToSpeechConfig: @patch.object(VertexAITextToSpeechConfig, "_ensure_access_token") @patch.object(VertexAITextToSpeechConfig, "_get_token_and_url") - def test_transform_text_to_speech_request_body( - self, mock_get_token, mock_ensure_token - ): + def test_transform_text_to_speech_request_body(self, mock_get_token, mock_ensure_token): """Test that transform_text_to_speech_request generates correct request body""" # Mock authentication mock_ensure_token.return_value = ("mock-token", "test-project") @@ -104,9 +104,7 @@ class TestVertexAITextToSpeechConfig: config = VertexAITextToSpeechConfig() # Test with a Chirp3 HD voice - voice_str, voice_dict = config._map_voice_to_vertex_format( - "en-US-Chirp3-HD-Charon" - ) + voice_str, voice_dict = config._map_voice_to_vertex_format("en-US-Chirp3-HD-Charon") assert voice_str == "en-US-Chirp3-HD-Charon" assert voice_dict is not None @@ -169,6 +167,284 @@ def test_transform_text_to_speech_response_leaves_unknown_bytes_unlabeled(): assert result.response.content == raw_pcm +class TestVertexAILyriaTextToSpeechConfig: + @pytest.mark.parametrize( + "model", + ["lyria-002", "vertex_ai/lyria-3-clip-preview", "lyria-3-pro-preview"], + ) + def test_provider_config_manager_selects_lyria_config(self, model): + config = ProviderConfigManager.get_provider_text_to_speech_config( + model=model, + provider=LlmProviders.VERTEX_AI, + ) + + assert isinstance(config, VertexAILyriaTextToSpeechConfig) + + def test_get_complete_url_for_lyria_2(self): + config = VertexAILyriaTextToSpeechConfig() + + url = config.get_complete_url( + model="lyria-002", + api_base=None, + litellm_params={ + "vertex_project": "music-project", + "vertex_location": "europe-west4", + }, + ) + + assert url == ( + "https://europe-west4-aiplatform.googleapis.com/v1/projects/music-project/" + "locations/europe-west4/publishers/google/models/lyria-002:predict" + ) + + def test_get_complete_url_for_lyria_3(self): + config = VertexAILyriaTextToSpeechConfig() + + url = config.get_complete_url( + model="lyria-3-pro-preview", + api_base=None, + litellm_params={"vertex_project": "music-project"}, + ) + + assert url == ("https://aiplatform.googleapis.com/v1beta1/projects/music-project/locations/global/interactions") + + @pytest.mark.parametrize( + ("model", "response_format", "expected_body"), + [ + ( + "lyria-002", + "wav", + { + "instances": [{"prompt": "A bright synth track"}], + "parameters": {"sample_count": 1}, + }, + ), + ( + "lyria-3-clip-preview", + "mp3", + { + "model": "lyria-3-clip-preview", + "input": "A bright synth track", + }, + ), + ( + "lyria-3-pro-preview", + "wav", + { + "model": "lyria-3-pro-preview", + "input": "A bright synth track", + "response_format": { + "type": "audio", + "mime_type": "audio/wav", + }, + }, + ), + ], + ) + @patch.object(VertexAILyriaTextToSpeechConfig, "_ensure_access_token") + def test_transform_request( + self, + mock_ensure_token, + model, + response_format, + expected_body, + ): + mock_ensure_token.return_value = ("mock-token", "music-project") + config = VertexAILyriaTextToSpeechConfig() + + request = config.transform_text_to_speech_request( + model=model, + input="A bright synth track", + voice="alloy", + optional_params={"response_format": response_format}, + litellm_params={"vertex_project": "music-project"}, + headers={}, + ) + + assert request["dict_body"] == expected_body + assert request["headers"]["Authorization"] == "Bearer mock-token" + assert request["headers"]["x-goog-user-project"] == "music-project" + + @pytest.mark.parametrize( + ("model", "response_json", "expected_audio", "expected_mime_type"), + [ + ( + "lyria-002", + { + "predictions": [ + { + "bytesBase64Encoded": "bHlyaWEtMi1hdWRpbw==", + } + ] + }, + b"lyria-2-audio", + "audio/wav", + ), + ( + "lyria-3-pro-preview", + { + "steps": [ + { + "type": "model_output", + "content": [ + {"type": "text", "text": "Generated lyrics"}, + { + "type": "audio", + "data": "bHlyaWEtMy1hdWRpbw==", + "mime_type": "audio/mpeg", + }, + ], + } + ] + }, + b"lyria-3-audio", + "audio/mpeg", + ), + ( + "lyria-3-clip-preview", + { + "outputs": [ + {"type": "text", "text": "Generated lyrics"}, + { + "type": "audio", + "data": "bHlyaWEtMy1hdWRpbw==", + "mime_type": "audio/mpeg", + }, + ] + }, + b"lyria-3-audio", + "audio/mpeg", + ), + ], + ) + def test_transform_response( + self, + model, + response_json, + expected_audio, + expected_mime_type, + ): + config = VertexAILyriaTextToSpeechConfig() + raw_response = httpx.Response(200, json=response_json) + + response = config.transform_text_to_speech_response( + model=model, + raw_response=raw_response, + logging_obj=MagicMock(), + ) + + assert response.content == expected_audio + assert response._hidden_params["audio_mime_type"] == expected_mime_type + + @pytest.mark.parametrize( + ("model", "response_format"), + [ + ("lyria-002", "mp3"), + ("lyria-3-clip-preview", "wav"), + ("lyria-3-pro-preview", "opus"), + ], + ) + def test_rejects_unsupported_response_format(self, model, response_format): + config = VertexAILyriaTextToSpeechConfig() + + with pytest.raises(litellm.UnsupportedParamsError): + config.map_openai_params( + model=model, + optional_params={"response_format": response_format}, + ) + + @pytest.mark.parametrize("param", ["speed", "instructions"]) + def test_rejects_unsupported_openai_params(self, param): + config = VertexAILyriaTextToSpeechConfig() + + with pytest.raises(litellm.UnsupportedParamsError): + config.map_openai_params( + model="lyria-3-pro-preview", + optional_params={param: "unsupported"}, + ) + + @pytest.mark.parametrize( + ("model", "response_format", "response_json", "expected_url", "expected_body"), + [ + ( + "lyria-002", + "wav", + { + "predictions": [ + { + "audioContent": "bHlyaWEtMi1hdWRpbw==", + "mimeType": "audio/wav", + } + ] + }, + "https://us-central1-aiplatform.googleapis.com/v1/projects/music-project/locations/us-central1/publishers/google/models/lyria-002:predict", + { + "instances": [{"prompt": "A bright synth track"}], + "parameters": {"sample_count": 1}, + }, + ), + ( + "lyria-3-pro-preview", + "mp3", + { + "steps": [ + { + "type": "model_output", + "content": [ + { + "type": "audio", + "data": "bHlyaWEtMy1hdWRpbw==", + "mime_type": "audio/mpeg", + } + ], + } + ] + }, + "https://aiplatform.googleapis.com/v1beta1/projects/music-project/locations/global/interactions", + { + "model": "lyria-3-pro-preview", + "input": "A bright synth track", + }, + ), + ], + ) + def test_litellm_speech_dispatches_to_lyria_api( + self, + model, + response_format, + response_json, + expected_url, + expected_body, + ): + mock_response = Mock(spec=httpx.Response) + mock_response.status_code = 200 + mock_response.json.return_value = response_json + with ( + patch.object( + VertexAILyriaTextToSpeechConfig, + "_ensure_access_token", + return_value=("mock-token", "music-project"), + ), + patch( + "litellm.llms.custom_httpx.llm_http_handler.HTTPHandler.post", + return_value=mock_response, + ) as mock_post, + ): + response = litellm.speech( + model=f"vertex_ai/{model}", + input="A bright synth track", + voice="alloy", + response_format=response_format, + vertex_project="music-project", + vertex_location="us-central1", + ) + + assert response.content in {b"lyria-2-audio", b"lyria-3-audio"} + mock_post.assert_called_once() + assert mock_post.call_args.kwargs["url"] == expected_url + assert mock_post.call_args.kwargs["json"] == expected_body + + @patch("litellm.llms.custom_httpx.llm_http_handler.HTTPHandler.post") @patch.object(VertexAITextToSpeechConfig, "_ensure_access_token") @patch.object(VertexAITextToSpeechConfig, "_get_token_and_url") @@ -182,9 +458,7 @@ def test_litellm_speech_vertex_ai_chirp(mock_get_token, mock_ensure_token, mock_ # Mock HTTP response mock_response = Mock(spec=httpx.Response) - mock_response.content = ( - b'{"audioContent": "SGVsbG8gV29ybGQ="}' # base64 encoded "Hello World" - ) + mock_response.content = b'{"audioContent": "SGVsbG8gV29ybGQ="}' # base64 encoded "Hello World" mock_response.status_code = 200 mock_response.headers = {"content-type": "application/json"} mock_response.json.return_value = {"audioContent": "SGVsbG8gV29ybGQ="} @@ -203,9 +477,7 @@ def test_litellm_speech_vertex_ai_chirp(mock_get_token, mock_ensure_token, mock_ call_kwargs = mock_post.call_args.kwargs # Verify the URL is the Google Cloud TTS API - assert ( - call_kwargs["url"] == "https://texttospeech.googleapis.com/v1/text:synthesize" - ) + assert call_kwargs["url"] == "https://texttospeech.googleapis.com/v1/text:synthesize" # Verify request body structure assert "json" in call_kwargs diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 7c2174018e8..ffc71c76d03 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -146,6 +146,24 @@ def test_cost_calculator_with_response_cost_in_additional_headers(): assert result == 1000 +@pytest.mark.parametrize( + ("model", "expected_cost"), + [ + ("vertex_ai/lyria-002", 0.06), + ("vertex_ai/lyria-3-clip-preview", 0.04), + ("vertex_ai/lyria-3-pro-preview", 0.08), + ], +) +def test_vertex_lyria_speech_cost(model, expected_cost, _local_model_cost_map): + cost = completion_cost( + model=model, + prompt="A bright synth track", + call_type="speech", + ) + + assert cost == pytest.approx(expected_cost) + + def test_baseten_model_api_pricing_entries(_local_model_cost_map): expected_pricing = { diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 7bec649099a..1b0b27032ce 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -2855,15 +2855,25 @@ def test_vertex_ai_lyria_models_in_cost_map(): assert lyria_2["litellm_provider"] == "vertex_ai" assert clip["litellm_provider"] == "vertex_ai" assert pro["litellm_provider"] == "vertex_ai" + assert lyria_2["mode"] == "audio_speech" + assert clip["mode"] == "audio_speech" + assert pro["mode"] == "audio_speech" assert lyria_2["audio_seconds_per_prediction"] == 30 assert lyria_2["output_cost_per_second"] == 0.002 assert lyria_2["supported_modalities"] == ["text"] assert lyria_2["supported_output_modalities"] == ["audio"] assert lyria_2["supports_audio_output"] is True + assert lyria_2["supported_endpoints"] == ["/v1/audio/speech"] assert clip["output_cost_per_image"] == 0.04 assert pro["output_cost_per_image"] == 0.08 - assert clip["supported_endpoints"] == ["/v1beta/interactions"] - assert pro["supported_endpoints"] == ["/v1beta/interactions"] + assert clip["supported_endpoints"] == [ + "/v1beta/interactions", + "/v1/audio/speech", + ] + assert pro["supported_endpoints"] == [ + "/v1beta/interactions", + "/v1/audio/speech", + ] assert clip["supported_modalities"] == ["text", "image"] assert pro["supported_modalities"] == ["text", "image"] assert clip["supported_regions"] == ["global"] @@ -2873,7 +2883,6 @@ def test_vertex_ai_lyria_models_in_cost_map(): assert clip["supports_image_input"] is True assert pro["supports_image_input"] is True - def test_model_info_for_fireworks_short_form_models(): """ Test that fireworks_ai short-form model entries (fireworks_ai/)