diff --git a/ci_cd/generate_model_prices_schema.py b/ci_cd/generate_model_prices_schema.py index 57cc742d5c4..ab29b70bdd4 100644 --- a/ci_cd/generate_model_prices_schema.py +++ b/ci_cd/generate_model_prices_schema.py @@ -58,6 +58,11 @@ OBJECT_KEYS: dict[str, JsonSchema] = { } ARRAY_KEYS: dict[str, JsonSchema] = { + "supported_audio_formats": { + "type": "array", + "description": "Audio container formats the model can return.", + "items": {"type": "string", "enum": ["mp3", "wav"]}, + }, "supported_endpoints": { "type": "array", "description": "OpenAI-style API routes this model can be called through, e.g. /v1/chat/completions.", @@ -231,6 +236,10 @@ def string_key_schemas(modes: tuple) -> dict[str, JsonSchema]: }, "comment": STRING, "audio_transcription_config": STRING, + "vertex_ai_audio_api": { + "type": "string", + "enum": ["lyria_predict", "lyria_interactions"], + }, } diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index cb7da32f857..9a9d2ceda03 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -81,6 +81,7 @@ from litellm.llms.together_ai.cost_calculator import ( get_model_params_and_category, has_together_registry_pricing, ) +from litellm.llms.vertex_ai.common_utils import get_vertex_ai_lyria_generation_cost from litellm.llms.vertex_ai.cost_calculator import ( cost_per_character as google_cost_per_character, ) @@ -496,6 +497,13 @@ def cost_per_token( # see this https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models if call_type == "speech" or call_type == "aspeech": + lyria_generation_cost: Final = ( + get_vertex_ai_lyria_generation_cost(model=model_without_prefix) + if custom_llm_provider in ("vertex_ai", "vertex_ai_beta") + else None + ) + if lyria_generation_cost is not None: + return 0.0, lyria_generation_cost speech_model_info = litellm.get_model_info(model=model_without_prefix, custom_llm_provider=custom_llm_provider) cost_metric: Final = select_cost_metric_for_model(speech_model_info) prompt_cost: float = 0.0 diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index 970759479fe..fe2e0ab6c06 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -1,9 +1,14 @@ import re +from collections.abc import Mapping from copy import deepcopy from enum import Enum +from functools import lru_cache +from types import MappingProxyType from typing import Any, Final, Literal, cast, get_type_hints import httpx +from pydantic import TypeAdapter, ValidationError +from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger @@ -21,6 +26,61 @@ from litellm.types.utils import TokenCountResponse from litellm.utils import supports_response_schema, supports_system_messages +class VertexAILyriaModelInfo(TypedDict): + vertex_ai_audio_api: ReadOnly[Literal["lyria_predict", "lyria_interactions"]] + supported_audio_formats: ReadOnly[tuple[Literal["mp3", "wav"], ...]] + output_cost_per_image: NotRequired[ReadOnly[float]] + + +_VERTEX_AI_LYRIA_MODEL_INFO_ADAPTER: Final = TypeAdapter(VertexAILyriaModelInfo) + + +def _validate_vertex_ai_lyria_model_info(raw_model_info: object) -> VertexAILyriaModelInfo | None: + if raw_model_info is None: + return None + try: + return _VERTEX_AI_LYRIA_MODEL_INFO_ADAPTER.validate_python(raw_model_info) + except ValidationError: + return None + + +@lru_cache(maxsize=1) +def _bundled_vertex_ai_lyria_model_infos() -> Mapping[str, VertexAILyriaModelInfo]: + from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap + + return MappingProxyType( + { + model_key: lyria_model_info + for model_key, raw_model_info in GetModelCostMap.load_local_model_cost_map().items() + if (lyria_model_info := _validate_vertex_ai_lyria_model_info(raw_model_info)) is not None + } + ) + + +def _vertex_ai_lyria_model_key(model: str) -> str: + return model if model.startswith("vertex_ai/") else f"vertex_ai/{model}" + + +def _vertex_ai_lyria_generation_cost(model_info: VertexAILyriaModelInfo | None) -> float | None: + return None if model_info is None else model_info.get("output_cost_per_image") + + +def get_vertex_ai_lyria_model_info(model: str) -> VertexAILyriaModelInfo | None: + model_key: Final = _vertex_ai_lyria_model_key(model) + runtime_model_info: Final = _validate_vertex_ai_lyria_model_info(litellm.model_cost.get(model_key)) + return runtime_model_info or _bundled_vertex_ai_lyria_model_infos().get(model_key) + + +def get_vertex_ai_lyria_generation_cost(model: str) -> float | None: + model_key: Final = _vertex_ai_lyria_model_key(model) + runtime_cost: Final = _vertex_ai_lyria_generation_cost( + _validate_vertex_ai_lyria_model_info(litellm.model_cost.get(model_key)) + ) + if runtime_cost is not None: + return runtime_cost + return _vertex_ai_lyria_generation_cost(_bundled_vertex_ai_lyria_model_infos().get(model_key)) + + class VertexAIError(BaseLLMException): def __init__( self, diff --git a/litellm/llms/vertex_ai/text_to_speech/transformation.py b/litellm/llms/vertex_ai/text_to_speech/transformation.py index d7b4ad22a01..d382f43495f 100644 --- a/litellm/llms/vertex_ai/text_to_speech/transformation.py +++ b/litellm/llms/vertex_ai/text_to_speech/transformation.py @@ -8,17 +8,25 @@ Reference: https://cloud.google.com/text-to-speech/docs/reference/rest/v1/text/s import base64 from collections.abc import Coroutine from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Union +from typing import TYPE_CHECKING, Any, Final, TypeAlias, Union import httpx +import litellm +from litellm.exceptions import UnsupportedParamsError from litellm.litellm_core_utils.audio_utils.utils import ( + DEFAULT_SPEECH_MEDIA_TYPE, speech_media_type_from_audio_bytes, ) +from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.base_llm.text_to_speech.transformation import ( BaseTextToSpeechConfig, TextToSpeechRequestData, ) +from litellm.llms.vertex_ai.common_utils import ( + VertexAILyriaModelInfo, + get_vertex_ai_lyria_model_info, +) from litellm.llms.vertex_ai.vertex_llm_base import VertexBase from litellm.types.llms.vertex_ai import VERTEX_CREDENTIALS_TYPES from litellm.types.llms.vertex_ai_text_to_speech import ( @@ -35,6 +43,10 @@ else: LiteLLMLoggingObj = Any HttpxBinaryResponseContent = Any +_LyriaVoice: TypeAlias = ( + str | dict | None +) # mutable-ok: inherited interface supports structured provider voice dictionaries + class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): """ @@ -472,3 +484,209 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): # Initialize the HttpxBinaryResponseContent instance return HttpxBinaryResponseContent(response) + + +class VertexAILyriaTextToSpeechConfig(VertexAITextToSpeechConfig): + @classmethod + def is_lyria_model(cls, model: str) -> bool: + return get_vertex_ai_lyria_model_info(model=model) is not None + + @staticmethod + def _get_model_info(model: str) -> VertexAILyriaModelInfo: + model_info: Final = get_vertex_ai_lyria_model_info(model=model) + if model_info is None: + raise ValueError(f"Vertex AI model {model!r} does not declare a Lyria audio API") + return model_info + + def get_supported_openai_params( + self, model: str + ) -> list: # mutable-ok: inherited provider interface returns a concrete parameter list + return [ # mutable-ok: inherited provider interface requires a concrete parameter list + "response_format" + ] + + def map_openai_params( + self, + model: str, + optional_params: dict, # mutable-ok: inherited provider interface accepts a concrete parameter dictionary + voice: _LyriaVoice = None, + drop_params: bool = False, + kwargs: dict | None = None, # mutable-ok: inherited provider interface accepts a concrete keyword dictionary + ) -> tuple[str | None, dict]: # mutable-ok: inherited provider interface returns concrete mapped parameters + mapped_params: Final = dict( # mutable-ok: mapping drops unsupported parameters before provider dispatch + optional_params + ) + base_model: Final = model.removeprefix("vertex_ai/") + model_info: Final = self._get_model_info(model=model) + unsupported_params: Final = tuple( + param for param in ("speed", "instructions") if mapped_params.get(param) is not None + ) + if unsupported_params: + if drop_params or litellm.drop_params: + for param in unsupported_params: + mapped_params.pop(param, None) + else: + raise UnsupportedParamsError( + status_code=400, + message=( + f"Vertex AI {base_model} does not support the OpenAI parameters: " + f"{', '.join(unsupported_params)}. To drop unsupported openai params " + "from the call, set `litellm.drop_params = True`" + ), + ) + response_format: Final = mapped_params.get("response_format") + supported_formats: Final = frozenset(model_info["supported_audio_formats"]) + if response_format is not None and response_format not in supported_formats: + if drop_params or litellm.drop_params: + mapped_params.pop("response_format", None) + else: + raise UnsupportedParamsError( + status_code=400, + message=( + f"Vertex AI {base_model} does not support response_format={response_format!r}. " + f"Supported values: {', '.join(sorted(supported_formats))}. " + "To drop unsupported openai params from the call, set `litellm.drop_params = True`" + ), + ) + return voice if isinstance(voice, str) else None, mapped_params + + def get_complete_url( + self, + model: str, + api_base: str | None, + litellm_params: dict, # mutable-ok: inherited provider interface accepts concrete LiteLLM parameters + ) -> str: + base_model: Final = model.removeprefix("vertex_ai/") + model_info: Final = self._get_model_info(model=model) + configured_project: Final = self.safe_get_vertex_ai_project(litellm_params) + project: Final = ( + self._ensure_access_token( + credentials=self.safe_get_vertex_ai_credentials(litellm_params), + project_id=None, + custom_llm_provider="vertex_ai", + )[1] + if configured_project is None + else configured_project + ) + if model_info["vertex_ai_audio_api"] == "lyria_interactions": + from litellm.llms.vertex_ai.interactions.transformation import ( + VertexAIInteractionsConfig, + ) + + def mint_access_token( + _credentials: VERTEX_CREDENTIALS_TYPES | None, + project_id: str | None, + ) -> tuple[str, str]: + return "", project_id or project + + return VertexAIInteractionsConfig(mint_access_token=mint_access_token).get_complete_url( + api_base=api_base, + model=base_model, + litellm_params={ # mutable-ok: interactions dispatch expects a concrete parameter dictionary + **litellm_params, + "vertex_project": project, + "vertex_location": "global", + }, + ) + location: Final = self.safe_get_vertex_ai_location(litellm_params) or self.get_default_vertex_location() + base_url: Final = self.get_api_base(api_base=api_base, vertex_location=location).rstrip("/") + encoded_project: Final = encode_url_path_segment(project, field_name="project") + encoded_location: Final = encode_url_path_segment(location, field_name="location") + encoded_model: Final = encode_url_path_segment(base_model, field_name="model") + return ( + f"{base_url}/v1/projects/{encoded_project}/locations/{encoded_location}" + f"/publishers/google/models/{encoded_model}:predict" + ) + + def transform_text_to_speech_request( + self, + model: str, + input: str, + voice: str | None, + optional_params: dict, # mutable-ok: inherited provider interface accepts concrete mapped parameters + litellm_params: dict, # mutable-ok: inherited provider interface accepts concrete LiteLLM parameters + headers: dict, # mutable-ok: inherited provider interface accepts and updates concrete HTTP headers + ) -> TextToSpeechRequestData: + access_token, project = self._ensure_access_token( + credentials=self.safe_get_vertex_ai_credentials(litellm_params), + project_id=self.safe_get_vertex_ai_project(litellm_params), + custom_llm_provider="vertex_ai", + ) + headers.update( + { # mutable-ok: HTTP dispatch requires a concrete header dictionary + "Authorization": f"Bearer {access_token}", + "x-goog-user-project": project, + "Content-Type": "application/json", + } + ) + base_model: Final = model.removeprefix("vertex_ai/") + model_info: Final = self._get_model_info(model=model) + request_body: Final[dict[str, object]] = ( # mutable-ok: HTTP dispatch requires a concrete provider payload + { # mutable-ok: predict dispatch requires a concrete provider request dictionary + "instances": [ # mutable-ok: predict dispatch requires a concrete instances list + {"prompt": input} # mutable-ok: predict dispatch requires a concrete instance dictionary + ], + "parameters": { # mutable-ok: predict dispatch requires a concrete parameters dictionary + "sample_count": 1 + }, + } + if model_info["vertex_ai_audio_api"] == "lyria_predict" + else { # mutable-ok: interactions dispatch requires a concrete provider request dictionary + "model": base_model, + "input": input, + **( + { # mutable-ok: interactions dispatch requires a nested response-format dictionary + "response_format": { # mutable-ok: interactions response format is a concrete provider payload + "type": "audio", + "mime_type": "audio/wav", + } + } + if optional_params.get("response_format") == "wav" + else {} # mutable-ok: no response override is merged for non-WAV output + ), + } + ) + return TextToSpeechRequestData(dict_body=request_body, headers=headers) + + def transform_text_to_speech_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: "LiteLLMLoggingObj", + ) -> "HttpxBinaryResponseContent": + from litellm.types.llms.openai import HttpxBinaryResponseContent + + response_json: Final = raw_response.json() + base_model: Final = model.removeprefix("vertex_ai/") + model_info: Final = self._get_model_info(model=model) + audio_data: str | None = None # rebind-ok: response parsing discovers audio data in provider-specific shapes + mime_type: str | None = None # rebind-ok: response parsing discovers the MIME type beside the audio payload + if model_info["vertex_ai_audio_api"] == "lyria_predict": + predictions: Final = response_json.get("predictions") or () + if predictions: + audio_data = predictions[0].get("audioContent") or predictions[0].get( + "bytesBase64Encoded" + ) # rebind-ok: predict response supplies the generated audio value + mime_type = predictions[0].get("mimeType") # rebind-ok: predict response supplies its audio MIME type + else: + for step in response_json.get("steps") or response_json.get("outputs") or (): + content_items = step.get("content") or () if step.get("type") == "model_output" else (step,) + for content in content_items: + if content.get("type") == "audio" and content.get("data"): + audio_data = content[ + "data" + ] # rebind-ok: interactions response supplies the generated audio value + mime_type = content.get( + "mime_type" + ) # rebind-ok: interactions response supplies its audio MIME type + if audio_data is None: + raise ValueError(f"No generated audio found in Vertex AI {base_model} response") + binary_data: Final = base64.b64decode(audio_data) + media_type: Final = mime_type or speech_media_type_from_audio_bytes(binary_data) or DEFAULT_SPEECH_MEDIA_TYPE + return HttpxBinaryResponseContent( + httpx.Response( + status_code=raw_response.status_code, + content=binary_data, + headers=MappingProxyType({"content-type": media_type}), + ) + ) diff --git a/litellm/main.py b/litellm/main.py index 2929790f2bd..253c9381337 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -8247,6 +8247,7 @@ def speech( ) elif custom_llm_provider == "vertex_ai" or custom_llm_provider == "vertex_ai_beta": from litellm.llms.vertex_ai.text_to_speech.transformation import ( + VertexAILyriaTextToSpeechConfig, VertexAITextToSpeechConfig, ) @@ -8271,7 +8272,11 @@ def speech( # Vertex AI Text-to-Speech (Google Cloud TTS) if text_to_speech_provider_config is None: - text_to_speech_provider_config = VertexAITextToSpeechConfig() + text_to_speech_provider_config = ( # rebind-ok: model metadata selects the Vertex TTS implementation + VertexAILyriaTextToSpeechConfig() + if VertexAILyriaTextToSpeechConfig.is_lyria_model(model) + else VertexAITextToSpeechConfig() + ) # Cast to specific Vertex AI config type to access dispatch method vertex_config: Final = cast(VertexAITextToSpeechConfig, text_to_speech_provider_config) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index f607c1e298e..2c55828c1e2 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -47141,6 +47141,99 @@ "output_cost_per_token": 4e-07, "supports_tool_choice": true }, + "vertex_ai/lyria-002": { + "litellm_provider": "vertex_ai", + "mode": "audio_speech", + "output_cost_per_image": 0.06, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria", + "supported_audio_formats": [ + "wav" + ], + "supported_endpoints": [ + "/v1/audio/speech" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "audio" + ], + "supports_audio_output": true, + "vertex_ai_audio_api": "lyria_predict" + }, + "vertex_ai/lyria-3-clip-preview": { + "input_cost_per_token": 0, + "litellm_provider": "vertex_ai", + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "audio_speech", + "output_cost_per_image": 0.04, + "output_cost_per_token": 0, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria", + "supported_audio_formats": [ + "mp3" + ], + "supported_endpoints": [ + "/v1beta/interactions", + "/v1/audio/speech" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "audio" + ], + "supported_regions": [ + "global" + ], + "supports_audio_input": false, + "supports_audio_output": true, + "supports_function_calling": false, + "supports_prompt_caching": false, + "supports_response_schema": false, + "supports_system_messages": false, + "supports_vision": false, + "supports_web_search": false, + "vertex_ai_audio_api": "lyria_interactions" + }, + "vertex_ai/lyria-3-pro-preview": { + "input_cost_per_token": 0, + "litellm_provider": "vertex_ai", + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "audio_speech", + "output_cost_per_image": 0.08, + "output_cost_per_token": 0, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria", + "supported_audio_formats": [ + "mp3", + "wav" + ], + "supported_endpoints": [ + "/v1beta/interactions", + "/v1/audio/speech" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "audio" + ], + "supported_regions": [ + "global" + ], + "supports_audio_input": false, + "supports_audio_output": true, + "supports_function_calling": false, + "supports_prompt_caching": false, + "supports_response_schema": false, + "supports_system_messages": false, + "supports_vision": false, + "supports_web_search": false, + "vertex_ai_audio_api": "lyria_interactions" + }, "vertex_ai/meta/llama-3.1-405b-instruct-maas": { "input_cost_per_token": 5e-06, "litellm_provider": "vertex_ai-llama_models", diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py index 49ec18013b5..119a53c2411 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py @@ -10,7 +10,10 @@ import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import VERTEX_BATCH_PREDICTION_JOBS_ROUTE from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.llms.vertex_ai.common_utils import get_vertex_location_from_url +from litellm.llms.vertex_ai.common_utils import ( + get_vertex_ai_lyria_generation_cost, + get_vertex_location_from_url, +) from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( ModelResponseIterator as VertexModelResponseIterator, ) @@ -44,7 +47,6 @@ else: PassThroughEndpointLogging = Any LiteLLMBatch = Any -# Define EndpointType locally to avoid import issues EndpointType = Any @@ -270,6 +272,16 @@ class VertexPassthroughLoggingHandler: _json_response: Final[dict[str, object]] = httpx_response.json() litellm_prediction_response: ModelResponse | EmbeddingResponse | ImageResponse = ModelResponse() + if VertexPassthroughLoggingHandler._is_audio_predict_response( + model=model, + json_response=_json_response, + ): + return VertexPassthroughLoggingHandler._handle_audio_predict_response( + json_response=_json_response, + logging_obj=logging_obj, + model=model, + kwargs=kwargs, + ) if vertex_image_generation_class.is_image_generation_response(_json_response): litellm_prediction_response = vertex_image_generation_class.process_image_generation_response( _json_response, @@ -323,6 +335,71 @@ class VertexPassthroughLoggingHandler: "kwargs": kwargs, } + @staticmethod + def _handle_audio_predict_response( + json_response: dict, # mutable-ok: passthrough logging receives the decoded provider response dictionary + logging_obj: LiteLLMLoggingObj, + model: str, + kwargs: dict, # mutable-ok: passthrough logging enriches the shared callback metadata dictionary + ) -> PassThroughEndpointLoggingTypedDict: + prediction_count: Final = VertexPassthroughLoggingHandler._get_audio_prediction_count( + json_response=json_response + ) + response_cost: Final = (get_vertex_ai_lyria_generation_cost(model=model) or 0.0) * prediction_count + + logging_obj.model = model # rebind-ok: passthrough attribution records the resolved Vertex model + logging_obj.model_call_details[ # rebind-ok: passthrough attribution enriches callback metadata + "model" + ] = model + logging_obj.model_call_details[ # rebind-ok: passthrough attribution enriches callback metadata + "custom_llm_provider" + ] = "vertex_ai" + logging_obj.custom_llm_provider = ( # rebind-ok: attribution records the resolved provider + "vertex_ai" + ) + logging_obj.model_call_details[ # rebind-ok: passthrough attribution enriches callback metadata + "response_cost" + ] = response_cost + + kwargs[ # rebind-ok: callback metadata is enriched for downstream hooks + "response_cost" + ] = response_cost + kwargs["model"] = model # rebind-ok: callback metadata records the resolved model + kwargs["custom_llm_provider"] = "vertex_ai" # rebind-ok: callback metadata records the resolved provider + + standard_pass_through_response_object: Final[ + StandardPassThroughResponseObject + ] = { # mutable-ok: callback contract requires a concrete response dictionary + "response": json_response, + } + return { # mutable-ok: passthrough logging contract requires a concrete result dictionary + "result": standard_pass_through_response_object, + "kwargs": kwargs, + } + + @staticmethod + def _is_audio_predict_response( + model: str, + json_response: dict, # mutable-ok: predicate inspects the decoded provider response dictionary without mutation + ) -> bool: + return ( + VertexPassthroughLoggingHandler._get_audio_prediction_count(json_response=json_response) > 0 + and get_vertex_ai_lyria_generation_cost(model=model) is not None + ) + + @staticmethod + def _get_audio_prediction_count( + json_response: dict, # mutable-ok: counter inspects the decoded provider response dictionary without mutation + ) -> int: + predictions: Final = json_response.get("predictions") + if not isinstance(predictions, list): + return 0 + return sum( + 1 + for prediction in predictions + if isinstance(prediction, dict) and (prediction.get("audioContent") or prediction.get("bytesBase64Encoded")) + ) + @staticmethod def _extract_embed_content_input(request_body: dict | None, batch: bool) -> str: """Extract raw input text from an :embedContent or :batchEmbedContents request body for token counting.""" diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 61c2fc8c5a5..78ef6edfb19 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -168,6 +168,8 @@ class ProviderSpecificModelInfo(TypedDict, total=False): default_reasoning_effort: ReadOnly[Literal["none", "minimal", "low", "medium", "high", "xhigh"] | None] supports_output_config: bool | None supports_image_size: bool | None + supported_audio_formats: ReadOnly[Sequence[Literal["mp3", "wav"]] | None] + vertex_ai_audio_api: ReadOnly[Literal["lyria_predict", "lyria_interactions"] | None] bedrock_output_config_effort_ceiling: Literal["low", "medium", "high", "max", "xhigh"] | None bedrock_converse_supports_strict_tools: bool | None @@ -335,6 +337,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): "image_generation", "chat", "audio_transcription", + "audio_speech", "responses", "ocr", "realtime", diff --git a/litellm/utils.py b/litellm/utils.py index 52c1859b525..bc2f4a86f12 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5946,6 +5946,8 @@ def _get_model_info_helper( provider_specific_entry=_model_info.get("provider_specific_entry", None), uses_embed_content=_model_info.get("uses_embed_content", None), supports_image_size=_model_info.get("supports_image_size", None), + supported_audio_formats=_model_info.get("supported_audio_formats", None), + vertex_ai_audio_api=_model_info.get("vertex_ai_audio_api", None), ) for cost_key, cost_value in _model_info.items(): if cost_key not in returned_model_info and _ABOVE_THRESHOLD_COST_KEY.search(cost_key) is not None: @@ -9436,9 +9438,12 @@ class ProviderConfigManager: # mapping would drop response_format before the bridge sees it (LIT-6501) return None from litellm.llms.vertex_ai.text_to_speech.transformation import ( + VertexAILyriaTextToSpeechConfig, VertexAITextToSpeechConfig, ) + if VertexAILyriaTextToSpeechConfig.is_lyria_model(model): + return VertexAILyriaTextToSpeechConfig() return VertexAITextToSpeechConfig() elif litellm.LlmProviders.MINIMAX == provider: from litellm.llms.minimax.text_to_speech.transformation import ( diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index f607c1e298e..2c55828c1e2 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -47141,6 +47141,99 @@ "output_cost_per_token": 4e-07, "supports_tool_choice": true }, + "vertex_ai/lyria-002": { + "litellm_provider": "vertex_ai", + "mode": "audio_speech", + "output_cost_per_image": 0.06, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria", + "supported_audio_formats": [ + "wav" + ], + "supported_endpoints": [ + "/v1/audio/speech" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "audio" + ], + "supports_audio_output": true, + "vertex_ai_audio_api": "lyria_predict" + }, + "vertex_ai/lyria-3-clip-preview": { + "input_cost_per_token": 0, + "litellm_provider": "vertex_ai", + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "audio_speech", + "output_cost_per_image": 0.04, + "output_cost_per_token": 0, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria", + "supported_audio_formats": [ + "mp3" + ], + "supported_endpoints": [ + "/v1beta/interactions", + "/v1/audio/speech" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "audio" + ], + "supported_regions": [ + "global" + ], + "supports_audio_input": false, + "supports_audio_output": true, + "supports_function_calling": false, + "supports_prompt_caching": false, + "supports_response_schema": false, + "supports_system_messages": false, + "supports_vision": false, + "supports_web_search": false, + "vertex_ai_audio_api": "lyria_interactions" + }, + "vertex_ai/lyria-3-pro-preview": { + "input_cost_per_token": 0, + "litellm_provider": "vertex_ai", + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "audio_speech", + "output_cost_per_image": 0.08, + "output_cost_per_token": 0, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria", + "supported_audio_formats": [ + "mp3", + "wav" + ], + "supported_endpoints": [ + "/v1beta/interactions", + "/v1/audio/speech" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "audio" + ], + "supported_regions": [ + "global" + ], + "supports_audio_input": false, + "supports_audio_output": true, + "supports_function_calling": false, + "supports_prompt_caching": false, + "supports_response_schema": false, + "supports_system_messages": false, + "supports_vision": false, + "supports_web_search": false, + "vertex_ai_audio_api": "lyria_interactions" + }, "vertex_ai/meta/llama-3.1-405b-instruct-maas": { "input_cost_per_token": 5e-06, "litellm_provider": "vertex_ai-llama_models", diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index a51149bf958..47a1934a703 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -623,6 +623,17 @@ "type": "string", "description": "URL of the provider pricing/model page this entry was taken from." }, + "supported_audio_formats": { + "type": "array", + "description": "Audio container formats the model can return.", + "items": { + "type": "string", + "enum": [ + "mp3", + "wav" + ] + } + }, "supported_endpoints": { "type": "array", "description": "OpenAI-style API routes this model can be called through, e.g. /v1/chat/completions.", @@ -846,6 +857,13 @@ "uses_embed_content": { "type": "boolean" }, + "vertex_ai_audio_api": { + "type": "string", + "enum": [ + "lyria_predict", + "lyria_interactions" + ] + }, "web_search_billing_unit": { "type": "string", "description": "Whether web search is billed per query or per prompt.", diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py index dddc95bf54a..c206fcec420 100644 --- a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py @@ -1738,3 +1738,44 @@ def test_vertex_text_embedding_request_includes_labels_from_metadata(): }, ) assert req.get("labels") == {"project_id": "cost-center-1"} + + +@pytest.mark.parametrize( + ("model", "expected_api"), + [ + ("lyria-002", "lyria_predict"), + ("vertex_ai/lyria-002", "lyria_predict"), + ("lyria-3-clip-preview", "lyria_interactions"), + ("lyria-3-pro-preview", "lyria_interactions"), + ], +) +def test_get_vertex_ai_lyria_model_info_resolves_audio_api(model, expected_api): + from litellm.llms.vertex_ai.common_utils import get_vertex_ai_lyria_model_info + + model_info = get_vertex_ai_lyria_model_info(model=model) + + assert model_info is not None + assert model_info["vertex_ai_audio_api"] == expected_api + + +@pytest.mark.parametrize("model", ["en-US-Studio-O", "gemini-2.5-flash-preview-tts", "chirp-3-hd-charon"]) +def test_get_vertex_ai_lyria_model_info_is_none_for_non_lyria_speech_models(model): + from litellm.llms.vertex_ai.common_utils import get_vertex_ai_lyria_model_info + + assert get_vertex_ai_lyria_model_info(model=model) is None + + +def test_get_vertex_ai_lyria_model_info_falls_back_to_bundled_map(monkeypatch): + import litellm + from litellm.llms.vertex_ai.common_utils import get_vertex_ai_lyria_model_info + + stale_runtime_model_cost = { + key: value for key, value in litellm.model_cost.items() if not key.startswith("vertex_ai/lyria") + } + monkeypatch.setattr(litellm, "model_cost", stale_runtime_model_cost) + + model_info = get_vertex_ai_lyria_model_info(model="lyria-3-pro-preview") + + assert model_info is not None + assert model_info["vertex_ai_audio_api"] == "lyria_interactions" + assert model_info["supported_audio_formats"] == ("mp3", "wav") diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py b/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py new file mode 100644 index 00000000000..98010021bca --- /dev/null +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py @@ -0,0 +1,230 @@ +from datetime import datetime +from typing import Final +from unittest.mock import MagicMock + +import httpx +import pytest + +import litellm +from litellm.proxy.pass_through_endpoints.llm_provider_handlers.vertex_passthrough_logging_handler import ( + VertexPassthroughLoggingHandler, +) +from litellm.types.utils import PassthroughCallTypes + + +def test_lyria_predict_response_preserves_audio_response_and_logs_cost( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setitem( + litellm.model_cost, + "vertex_ai/lyria-002", + { + "vertex_ai_audio_api": "lyria_predict", + "supported_audio_formats": ["wav"], + "output_cost_per_image": 0.06, + }, + ) + logging_obj = MagicMock() + logging_obj.model_call_details = {} + response = httpx.Response( + status_code=200, + json={ + "predictions": [ + { + "audioContent": "clip-1", + "mimeType": "audio/wav", + }, + { + "audioContent": "clip-2", + "mimeType": "audio/wav", + }, + ] + }, + ) + + result = VertexPassthroughLoggingHandler.vertex_passthrough_handler( + httpx_response=response, + logging_obj=logging_obj, + url_route="/v1/projects/test/locations/us-central1/publishers/google/models/lyria-002:predict", + result=response.text, + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + request_body={"instances": [{"prompt": "ambient piano"}]}, + ) + + assert result["result"] == { + "response": { + "predictions": [ + { + "audioContent": "clip-1", + "mimeType": "audio/wav", + }, + { + "audioContent": "clip-2", + "mimeType": "audio/wav", + }, + ] + } + } + assert result["kwargs"]["model"] == "lyria-002" + assert result["kwargs"]["custom_llm_provider"] == "vertex_ai" + assert result["kwargs"]["response_cost"] == pytest.approx(0.12) + assert logging_obj.model == "lyria-002" + assert logging_obj.model_call_details["response_cost"] == pytest.approx(0.12) + + +def test_audio_predict_response_uses_model_map_metadata( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setitem( + litellm.model_cost, + "vertex_ai/music-audio-preview", + { + "vertex_ai_audio_api": "lyria_predict", + "supported_audio_formats": ["wav"], + "output_cost_per_image": 0.5, + }, + ) + logging_obj = MagicMock() + logging_obj.model_call_details = {} + response = httpx.Response( + status_code=200, + json={ + "predictions": [ + { + "audioContent": "clip", + "mimeType": "audio/wav", + } + ] + }, + ) + + result = VertexPassthroughLoggingHandler.vertex_passthrough_handler( + httpx_response=response, + logging_obj=logging_obj, + url_route="/v1/projects/test/locations/us-central1/publishers/google/models/music-audio-preview:predict", + result=response.text, + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + request_body={"instances": [{"prompt": "ambient piano"}]}, + ) + + assert result["kwargs"]["model"] == "music-audio-preview" + assert result["kwargs"]["response_cost"] == pytest.approx(0.5) + assert logging_obj.model_call_details["response_cost"] == pytest.approx(0.5) + + +def test_audio_predict_response_supports_bytes_base64_encoded( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setitem( + litellm.model_cost, + "vertex_ai/lyria-002", + { + "vertex_ai_audio_api": "lyria_predict", + "supported_audio_formats": ["wav"], + "output_cost_per_image": 0.06, + }, + ) + logging_obj = MagicMock() + logging_obj.model_call_details = {} + response = httpx.Response( + status_code=200, + json={"predictions": [{"bytesBase64Encoded": "clip"}]}, + ) + + result = VertexPassthroughLoggingHandler.vertex_passthrough_handler( + httpx_response=response, + logging_obj=logging_obj, + url_route="/v1/projects/test/locations/us-central1/publishers/google/models/lyria-002:predict", + result=response.text, + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + request_body={"instances": [{"prompt": "ambient piano"}]}, + ) + + assert result["kwargs"]["response_cost"] == pytest.approx(0.06) + assert logging_obj.model_call_details["response_cost"] == pytest.approx(0.06) + + +@pytest.mark.parametrize("runtime_entry_is_missing", (True, False)) +def test_lyria_predict_cost_falls_back_to_bundled_map_when_runtime_metadata_is_incomplete( + monkeypatch: pytest.MonkeyPatch, + runtime_entry_is_missing: bool, + local_model_cost_map: None, +) -> None: + if runtime_entry_is_missing: + monkeypatch.delitem(litellm.model_cost, "vertex_ai/lyria-002") + else: + monkeypatch.setitem( + litellm.model_cost, + "vertex_ai/lyria-002", + { + key: value + for key, value in litellm.model_cost["vertex_ai/lyria-002"].items() + if key != "output_cost_per_image" + }, + ) + logging_obj = MagicMock() + logging_obj.model_call_details = {} + response = httpx.Response( + status_code=200, + json={ + "predictions": [ + { + "audioContent": "clip", + "mimeType": "audio/wav", + } + ] + }, + ) + + result = VertexPassthroughLoggingHandler.vertex_passthrough_handler( + httpx_response=response, + logging_obj=logging_obj, + url_route="/v1/projects/test/locations/us-central1/publishers/google/models/lyria-002:predict", + result=response.text, + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + request_body={"instances": [{"prompt": "ambient piano"}]}, + ) + + if runtime_entry_is_missing: + assert "vertex_ai/lyria-002" not in litellm.model_cost + assert result["kwargs"]["model"] == "lyria-002" + assert result["kwargs"]["response_cost"] == pytest.approx(0.06) + assert logging_obj.model_call_details["response_cost"] == pytest.approx(0.06) + + +def test_image_predict_response_is_not_billed_as_audio( + local_model_cost_map: None, +) -> None: + logging_obj = MagicMock() + logging_obj.model_call_details = {} + response = httpx.Response( + status_code=200, + json={"predictions": [{"bytesBase64Encoded": "frame", "mimeType": "image/png"}]}, + ) + + result = VertexPassthroughLoggingHandler.vertex_passthrough_handler( + httpx_response=response, + logging_obj=logging_obj, + url_route=( + "/v1/projects/test/locations/us-central1/publishers/google/models/imagen-4.0-generate-001:predict" + ), + result=response.text, + start_time=datetime.now(), + end_time=datetime.now(), + cache_hit=False, + request_body={"instances": [{"prompt": "a red cube"}]}, + ) + + assert isinstance(result["result"], litellm.ImageResponse) + assert logging_obj.call_type == PassthroughCallTypes.passthrough_image_generation.value + assert result["kwargs"]["response_cost"] == pytest.approx( + litellm.model_cost["vertex_ai/imagen-4.0-generate-001"]["output_cost_per_image"] + ) diff --git a/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py b/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py index fba337b5f2c..b5eec42b569 100644 --- a/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py @@ -1,14 +1,17 @@ import base64 +from typing import Final from unittest.mock import MagicMock, Mock, patch import httpx import pytest - import litellm from litellm.llms.vertex_ai.text_to_speech.transformation import ( + VertexAILyriaTextToSpeechConfig, VertexAITextToSpeechConfig, ) +from litellm.types.utils import LlmProviders +from litellm.utils import ProviderConfigManager class TestVertexAITextToSpeechConfig: @@ -41,9 +44,7 @@ class TestVertexAITextToSpeechConfig: @patch.object(VertexAITextToSpeechConfig, "_ensure_access_token") @patch.object(VertexAITextToSpeechConfig, "_get_token_and_url") - def test_transform_text_to_speech_request_body( - self, mock_get_token, mock_ensure_token - ): + def test_transform_text_to_speech_request_body(self, mock_get_token, mock_ensure_token): """Test that transform_text_to_speech_request generates correct request body""" # Mock authentication mock_ensure_token.return_value = ("mock-token", "test-project") @@ -104,9 +105,7 @@ class TestVertexAITextToSpeechConfig: config = VertexAITextToSpeechConfig() # Test with a Chirp3 HD voice - voice_str, voice_dict = config._map_voice_to_vertex_format( - "en-US-Chirp3-HD-Charon" - ) + voice_str, voice_dict = config._map_voice_to_vertex_format("en-US-Chirp3-HD-Charon") assert voice_str == "en-US-Chirp3-HD-Charon" assert voice_dict is not None @@ -169,6 +168,391 @@ def test_transform_text_to_speech_response_leaves_unknown_bytes_unlabeled(): assert result.response.content == raw_pcm +class TestVertexAILyriaTextToSpeechConfig: + @pytest.mark.parametrize( + "model", + ["lyria-002", "vertex_ai/lyria-3-clip-preview", "lyria-3-pro-preview"], + ) + def test_provider_config_manager_selects_lyria_config(self, model): + config = ProviderConfigManager.get_provider_text_to_speech_config( + model=model, + provider=LlmProviders.VERTEX_AI, + ) + + assert isinstance(config, VertexAILyriaTextToSpeechConfig) + + @pytest.mark.parametrize( + ("model", "vertex_ai_audio_api", "supported_audio_formats", "expected_url"), + [ + ( + "future-lyria-predict", + "lyria_predict", + ["wav"], + "https://us-central1-aiplatform.googleapis.com/v1/projects/music-project/locations/" + "us-central1/publishers/google/models/future-lyria-predict:predict", + ), + ( + "future-music-interactions", + "lyria_interactions", + ["mp3", "wav"], + "https://aiplatform.googleapis.com/v1beta1/projects/music-project/locations/global/interactions", + ), + ], + ) + def test_dispatches_from_model_metadata( + self, + monkeypatch, + model, + vertex_ai_audio_api, + supported_audio_formats, + expected_url, + ): + monkeypatch.setitem( + litellm.model_cost, + f"vertex_ai/{model}", + { + "vertex_ai_audio_api": vertex_ai_audio_api, + "supported_audio_formats": supported_audio_formats, + }, + ) + + config = ProviderConfigManager.get_provider_text_to_speech_config( + model=model, + provider=LlmProviders.VERTEX_AI, + ) + + assert isinstance(config, VertexAILyriaTextToSpeechConfig) + assert ( + config.get_complete_url( + model=model, + api_base=None, + litellm_params={ + "vertex_project": "music-project", + "vertex_location": "us-central1", + }, + ) + == expected_url + ) + + def test_vertex_chirp_does_not_select_lyria_config(self): + config = ProviderConfigManager.get_provider_text_to_speech_config( + model="chirp", + provider=LlmProviders.VERTEX_AI, + ) + + assert isinstance(config, VertexAITextToSpeechConfig) + assert not isinstance(config, VertexAILyriaTextToSpeechConfig) + + def test_get_complete_url_for_lyria_2(self): + config = VertexAILyriaTextToSpeechConfig() + + url = config.get_complete_url( + model="lyria-002", + api_base=None, + litellm_params={ + "vertex_project": "music-project", + "vertex_location": "europe-west4", + }, + ) + + assert url == ( + "https://europe-west4-aiplatform.googleapis.com/v1/projects/music-project/" + "locations/europe-west4/publishers/google/models/lyria-002:predict" + ) + + def test_get_complete_url_encodes_injected_predict_path_segments(self, monkeypatch: pytest.MonkeyPatch) -> None: + injected: Final = ( + "victim-project/locations/us-central1/publishers/google/models/other-model:predict?ignored=" + ) + encoded: Final = ( + "victim-project%2Flocations%2Fus-central1%2Fpublishers%2Fgoogle" + "%2Fmodels%2Fother-model%3Apredict%3Fignored%3D" + ) + monkeypatch.setitem( + litellm.model_cost, + f"vertex_ai/{injected}", + { + "vertex_ai_audio_api": "lyria_predict", + "supported_audio_formats": ["wav"], + }, + ) + + url: Final = VertexAILyriaTextToSpeechConfig().get_complete_url( + model=injected, + api_base="https://us-central1-aiplatform.googleapis.com", + litellm_params={ + "vertex_project": injected, + "vertex_location": injected, + }, + ) + + assert url == ( + "https://us-central1-aiplatform.googleapis.com" + f"/v1/projects/{encoded}/locations/{encoded}/publishers/google/models/{encoded}:predict" + ) + + def test_get_complete_url_for_lyria_3(self): + config = VertexAILyriaTextToSpeechConfig() + + url = config.get_complete_url( + model="lyria-3-pro-preview", + api_base=None, + litellm_params={"vertex_project": "music-project"}, + ) + + assert url == ("https://aiplatform.googleapis.com/v1beta1/projects/music-project/locations/global/interactions") + + @pytest.mark.parametrize( + ("model", "response_format", "expected_body"), + [ + ( + "lyria-002", + "wav", + { + "instances": [{"prompt": "A bright synth track"}], + "parameters": {"sample_count": 1}, + }, + ), + ( + "lyria-3-clip-preview", + "mp3", + { + "model": "lyria-3-clip-preview", + "input": "A bright synth track", + }, + ), + ( + "lyria-3-pro-preview", + "wav", + { + "model": "lyria-3-pro-preview", + "input": "A bright synth track", + "response_format": { + "type": "audio", + "mime_type": "audio/wav", + }, + }, + ), + ], + ) + def test_transform_request( + self, + model, + response_format, + expected_body, + ): + class _LyriaConfig(VertexAILyriaTextToSpeechConfig): + def _ensure_access_token(self, *args: object, **kwargs: object) -> tuple[str, str]: + return "mock-token", "music-project" + + config = _LyriaConfig() + + request = config.transform_text_to_speech_request( + model=model, + input="A bright synth track", + voice="alloy", + optional_params={"response_format": response_format}, + litellm_params={"vertex_project": "music-project"}, + headers={}, + ) + + assert request["dict_body"] == expected_body + assert request["headers"]["Authorization"] == "Bearer mock-token" + assert request["headers"]["x-goog-user-project"] == "music-project" + + @pytest.mark.parametrize( + ("model", "response_json", "expected_audio", "expected_mime_type"), + [ + ( + "lyria-002", + { + "predictions": [ + { + "bytesBase64Encoded": "UklGRiQAAABXQVZFZm10IA==", + } + ] + }, + b"RIFF$\x00\x00\x00WAVEfmt ", + "audio/wav", + ), + ( + "lyria-3-pro-preview", + { + "steps": [ + { + "type": "model_output", + "content": [ + {"type": "text", "text": "Generated lyrics"}, + { + "type": "audio", + "data": "bHlyaWEtMy1hdWRpbw==", + "mime_type": "audio/mpeg", + }, + ], + } + ] + }, + b"lyria-3-audio", + "audio/mpeg", + ), + ( + "lyria-3-clip-preview", + { + "outputs": [ + {"type": "text", "text": "Generated lyrics"}, + { + "type": "audio", + "data": "bHlyaWEtMy1hdWRpbw==", + "mime_type": "audio/mpeg", + }, + ] + }, + b"lyria-3-audio", + "audio/mpeg", + ), + ( + "lyria-3-pro-preview", + { + "outputs": [ + { + "type": "audio", + "data": "UklGRiQAAABXQVZFZm10IA==", + } + ] + }, + b"RIFF$\x00\x00\x00WAVEfmt ", + "audio/wav", + ), + ], + ) + def test_transform_response( + self, + model, + response_json, + expected_audio, + expected_mime_type, + ): + config = VertexAILyriaTextToSpeechConfig() + raw_response = httpx.Response(200, json=response_json) + + response = config.transform_text_to_speech_response( + model=model, + raw_response=raw_response, + logging_obj=MagicMock(), + ) + + assert response.content == expected_audio + assert response.response.headers["content-type"] == expected_mime_type + + @pytest.mark.parametrize( + ("model", "response_format"), + [ + ("lyria-002", "mp3"), + ("lyria-3-clip-preview", "wav"), + ("lyria-3-pro-preview", "opus"), + ], + ) + def test_rejects_unsupported_response_format(self, model, response_format): + config = VertexAILyriaTextToSpeechConfig() + + with pytest.raises(litellm.UnsupportedParamsError): + config.map_openai_params( + model=model, + optional_params={"response_format": response_format}, + ) + + @pytest.mark.parametrize("param", ["speed", "instructions"]) + def test_rejects_unsupported_openai_params(self, param): + config = VertexAILyriaTextToSpeechConfig() + + with pytest.raises(litellm.UnsupportedParamsError): + config.map_openai_params( + model="lyria-3-pro-preview", + optional_params={param: "unsupported"}, + ) + + @pytest.mark.parametrize( + ("model", "response_format", "response_json", "expected_url", "expected_body"), + [ + ( + "lyria-002", + "wav", + { + "predictions": [ + { + "audioContent": "bHlyaWEtMi1hdWRpbw==", + "mimeType": "audio/wav", + } + ] + }, + "https://us-central1-aiplatform.googleapis.com/v1/projects/music-project/locations/us-central1/publishers/google/models/lyria-002:predict", + { + "instances": [{"prompt": "A bright synth track"}], + "parameters": {"sample_count": 1}, + }, + ), + ( + "lyria-3-pro-preview", + "mp3", + { + "steps": [ + { + "type": "model_output", + "content": [ + { + "type": "audio", + "data": "bHlyaWEtMy1hdWRpbw==", + "mime_type": "audio/mpeg", + } + ], + } + ] + }, + "https://aiplatform.googleapis.com/v1beta1/projects/music-project/locations/global/interactions", + { + "model": "lyria-3-pro-preview", + "input": "A bright synth track", + }, + ), + ], + ) + def test_litellm_speech_dispatches_to_lyria_api( + self, + model, + response_format, + response_json, + expected_url, + expected_body, + ): + mock_response = Mock(spec=httpx.Response) + mock_response.status_code = 200 + mock_response.json.return_value = response_json + with ( + patch.object( # test-quality-ok: litellm.speech has no seam for Vertex token minting + VertexAILyriaTextToSpeechConfig, + "_ensure_access_token", + return_value=("mock-token", "music-project"), + ), + patch( # test-quality-ok: litellm.speech has no seam for the HTTP handler + "litellm.llms.custom_httpx.llm_http_handler.HTTPHandler.post", + return_value=mock_response, + ) as mock_post, + ): + response = litellm.speech( + model=f"vertex_ai/{model}", + input="A bright synth track", + voice="alloy", + response_format=response_format, + vertex_project="music-project", + vertex_location="us-central1", + ) + + assert response.content in {b"lyria-2-audio", b"lyria-3-audio"} + mock_post.assert_called_once() + assert mock_post.call_args.kwargs["url"] == expected_url + assert mock_post.call_args.kwargs["json"] == expected_body + + @patch("litellm.llms.custom_httpx.llm_http_handler.HTTPHandler.post") @patch.object(VertexAITextToSpeechConfig, "_ensure_access_token") @patch.object(VertexAITextToSpeechConfig, "_get_token_and_url") @@ -182,9 +566,7 @@ def test_litellm_speech_vertex_ai_chirp(mock_get_token, mock_ensure_token, mock_ # Mock HTTP response mock_response = Mock(spec=httpx.Response) - mock_response.content = ( - b'{"audioContent": "SGVsbG8gV29ybGQ="}' # base64 encoded "Hello World" - ) + mock_response.content = b'{"audioContent": "SGVsbG8gV29ybGQ="}' # base64 encoded "Hello World" mock_response.status_code = 200 mock_response.headers = {"content-type": "application/json"} mock_response.json.return_value = {"audioContent": "SGVsbG8gV29ybGQ="} @@ -203,9 +585,7 @@ def test_litellm_speech_vertex_ai_chirp(mock_get_token, mock_ensure_token, mock_ call_kwargs = mock_post.call_args.kwargs # Verify the URL is the Google Cloud TTS API - assert ( - call_kwargs["url"] == "https://texttospeech.googleapis.com/v1/text:synthesize" - ) + assert call_kwargs["url"] == "https://texttospeech.googleapis.com/v1/text:synthesize" # Verify request body structure assert "json" in call_kwargs diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 2046695f151..1945e5ffac5 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -1,6 +1,7 @@ import json from pathlib import Path +from typing import Final import pytest @@ -146,6 +147,51 @@ def test_cost_calculator_with_response_cost_in_additional_headers(): assert result == 1000 +@pytest.mark.parametrize( + ("model", "expected_cost"), + [ + ("vertex_ai/lyria-002", 0.06), + ("vertex_ai/lyria-3-clip-preview", 0.04), + ("vertex_ai/lyria-3-pro-preview", 0.08), + ], +) +@pytest.mark.parametrize("runtime_state", ("complete", "missing", "routing_only", "custom_zero", "custom_price")) +@pytest.mark.parametrize("call_type", ("speech", "aspeech")) +def test_vertex_lyria_speech_cost( + model: str, + expected_cost: float, + _local_model_cost_map: None, + monkeypatch: pytest.MonkeyPatch, + runtime_state: str, + call_type: str, +) -> None: + model_info: Final = litellm.model_cost[model] + if runtime_state == "missing": + monkeypatch.delitem(litellm.model_cost, model) + elif runtime_state == "routing_only": + monkeypatch.setitem( + litellm.model_cost, + model, + {key: value for key, value in model_info.items() if key != "output_cost_per_image"}, + ) + elif runtime_state in ("custom_zero", "custom_price"): + multiplier: Final = 0 if runtime_state == "custom_zero" else 2 + monkeypatch.setitem( + litellm.model_cost, + model, + {**model_info, "output_cost_per_image": model_info["output_cost_per_image"] * multiplier}, + ) + + cost: Final = completion_cost( + model=model, + prompt="A bright synth track", + call_type=call_type, + ) + + expected: Final = 0 if runtime_state == "custom_zero" else expected_cost * (2 if runtime_state == "custom_price" else 1) + assert cost == pytest.approx(expected) + + def test_baseten_model_api_pricing_entries(_local_model_cost_map): expected_pricing = { diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 14907e17b1b..d686b032ee0 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -1091,6 +1091,17 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "supports_sampling_params": {"type": "boolean"}, "supports_output_config": {"type": "boolean"}, "supports_speed": {"type": "boolean"}, + "supported_audio_formats": { + "type": "array", + "items": { + "type": "string", + "enum": ["mp3", "wav"], + }, + }, + "vertex_ai_audio_api": { + "type": "string", + "enum": ["lyria_predict", "lyria_interactions"], + }, "bedrock_output_config_effort_ceiling": { "type": "string", "enum": ["low", "medium", "high", "max", "xhigh"], @@ -1113,6 +1124,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "/v1/images/variations", "/v1/images/edits", "/v1/batch", + "/v1beta/interactions", "/v1/audio/transcriptions", "/v1/audio/speech", "/v1/ocr", @@ -2879,6 +2891,60 @@ def test_gemini_lyria_3_preview_models_in_cost_map(): assert clip["output_cost_per_image"] == 0.04 +def test_vertex_ai_lyria_models_in_cost_map(): + import json + from pathlib import Path + + json_path = Path(__file__).parents[2] / "model_prices_and_context_window.json" + with open(json_path) as f: + model_cost = json.load(f) + + lyria_2 = model_cost.get("vertex_ai/lyria-002") + clip = model_cost.get("vertex_ai/lyria-3-clip-preview") + pro = model_cost.get("vertex_ai/lyria-3-pro-preview") + + assert lyria_2 is not None + assert clip is not None + assert pro is not None + assert lyria_2["litellm_provider"] == "vertex_ai" + assert clip["litellm_provider"] == "vertex_ai" + assert pro["litellm_provider"] == "vertex_ai" + assert lyria_2["mode"] == "audio_speech" + assert clip["mode"] == "audio_speech" + assert pro["mode"] == "audio_speech" + assert lyria_2["output_cost_per_image"] == 0.06 + assert lyria_2["supported_modalities"] == ["text"] + assert lyria_2["supported_output_modalities"] == ["audio"] + assert lyria_2["supports_audio_output"] is True + assert lyria_2["supported_audio_formats"] == ["wav"] + assert lyria_2["vertex_ai_audio_api"] == "lyria_predict" + assert lyria_2["supported_endpoints"] == ["/v1/audio/speech"] + assert clip["output_cost_per_image"] == 0.04 + assert pro["output_cost_per_image"] == 0.08 + assert clip["supported_audio_formats"] == ["mp3"] + assert pro["supported_audio_formats"] == ["mp3", "wav"] + assert clip["vertex_ai_audio_api"] == "lyria_interactions" + assert pro["vertex_ai_audio_api"] == "lyria_interactions" + assert clip["supported_endpoints"] == [ + "/v1beta/interactions", + "/v1/audio/speech", + ] + assert pro["supported_endpoints"] == [ + "/v1beta/interactions", + "/v1/audio/speech", + ] + assert clip["supported_modalities"] == ["text"] + assert pro["supported_modalities"] == ["text"] + assert clip["supports_vision"] is False + assert pro["supports_vision"] is False + assert "supports_image_input" not in clip + assert "supports_image_input" not in pro + assert clip["supported_regions"] == ["global"] + assert pro["supported_regions"] == ["global"] + assert clip["supports_audio_output"] is True + assert pro["supports_audio_output"] is True + + def test_model_info_for_fireworks_short_form_models(): """ Test that fireworks_ai short-form model entries (fireworks_ai/)