scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-09 22:31:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 87

feat(vertex): expose Lyria through audio speech

Browse source
This commit is contained in:
Emerson Gomes 2026-07-15 19:56:53 -05:00
parent e00fe023a9
commit b96844dd0c
No known key found for this signature in database
GPG key ID: D3DF28AB5D1B5E17
13 changed files with 565 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
litellm/cost_calculator.py
View file

@ -496,9 +496,19 @@ def cost_per_token(
# see this https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/models
if call_type == "speech" or call_type == "aspeech":
speech_model_info = litellm.get_model_info(model=model_without_prefix, custom_llm_provider=custom_llm_provider)
cost_metric: Final = select_cost_metric_for_model(speech_model_info)
prompt_cost: float = 0.0
completion_cost: float = 0.0
if not speech_model_info.get("input_cost_per_character") and not speech_model_info.get(
"input_cost_per_token"
):
output_cost_per_generation: Final = speech_model_info.get("output_cost_per_image")
output_cost_per_second: Final = speech_model_info.get("output_cost_per_second")
audio_seconds_per_prediction: Final = speech_model_info.get("audio_seconds_per_prediction")
if output_cost_per_generation is not None:
return prompt_cost, float(output_cost_per_generation)
if output_cost_per_second is not None and audio_seconds_per_prediction is not None:
return prompt_cost, float(output_cost_per_second) * float(audio_seconds_per_prediction)
cost_metric: Final = select_cost_metric_for_model(speech_model_info)
if cost_metric == "cost_per_character":
if prompt_characters is None:
raise ValueError(

160
litellm/llms/vertex_ai/text_to_speech/transformation.py
View file

@ -12,6 +12,8 @@ from typing import TYPE_CHECKING, Any, Final, Union
import httpx
import litellm
from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.audio_utils.utils import (
speech_media_type_from_audio_bytes,
)
@ -471,3 +473,161 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase):
# Initialize the HttpxBinaryResponseContent instance
return HttpxBinaryResponseContent(response)
class VertexAILyriaTextToSpeechConfig(VertexAITextToSpeechConfig):
LYRIA_MODELS = {
"lyria-002",
"lyria-3-clip-preview",
"lyria-3-pro-preview",
}
@classmethod
def is_lyria_model(cls, model: str) -> bool:
return model.removeprefix("vertex_ai/") in cls.LYRIA_MODELS
def get_supported_openai_params(self, model: str) -> list:
return ["response_format"]
def map_openai_params(
self,
model: str,
optional_params: dict,
voice: str | dict | None = None,
drop_params: bool = False,
kwargs: dict = {},
) -> tuple[str | None, dict]:
mapped_params = dict(optional_params)
base_model = model.removeprefix("vertex_ai/")
unsupported_params = [param for param in ("speed", "instructions") if mapped_params.get(param) is not None]
if unsupported_params:
if drop_params or litellm.drop_params:
for param in unsupported_params:
mapped_params.pop(param, None)
else:
raise UnsupportedParamsError(
status_code=400,
message=(
f"Vertex AI {base_model} does not support the OpenAI parameters: "
f"{', '.join(unsupported_params)}. To drop unsupported openai params "
"from the call, set `litellm.drop_params = True`"
),
)
response_format = mapped_params.get("response_format")
supported_formats = (
{"wav"} if base_model == "lyria-002" else {"mp3", "wav"} if base_model == "lyria-3-pro-preview" else {"mp3"}
)
if response_format is not None and response_format not in supported_formats:
if drop_params or litellm.drop_params:
mapped_params.pop("response_format", None)
else:
raise UnsupportedParamsError(
status_code=400,
message=(
f"Vertex AI {base_model} does not support response_format={response_format!r}. "
f"Supported values: {', '.join(sorted(supported_formats))}. "
"To drop unsupported openai params from the call, set `litellm.drop_params = True`"
),
)
return voice if isinstance(voice, str) else None, mapped_params
def get_complete_url(
self,
model: str,
api_base: str | None,
litellm_params: dict,
) -> str:
base_model = model.removeprefix("vertex_ai/")
project = self.safe_get_vertex_ai_project(litellm_params)
if project is None:
_, project = self._ensure_access_token(
credentials=self.safe_get_vertex_ai_credentials(litellm_params),
project_id=None,
custom_llm_provider="vertex_ai",
)
if base_model.startswith("lyria-3-"):
from litellm.llms.vertex_ai.interactions.transformation import (
VertexAIInteractionsConfig,
)
return VertexAIInteractionsConfig().get_complete_url(
api_base=api_base,
model=base_model,
litellm_params={**litellm_params, "vertex_project": project},
)
location = self.safe_get_vertex_ai_location(litellm_params) or self.get_default_vertex_location()
base_url = self.get_api_base(api_base=api_base, vertex_location=location).rstrip("/")
return f"{base_url}/v1/projects/{project}/locations/{location}/publishers/google/models/{base_model}:predict"
def transform_text_to_speech_request(
self,
model: str,
input: str,
voice: str | None,
optional_params: dict,
litellm_params: dict,
headers: dict,
) -> TextToSpeechRequestData:
access_token, project = self._ensure_access_token(
credentials=self.safe_get_vertex_ai_credentials(litellm_params),
project_id=self.safe_get_vertex_ai_project(litellm_params),
custom_llm_provider="vertex_ai",
)
headers.update(
{
"Authorization": f"Bearer {access_token}",
"x-goog-user-project": project,
"Content-Type": "application/json",
}
)
base_model = model.removeprefix("vertex_ai/")
if base_model == "lyria-002":
request_body = {
"instances": [{"prompt": input}],
"parameters": {"sample_count": 1},
}
else:
request_body = {"model": base_model, "input": input}
if optional_params.get("response_format") == "wav":
request_body["response_format"] = {
"type": "audio",
"mime_type": "audio/wav",
}
return TextToSpeechRequestData(dict_body=request_body, headers=headers)
def transform_text_to_speech_response(
self,
model: str,
raw_response: httpx.Response,
logging_obj: "LiteLLMLoggingObj",
) -> "HttpxBinaryResponseContent":
from litellm.types.llms.openai import HttpxBinaryResponseContent
response_json = raw_response.json()
base_model = model.removeprefix("vertex_ai/")
audio_data: str | None = None
mime_type: str | None = None
if base_model == "lyria-002":
predictions = response_json.get("predictions") or []
if predictions:
audio_data = predictions[0].get("audioContent") or predictions[0].get("bytesBase64Encoded")
mime_type = predictions[0].get("mimeType")
else:
for step in response_json.get("steps") or response_json.get("outputs") or []:
content_items = step.get("content") or [] if step.get("type") == "model_output" else [step]
for content in content_items:
if content.get("type") == "audio" and content.get("data"):
audio_data = content["data"]
mime_type = content.get("mime_type")
if audio_data is None:
raise ValueError(f"No generated audio found in Vertex AI {base_model} response")
mime_type = mime_type or ("audio/wav" if base_model == "lyria-002" else "audio/mpeg")
response = HttpxBinaryResponseContent(
httpx.Response(
status_code=raw_response.status_code,
content=base64.b64decode(audio_data),
headers={"content-type": mime_type},
)
)
response._hidden_params = {"audio_mime_type": mime_type}
return response

6
litellm/main.py
View file

@ -8235,6 +8235,7 @@ def speech(
)
elif custom_llm_provider == "vertex_ai" or custom_llm_provider == "vertex_ai_beta":
from litellm.llms.vertex_ai.text_to_speech.transformation import (
VertexAILyriaTextToSpeechConfig,
VertexAITextToSpeechConfig,
)
@ -8259,7 +8260,10 @@ def speech(
# Vertex AI Text-to-Speech (Google Cloud TTS)
if text_to_speech_provider_config is None:
text_to_speech_provider_config = VertexAITextToSpeechConfig()
if VertexAILyriaTextToSpeechConfig.is_lyria_model(model):
text_to_speech_provider_config = VertexAILyriaTextToSpeechConfig()
else:
text_to_speech_provider_config = VertexAITextToSpeechConfig()
# Cast to specific Vertex AI config type to access dispatch method
vertex_config: Final = cast(VertexAITextToSpeechConfig, text_to_speech_provider_config)

15
litellm/model_prices_and_context_window_backup.json
View file

@ -45516,9 +45516,12 @@
"litellm_provider": "vertex_ai",
"max_audio_length_hours": 0.009111111111111111,
"max_audio_per_prompt": 4,
"mode": "chat",
"mode": "audio_speech",
"output_cost_per_second": 0.002,
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria",
"supported_endpoints": [
"/v1/audio/speech"
],
"supported_modalities": [
"text"
],
@ -45533,12 +45536,13 @@
"max_input_tokens": 131072,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"mode": "audio_speech",
"output_cost_per_image": 0.04,
"output_cost_per_token": 0,
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria",
"supported_endpoints": [
"/v1beta/interactions"
"/v1beta/interactions",
"/v1/audio/speech"
],
"supported_modalities": [
"text",
@ -45566,12 +45570,13 @@
"max_input_tokens": 131072,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"mode": "audio_speech",
"output_cost_per_image": 0.08,
"output_cost_per_token": 0,
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria",
"supported_endpoints": [
"/v1beta/interactions"
"/v1beta/interactions",
"/v1/audio/speech"
],
"supported_modalities": [
"text",

12
litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py
View file

@ -368,14 +368,8 @@ class VertexPassthroughLoggingHandler:
@staticmethod
def _is_audio_predict_response(model: str, json_response: dict) -> bool:
return (
VertexPassthroughLoggingHandler._get_audio_prediction_count(
json_response=json_response
)
> 0
and VertexPassthroughLoggingHandler._get_audio_prediction_unit_cost(
model=model
)
is not None
VertexPassthroughLoggingHandler._get_audio_prediction_count(json_response=json_response) > 0
and VertexPassthroughLoggingHandler._get_audio_prediction_unit_cost(model=model) is not None
)
@staticmethod
@ -397,7 +391,7 @@ class VertexPassthroughLoggingHandler:
return sum(
1
for prediction in predictions
if isinstance(prediction, dict) and prediction.get("audioContent")
if isinstance(prediction, dict) and (prediction.get("audioContent") or prediction.get("bytesBase64Encoded"))
)
@staticmethod

11
litellm/proxy/proxy_server.py
View file

@ -11176,9 +11176,14 @@ async def audio_speech(
upstream_content_type: Final = (
response.response.headers.get("content-type") if isinstance(response, HttpxBinaryResponseContent) else None
)
media_type: Final = resolve_speech_media_type(
upstream_content_type=upstream_content_type,
response_format=requested_format if isinstance(requested_format, str) else None,
hidden_audio_mime_type: Final = hidden_params.get("audio_mime_type")
media_type: Final = (
hidden_audio_mime_type
if isinstance(hidden_audio_mime_type, str)
else resolve_speech_media_type(
upstream_content_type=upstream_content_type,
response_format=requested_format if isinstance(requested_format, str) else None,
)
)
return StreamingResponse(

4
litellm/types/utils.py
View file

@ -310,6 +310,9 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
output_cost_per_video_per_second: float | None # only for vertex ai models
output_cost_per_audio_per_second: float | None # only for vertex ai models
output_cost_per_second: float | None # for OpenAI Speech models
audio_seconds_per_prediction: float | None
max_audio_length_hours: float | None
max_audio_per_prompt: int | None
output_cost_per_second_1080p: (
float | None
) # video_generation tier: key output_cost_per_second_<resolution> (e.g. 1080p, 720p)
@ -333,6 +336,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
"image_generation",
"chat",
"audio_transcription",
"audio_speech",
"responses",
"ocr",
"realtime",

6
litellm/utils.py
View file

@ -5880,6 +5880,9 @@ def _get_model_info_helper(
"output_cost_per_token_above_512k_tokens", None
),
output_cost_per_second=_model_info.get("output_cost_per_second", None),
audio_seconds_per_prediction=_model_info.get("audio_seconds_per_prediction", None),
max_audio_length_hours=_model_info.get("max_audio_length_hours", None),
max_audio_per_prompt=_model_info.get("max_audio_per_prompt", None),
output_cost_per_second_1080p=_model_info.get("output_cost_per_second_1080p", None),
output_cost_per_second_480p=_model_info.get("output_cost_per_second_480p", None),
output_cost_per_second_4k=_model_info.get("output_cost_per_second_4k", None),
@ -9415,9 +9418,12 @@ class ProviderConfigManager:
# mapping would drop response_format before the bridge sees it (LIT-6501)
return None
from litellm.llms.vertex_ai.text_to_speech.transformation import (
VertexAILyriaTextToSpeechConfig,
VertexAITextToSpeechConfig,
)
if VertexAILyriaTextToSpeechConfig.is_lyria_model(model):
return VertexAILyriaTextToSpeechConfig()
return VertexAITextToSpeechConfig()
elif litellm.LlmProviders.MINIMAX == provider:
from litellm.llms.minimax.text_to_speech.transformation import (

15
model_prices_and_context_window.json
View file

@ -45516,9 +45516,12 @@
"litellm_provider": "vertex_ai",
"max_audio_length_hours": 0.009111111111111111,
"max_audio_per_prompt": 4,
"mode": "chat",
"mode": "audio_speech",
"output_cost_per_second": 0.002,
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria",
"supported_endpoints": [
"/v1/audio/speech"
],
"supported_modalities": [
"text"
],
@ -45533,12 +45536,13 @@
"max_input_tokens": 131072,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"mode": "audio_speech",
"output_cost_per_image": 0.04,
"output_cost_per_token": 0,
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria",
"supported_endpoints": [
"/v1beta/interactions"
"/v1beta/interactions",
"/v1/audio/speech"
],
"supported_modalities": [
"text",
@ -45566,12 +45570,13 @@
"max_input_tokens": 131072,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"mode": "audio_speech",
"output_cost_per_image": 0.08,
"output_cost_per_token": 0,
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#lyria",
"supported_endpoints": [
"/v1beta/interactions"
"/v1beta/interactions",
"/v1/audio/speech"
],
"supported_modalities": [
"text",

33
tests/test_litellm/llms/vertex_ai/test_vertex_passthrough_logging_handler.py
View file

@ -110,3 +110,36 @@ def test_audio_predict_response_uses_model_map_metadata(
assert result["kwargs"]["model"] == "music-audio-preview"
assert result["kwargs"]["response_cost"] == pytest.approx(6.0)
assert logging_obj.model_call_details["response_cost"] == pytest.approx(6.0)
def test_audio_predict_response_supports_bytes_base64_encoded(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setitem(
litellm.model_cost,
"vertex_ai/lyria-002",
{
"audio_seconds_per_prediction": 30,
"output_cost_per_second": 0.002,
},
)
logging_obj = MagicMock()
logging_obj.model_call_details = {}
response = httpx.Response(
status_code=200,
json={"predictions": [{"bytesBase64Encoded": "clip"}]},
)
result = VertexPassthroughLoggingHandler.vertex_passthrough_handler(
httpx_response=response,
logging_obj=logging_obj,
url_route="/v1/projects/test/locations/us-central1/publishers/google/models/lyria-002:predict",
result=response.text,
start_time=datetime.now(),
end_time=datetime.now(),
cache_hit=False,
request_body={"instances": [{"prompt": "ambient piano"}]},
)
assert result["kwargs"]["response_cost"] == pytest.approx(0.06)
assert logging_obj.model_call_details["response_cost"] == pytest.approx(0.06)

298
tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py
View file

@ -4,11 +4,13 @@ from unittest.mock import MagicMock, Mock, patch
import httpx
import pytest
import litellm
from litellm.llms.vertex_ai.text_to_speech.transformation import (
VertexAILyriaTextToSpeechConfig,
VertexAITextToSpeechConfig,
)
from litellm.types.utils import LlmProviders
from litellm.utils import ProviderConfigManager
class TestVertexAITextToSpeechConfig:
@ -41,9 +43,7 @@ class TestVertexAITextToSpeechConfig:
@patch.object(VertexAITextToSpeechConfig, "_ensure_access_token")
@patch.object(VertexAITextToSpeechConfig, "_get_token_and_url")
def test_transform_text_to_speech_request_body(
self, mock_get_token, mock_ensure_token
):
def test_transform_text_to_speech_request_body(self, mock_get_token, mock_ensure_token):
"""Test that transform_text_to_speech_request generates correct request body"""
# Mock authentication
mock_ensure_token.return_value = ("mock-token", "test-project")
@ -104,9 +104,7 @@ class TestVertexAITextToSpeechConfig:
config = VertexAITextToSpeechConfig()
# Test with a Chirp3 HD voice
voice_str, voice_dict = config._map_voice_to_vertex_format(
"en-US-Chirp3-HD-Charon"
)
voice_str, voice_dict = config._map_voice_to_vertex_format("en-US-Chirp3-HD-Charon")
assert voice_str == "en-US-Chirp3-HD-Charon"
assert voice_dict is not None
@ -169,6 +167,284 @@ def test_transform_text_to_speech_response_leaves_unknown_bytes_unlabeled():
assert result.response.content == raw_pcm
class TestVertexAILyriaTextToSpeechConfig:
@pytest.mark.parametrize(
"model",
["lyria-002", "vertex_ai/lyria-3-clip-preview", "lyria-3-pro-preview"],
)
def test_provider_config_manager_selects_lyria_config(self, model):
config = ProviderConfigManager.get_provider_text_to_speech_config(
model=model,
provider=LlmProviders.VERTEX_AI,
)
assert isinstance(config, VertexAILyriaTextToSpeechConfig)
def test_get_complete_url_for_lyria_2(self):
config = VertexAILyriaTextToSpeechConfig()
url = config.get_complete_url(
model="lyria-002",
api_base=None,
litellm_params={
"vertex_project": "music-project",
"vertex_location": "europe-west4",
},
)
assert url == (
"https://europe-west4-aiplatform.googleapis.com/v1/projects/music-project/"
"locations/europe-west4/publishers/google/models/lyria-002:predict"
)
def test_get_complete_url_for_lyria_3(self):
config = VertexAILyriaTextToSpeechConfig()
url = config.get_complete_url(
model="lyria-3-pro-preview",
api_base=None,
litellm_params={"vertex_project": "music-project"},
)
assert url == ("https://aiplatform.googleapis.com/v1beta1/projects/music-project/locations/global/interactions")
@pytest.mark.parametrize(
("model", "response_format", "expected_body"),
[
(
"lyria-002",
"wav",
{
"instances": [{"prompt": "A bright synth track"}],
"parameters": {"sample_count": 1},
},
),
(
"lyria-3-clip-preview",
"mp3",
{
"model": "lyria-3-clip-preview",
"input": "A bright synth track",
},
),
(
"lyria-3-pro-preview",
"wav",
{
"model": "lyria-3-pro-preview",
"input": "A bright synth track",
"response_format": {
"type": "audio",
"mime_type": "audio/wav",
},
},
),
],
)
@patch.object(VertexAILyriaTextToSpeechConfig, "_ensure_access_token")
def test_transform_request(
self,
mock_ensure_token,
model,
response_format,
expected_body,
):
mock_ensure_token.return_value = ("mock-token", "music-project")
config = VertexAILyriaTextToSpeechConfig()
request = config.transform_text_to_speech_request(
model=model,
input="A bright synth track",
voice="alloy",
optional_params={"response_format": response_format},
litellm_params={"vertex_project": "music-project"},
headers={},
)
assert request["dict_body"] == expected_body
assert request["headers"]["Authorization"] == "Bearer mock-token"
assert request["headers"]["x-goog-user-project"] == "music-project"
@pytest.mark.parametrize(
("model", "response_json", "expected_audio", "expected_mime_type"),
[
(
"lyria-002",
{
"predictions": [
{
"bytesBase64Encoded": "bHlyaWEtMi1hdWRpbw==",
}
]
},
b"lyria-2-audio",
"audio/wav",
),
(
"lyria-3-pro-preview",
{
"steps": [
{
"type": "model_output",
"content": [
{"type": "text", "text": "Generated lyrics"},
{
"type": "audio",
"data": "bHlyaWEtMy1hdWRpbw==",
"mime_type": "audio/mpeg",
},
],
}
]
},
b"lyria-3-audio",
"audio/mpeg",
),
(
"lyria-3-clip-preview",
{
"outputs": [
{"type": "text", "text": "Generated lyrics"},
{
"type": "audio",
"data": "bHlyaWEtMy1hdWRpbw==",
"mime_type": "audio/mpeg",
},
]
},
b"lyria-3-audio",
"audio/mpeg",
),
],
)
def test_transform_response(
self,
model,
response_json,
expected_audio,
expected_mime_type,
):
config = VertexAILyriaTextToSpeechConfig()
raw_response = httpx.Response(200, json=response_json)
response = config.transform_text_to_speech_response(
model=model,
raw_response=raw_response,
logging_obj=MagicMock(),
)
assert response.content == expected_audio
assert response._hidden_params["audio_mime_type"] == expected_mime_type
@pytest.mark.parametrize(
("model", "response_format"),
[
("lyria-002", "mp3"),
("lyria-3-clip-preview", "wav"),
("lyria-3-pro-preview", "opus"),
],
)
def test_rejects_unsupported_response_format(self, model, response_format):
config = VertexAILyriaTextToSpeechConfig()
with pytest.raises(litellm.UnsupportedParamsError):
config.map_openai_params(
model=model,
optional_params={"response_format": response_format},
)
@pytest.mark.parametrize("param", ["speed", "instructions"])
def test_rejects_unsupported_openai_params(self, param):
config = VertexAILyriaTextToSpeechConfig()
with pytest.raises(litellm.UnsupportedParamsError):
config.map_openai_params(
model="lyria-3-pro-preview",
optional_params={param: "unsupported"},
)
@pytest.mark.parametrize(
("model", "response_format", "response_json", "expected_url", "expected_body"),
[
(
"lyria-002",
"wav",
{
"predictions": [
{
"audioContent": "bHlyaWEtMi1hdWRpbw==",
"mimeType": "audio/wav",
}
]
},
"https://us-central1-aiplatform.googleapis.com/v1/projects/music-project/locations/us-central1/publishers/google/models/lyria-002:predict",
{
"instances": [{"prompt": "A bright synth track"}],
"parameters": {"sample_count": 1},
},
),
(
"lyria-3-pro-preview",
"mp3",
{
"steps": [
{
"type": "model_output",
"content": [
{
"type": "audio",
"data": "bHlyaWEtMy1hdWRpbw==",
"mime_type": "audio/mpeg",
}
],
}
]
},
"https://aiplatform.googleapis.com/v1beta1/projects/music-project/locations/global/interactions",
{
"model": "lyria-3-pro-preview",
"input": "A bright synth track",
},
),
],
)
def test_litellm_speech_dispatches_to_lyria_api(
self,
model,
response_format,
response_json,
expected_url,
expected_body,
):
mock_response = Mock(spec=httpx.Response)
mock_response.status_code = 200
mock_response.json.return_value = response_json
with (
patch.object(
VertexAILyriaTextToSpeechConfig,
"_ensure_access_token",
return_value=("mock-token", "music-project"),
),
patch(
"litellm.llms.custom_httpx.llm_http_handler.HTTPHandler.post",
return_value=mock_response,
) as mock_post,
):
response = litellm.speech(
model=f"vertex_ai/{model}",
input="A bright synth track",
voice="alloy",
response_format=response_format,
vertex_project="music-project",
vertex_location="us-central1",
)
assert response.content in {b"lyria-2-audio", b"lyria-3-audio"}
mock_post.assert_called_once()
assert mock_post.call_args.kwargs["url"] == expected_url
assert mock_post.call_args.kwargs["json"] == expected_body
@patch("litellm.llms.custom_httpx.llm_http_handler.HTTPHandler.post")
@patch.object(VertexAITextToSpeechConfig, "_ensure_access_token")
@patch.object(VertexAITextToSpeechConfig, "_get_token_and_url")
@ -182,9 +458,7 @@ def test_litellm_speech_vertex_ai_chirp(mock_get_token, mock_ensure_token, mock_
# Mock HTTP response
mock_response = Mock(spec=httpx.Response)
mock_response.content = (
b'{"audioContent": "SGVsbG8gV29ybGQ="}' # base64 encoded "Hello World"
)
mock_response.content = b'{"audioContent": "SGVsbG8gV29ybGQ="}' # base64 encoded "Hello World"
mock_response.status_code = 200
mock_response.headers = {"content-type": "application/json"}
mock_response.json.return_value = {"audioContent": "SGVsbG8gV29ybGQ="}
@ -203,9 +477,7 @@ def test_litellm_speech_vertex_ai_chirp(mock_get_token, mock_ensure_token, mock_
call_kwargs = mock_post.call_args.kwargs
# Verify the URL is the Google Cloud TTS API
assert (
call_kwargs["url"] == "https://texttospeech.googleapis.com/v1/text:synthesize"
)
assert call_kwargs["url"] == "https://texttospeech.googleapis.com/v1/text:synthesize"
# Verify request body structure
assert "json" in call_kwargs

18
tests/test_litellm/test_cost_calculator.py
View file

@ -146,6 +146,24 @@ def test_cost_calculator_with_response_cost_in_additional_headers():
assert result == 1000
@pytest.mark.parametrize(
("model", "expected_cost"),
[
("vertex_ai/lyria-002", 0.06),
("vertex_ai/lyria-3-clip-preview", 0.04),
("vertex_ai/lyria-3-pro-preview", 0.08),
],
)
def test_vertex_lyria_speech_cost(model, expected_cost, _local_model_cost_map):
cost = completion_cost(
model=model,
prompt="A bright synth track",
call_type="speech",
)
assert cost == pytest.approx(expected_cost)
def test_baseten_model_api_pricing_entries(_local_model_cost_map):
expected_pricing = {

15
tests/test_litellm/test_utils.py
View file

@ -2855,15 +2855,25 @@ def test_vertex_ai_lyria_models_in_cost_map():
assert lyria_2["litellm_provider"] == "vertex_ai"
assert clip["litellm_provider"] == "vertex_ai"
assert pro["litellm_provider"] == "vertex_ai"
assert lyria_2["mode"] == "audio_speech"
assert clip["mode"] == "audio_speech"
assert pro["mode"] == "audio_speech"
assert lyria_2["audio_seconds_per_prediction"] == 30
assert lyria_2["output_cost_per_second"] == 0.002
assert lyria_2["supported_modalities"] == ["text"]
assert lyria_2["supported_output_modalities"] == ["audio"]
assert lyria_2["supports_audio_output"] is True
assert lyria_2["supported_endpoints"] == ["/v1/audio/speech"]
assert clip["output_cost_per_image"] == 0.04
assert pro["output_cost_per_image"] == 0.08
assert clip["supported_endpoints"] == ["/v1beta/interactions"]
assert pro["supported_endpoints"] == ["/v1beta/interactions"]
assert clip["supported_endpoints"] == [
"/v1beta/interactions",
"/v1/audio/speech",
]
assert pro["supported_endpoints"] == [
"/v1beta/interactions",
"/v1/audio/speech",
]
assert clip["supported_modalities"] == ["text", "image"]
assert pro["supported_modalities"] == ["text", "image"]
assert clip["supported_regions"] == ["global"]
@ -2873,7 +2883,6 @@ def test_vertex_ai_lyria_models_in_cost_map():
assert clip["supports_image_input"] is True
assert pro["supports_image_input"] is True
def test_model_info_for_fireworks_short_form_models():
"""
Test that fireworks_ai short-form model entries (fireworks_ai/<model>)