mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-09 22:31:41 +00:00
Merge pull request #38755 from BerriAI/litellm_mistral_voxtral_tts_speech
feat(mistral): add text-to-speech support for /v1/audio/speech
This commit is contained in:
Mateo Wang
GitHub
commit
1c7b13bdbf
10 changed files with 582 additions and 7 deletions
210
litellm/llms/mistral/audio_speech/transformation.py
Normal file
210
litellm/llms/mistral/audio_speech/transformation.py
Normal file
|
|
@ -0,0 +1,210 @@
|
|||
"""
|
||||
Support for Mistral Voxtral text-to-speech via ``/v1/audio/speech``.
|
||||
|
||||
API reference: https://docs.mistral.ai/api/#tag/audio/operation/audio_speech_v1_audio_speech_post
|
||||
"""
|
||||
|
||||
import base64
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
import httpx
|
||||
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.base_llm.text_to_speech.transformation import (
|
||||
BaseTextToSpeechConfig,
|
||||
TextToSpeechRequestData,
|
||||
)
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.types.llms.openai import HttpxBinaryResponseContent
|
||||
|
||||
|
||||
class MistralTextToSpeechException(BaseLLMException):
|
||||
pass
|
||||
|
||||
|
||||
class MistralTextToSpeechConfig(BaseTextToSpeechConfig):
|
||||
TTS_BASE_URL: Final[str] = "https://api.mistral.ai/v1"
|
||||
AUDIO_CONTENT_TYPES: Final[MappingProxyType[str, str]] = MappingProxyType(
|
||||
{
|
||||
"mp3": "audio/mpeg",
|
||||
"wav": "audio/wav",
|
||||
"pcm": "audio/pcm",
|
||||
"flac": "audio/flac",
|
||||
"opus": "audio/ogg",
|
||||
}
|
||||
)
|
||||
DROPPED_RESPONSE_HEADERS: Final[frozenset[str]] = frozenset(
|
||||
{"content-encoding", "transfer-encoding", "content-length", "content-type"}
|
||||
)
|
||||
OPENAI_VOICE_ALIASES: Final[MappingProxyType[str, str]] = MappingProxyType(
|
||||
{
|
||||
"alloy": "en_paul_neutral",
|
||||
"echo": "gb_oliver_neutral",
|
||||
"fable": "en_paul_cheerful",
|
||||
"onyx": "en_paul_confident",
|
||||
"nova": "gb_jane_sarcasm",
|
||||
"shimmer": "gb_jane_sarcasm",
|
||||
}
|
||||
)
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list: # mutable-ok: base class contract returns a plain list
|
||||
return ["voice", "response_format"] # mutable-ok: base class contract returns a plain list
|
||||
|
||||
def _map_openai_voice(self, voice_id: str) -> str:
|
||||
return self.OPENAI_VOICE_ALIASES.get(voice_id.lower(), voice_id)
|
||||
|
||||
def _resolve_voice_id(self, voice: object) -> str | None:
|
||||
if isinstance(voice, str) and voice.strip():
|
||||
return self._map_openai_voice(voice.strip())
|
||||
if isinstance(voice, Mapping):
|
||||
candidates: Final = (voice.get(key) for key in ("voice_id", "id", "name"))
|
||||
resolved: Final = next(
|
||||
(candidate.strip() for candidate in candidates if isinstance(candidate, str) and candidate.strip()),
|
||||
None,
|
||||
)
|
||||
return self._map_openai_voice(resolved) if resolved else None
|
||||
return None
|
||||
|
||||
def map_openai_params(
|
||||
self,
|
||||
model: str,
|
||||
optional_params: Mapping[str, object],
|
||||
voice: object = None,
|
||||
drop_params: bool = False,
|
||||
kwargs: Mapping[str, object] | None = None,
|
||||
) -> tuple[str | None, dict]: # mutable-ok: base class contract returns a plain dict
|
||||
response_format: Final = optional_params.get("response_format")
|
||||
ref_audio: Final = kwargs.get("ref_audio") if kwargs else None
|
||||
voice_id_kwarg: Final = kwargs.get("voice_id") if kwargs else None
|
||||
mapped_voice: Final = self._resolve_voice_id(voice) or self._resolve_voice_id(voice_id_kwarg)
|
||||
mapped_params: Final = { # mutable-ok: base class contract returns a plain dict
|
||||
key: value
|
||||
for key, value in (("response_format", response_format), ("ref_audio", ref_audio))
|
||||
if isinstance(value, str)
|
||||
}
|
||||
return mapped_voice, mapped_params
|
||||
|
||||
def validate_environment(
|
||||
self,
|
||||
headers: Mapping[str, str],
|
||||
model: str,
|
||||
api_key: str | None = None,
|
||||
api_base: str | None = None,
|
||||
) -> dict: # mutable-ok: base class contract returns a plain dict
|
||||
resolved_key: Final = api_key or get_secret_str("MISTRAL_API_KEY")
|
||||
if resolved_key is None:
|
||||
raise MistralTextToSpeechException(
|
||||
status_code=401,
|
||||
message="Mistral API key is required. Set MISTRAL_API_KEY or pass api_key.",
|
||||
)
|
||||
return { # mutable-ok: base class contract returns a plain dict
|
||||
**headers,
|
||||
"Authorization": f"Bearer {resolved_key}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
def get_complete_url(
|
||||
self,
|
||||
model: str,
|
||||
api_base: str | None,
|
||||
litellm_params: Mapping[str, object],
|
||||
) -> str:
|
||||
configured_base: Final = (api_base or self.TTS_BASE_URL).rstrip("/")
|
||||
versioned_base: Final = configured_base if configured_base.endswith("/v1") else f"{configured_base}/v1"
|
||||
return f"{versioned_base}/audio/speech"
|
||||
|
||||
def transform_text_to_speech_request(
|
||||
self,
|
||||
model: str,
|
||||
input: str,
|
||||
voice: str | None,
|
||||
optional_params: Mapping[str, object],
|
||||
litellm_params: Mapping[str, object],
|
||||
headers: Mapping[str, str],
|
||||
) -> TextToSpeechRequestData:
|
||||
response_format: Final = optional_params.get("response_format")
|
||||
ref_audio: Final = optional_params.get("ref_audio")
|
||||
request_data: Final[TextToSpeechRequestData] = {
|
||||
"dict_body": {
|
||||
"model": model,
|
||||
"input": input,
|
||||
**({"voice_id": voice} if voice else {}),
|
||||
**({"response_format": response_format} if isinstance(response_format, str) else {}),
|
||||
**({"ref_audio": ref_audio} if isinstance(ref_audio, str) else {}),
|
||||
},
|
||||
"headers": {"Content-Type": "application/json"},
|
||||
}
|
||||
return request_data
|
||||
|
||||
def _requested_content_type(self, request: httpx.Request) -> str:
|
||||
request_body: Final = json.loads(request.content or b"{}")
|
||||
requested_format: Final = request_body.get("response_format")
|
||||
if not isinstance(requested_format, str):
|
||||
return "audio/mpeg"
|
||||
return self.AUDIO_CONTENT_TYPES.get(requested_format, "audio/mpeg")
|
||||
|
||||
def transform_text_to_speech_response(
|
||||
self,
|
||||
model: str,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: "LiteLLMLoggingObj",
|
||||
) -> "HttpxBinaryResponseContent":
|
||||
from litellm.types.llms.openai import HttpxBinaryResponseContent
|
||||
|
||||
try:
|
||||
response_json: Final = raw_response.json()
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
raise MistralTextToSpeechException(
|
||||
status_code=raw_response.status_code,
|
||||
message=f"Non-JSON response from Mistral speech API: {raw_response.text[:500]}",
|
||||
headers=raw_response.headers,
|
||||
)
|
||||
audio_b64: Final = response_json.get("audio_data")
|
||||
if not isinstance(audio_b64, str) or not audio_b64:
|
||||
raise MistralTextToSpeechException(
|
||||
status_code=500,
|
||||
message=f"No audio_data in Mistral speech response. Response keys: {tuple(response_json.keys())}",
|
||||
headers=raw_response.headers,
|
||||
)
|
||||
try:
|
||||
audio_bytes: Final = base64.b64decode(audio_b64, validate=True)
|
||||
except ValueError:
|
||||
raise MistralTextToSpeechException(
|
||||
status_code=500,
|
||||
message="Invalid base64 audio_data in Mistral speech response.",
|
||||
headers=raw_response.headers,
|
||||
)
|
||||
retained_headers: Final = tuple(
|
||||
(key, value)
|
||||
for key, value in raw_response.headers.items()
|
||||
if key.lower() not in self.DROPPED_RESPONSE_HEADERS
|
||||
)
|
||||
response_headers: Final = retained_headers + (
|
||||
("content-length", str(len(audio_bytes))),
|
||||
("content-type", self._requested_content_type(raw_response.request)),
|
||||
)
|
||||
binary_response: Final = httpx.Response(
|
||||
status_code=200,
|
||||
headers=response_headers,
|
||||
content=audio_bytes,
|
||||
request=raw_response.request,
|
||||
)
|
||||
return HttpxBinaryResponseContent(binary_response)
|
||||
|
||||
def get_error_class(
|
||||
self,
|
||||
error_message: str,
|
||||
status_code: int,
|
||||
headers: dict | httpx.Headers, # mutable-ok: BaseLLMException takes a plain dict or httpx.Headers
|
||||
) -> BaseLLMException:
|
||||
return MistralTextToSpeechException(
|
||||
message=error_message,
|
||||
status_code=status_code,
|
||||
headers=headers,
|
||||
)
|
||||
|
|
@ -8389,6 +8389,34 @@ def speech(
|
|||
client=client,
|
||||
_is_async=aspeech or False,
|
||||
)
|
||||
elif custom_llm_provider == "mistral":
|
||||
from litellm.llms.mistral.audio_speech.transformation import (
|
||||
MistralTextToSpeechConfig,
|
||||
)
|
||||
|
||||
mistral_tts_config: Final = text_to_speech_provider_config or MistralTextToSpeechConfig()
|
||||
|
||||
if api_base is not None:
|
||||
litellm_params_dict["api_base"] = api_base
|
||||
if api_key is not None:
|
||||
litellm_params_dict["api_key"] = api_key
|
||||
|
||||
mistral_voice: Final[str | None] = voice if isinstance(voice, str) else None
|
||||
|
||||
response = base_llm_http_handler.text_to_speech_handler(
|
||||
model=model,
|
||||
input=input,
|
||||
voice=mistral_voice,
|
||||
text_to_speech_provider_config=mistral_tts_config,
|
||||
text_to_speech_optional_params=optional_params,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
litellm_params=litellm_params_dict,
|
||||
logging_obj=logging_obj,
|
||||
timeout=timeout,
|
||||
extra_headers=extra_headers,
|
||||
client=client,
|
||||
_is_async=aspeech or False,
|
||||
)
|
||||
elif custom_llm_provider == "aws_polly":
|
||||
from litellm.llms.aws_polly.text_to_speech.transformation import (
|
||||
AWSPollyTextToSpeechConfig,
|
||||
|
|
|
|||
|
|
@ -34947,9 +34947,9 @@
|
|||
"supports_audio_input": true
|
||||
},
|
||||
"mistral/voxtral-mini-tts-latest": {
|
||||
"input_cost_per_character": 1.6e-05,
|
||||
"litellm_provider": "mistral",
|
||||
"mode": "audio_speech",
|
||||
"output_cost_per_character": 1.6e-05,
|
||||
"source": "https://docs.mistral.ai/models/model-cards/voxtral-tts-26-03",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
|
|
@ -56516,9 +56516,9 @@
|
|||
"supports_audio_input": true
|
||||
},
|
||||
"mistral/voxtral-mini-tts-2603": {
|
||||
"input_cost_per_character": 1.6e-05,
|
||||
"litellm_provider": "mistral",
|
||||
"mode": "audio_speech",
|
||||
"output_cost_per_character": 1.6e-05,
|
||||
"source": "https://docs.mistral.ai/models/model-cards/voxtral-tts-26-03",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
|
|
|
|||
|
|
@ -4453,7 +4453,7 @@ class Router:
|
|||
self.fail_calls[model_name] += 1
|
||||
raise e
|
||||
|
||||
async def aspeech(self, model: str, input: str, voice: str, **kwargs):
|
||||
async def aspeech(self, model: str, input: str, voice: str | None = None, **kwargs):
|
||||
"""
|
||||
Example Usage:
|
||||
|
||||
|
|
@ -4505,7 +4505,7 @@ class Router:
|
|||
)
|
||||
raise e
|
||||
|
||||
async def _aspeech(self, model: str, input: str, voice: str, **kwargs):
|
||||
async def _aspeech(self, model: str, input: str, voice: str | None = None, **kwargs):
|
||||
model_name: Final = model
|
||||
try:
|
||||
verbose_router_logger.debug("Inside _aspeech()- model: %s; kwargs: %s", model, kwargs)
|
||||
|
|
@ -4529,7 +4529,7 @@ class Router:
|
|||
**{
|
||||
**data,
|
||||
"input": input,
|
||||
"voice": voice,
|
||||
"voice": data.get("voice") if voice is None else voice,
|
||||
"client": model_client,
|
||||
**kwargs,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9453,6 +9453,12 @@ class ProviderConfigManager:
|
|||
)
|
||||
|
||||
return MinimaxTextToSpeechConfig()
|
||||
elif litellm.LlmProviders.MISTRAL == provider:
|
||||
from litellm.llms.mistral.audio_speech.transformation import (
|
||||
MistralTextToSpeechConfig,
|
||||
)
|
||||
|
||||
return MistralTextToSpeechConfig()
|
||||
elif litellm.LlmProviders.AWS_POLLY == provider:
|
||||
from litellm.llms.aws_polly.text_to_speech.transformation import (
|
||||
AWSPollyTextToSpeechConfig,
|
||||
|
|
|
|||
|
|
@ -34947,9 +34947,9 @@
|
|||
"supports_audio_input": true
|
||||
},
|
||||
"mistral/voxtral-mini-tts-latest": {
|
||||
"input_cost_per_character": 1.6e-05,
|
||||
"litellm_provider": "mistral",
|
||||
"mode": "audio_speech",
|
||||
"output_cost_per_character": 1.6e-05,
|
||||
"source": "https://docs.mistral.ai/models/model-cards/voxtral-tts-26-03",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
|
|
@ -56516,9 +56516,9 @@
|
|||
"supports_audio_input": true
|
||||
},
|
||||
"mistral/voxtral-mini-tts-2603": {
|
||||
"input_cost_per_character": 1.6e-05,
|
||||
"litellm_provider": "mistral",
|
||||
"mode": "audio_speech",
|
||||
"output_cost_per_character": 1.6e-05,
|
||||
"source": "https://docs.mistral.ai/models/model-cards/voxtral-tts-26-03",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,197 @@
|
|||
import base64
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.llms.base_llm.text_to_speech.transformation import BaseTextToSpeechConfig
|
||||
from litellm.llms.mistral.audio_speech.transformation import (
|
||||
MistralTextToSpeechConfig,
|
||||
MistralTextToSpeechException,
|
||||
)
|
||||
from litellm.utils import ProviderConfigManager
|
||||
|
||||
SPEECH_URL: Final = "https://api.mistral.ai/v1/audio/speech"
|
||||
|
||||
|
||||
def test_mistral_text_to_speech_config_installed():
|
||||
config: Final = ProviderConfigManager.get_provider_text_to_speech_config(
|
||||
model="voxtral-mini-tts-2603",
|
||||
provider=litellm.LlmProviders.MISTRAL,
|
||||
)
|
||||
assert isinstance(config, BaseTextToSpeechConfig)
|
||||
assert isinstance(config, MistralTextToSpeechConfig)
|
||||
|
||||
|
||||
def test_map_openai_params_drops_speed_and_instructions():
|
||||
config: Final = MistralTextToSpeechConfig()
|
||||
voice, params = config.map_openai_params(
|
||||
model="voxtral-mini-tts-2603",
|
||||
optional_params={"response_format": "wav", "speed": 1.5, "instructions": "sound cheerful"},
|
||||
voice="en_paul_neutral",
|
||||
)
|
||||
assert voice == "en_paul_neutral"
|
||||
assert params == {"response_format": "wav"}
|
||||
|
||||
|
||||
def test_map_openai_params_accepts_voice_dict_and_ref_audio():
|
||||
config: Final = MistralTextToSpeechConfig()
|
||||
voice, params = config.map_openai_params(
|
||||
model="voxtral-mini-tts-2603",
|
||||
optional_params={},
|
||||
voice={"voice_id": "1f3a8b0c-voice-uuid"},
|
||||
kwargs={"ref_audio": "bXktdm9pY2Utc2FtcGxl"},
|
||||
)
|
||||
assert voice == "1f3a8b0c-voice-uuid"
|
||||
assert params == {"ref_audio": "bXktdm9pY2Utc2FtcGxl"}
|
||||
|
||||
|
||||
def test_transform_request_builds_mistral_body():
|
||||
config: Final = MistralTextToSpeechConfig()
|
||||
data: Final = config.transform_text_to_speech_request(
|
||||
model="voxtral-mini-tts-2603",
|
||||
input="hello from litellm",
|
||||
voice="en_paul_neutral",
|
||||
optional_params={"response_format": "wav"},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
assert data["dict_body"] == {
|
||||
"model": "voxtral-mini-tts-2603",
|
||||
"input": "hello from litellm",
|
||||
"voice_id": "en_paul_neutral",
|
||||
"response_format": "wav",
|
||||
}
|
||||
assert data["headers"] == {"Content-Type": "application/json"}
|
||||
|
||||
|
||||
def test_transform_request_omits_voice_for_ref_audio_cloning():
|
||||
config: Final = MistralTextToSpeechConfig()
|
||||
data: Final = config.transform_text_to_speech_request(
|
||||
model="voxtral-mini-tts-2603",
|
||||
input="clone me",
|
||||
voice=None,
|
||||
optional_params={"ref_audio": "bXktdm9pY2Utc2FtcGxl"},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
assert data["dict_body"] == {
|
||||
"model": "voxtral-mini-tts-2603",
|
||||
"input": "clone me",
|
||||
"ref_audio": "bXktdm9pY2Utc2FtcGxl",
|
||||
}
|
||||
|
||||
|
||||
def test_get_complete_url_default_base():
|
||||
config: Final = MistralTextToSpeechConfig()
|
||||
url: Final = config.get_complete_url(model="voxtral-mini-tts-2603", api_base=None, litellm_params={})
|
||||
assert url == SPEECH_URL
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"api_base",
|
||||
["https://custom.api.example.com/v1/", "https://custom.api.example.com/v1", "https://custom.api.example.com"],
|
||||
)
|
||||
def test_get_complete_url_custom_base_always_versioned(api_base: str):
|
||||
config: Final = MistralTextToSpeechConfig()
|
||||
url: Final = config.get_complete_url(model="voxtral-mini-tts-2603", api_base=api_base, litellm_params={})
|
||||
assert url == "https://custom.api.example.com/v1/audio/speech"
|
||||
|
||||
|
||||
def test_validate_environment_sets_bearer_header():
|
||||
config: Final = MistralTextToSpeechConfig()
|
||||
headers: Final = config.validate_environment(
|
||||
headers={"x-custom": "1"},
|
||||
model="voxtral-mini-tts-2603",
|
||||
api_key="sk-mistral-test",
|
||||
)
|
||||
assert headers == {
|
||||
"x-custom": "1",
|
||||
"Authorization": "Bearer sk-mistral-test",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
|
||||
def test_validate_environment_requires_key(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.delenv("MISTRAL_API_KEY", raising=False)
|
||||
config: Final = MistralTextToSpeechConfig()
|
||||
with pytest.raises(MistralTextToSpeechException, match="MISTRAL_API_KEY"):
|
||||
config.validate_environment(headers={}, model="voxtral-mini-tts-2603")
|
||||
|
||||
|
||||
def test_transform_response_decodes_base64_audio():
|
||||
config: Final = MistralTextToSpeechConfig()
|
||||
audio_bytes: Final = b"RIFF-fake-wav-bytes"
|
||||
raw_response: Final = httpx.Response(
|
||||
200,
|
||||
json={"audio_data": base64.b64encode(audio_bytes).decode()},
|
||||
headers={"x-request-id": "req-123"},
|
||||
request=httpx.Request(
|
||||
"POST",
|
||||
SPEECH_URL,
|
||||
json={"model": "voxtral-mini-tts-2603", "input": "hi", "response_format": "wav"},
|
||||
),
|
||||
)
|
||||
result: Final = config.transform_text_to_speech_response(
|
||||
model="voxtral-mini-tts-2603",
|
||||
raw_response=raw_response,
|
||||
logging_obj=MagicMock(),
|
||||
)
|
||||
assert result.content == audio_bytes
|
||||
assert result.response.headers["content-type"] == "audio/wav"
|
||||
assert result.response.headers["content-length"] == str(len(audio_bytes))
|
||||
assert result.response.headers["x-request-id"] == "req-123"
|
||||
|
||||
|
||||
def test_transform_response_missing_audio_data_raises():
|
||||
config: Final = MistralTextToSpeechConfig()
|
||||
raw_response: Final = httpx.Response(
|
||||
200,
|
||||
json={"detail": "unexpected"},
|
||||
request=httpx.Request("POST", SPEECH_URL, json={"model": "voxtral-mini-tts-2603", "input": "hi"}),
|
||||
)
|
||||
with pytest.raises(MistralTextToSpeechException, match="audio_data"):
|
||||
config.transform_text_to_speech_response(
|
||||
model="voxtral-mini-tts-2603",
|
||||
raw_response=raw_response,
|
||||
logging_obj=MagicMock(),
|
||||
)
|
||||
|
||||
|
||||
def test_map_openai_params_maps_openai_voice_aliases():
|
||||
config: Final = MistralTextToSpeechConfig()
|
||||
alloy_voice, _ = config.map_openai_params(
|
||||
model="voxtral-mini-tts-2603",
|
||||
optional_params={},
|
||||
voice="alloy",
|
||||
)
|
||||
nova_voice, _ = config.map_openai_params(
|
||||
model="voxtral-mini-tts-2603",
|
||||
optional_params={},
|
||||
voice="Nova",
|
||||
)
|
||||
passthrough_voice, _ = config.map_openai_params(
|
||||
model="voxtral-mini-tts-2603",
|
||||
optional_params={},
|
||||
voice="en_paul_happy",
|
||||
)
|
||||
assert alloy_voice == "en_paul_neutral"
|
||||
assert nova_voice == "gb_jane_sarcasm"
|
||||
assert passthrough_voice == "en_paul_happy"
|
||||
|
||||
|
||||
def test_transform_response_invalid_base64_raises():
|
||||
config: Final = MistralTextToSpeechConfig()
|
||||
raw_response: Final = httpx.Response(
|
||||
status_code=200,
|
||||
json={"audio_data": "QUJD!QUJD"},
|
||||
request=httpx.Request("POST", SPEECH_URL),
|
||||
)
|
||||
with pytest.raises(MistralTextToSpeechException, match="base64"):
|
||||
config.transform_text_to_speech_response(
|
||||
model="voxtral-mini-tts-2603",
|
||||
raw_response=raw_response,
|
||||
logging_obj=MagicMock(),
|
||||
)
|
||||
|
|
@ -4526,6 +4526,18 @@ def test_explicit_pricing_precedes_private_provider_response_model(
|
|||
assert selected == expected
|
||||
|
||||
|
||||
def test_cost_per_token_mistral_voxtral_tts_bills_per_input_character(_local_model_cost_map):
|
||||
prompt_usd, completion_usd = cost_per_token(
|
||||
model="voxtral-mini-tts-2603",
|
||||
custom_llm_provider="mistral",
|
||||
call_type="speech",
|
||||
prompt_characters=1000,
|
||||
)
|
||||
|
||||
assert prompt_usd == pytest.approx(1000 * 1.6e-05)
|
||||
assert completion_usd == 0.0
|
||||
|
||||
|
||||
def test_batch_cost_calculator_gpt_6_astra_bills_half_the_standard_rate(_local_model_cost_map):
|
||||
"""gpt-6-astra batch pricing is 50% off the standard $10 input and $50 output rates per 1M tokens."""
|
||||
from litellm.cost_calculator import batch_cost_calculator
|
||||
|
|
|
|||
|
|
@ -3351,6 +3351,52 @@ def test_stream_chunk_builder_leaves_xai_reported_cost_to_the_calculator(monkeyp
|
|||
assert logging_obj._response_cost_calculator(result=response) == pytest.approx(0.63)
|
||||
|
||||
|
||||
def test_speech_mistral_dispatches_and_decodes_audio(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setenv("MISTRAL_API_KEY", "sk-mistral-test")
|
||||
audio_bytes: Final = b"ID3-fake-mp3-bytes"
|
||||
mock_route: Final = respx_mock.post("https://api.mistral.ai/v1/audio/speech").mock(
|
||||
return_value=httpx.Response(200, json={"audio_data": base64.b64encode(audio_bytes).decode()})
|
||||
)
|
||||
|
||||
response: Final = litellm.speech(
|
||||
model="mistral/voxtral-mini-tts-2603",
|
||||
input="hello from litellm",
|
||||
voice="en_paul_neutral",
|
||||
response_format="wav",
|
||||
speed=2,
|
||||
instructions="sound cheerful",
|
||||
)
|
||||
|
||||
assert mock_route.called
|
||||
request_body: Final = json.loads(mock_route.calls.last.request.content)
|
||||
assert request_body == {
|
||||
"model": "voxtral-mini-tts-2603",
|
||||
"input": "hello from litellm",
|
||||
"voice_id": "en_paul_neutral",
|
||||
"response_format": "wav",
|
||||
}
|
||||
assert mock_route.calls.last.request.headers["authorization"] == "Bearer sk-mistral-test"
|
||||
assert response.content == audio_bytes
|
||||
|
||||
|
||||
def test_speech_mistral_routes_to_configured_api_base(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setenv("MISTRAL_API_KEY", "sk-mistral-test")
|
||||
audio_bytes: Final = b"ID3-gateway-bytes"
|
||||
gateway_route: Final = respx_mock.post("https://mistral.gateway.internal/v1/audio/speech").mock(
|
||||
return_value=httpx.Response(200, json={"audio_data": base64.b64encode(audio_bytes).decode()})
|
||||
)
|
||||
|
||||
response: Final = litellm.speech(
|
||||
model="mistral/voxtral-mini-tts-2603",
|
||||
input="hello from litellm",
|
||||
voice="en_paul_neutral",
|
||||
api_base="https://mistral.gateway.internal",
|
||||
)
|
||||
|
||||
assert gateway_route.called
|
||||
assert response.content == audio_bytes
|
||||
|
||||
|
||||
FOUNDRY_HOST: Final = "https://my-project.services.ai.azure.com"
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -12806,6 +12806,82 @@ class TestTierParamsTheTargetAccepts:
|
|||
assert accepted == {"reasoning_effort": "max"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_router_aspeech_without_voice_dispatches_ref_audio_cloning(respx_mock, monkeypatch):
|
||||
import base64
|
||||
|
||||
monkeypatch.setenv("MISTRAL_API_KEY", "sk-mistral-test")
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
audio_bytes = b"RIFFfake-wav-bytes"
|
||||
respx_mock.post("https://api.mistral.ai/v1/audio/speech").respond(
|
||||
json={"audio_data": base64.b64encode(audio_bytes).decode()}
|
||||
)
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "voxtral-tts",
|
||||
"litellm_params": {"model": "mistral/voxtral-mini-tts-2603"},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
response = await router.aspeech(model="voxtral-tts", input="clone me", ref_audio="ZmFrZQ==")
|
||||
|
||||
request_body = json.loads(respx_mock.calls.last.request.content)
|
||||
assert request_body == {"model": "voxtral-mini-tts-2603", "input": "clone me", "ref_audio": "ZmFrZQ=="}
|
||||
assert response.content == audio_bytes
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_router_aspeech_without_voice_keeps_deployment_default_voice(respx_mock, monkeypatch):
|
||||
import base64
|
||||
|
||||
monkeypatch.setenv("MISTRAL_API_KEY", "sk-mistral-test")
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
audio_bytes = b"RIFFfake-wav-bytes"
|
||||
respx_mock.post("https://api.mistral.ai/v1/audio/speech").respond(
|
||||
json={"audio_data": base64.b64encode(audio_bytes).decode()}
|
||||
)
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "voxtral-tts",
|
||||
"litellm_params": {"model": "mistral/voxtral-mini-tts-2603", "voice": "en_paul_neutral"},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
await router.aspeech(model="voxtral-tts", input="use my default")
|
||||
|
||||
request_body = json.loads(respx_mock.calls.last.request.content)
|
||||
assert request_body["voice_id"] == "en_paul_neutral"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_router_aspeech_request_voice_overrides_deployment_default(respx_mock, monkeypatch):
|
||||
import base64
|
||||
|
||||
monkeypatch.setenv("MISTRAL_API_KEY", "sk-mistral-test")
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
audio_bytes = b"RIFFfake-wav-bytes"
|
||||
respx_mock.post("https://api.mistral.ai/v1/audio/speech").respond(
|
||||
json={"audio_data": base64.b64encode(audio_bytes).decode()}
|
||||
)
|
||||
router = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "voxtral-tts",
|
||||
"litellm_params": {"model": "mistral/voxtral-mini-tts-2603", "voice": "en_paul_neutral"},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
await router.aspeech(model="voxtral-tts", input="override me", voice="gb_oliver_neutral")
|
||||
|
||||
request_body = json.loads(respx_mock.calls.last.request.content)
|
||||
assert request_body["voice_id"] == "gb_oliver_neutral"
|
||||
|
||||
|
||||
class TestRequestReasoningEffortOverride:
|
||||
def test_drop_effort_from_nested_carrier_preserves_other_nested_values(self):
|
||||
params: dict[str, object] = {"output_config": {"effort": "high", "format": "json"}}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue