mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-05 08:07:05 +00:00
Merge pull request #38868 from BerriAI/litellm_fix_gemini_tts_container
fix(speech): honor pcm/wav response_format for Gemini TTS and reject unsupported containers
This commit is contained in:
Mateo Wang
GitHub
commit
d22a3e847d
12 changed files with 402 additions and 40 deletions
|
|
@ -115,9 +115,11 @@ class SpeechToCompletionBridgeHandler:
|
|||
**request_data,
|
||||
)
|
||||
|
||||
requested_response_format: Final = optional_params.get("response_format")
|
||||
if isinstance(result, ModelResponse):
|
||||
return self.transformation_handler.transform_response(
|
||||
model_response=result,
|
||||
response_format=requested_response_format if isinstance(requested_response_format, str) else None,
|
||||
)
|
||||
else:
|
||||
raise Exception(f"Unmapped response type. Got type: {type(result)}")
|
||||
|
|
|
|||
|
|
@ -21,6 +21,8 @@ def _completion_response_cost(model_response: "ModelResponse") -> float | None:
|
|||
|
||||
|
||||
GEMINI_TTS_CHAT_AUDIO_FORMAT: Final = "pcm16"
|
||||
GEMINI_TTS_RAW_RESPONSE_FORMAT: Final = "pcm"
|
||||
GEMINI_TTS_SUPPORTED_RESPONSE_FORMATS: Final = frozenset({"wav", GEMINI_TTS_RAW_RESPONSE_FORMAT})
|
||||
|
||||
|
||||
class ChatAudioParam(TypedDict):
|
||||
|
|
@ -29,6 +31,26 @@ class ChatAudioParam(TypedDict):
|
|||
|
||||
|
||||
class SpeechToCompletionBridgeTransformationHandler:
|
||||
def _validate_response_format(
|
||||
self, model: str, custom_llm_provider: str, optional_params: Mapping[str, object]
|
||||
) -> None:
|
||||
if not self._is_gemini_tts_model(model):
|
||||
return
|
||||
response_format: Final = optional_params.get("response_format")
|
||||
if not isinstance(response_format, str) or response_format in GEMINI_TTS_SUPPORTED_RESPONSE_FORMATS:
|
||||
return
|
||||
from litellm.exceptions import BadRequestError
|
||||
|
||||
supported: Final = ", ".join(sorted(GEMINI_TTS_SUPPORTED_RESPONSE_FORMATS))
|
||||
raise BadRequestError(
|
||||
message=(
|
||||
f"Gemini TTS only produces raw PCM16 audio, so response_format='{response_format}'"
|
||||
f" is not supported. Supported response formats: {supported}."
|
||||
),
|
||||
model=model,
|
||||
llm_provider=custom_llm_provider,
|
||||
)
|
||||
|
||||
def _chat_completion_params(self, optional_params: Mapping[str, object]) -> Mapping[str, object]:
|
||||
return MappingProxyType(
|
||||
{
|
||||
|
|
@ -67,6 +89,7 @@ class SpeechToCompletionBridgeTransformationHandler:
|
|||
litellm_logging_obj: "LiteLLMLoggingObj",
|
||||
custom_llm_provider: str,
|
||||
) -> dict:
|
||||
self._validate_response_format(model, custom_llm_provider, optional_params)
|
||||
user_message: Final[ChatCompletionUserMessage] = {"role": "user", "content": input}
|
||||
return_kwargs: Final = {
|
||||
"model": model,
|
||||
|
|
@ -125,7 +148,14 @@ class SpeechToCompletionBridgeTransformationHandler:
|
|||
"""Check if the model is a Gemini TTS model that returns PCM16 data."""
|
||||
return "gemini" in model.lower() and ("tts" in model.lower() or "preview-tts" in model.lower())
|
||||
|
||||
def transform_response(self, model_response: "ModelResponse") -> "HttpxBinaryResponseContent":
|
||||
def _gemini_tts_response_body(self, decoded_audio: bytes, response_format: str | None) -> tuple[bytes, str]:
|
||||
if response_format == GEMINI_TTS_RAW_RESPONSE_FORMAT:
|
||||
return decoded_audio, "audio/pcm"
|
||||
return self._convert_pcm16_to_wav(decoded_audio), "audio/wav"
|
||||
|
||||
def transform_response(
|
||||
self, model_response: "ModelResponse", response_format: str | None
|
||||
) -> "HttpxBinaryResponseContent":
|
||||
import base64
|
||||
|
||||
import httpx
|
||||
|
|
@ -136,23 +166,17 @@ class SpeechToCompletionBridgeTransformationHandler:
|
|||
audio_part: Final = cast(Choices, model_response.choices[0]).message.audio
|
||||
if audio_part is None:
|
||||
raise ValueError("No audio part found in the response")
|
||||
audio_content: Final = audio_part.data
|
||||
decoded_audio: Final = base64.b64decode(audio_part.data)
|
||||
|
||||
# Decode base64 to get binary content
|
||||
binary_data = base64.b64decode(audio_content)
|
||||
|
||||
# Check if this is a Gemini TTS model that returns raw PCM16 data
|
||||
model: Final = getattr(model_response, "model", "")
|
||||
headers: Final = {}
|
||||
if self._is_gemini_tts_model(model):
|
||||
# Convert PCM16 to WAV format for proper audio file playback
|
||||
binary_data = self._convert_pcm16_to_wav(binary_data)
|
||||
headers["Content-Type"] = "audio/wav"
|
||||
else:
|
||||
headers["Content-Type"] = "audio/mpeg"
|
||||
|
||||
# Create an httpx.Response object
|
||||
response: Final = httpx.Response(status_code=200, content=binary_data, headers=headers)
|
||||
content, content_type = (
|
||||
self._gemini_tts_response_body(decoded_audio, response_format)
|
||||
if self._is_gemini_tts_model(model)
|
||||
else (decoded_audio, "audio/mpeg")
|
||||
)
|
||||
response: Final = httpx.Response(
|
||||
status_code=200, content=content, headers=MappingProxyType({"Content-Type": content_type})
|
||||
)
|
||||
binary_response: Final = HttpxBinaryResponseContent(response)
|
||||
binary_response.set_response_cost(_completion_response_cost(model_response))
|
||||
return binary_response
|
||||
|
|
|
|||
|
|
@ -7,7 +7,13 @@ import os
|
|||
from dataclasses import dataclass
|
||||
from typing import Final
|
||||
|
||||
from litellm.types.files import get_file_mime_type_from_extension
|
||||
from litellm.types.files import (
|
||||
AUDIO_FILE_TYPES,
|
||||
FILE_EXTENSIONS,
|
||||
FILE_MIME_TYPES,
|
||||
FileType,
|
||||
get_file_mime_type_from_extension,
|
||||
)
|
||||
from litellm.types.utils import FileTypes
|
||||
|
||||
|
||||
|
|
@ -323,3 +329,75 @@ def calculate_request_duration(file: FileTypes) -> float | None:
|
|||
except Exception:
|
||||
# Silently fail if duration extraction fails
|
||||
return None
|
||||
|
||||
|
||||
DEFAULT_SPEECH_MEDIA_TYPE: Final = "audio/mpeg"
|
||||
|
||||
|
||||
def _speech_media_type_for_response_format(response_format: str) -> str | None:
|
||||
file_type: Final = next(
|
||||
(candidate for candidate, extensions in FILE_EXTENSIONS.items() if response_format.lower() in extensions),
|
||||
None,
|
||||
)
|
||||
if file_type is None or file_type not in AUDIO_FILE_TYPES:
|
||||
return None
|
||||
return FILE_MIME_TYPES[file_type]
|
||||
|
||||
|
||||
def resolve_speech_media_type(upstream_content_type: str | None, response_format: str | None) -> str:
|
||||
upstream_media_type: Final = (upstream_content_type or "").split(";", 1)[0].strip().lower()
|
||||
if upstream_media_type.startswith("audio/"):
|
||||
return upstream_media_type
|
||||
requested_media_type: Final = (
|
||||
None if response_format is None else _speech_media_type_for_response_format(response_format)
|
||||
)
|
||||
return requested_media_type or DEFAULT_SPEECH_MEDIA_TYPE
|
||||
|
||||
|
||||
_OGG_OPUS_HEAD_WINDOW: Final = 64
|
||||
_ADTS_SYNC_AND_LAYER_MASK: Final = 0xF6
|
||||
_ADTS_SYNC_AND_LAYER: Final = 0xF0
|
||||
_ADTS_SAMPLE_RATE_INDEX_LIMIT: Final = 13
|
||||
_MPEG_SYNC_MASK: Final = 0xE0
|
||||
_MPEG_LAYER_MASK: Final = 0x06
|
||||
_MPEG_RESERVED_VERSION: Final = 0x01
|
||||
_MPEG_INVALID_BITRATE_INDEX: Final = 0x0F
|
||||
_MPEG_RESERVED_SAMPLE_RATE_INDEX: Final = 0x03
|
||||
|
||||
|
||||
def _adts_aac_frame_media_type(header: bytes) -> str | None:
|
||||
sample_rate_index: Final = (header[2] >> 2) & 0x0F
|
||||
return FILE_MIME_TYPES[FileType.AAC] if sample_rate_index < _ADTS_SAMPLE_RATE_INDEX_LIMIT else None
|
||||
|
||||
|
||||
def _mpeg_audio_frame_media_type(header: bytes) -> str | None:
|
||||
version: Final = (header[1] >> 3) & 0x03
|
||||
layer: Final = header[1] & _MPEG_LAYER_MASK
|
||||
bitrate_index: Final = header[2] >> 4
|
||||
sample_rate_index: Final = (header[2] >> 2) & 0x03
|
||||
if (
|
||||
(header[1] & _MPEG_SYNC_MASK) != _MPEG_SYNC_MASK
|
||||
or version == _MPEG_RESERVED_VERSION
|
||||
or layer == 0
|
||||
or bitrate_index == _MPEG_INVALID_BITRATE_INDEX
|
||||
or sample_rate_index == _MPEG_RESERVED_SAMPLE_RATE_INDEX
|
||||
):
|
||||
return None
|
||||
return FILE_MIME_TYPES[FileType.MP3]
|
||||
|
||||
|
||||
def speech_media_type_from_audio_bytes(audio: bytes) -> str | None:
|
||||
if audio[:4] == b"RIFF" and audio[8:12] == b"WAVE":
|
||||
return FILE_MIME_TYPES[FileType.WAV]
|
||||
if audio[:4] == b"fLaC":
|
||||
return FILE_MIME_TYPES[FileType.FLAC]
|
||||
if audio[:4] == b"OggS":
|
||||
is_opus: Final = b"OpusHead" in audio[:_OGG_OPUS_HEAD_WINDOW]
|
||||
return FILE_MIME_TYPES[FileType.OPUS if is_opus else FileType.OGG]
|
||||
if audio[:3] == b"ID3":
|
||||
return FILE_MIME_TYPES[FileType.MP3]
|
||||
if len(audio) < 3 or audio[0] != 0xFF:
|
||||
return None
|
||||
if (audio[1] & _ADTS_SYNC_AND_LAYER_MASK) == _ADTS_SYNC_AND_LAYER:
|
||||
return _adts_aac_frame_media_type(audio)
|
||||
return _mpeg_audio_frame_media_type(audio)
|
||||
|
|
|
|||
|
|
@ -7,10 +7,14 @@ Reference: https://cloud.google.com/text-to-speech/docs/reference/rest/v1/text/s
|
|||
|
||||
import base64
|
||||
from collections.abc import Coroutine
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, Union
|
||||
|
||||
import httpx
|
||||
|
||||
from litellm.litellm_core_utils.audio_utils.utils import (
|
||||
speech_media_type_from_audio_bytes,
|
||||
)
|
||||
from litellm.llms.base_llm.text_to_speech.transformation import (
|
||||
BaseTextToSpeechConfig,
|
||||
TextToSpeechRequestData,
|
||||
|
|
@ -457,12 +461,11 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase):
|
|||
if not response_content:
|
||||
raise ValueError("No audioContent in Vertex AI TTS response")
|
||||
|
||||
# Decode base64 to get binary content
|
||||
binary_data: Final = base64.b64decode(response_content)
|
||||
|
||||
# Create an httpx.Response object with the binary data
|
||||
media_type: Final = speech_media_type_from_audio_bytes(binary_data)
|
||||
response: Final = httpx.Response(
|
||||
status_code=200,
|
||||
headers=None if media_type is None else MappingProxyType({"content-type": media_type}),
|
||||
content=binary_data,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -263,6 +263,7 @@ from litellm.litellm_core_utils.agentic_loop_settings import (
|
|||
validated_max_agentic_loops,
|
||||
)
|
||||
from litellm.litellm_core_utils.asyncify import asyncify
|
||||
from litellm.litellm_core_utils.audio_utils.utils import resolve_speech_media_type
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
_get_parent_otel_span_from_kwargs,
|
||||
get_litellm_metadata_from_kwargs,
|
||||
|
|
@ -11061,15 +11062,14 @@ async def audio_speech(
|
|||
if callback_headers:
|
||||
custom_headers.update(callback_headers)
|
||||
|
||||
# Determine media type based on model type
|
||||
media_type = "audio/mpeg" # Default for OpenAI TTS
|
||||
request_model: Final = data.get("model", "")
|
||||
if request_model:
|
||||
request_model_lower: Final = request_model.lower()
|
||||
if "gemini" in request_model_lower and (
|
||||
"tts" in request_model_lower or "preview-tts" in request_model_lower
|
||||
):
|
||||
media_type = "audio/wav" # Gemini TTS returns WAV format after conversion
|
||||
requested_format: Final = data.get("response_format")
|
||||
upstream_content_type: Final = (
|
||||
response.response.headers.get("content-type") if isinstance(response, HttpxBinaryResponseContent) else None
|
||||
)
|
||||
media_type: Final = resolve_speech_media_type(
|
||||
upstream_content_type=upstream_content_type,
|
||||
response_format=requested_format if isinstance(requested_format, str) else None,
|
||||
)
|
||||
|
||||
return StreamingResponse(
|
||||
_audio_speech_chunk_generator(response),
|
||||
|
|
@ -11085,7 +11085,15 @@ async def audio_speech(
|
|||
)
|
||||
verbose_proxy_logger.error("litellm.proxy.proxy_server.audio_speech(): Exception occured - %s", e)
|
||||
verbose_proxy_logger.debug(traceback.format_exc())
|
||||
raise e
|
||||
if isinstance(e, (ProxyException, HTTPException)):
|
||||
raise e
|
||||
raise ProxyException(
|
||||
message=getattr(e, "message", f"{e}"),
|
||||
type=getattr(e, "type", "None"),
|
||||
param=getattr(e, "param", "None"),
|
||||
openai_code=getattr(e, "code", None),
|
||||
code=getattr(e, "status_code", 500),
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
|
|
|
|||
|
|
@ -9433,6 +9433,10 @@ class ProviderConfigManager:
|
|||
|
||||
return RunwayMLTextToSpeechConfig()
|
||||
elif litellm.LlmProviders.VERTEX_AI == provider:
|
||||
if "gemini" in model:
|
||||
# Gemini TTS uses the speech_to_completion bridge, and Google Cloud TTS param
|
||||
# mapping would drop response_format before the bridge sees it (LIT-6501)
|
||||
return None
|
||||
from litellm.llms.vertex_ai.text_to_speech.transformation import (
|
||||
VertexAITextToSpeechConfig,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import base64
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
|
|
@ -8,8 +9,17 @@ from litellm.constants import OPENAI_CHAT_COMPLETION_PARAMS
|
|||
from litellm.endpoints.speech.speech_to_completion_bridge.transformation import (
|
||||
SpeechToCompletionBridgeTransformationHandler,
|
||||
)
|
||||
from litellm.types.utils import ChatCompletionAudioResponse, Choices, Message, ModelResponse
|
||||
|
||||
GEMINI_TTS_MODEL: Final = "gemini-3.1-flash-tts-preview"
|
||||
PCM_BYTES: Final = b"\x01\x02\x03\x04" * 6
|
||||
|
||||
|
||||
def _model_response(model: str, pcm: bytes) -> ModelResponse:
|
||||
audio: Final = ChatCompletionAudioResponse(
|
||||
data=base64.b64encode(pcm).decode(), expires_at=0, transcript="hello"
|
||||
)
|
||||
return ModelResponse(model=model, choices=[Choices(message=Message(content=None, audio=audio))])
|
||||
|
||||
|
||||
def _bridge_request(response_format: str | None) -> dict:
|
||||
|
|
@ -28,7 +38,7 @@ def _bridge_request(response_format: str | None) -> dict:
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("response_format", ["wav", "mp3", "pcm", None])
|
||||
@pytest.mark.parametrize("response_format", ["wav", "pcm", None])
|
||||
def test_gemini_tts_request_keeps_speech_response_format_out_of_chat_params(response_format: str | None) -> None:
|
||||
request: Final = _bridge_request(response_format)
|
||||
|
||||
|
|
@ -60,3 +70,48 @@ def test_non_gemini_request_forwards_speech_response_format_as_audio_format() ->
|
|||
|
||||
assert "response_format" not in request
|
||||
assert request["audio"] == {"voice": "alloy", "format": "wav"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("response_format", ["mp3", "flac", "opus", "aac"])
|
||||
def test_gemini_tts_request_rejects_formats_gemini_cannot_produce(response_format: str) -> None:
|
||||
with pytest.raises(litellm.BadRequestError) as excinfo:
|
||||
_bridge_request(response_format)
|
||||
|
||||
assert excinfo.value.status_code == 400
|
||||
assert response_format in str(excinfo.value)
|
||||
assert "pcm" in str(excinfo.value)
|
||||
assert "wav" in str(excinfo.value)
|
||||
|
||||
|
||||
def test_gemini_tts_pcm_response_returns_raw_pcm_bytes() -> None:
|
||||
response: Final = SpeechToCompletionBridgeTransformationHandler().transform_response(
|
||||
model_response=_model_response(GEMINI_TTS_MODEL, PCM_BYTES),
|
||||
response_format="pcm",
|
||||
)
|
||||
|
||||
assert response.response.content == PCM_BYTES
|
||||
assert response.response.headers["content-type"] == "audio/pcm"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("response_format", ["wav", None])
|
||||
def test_gemini_tts_wav_and_default_responses_wrap_pcm_in_wav(response_format: str | None) -> None:
|
||||
response: Final = SpeechToCompletionBridgeTransformationHandler().transform_response(
|
||||
model_response=_model_response(GEMINI_TTS_MODEL, PCM_BYTES),
|
||||
response_format=response_format,
|
||||
)
|
||||
|
||||
body: Final = response.response.content
|
||||
assert body[:4] == b"RIFF"
|
||||
assert body[8:12] == b"WAVE"
|
||||
assert body[44:] == PCM_BYTES
|
||||
assert response.response.headers["content-type"] == "audio/wav"
|
||||
|
||||
|
||||
def test_non_gemini_response_keeps_original_bytes_and_mpeg_content_type() -> None:
|
||||
response: Final = SpeechToCompletionBridgeTransformationHandler().transform_response(
|
||||
model_response=_model_response("gpt-4o-audio-preview", PCM_BYTES),
|
||||
response_format="mp3",
|
||||
)
|
||||
|
||||
assert response.response.content == PCM_BYTES
|
||||
assert response.response.headers["content-type"] == "audio/mpeg"
|
||||
|
|
|
|||
|
|
@ -347,3 +347,65 @@ class TestNormalizeTranscriptionLanguageToBcp47:
|
|||
)
|
||||
|
||||
assert normalize_transcription_language_to_bcp47(language) == expected
|
||||
|
||||
|
||||
class TestResolveSpeechMediaType:
|
||||
@pytest.mark.parametrize(
|
||||
("upstream_content_type", "response_format", "expected"),
|
||||
[
|
||||
("audio/wav", None, "audio/wav"),
|
||||
("AUDIO/WAV", None, "audio/wav"),
|
||||
("audio/flac; charset=binary", "mp3", "audio/flac"),
|
||||
("application/json", "flac", "audio/flac"),
|
||||
("application/octet-stream", "pcm", "audio/pcm"),
|
||||
(None, "wav", "audio/wav"),
|
||||
(None, "WAV", "audio/wav"),
|
||||
(None, "opus", "audio/opus"),
|
||||
(None, "aac", "audio/aac"),
|
||||
(None, "mp3", "audio/mpeg"),
|
||||
(None, "mp4", "audio/mpeg"),
|
||||
(None, "bogus", "audio/mpeg"),
|
||||
(None, None, "audio/mpeg"),
|
||||
("", None, "audio/mpeg"),
|
||||
],
|
||||
)
|
||||
def test_resolution(self, upstream_content_type, response_format, expected):
|
||||
from litellm.litellm_core_utils.audio_utils.utils import resolve_speech_media_type
|
||||
|
||||
resolved = resolve_speech_media_type(
|
||||
upstream_content_type=upstream_content_type,
|
||||
response_format=response_format,
|
||||
)
|
||||
assert resolved == expected
|
||||
|
||||
|
||||
class TestSpeechMediaTypeFromAudioBytes:
|
||||
@pytest.mark.parametrize(
|
||||
("audio", "expected"),
|
||||
[
|
||||
(b"RIFF\x24\x00\x00\x00WAVEfmt ", "audio/wav"),
|
||||
(b"fLaC\x00\x00\x00\x22", "audio/flac"),
|
||||
(b"OggS" + b"\x00" * 24 + b"OpusHead", "audio/opus"),
|
||||
(b"OggS" + b"\x00" * 24 + b"\x01vorbis", "audio/ogg"),
|
||||
(b"ID3\x04\x00\x00\x00\x00\x00\x00", "audio/mpeg"),
|
||||
(b"\xff\xfb\x90\x64", "audio/mpeg"),
|
||||
(b"\xff\xf3\x80\x00", "audio/mpeg"),
|
||||
(b"\xff\xf1\x50\x80", "audio/aac"),
|
||||
(b"\xff\xf9\x50\x80", "audio/aac"),
|
||||
(b"RIFF\x24\x00\x00\x00AVI LIST", None),
|
||||
(b"\xff\xff\xff\xff\xff\xff", None),
|
||||
(b"\xff\xfb\xf0\x00", None),
|
||||
(b"\xff\xfb\x9c\x00", None),
|
||||
(b"\xff\xeb\x90\x00", None),
|
||||
(b"\xff\xf1\xf4\x80", None),
|
||||
(b"\xff\x00\x00\x00", None),
|
||||
(b"\x00\x01\x02\x03\x04\x05", None),
|
||||
(b"\xff\xfb", None),
|
||||
(b"\xff", None),
|
||||
(b"", None),
|
||||
],
|
||||
)
|
||||
def test_sniffing(self, audio, expected):
|
||||
from litellm.litellm_core_utils.audio_utils.utils import speech_media_type_from_audio_bytes
|
||||
|
||||
assert speech_media_type_from_audio_bytes(audio) == expected
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import base64
|
||||
from unittest.mock import MagicMock, Mock, patch
|
||||
|
||||
import httpx
|
||||
|
|
@ -126,6 +127,48 @@ class TestVertexAITextToSpeechConfig:
|
|||
assert voice_dict == voice_input
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("audio", "expected_content_type"),
|
||||
[
|
||||
(b"RIFF\x24\x00\x00\x00WAVEfmt \x10\x00\x00\x00", "audio/wav"),
|
||||
(b"\xff\xfb\x90\x64\x00\x00\x00\x00", "audio/mpeg"),
|
||||
(b"OggS" + b"\x00" * 24 + b"OpusHead", "audio/opus"),
|
||||
(b"fLaC\x00\x00\x00\x22", "audio/flac"),
|
||||
],
|
||||
)
|
||||
def test_transform_text_to_speech_response_labels_content_type(audio, expected_content_type):
|
||||
raw_response = httpx.Response(
|
||||
status_code=200,
|
||||
json={"audioContent": base64.b64encode(audio).decode()},
|
||||
)
|
||||
|
||||
result = VertexAITextToSpeechConfig().transform_text_to_speech_response(
|
||||
model="vertex_ai/chirp",
|
||||
raw_response=raw_response,
|
||||
logging_obj=MagicMock(),
|
||||
)
|
||||
|
||||
assert result.response.headers["content-type"] == expected_content_type
|
||||
assert result.response.content == audio
|
||||
|
||||
|
||||
def test_transform_text_to_speech_response_leaves_unknown_bytes_unlabeled():
|
||||
raw_pcm = b"\x00\x01\x02\x03\x04\x05\x06\x07"
|
||||
raw_response = httpx.Response(
|
||||
status_code=200,
|
||||
json={"audioContent": base64.b64encode(raw_pcm).decode()},
|
||||
)
|
||||
|
||||
result = VertexAITextToSpeechConfig().transform_text_to_speech_response(
|
||||
model="vertex_ai/chirp",
|
||||
raw_response=raw_response,
|
||||
logging_obj=MagicMock(),
|
||||
)
|
||||
|
||||
assert "content-type" not in result.response.headers
|
||||
assert result.response.content == raw_pcm
|
||||
|
||||
|
||||
@patch("litellm.llms.custom_httpx.llm_http_handler.HTTPHandler.post")
|
||||
@patch.object(VertexAITextToSpeechConfig, "_ensure_access_token")
|
||||
@patch.object(VertexAITextToSpeechConfig, "_get_token_and_url")
|
||||
|
|
|
|||
|
|
@ -12,13 +12,16 @@ from __future__ import annotations
|
|||
import io
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.types.llms.openai import HttpxBinaryResponseContent
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def patched_speech(monkeypatch):
|
||||
def patched_speech(monkeypatch, request):
|
||||
upstream_content_type = getattr(request, "param", "audio/mpeg")
|
||||
monkeypatch.setattr(proxy_server, "llm_router", MagicMock())
|
||||
monkeypatch.setattr(
|
||||
proxy_server,
|
||||
|
|
@ -36,15 +39,14 @@ def patched_speech(monkeypatch):
|
|||
|
||||
monkeypatch.setattr(proxy_server, "add_litellm_data_to_request", _add_data)
|
||||
|
||||
class _FakeBinaryResp:
|
||||
async def aiter_bytes(self, chunk_size: int = 8192):
|
||||
async def _gen():
|
||||
yield b"\x00\x01\x02"
|
||||
|
||||
return _gen()
|
||||
|
||||
async def _llm_call():
|
||||
return _FakeBinaryResp()
|
||||
return HttpxBinaryResponseContent(
|
||||
httpx.Response(
|
||||
status_code=200,
|
||||
headers={} if upstream_content_type is None else {"content-type": upstream_content_type},
|
||||
content=b"\x00\x01\x02",
|
||||
)
|
||||
)
|
||||
|
||||
async def _fake_route_request(*args, **kwargs):
|
||||
return _llm_call()
|
||||
|
|
@ -79,6 +81,24 @@ def patched_speech_error(monkeypatch):
|
|||
yield
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def patched_speech_provider_rejection(monkeypatch, patched_speech_error):
|
||||
import litellm
|
||||
|
||||
async def _raise(*args, **kwargs):
|
||||
raise litellm.BadRequestError(
|
||||
message=(
|
||||
"Gemini TTS only produces raw PCM16 audio, so response_format='mp3' is not supported."
|
||||
" Supported response formats: pcm, wav."
|
||||
),
|
||||
model="gemini-3.1-flash-tts-preview",
|
||||
llm_provider="gemini",
|
||||
)
|
||||
|
||||
monkeypatch.setattr(proxy_server, "route_request", _raise)
|
||||
yield
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def patched_transcription(monkeypatch):
|
||||
router = MagicMock()
|
||||
|
|
@ -152,6 +172,35 @@ def test_audio_speech_happy_path(client, auth_as, patched_speech, path):
|
|||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("patched_speech", "response_format", "expected_content_type"),
|
||||
[
|
||||
("audio/wav", "wav", "audio/wav"),
|
||||
("audio/flac", "flac", "audio/flac"),
|
||||
("audio/pcm", "pcm", "audio/pcm"),
|
||||
("audio/wav", "mp3", "audio/wav"),
|
||||
("application/json", "flac", "audio/flac"),
|
||||
(None, "wav", "audio/wav"),
|
||||
(None, None, "audio/mpeg"),
|
||||
],
|
||||
indirect=["patched_speech"],
|
||||
)
|
||||
def test_audio_speech_content_type_matches_audio_format(
|
||||
client, auth_as, patched_speech, response_format, expected_content_type
|
||||
):
|
||||
"""Regression for LIT-6482: /v1/audio/speech mislabeled wav/flac/pcm as audio/mpeg."""
|
||||
payload = {
|
||||
"model": "tts-1",
|
||||
"input": "Hi",
|
||||
"voice": "alloy",
|
||||
**({} if response_format is None else {"response_format": response_format}),
|
||||
}
|
||||
with auth_as():
|
||||
response = client.post("/v1/audio/speech", json=payload)
|
||||
assert response.status_code == 200
|
||||
assert response.headers.get("content-type", "").split(";")[0] == expected_content_type
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", ["/v1/audio/speech", "/audio/speech"])
|
||||
def test_audio_speech_error(client, auth_as, patched_speech_error, path):
|
||||
"""Pins ``POST /v1/audio/speech`` and ``POST /audio/speech`` (error)."""
|
||||
|
|
@ -162,6 +211,18 @@ def test_audio_speech_error(client, auth_as, patched_speech_error, path):
|
|||
assert len(response.content) > 0
|
||||
|
||||
|
||||
def test_audio_speech_bad_request_maps_to_400(client, auth_as, patched_speech_provider_rejection):
|
||||
"""Regression for LIT-6501: a BadRequestError from the speech path surfaced as a generic 500."""
|
||||
payload = {"model": "gemini-tts", "input": "Hi", "voice": "Kore", "response_format": "mp3"}
|
||||
with auth_as():
|
||||
response = client.post("/v1/audio/speech", json=payload)
|
||||
assert response.status_code == 400
|
||||
error = response.json()["error"]
|
||||
assert "response_format='mp3'" in error["message"]
|
||||
assert "pcm" in error["message"]
|
||||
assert "wav" in error["message"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", ["/v1/audio/transcriptions", "/audio/transcriptions"])
|
||||
def test_audio_transcription_happy_path(client, auth_as, patched_transcription, path):
|
||||
"""Pins ``POST /v1/audio/transcriptions`` / ``POST /audio/transcriptions`` (happy)."""
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import asyncio
|
|||
import os
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
|
|
@ -29,6 +30,7 @@ def _make_mock_tts_response():
|
|||
inner = MagicMock()
|
||||
inner.aiter_bytes = _aiter_bytes
|
||||
inner._hidden_params = {}
|
||||
inner.response = httpx.Response(status_code=200, headers={"content-type": "audio/mpeg"})
|
||||
|
||||
async def _resolver():
|
||||
return inner
|
||||
|
|
|
|||
|
|
@ -1416,6 +1416,26 @@ def test_get_provider_rerank_config():
|
|||
assert isinstance(config, HostedVLLMRerankConfig)
|
||||
|
||||
|
||||
def test_get_provider_text_to_speech_config_vertex_gemini_skips_cloud_tts():
|
||||
"""Regression for LIT-6501: mapping vertex Gemini TTS params through Google Cloud TTS
|
||||
dropped response_format before the speech_to_completion bridge could honor it."""
|
||||
from litellm.llms.vertex_ai.text_to_speech.transformation import VertexAITextToSpeechConfig
|
||||
from litellm.utils import LlmProviders
|
||||
|
||||
assert (
|
||||
ProviderConfigManager.get_provider_text_to_speech_config(
|
||||
model="gemini-2.5-flash-preview-tts", provider=LlmProviders.VERTEX_AI
|
||||
)
|
||||
is None
|
||||
)
|
||||
assert isinstance(
|
||||
ProviderConfigManager.get_provider_text_to_speech_config(
|
||||
model="en-US-Studio-O", provider=LlmProviders.VERTEX_AI
|
||||
),
|
||||
VertexAITextToSpeechConfig,
|
||||
)
|
||||
|
||||
|
||||
# Models that should be skipped during testing
|
||||
OLD_PROVIDERS = ["aleph_alpha", "palm"]
|
||||
SKIP_MODELS = [
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue