From 9c54ed898b64b0a2b99c20708080de3ebbe4053a Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Tue, 6 Oct 2026 09:05:57 -0500 Subject: [PATCH] fix(vertex-ai): preserve FLAC billing and wrapped audio headers --- .../llms/vertex_ai/text_to_speech/transformation.py | 8 ++++++++ litellm/proxy/proxy_server.py | 8 ++++++-- .../vertex_ai/text_to_speech/test_transformation.py | 13 ++++++++++--- tests/unit/proxy/proxy_server/test_routes_audio.py | 4 ++++ 4 files changed, 28 insertions(+), 5 deletions(-) diff --git a/litellm/llms/vertex_ai/text_to_speech/transformation.py b/litellm/llms/vertex_ai/text_to_speech/transformation.py index 261c5f09768..9ce4d39524f 100644 --- a/litellm/llms/vertex_ai/text_to_speech/transformation.py +++ b/litellm/llms/vertex_ai/text_to_speech/transformation.py @@ -68,6 +68,14 @@ def _fallback_gemini_tts_audio_duration(audio: bytes, encoding: str, sample_rate if byte_rate > 0 and data_size <= len(audio) - data_offset - 8: return data_size / byte_rate + if encoding == "FLAC" and audio[:4] == b"fLaC" and len(audio) >= 42: + if audio[4] & 0x7F != 0 or int.from_bytes(audio[5:8], "big") != 34: + return None + stream_info: Final = int.from_bytes(audio[18:26], "big") + flac_sample_rate: Final = stream_info >> 44 + total_samples: Final = stream_info & ((1 << 36) - 1) + return total_samples / flac_sample_rate if flac_sample_rate > 0 and total_samples > 0 else None + if encoding == "MP3" and speech_media_type_from_audio_bytes(audio) == "audio/mpeg": return len(audio) / 4000 diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 0ec86edc4a7..39e3fa2f2e9 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -12733,8 +12733,12 @@ async def audio_speech( upstream_content_type: Final = ( response.response.headers.get("content-type") if isinstance(response, HttpxBinaryResponseContent) else None ) - media_type: Final = extra_media_type or resolve_speech_media_type( - upstream_content_type=upstream_content_type, + media_type: Final = resolve_speech_media_type( + upstream_content_type=( + upstream_content_type + if upstream_content_type is not None and upstream_content_type.lower().startswith("audio/") + else extra_media_type or upstream_content_type + ), response_format=requested_format_str, ) diff --git a/tests/unit/llms/vertex_ai/text_to_speech/test_transformation.py b/tests/unit/llms/vertex_ai/text_to_speech/test_transformation.py index f4026d08df0..4a7fb7f0fad 100644 --- a/tests/unit/llms/vertex_ai/text_to_speech/test_transformation.py +++ b/tests/unit/llms/vertex_ai/text_to_speech/test_transformation.py @@ -2,6 +2,7 @@ import base64 import io import math import struct +import sys import wave from pathlib import Path from typing import Final @@ -9,6 +10,7 @@ from unittest.mock import MagicMock, Mock, patch import httpx import pytest +import soundfile from pydantic import ValidationError import litellm @@ -509,8 +511,8 @@ def test_transform_text_to_speech_response_leaves_unknown_bytes_unlabeled(): assert result.response.content == raw_pcm -@pytest.mark.parametrize("encoding", ["LINEAR16", "PCM", "MP3", "OGG_OPUS", "ALAW", "MULAW"]) -def test_gemini_cloud_tts_response_bills_text_and_audio(encoding: str): +@pytest.mark.parametrize("encoding", ["LINEAR16", "PCM", "MP3", "OGG_OPUS", "ALAW", "MULAW", "FLAC"]) +def test_gemini_cloud_tts_response_bills_text_and_audio(encoding: str, monkeypatch: pytest.MonkeyPatch) -> None: model: Final = "gemini-3.1-flash-tts-preview" input_text: Final = "Hello from Gemini text to speech" pcm_audio: Final = b"\x00\x00" * 24000 @@ -520,6 +522,8 @@ def test_gemini_cloud_tts_response_bills_text_and_audio(encoding: str): wav_file.setsampwidth(2) wav_file.setframerate(24000) wav_file.writeframes(pcm_audio) + flac_buffer: Final = io.BytesIO() + soundfile.write(flac_buffer, (0.0,) * 24000, 48000, format="FLAC", subtype="PCM_16") g711_audio: Final = ( struct.pack( "<4sI4s4sIHHIIHH4sI", @@ -547,6 +551,7 @@ def test_gemini_cloud_tts_response_bills_text_and_audio(encoding: str): "OGG_OPUS": (fixture_dir / "gemini_tts_speech.ogg").read_bytes(), "ALAW": g711_audio, "MULAW": g711_audio, + "FLAC": flac_buffer.getvalue(), } audio_bytes: Final = audio_by_encoding[encoding] logger: Final = MagicMock() @@ -571,6 +576,7 @@ def test_gemini_cloud_tts_response_bills_text_and_audio(encoding: str): "Google Cloud AudioEncoding and RFC 7845, checked 2026-09-23: " "https://cloud.google.com/text-to-speech/docs/reference/rest/v1/AudioEncoding " "https://www.rfc-editor.org/rfc/rfc7845.html" + " https://www.rfc-editor.org/rfc/rfc9639.html#section-8.2, checked 2026-10-06" ) assert usage.completion_tokens == math.ceil(duration * 25), ( "Google Cloud TTS pricing, 2026-09-23: https://cloud.google.com/text-to-speech/pricing" @@ -579,7 +585,8 @@ def test_gemini_cloud_tts_response_bills_text_and_audio(encoding: str): assert usage.total_tokens == usage.prompt_tokens + usage.completion_tokens assert usage.completion_tokens_details is not None assert usage.completion_tokens_details.audio_tokens == usage.completion_tokens - with patch("litellm.llms.vertex_ai.text_to_speech.transformation.calculate_request_duration", return_value=None): + with monkeypatch.context() as dependencies: + dependencies.setitem(sys.modules, "soundfile", None) sdk_result: Final = VertexAITextToSpeechConfig().transform_text_to_speech_response(model, raw_response, logger) assert sdk_result.usage == usage diff --git a/tests/unit/proxy/proxy_server/test_routes_audio.py b/tests/unit/proxy/proxy_server/test_routes_audio.py index f4230fe33aa..fc98266f139 100644 --- a/tests/unit/proxy/proxy_server/test_routes_audio.py +++ b/tests/unit/proxy/proxy_server/test_routes_audio.py @@ -201,6 +201,9 @@ def test_audio_speech_happy_path(client, auth_as, patched_speech, path): ("audio/flac", "flac", "audio/flac"), ("audio/pcm", "pcm", "audio/pcm"), ("audio/wav", "mp3", "audio/wav"), + ("audio/wav", "alaw", "audio/wav"), + ("audio/wav", "mulaw", "audio/wav"), + ("audio/wav", "pcm16", "audio/wav"), ("application/json", "flac", "audio/flac"), (None, "wav", "audio/wav"), (None, None, "audio/mpeg"), @@ -224,6 +227,7 @@ def test_audio_speech_content_type_matches_audio_format( @pytest.mark.parametrize("path", ["/v1/audio/speech", "/audio/speech"]) +@pytest.mark.parametrize("patched_speech", (None, "application/json"), indirect=True) @pytest.mark.parametrize( "response_format,expected_media_type", [