diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 7990832dc48..2e90c073780 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -972,7 +972,11 @@ def get_usage_object( ( completion_response.get("usage") if isinstance(completion_response, dict) - else getattr(completion_response, "get", lambda x: None)("usage") + else ( + completion_response.usage + if isinstance(completion_response, HttpxBinaryResponseContent) + else getattr(completion_response, "get", lambda x: None)("usage") + ) ), ) @@ -1415,8 +1419,10 @@ def completion_cost( verbose_logger.debug("selected model name for cost calculation: %s", model) if completion_response is not None and ( - isinstance(completion_response, BaseModel) or isinstance(completion_response, dict) - ): # tts returns a custom class + isinstance(completion_response, (BaseModel, dict)) + or isinstance(completion_response, HttpxBinaryResponseContent) + and completion_response.usage is not None + ): if isinstance(completion_response, dict): usage_obj: dict | Usage | None = completion_response.get("usage", {}) else: diff --git a/litellm/llms/vertex_ai/text_to_speech/transformation.py b/litellm/llms/vertex_ai/text_to_speech/transformation.py index 2b799145422..2fcb68b0f44 100644 --- a/litellm/llms/vertex_ai/text_to_speech/transformation.py +++ b/litellm/llms/vertex_ai/text_to_speech/transformation.py @@ -6,6 +6,7 @@ Reference: https://cloud.google.com/text-to-speech/docs/reference/rest/v1/text/s """ import base64 +import math from collections.abc import Coroutine, Mapping from types import MappingProxyType from typing import TYPE_CHECKING, Any, ClassVar, Final, TypeAlias, Union @@ -16,6 +17,7 @@ import litellm from litellm.exceptions import UnsupportedParamsError from litellm.litellm_core_utils.audio_utils.utils import ( DEFAULT_SPEECH_MEDIA_TYPE, + calculate_request_duration, speech_media_type_from_audio_bytes, ) from litellm.litellm_core_utils.url_utils import encode_url_path_segment @@ -665,8 +667,34 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): content=binary_data, ) - # Initialize the HttpxBinaryResponseContent instance - return HttpxBinaryResponseContent(response) + binary_response: Final = HttpxBinaryResponseContent(response) + if self._is_gemini_tts_model(model): + from litellm.types.utils import CompletionTokensDetailsWrapper, Usage + + request_body: Final = logging_obj.model_call_details["additional_args"]["complete_input_dict"]["dict_body"] + input_data: Final = request_body["input"] + audio_config: Final = request_body["audioConfig"] + container_duration: Final = calculate_request_duration(binary_data) + sample_rate: Final = audio_config.get("sampleRateHertz") or 24000 + duration: Final = ( + len(binary_data) / (2 * sample_rate) + if container_duration is None and audio_config["audioEncoding"] == "PCM" + else container_duration + ) + if duration is None: + raise ValueError("Cannot determine Gemini TTS output duration for cost calculation") + input_text: Final = " ".join( + value for value in (input_data.get("text"), input_data.get("ssml"), input_data.get("prompt")) if value + ) + prompt_tokens: Final = litellm.token_counter(model=model, text=input_text) + audio_tokens: Final = math.ceil(duration * 25) + binary_response.usage = Usage( + prompt_tokens=prompt_tokens, + completion_tokens=audio_tokens, + total_tokens=prompt_tokens + audio_tokens, + completion_tokens_details=CompletionTokensDetailsWrapper(audio_tokens=audio_tokens), + ) + return binary_response class VertexAILyriaTextToSpeechConfig(VertexAITextToSpeechConfig): diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 599b1a76249..b7a086d151d 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1,7 +1,7 @@ from collections.abc import Iterable, Mapping from enum import Enum from os import PathLike -from typing import IO, Any, Final, Literal, Optional, TypeAlias, Union +from typing import IO, TYPE_CHECKING, Any, Final, Literal, Optional, TypeAlias, Union import httpx from openai import Omit @@ -93,6 +93,9 @@ from litellm.types.responses.main import ( from .base import CachedTokensDetails +if TYPE_CHECKING: + from litellm.types.utils import Usage + FileContent = IO[bytes] | bytes | PathLike FileTypes = ( @@ -120,10 +123,12 @@ class BinaryResponseSummary(TypedDict): class HttpxBinaryResponseContent(_HttpxBinaryResponseContent): _hidden_params: dict + usage: "Usage | None" def __init__(self, response: httpx.Response) -> None: super().__init__(response) self._hidden_params = {} # mutable-ok: mutable-dict contract shared with ModelResponse logging consumers + self.usage = None def logging_summary(self) -> BinaryResponseSummary: return { diff --git a/pyproject.toml b/pyproject.toml index ba72378989a..55e68a59524 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ dependencies = [ "pydantic-settings>=2.14.1,<3.0", "jsonschema>=4.0.0,<5.0", "boto3>=1.43.1,<2.0", + "soundfile>=0.12.1,<1.0", ] [project.urls] @@ -82,7 +83,6 @@ proxy = [ "InquirerPy>=0.3.4,<1.0", "tomlkit>=0.13.3,<1.0", "polars>=1.38.1,<2.0", - "soundfile>=0.12.1,<1.0", "pyroscope-io>=0.8.16,<1.0; sys_platform != 'win32'", "expression>=5.6.0,<6.0", ] diff --git a/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py b/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py index 50a73e39c12..62916841ea5 100644 --- a/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/text_to_speech/test_transformation.py @@ -1,4 +1,8 @@ import base64 +import io +import math +import wave +from pathlib import Path from typing import Final from unittest.mock import MagicMock, Mock, patch @@ -10,6 +14,8 @@ from litellm.llms.vertex_ai.text_to_speech.transformation import ( VertexAILyriaTextToSpeechConfig, VertexAITextToSpeechConfig, ) +from litellm.cost_calculator import response_cost_calculator +from litellm.litellm_core_utils.audio_utils.utils import calculate_request_duration from litellm.types.utils import LlmProviders from litellm.utils import ProviderConfigManager @@ -407,6 +413,66 @@ def test_transform_text_to_speech_response_leaves_unknown_bytes_unlabeled(): assert result.response.content == raw_pcm +@pytest.mark.parametrize("encoding", ["LINEAR16", "PCM", "MP3"]) +def test_gemini_cloud_tts_response_bills_text_and_audio(encoding: str): + model: Final = "gemini-3.1-flash-tts-preview" + input_text: Final = "Hello from Gemini text to speech" + pcm_audio: Final = b"\x00\x00" * 24000 + wav_buffer: Final = io.BytesIO() + with wave.open(wav_buffer, "wb") as wav_file: + wav_file.setnchannels(1) + wav_file.setsampwidth(2) + wav_file.setframerate(24000) + wav_file.writeframes(pcm_audio) + audio_by_encoding: Final = { + "PCM": pcm_audio, + "LINEAR16": wav_buffer.getvalue(), + "MP3": (Path(__file__).resolve().parents[4] / "audio_tests/speech_vertex.mp3").read_bytes(), + } + audio_bytes: Final = audio_by_encoding[encoding] + logger: Final = MagicMock() + logger.model_call_details = { + "additional_args": { + "complete_input_dict": { + "dict_body": { + "input": {"text": input_text}, + "audioConfig": {"audioEncoding": encoding, "sampleRateHertz": 24000}, + } + } + } + } + raw_response: Final = httpx.Response(200, json={"audioContent": base64.b64encode(audio_bytes).decode()}) + + result: Final = VertexAITextToSpeechConfig().transform_text_to_speech_response(model, raw_response, logger) + usage: Final = result.usage + assert usage is not None + duration: Final = len(audio_bytes) / 48000 if encoding == "PCM" else calculate_request_duration(audio_bytes) + assert duration is not None + assert usage.completion_tokens == math.ceil(duration * 25), ( + "Google Cloud TTS pricing, 2026-09-23: https://cloud.google.com/text-to-speech/pricing" + ) + assert usage.prompt_tokens == litellm.token_counter(model=model, text=input_text) + assert usage.total_tokens == usage.prompt_tokens + usage.completion_tokens + assert usage.completion_tokens_details is not None + assert usage.completion_tokens_details.audio_tokens == usage.completion_tokens + + model_info: Final = litellm.get_model_info(model, custom_llm_provider="vertex_ai") + cost: Final = response_cost_calculator( + response_object=result, + model=model, + custom_llm_provider="vertex_ai", + call_type="speech", + optional_params={}, + prompt=input_text, + ) + expected_cost: Final = ( + usage.prompt_tokens * model_info["input_cost_per_token"] + + usage.completion_tokens * model_info["output_cost_per_audio_token"] + ) + assert cost == pytest.approx(expected_cost) + assert cost > usage.prompt_tokens * model_info["input_cost_per_token"] + + class TestVertexAILyriaTextToSpeechConfig: @pytest.mark.parametrize( "model", @@ -867,10 +933,11 @@ def test_litellm_speech_vertex_ai_gemini_tts_mp3_uses_cloud_tts(mock_get_token, mock_response = Mock(spec=httpx.Response) mock_response.status_code = 200 mock_response.headers = {"content-type": "application/json"} - mock_response.json.return_value = {"audioContent": "SGVsbG8gV29ybGQ="} + audio_bytes: Final = (Path(__file__).resolve().parents[4] / "audio_tests/speech_vertex.mp3").read_bytes() + mock_response.json.return_value = {"audioContent": base64.b64encode(audio_bytes).decode()} mock_post.return_value = mock_response - litellm.speech( + result: Final = litellm.speech( model="vertex_ai/gemini-3.1-flash-tts-preview", input="Ryan: Hi.\nKatie: Hello.", voice={ @@ -900,6 +967,11 @@ def test_litellm_speech_vertex_ai_gemini_tts_mp3_uses_cloud_tts(mock_get_token, vertex_location="global", ) + assert result.response.content == audio_bytes + assert result.usage is not None + assert result.usage.prompt_tokens > 0 + assert result.usage.completion_tokens > 0 + mock_post.assert_called_once() call_kwargs = mock_post.call_args.kwargs assert call_kwargs["url"] == "https://texttospeech.googleapis.com/v1/text:synthesize" diff --git a/uv.lock b/uv.lock index 85e2b6d4e52..590812613de 100644 --- a/uv.lock +++ b/uv.lock @@ -4518,6 +4518,7 @@ dependencies = [ { name = "pydantic-settings" }, { name = "python-dotenv" }, { name = "pyyaml" }, + { name = "soundfile" }, { name = "tiktoken" }, { name = "tokenizers" }, ] @@ -4592,7 +4593,6 @@ proxy = [ { name = "restrictedpython" }, { name = "rich" }, { name = "rq" }, - { name = "soundfile" }, { name = "starlette" }, { name = "tomlkit" }, { name = "uvicorn" }, @@ -4841,7 +4841,7 @@ requires-dist = [ { name = "rq", marker = "extra == 'proxy'", specifier = ">=2.7.0,<3.0" }, { name = "semantic-router", marker = "python_full_version < '3.14' and extra == 'semantic-router'", specifier = ">=0.1.15,<1.0" }, { name = "sentry-sdk", marker = "extra == 'proxy-runtime'", specifier = ">=2.21.0,<3.0" }, - { name = "soundfile", marker = "extra == 'proxy'", specifier = ">=0.12.1,<1.0" }, + { name = "soundfile", specifier = ">=0.12.1,<1.0" }, { name = "soundfile", marker = "extra == 'stt-nvidia-riva'", specifier = ">=0.12.1" }, { name = "starlette", marker = "extra == 'proxy'", specifier = ">=1.0.1,<2.0" }, { name = "tiktoken", marker = "python_full_version < '3.14'", specifier = ">=0.8.0,<1.0" },