From 1efca3905453244134eeab70cb1d18ed4edf235a Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 17 Nov 2026 11:35:30 -0800 Subject: [PATCH] fix(openai): bill the audio output tokens OpenAI reports for gpt-4o-mini-tts speech (#52317) * fix(openai): bill the audio output tokens OpenAI reports for token-priced speech models Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(openai): type the speech usage stream and expect stream_format on the wire test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(openai): flush the last speech event and ask for uncompressed events Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/llms/openai/openai.py | 13 +- litellm/llms/openai/speech_usage_stream.py | 193 +++++++++++++++ .../compatibility/test_audio_speech_wire.py | 2 +- .../cost_calculation/cases/openai.py | 11 +- .../llms/openai/test_openai_audio_speech.py | 224 +++++++++++++++++- 5 files changed, 433 insertions(+), 10 deletions(-) create mode 100644 litellm/llms/openai/speech_usage_stream.py diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index 1af6b3f29bb..3b3e554ab2b 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -79,6 +79,7 @@ from .common_utils import ( model_dump_provider_response, upstream_error_response, ) +from .speech_usage_stream import speech_response_with_usage, with_usage_stream from .workload_identity import ( build_async_openai_client, build_openai_client, @@ -1705,7 +1706,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): model: str, input: str, voice: str, - optional_params: dict, + optional_params: dict[str, object], api_key: str | None, api_base: str | None, organization: str | None, @@ -1759,13 +1760,13 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): }, ) - params: Final = speech_create_params(model, voice, input, optional_params) + params: Final = speech_create_params(model, voice, input, with_usage_stream(model, optional_params)) raw_response: Final = ( sync_client.audio.speech.with_streaming_response.create(**params).__enter__().http_response if stream else sync_client.audio.speech.create(**params).response ) - speech_response: Final = HttpxBinaryResponseContent(response=raw_response) + speech_response: Final = speech_response_with_usage(raw_response, model, optional_params) set_provider_response_headers_in_hidden_params(speech_response, raw_response.headers) return speech_response @@ -1774,7 +1775,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): model: str, input: str, voice: str, - optional_params: dict, + optional_params: dict[str, object], api_key: str | None, api_base: str | None, organization: str | None, @@ -1810,13 +1811,13 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): }, ) - params: Final = speech_create_params(model, voice, input, optional_params) + params: Final = speech_create_params(model, voice, input, with_usage_stream(model, optional_params)) raw_response: Final = ( (await openai_client.audio.speech.with_streaming_response.create(**params).__aenter__()).http_response if stream else (await openai_client.audio.speech.create(**params)).response ) - speech_response: Final = HttpxBinaryResponseContent(response=raw_response) + speech_response: Final = speech_response_with_usage(raw_response, model, optional_params) set_provider_response_headers_in_hidden_params(speech_response, raw_response.headers) return speech_response diff --git a/litellm/llms/openai/speech_usage_stream.py b/litellm/llms/openai/speech_usage_stream.py new file mode 100644 index 00000000000..f5a16615647 --- /dev/null +++ b/litellm/llms/openai/speech_usage_stream.py @@ -0,0 +1,193 @@ +import base64 +from collections.abc import AsyncIterator, Iterator, Mapping +from typing import Final + +import httpx +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError + +from litellm.litellm_core_utils import model_cost_table +from litellm.types.llms.openai import HttpxBinaryResponseContent +from litellm.types.utils import CompletionTokensDetailsWrapper, Usage + +_SSE_MEDIA_TYPE: Final = "text/event-stream" +_STREAM_FORMAT_KEY: Final = "stream_format" +_AUDIO_MEDIA_TYPES: Final[Mapping[str, str]] = { + "mp3": "audio/mpeg", + "opus": "audio/opus", + "aac": "audio/aac", + "flac": "audio/flac", + "wav": "audio/wav", + "pcm": "audio/pcm", +} + + +class _SpeechUsage(BaseModel): + model_config = ConfigDict(frozen=True) + + input_tokens: int + output_tokens: int + + +class _SpeechPricing(BaseModel): + model_config = ConfigDict(frozen=True) + + litellm_provider: str | None = None + output_cost_per_audio_token: float | None = None + + +_EXTRA_BODY: Final = TypeAdapter(dict[str, object]) + + +class _SpeechEvent(BaseModel): + model_config = ConfigDict(frozen=True) + + type: str + audio: str | None = None + usage: _SpeechUsage | None = None + + +def _parse_event(line: bytes) -> _SpeechEvent | None: + if not line.startswith(b"data:"): + return None + try: + return _SpeechEvent.model_validate_json(line.removeprefix(b"data:").strip()) + except ValidationError: + return None + + +class SpeechUsageDecoder: + """Turns OpenAI's speech SSE events back into the audio bytes and keeps the usage the last event reports.""" + + def __init__(self) -> None: + self._pending = b"" + self.usage: Usage | None = None + + def feed(self, chunk: bytes) -> bytes: + *lines, self._pending = (self._pending + chunk).split(b"\n") + return b"".join(self._audio(event) for line in lines if (event := _parse_event(line)) is not None) + + def _audio(self, event: _SpeechEvent) -> bytes: + if event.usage is not None: + self.usage = Usage( + prompt_tokens=event.usage.input_tokens, + completion_tokens=event.usage.output_tokens, + total_tokens=event.usage.input_tokens + event.usage.output_tokens, + completion_tokens_details=CompletionTokensDetailsWrapper(audio_tokens=event.usage.output_tokens), + ) + return b"" if event.audio is None else base64.b64decode(event.audio) + + +def _decoded_audio(decoder: SpeechUsageDecoder, speech_response: HttpxBinaryResponseContent, chunk: bytes) -> bytes: + audio: Final = decoder.feed(chunk) + if decoder.usage is not None and speech_response.usage is not decoder.usage: + speech_response.usage = decoder.usage # rebind-ok: the caller already holds this response while it streams + speech_response.set_response_cost(None) + return audio + + +class _UsageAsyncByteStream(httpx.AsyncByteStream): + def __init__(self, stream: httpx.AsyncByteStream, speech_response: HttpxBinaryResponseContent) -> None: + self._stream: Final = stream + self._speech_response: Final = speech_response + self._decoder: Final = SpeechUsageDecoder() + + async def __aiter__(self) -> AsyncIterator[bytes]: + async for chunk in self._stream: + if audio := _decoded_audio(self._decoder, self._speech_response, chunk): + yield audio + if last_audio := _decoded_audio(self._decoder, self._speech_response, b"\n"): + yield last_audio + + async def aclose(self) -> None: + await self._stream.aclose() + + +class _UsageSyncByteStream(httpx.SyncByteStream): + def __init__(self, stream: httpx.SyncByteStream, speech_response: HttpxBinaryResponseContent) -> None: + self._stream: Final = stream + self._speech_response: Final = speech_response + self._decoder: Final = SpeechUsageDecoder() + + def __iter__(self) -> Iterator[bytes]: + for chunk in self._stream: + if audio := _decoded_audio(self._decoder, self._speech_response, chunk): + yield audio + if last_audio := _decoded_audio(self._decoder, self._speech_response, b"\n"): + yield last_audio + + def close(self) -> None: + self._stream.close() + + +def bills_audio_output_tokens(model: str) -> bool: + row: Final = model_cost_table.published().get(model) + if row is None: + return False + try: + pricing: Final = _SpeechPricing.model_validate(row) + except ValidationError: + return False + return pricing.litellm_provider == "openai" and pricing.output_cost_per_audio_token is not None + + +def _caller_extra_body(optional_params: Mapping[str, object]) -> Mapping[str, object]: + return _EXTRA_BODY.validate_python(optional_params.get("extra_body") or {}) + + +def _caller_extra_headers(optional_params: Mapping[str, object]) -> Mapping[str, object]: + return _EXTRA_BODY.validate_python(optional_params.get("extra_headers") or {}) + + +def wants_usage_stream(model: str, optional_params: Mapping[str, object]) -> bool: + caller_asked: Final = _STREAM_FORMAT_KEY in optional_params or _STREAM_FORMAT_KEY in _caller_extra_body( + optional_params + ) + return bills_audio_output_tokens(model) and not caller_asked + + +def with_usage_stream(model: str, optional_params: Mapping[str, object]) -> Mapping[str, object]: + """Asks OpenAI for the SSE form of the audio, the only form that reports the audio tokens it bills.""" + if not wants_usage_stream(model, optional_params): + return optional_params + return { + **optional_params, + "extra_body": {**_caller_extra_body(optional_params), _STREAM_FORMAT_KEY: "sse"}, + "extra_headers": {**_caller_extra_headers(optional_params), "Accept-Encoding": "identity"}, + } + + +def _audio_headers(headers: httpx.Headers, response_format: object) -> httpx.Headers: + media_type: Final = _AUDIO_MEDIA_TYPES.get(str(response_format), _AUDIO_MEDIA_TYPES["mp3"]) + kept: Final = tuple( + (name, value) + for name, value in headers.multi_items() + if name.lower() not in ("content-type", "content-length", "content-encoding") + ) + return httpx.Headers((*kept, ("content-type", media_type))) + + +def speech_response_with_usage( + raw_response: httpx.Response, model: str, optional_params: Mapping[str, object] +) -> HttpxBinaryResponseContent: + """The speech response the caller gets. The SSE answer we asked for is decoded back to audio with its usage.""" + unchanged: Final = HttpxBinaryResponseContent(response=raw_response) + if not wants_usage_stream(model, optional_params) or not (unchanged.content_type or "").startswith(_SSE_MEDIA_TYPE): + return unchanged + headers: Final = _audio_headers(raw_response.headers, optional_params.get("response_format")) + if raw_response.is_stream_consumed: + decoder: Final = SpeechUsageDecoder() + audio: Final = decoder.feed(raw_response.content + b"\n") + return HttpxBinaryResponseContent( + response=httpx.Response( + status_code=raw_response.status_code, headers=headers, content=audio, request=raw_response.request + ), + usage=decoder.usage, + ) + raw_response.headers = headers # rebind-ok: the unread provider response is what streams the audio to the caller + speech_response: Final = HttpxBinaryResponseContent(response=raw_response) + raw_response.stream = ( # rebind-ok: the unread provider response is what streams the audio to the caller + _UsageAsyncByteStream(raw_response.stream, speech_response) + if isinstance(raw_response.stream, httpx.AsyncByteStream) + else _UsageSyncByteStream(raw_response.stream, speech_response) + ) + return speech_response diff --git a/tests/integration/compatibility/test_audio_speech_wire.py b/tests/integration/compatibility/test_audio_speech_wire.py index 91295cc1688..6bf248aa422 100644 --- a/tests/integration/compatibility/test_audio_speech_wire.py +++ b/tests/integration/compatibility/test_audio_speech_wire.py @@ -40,7 +40,7 @@ def test_audio_speech_forwards_the_full_json_body_and_streams_the_audio(gateway: assert request.target == "/v1/audio/speech" assert request.headers["authorization"] == "Bearer synthetic-openai-key" body: Final = json.loads(request.body) - assert body == _SPEECH_REQUEST + assert body == {**_SPEECH_REQUEST, "stream_format": "sse"} assert type(body["speed"]) is float return Reply(body=_SPEECH_BYTES, content_type="audio/wav") diff --git a/tests/integration/cost_calculation/cases/openai.py b/tests/integration/cost_calculation/cases/openai.py index a662677f804..77465e466f9 100644 --- a/tests/integration/cost_calculation/cases/openai.py +++ b/tests/integration/cost_calculation/cases/openai.py @@ -12467,9 +12467,16 @@ GPT_4O_MINI_TTS_SPEECH: Final = CostTrackingTestCase( model="gpt-4o-mini-tts", endpoint="/v1/audio/speech", request={"input": "The pricing harness says hello.", "voice": "alloy", "response_format": "mp3"}, - response=BinaryResponse(content_type="audio/mpeg", length=2048), + response=SseResponse( + content_type="text/event-stream", + frames=( + 'data: {"type":"speech.audio.delta","audio":"c2NyaXB0ZWQgYXVkaW8="}', + 'data: {"type":"speech.audio.done","usage":{"input_tokens":6,"output_tokens":50,"total_tokens":56}}', + "data: [DONE]", + ), + ), expected=ExactExpected( - spend=3.66e-06, input_cost=3.66e-06, output_cost=0.0, prompt_tokens=0, completion_tokens=0 + spend=0.00060366, input_cost=3.66e-06, output_cost=0.0006, prompt_tokens=6, completion_tokens=50 ), ) diff --git a/tests/unit/llms/openai/test_openai_audio_speech.py b/tests/unit/llms/openai/test_openai_audio_speech.py index 27ae0ffd543..383b5c7766b 100644 --- a/tests/unit/llms/openai/test_openai_audio_speech.py +++ b/tests/unit/llms/openai/test_openai_audio_speech.py @@ -2,13 +2,16 @@ from __future__ import annotations +import base64 +import json from typing import Final from unittest.mock import Mock import httpx import pytest -from openai import AsyncAzureOpenAI, AsyncOpenAI +from openai import AsyncAzureOpenAI, AsyncOpenAI, OpenAI +import litellm from litellm.llms.azure.azure import AzureChatCompletion from litellm.llms.openai.openai import OpenAIChatCompletion @@ -103,3 +106,222 @@ async def test_azure_audio_speech_yields_chunks_before_upstream_finishes(): assert await anext(audio) == b"one" assert produced == [b"one"] + + +_SPEECH_SSE: Final = ( + b'data: {"type":"speech.audio.delta","audio":"%s"}\n\n' + b'data: {"type":"speech.audio.delta","audio":"%s"}\n\n' + b'data: {"type":"speech.audio.done","usage":{"input_tokens":6,"output_tokens":50,"total_tokens":56}}\n\n' + b"data: [DONE]\n\n" +) % (base64.b64encode(b"first-half|"), base64.b64encode(b"second-half")) + + +class _SpeechSseStream(httpx.SyncByteStream, httpx.AsyncByteStream): + def __init__(self, frames: list[bytes]): + self._frames = frames + + def __iter__(self): + yield from self._frames + + async def __aiter__(self): + for frame in self._frames: + yield frame + + +def _sse_transport( + requests: list[dict[str, object]], frame_size: int, sse: bytes = _SPEECH_SSE, encodings: list[str] | None = None +) -> httpx.MockTransport: + def _respond(request: httpx.Request) -> httpx.Response: + requests.append(json.loads(request.content)) + if encodings is not None: + encodings.append(request.headers["accept-encoding"]) + frames: Final = [sse[start : start + frame_size] for start in range(0, len(sse), frame_size)] + return httpx.Response( + status_code=200, + headers={"content-type": "text/event-stream; charset=utf-8"}, + stream=_SpeechSseStream(frames), + ) + + return httpx.MockTransport(_respond) + + +async def _token_priced_speech( + requests: list[dict[str, object]], *, stream: bool, frame_size: int, optional_params: dict[str, object] +): + return await OpenAIChatCompletion().async_audio_speech( + model="gpt-4o-mini-tts", + input="hello", + voice="alloy", + optional_params=optional_params, + api_key="fake-key", + api_base=None, + organization=None, + project=None, + max_retries=0, + timeout=10.0, + logging_obj=Mock(), + client=AsyncOpenAI( + api_key="fake-key", http_client=httpx.AsyncClient(transport=_sse_transport(requests, frame_size)) + ), + stream=stream, + ) + + +def _audio_output_cost(response: litellm.HttpxBinaryResponseContent) -> float: + _, completion_cost = litellm.cost_per_token( + model="gpt-4o-mini-tts", custom_llm_provider="openai", usage_object=response.usage, call_type="aspeech" + ) + return completion_cost + + +@pytest.mark.asyncio +@pytest.mark.parametrize("frame_size", [7, 64, 4096]) +async def test_streamed_token_priced_speech_returns_audio_and_bills_the_audio_tokens_openai_reports(frame_size: int): + requests: Final[list[dict[str, object]]] = [] + + response: Final = await _token_priced_speech( + requests, stream=True, frame_size=frame_size, optional_params={"response_format": "wav"} + ) + audio: Final = b"".join([chunk async for chunk in await response.aiter_bytes()]) + + assert audio == b"first-half|second-half" + assert response.content_type == "audio/wav" + assert requests == [ + { + "model": "gpt-4o-mini-tts", + "voice": "alloy", + "input": "hello", + "response_format": "wav", + "stream_format": "sse", + } + ] + assert response.usage is not None + assert (response.usage.prompt_tokens, response.usage.completion_tokens) == (6, 50) + assert _audio_output_cost(response) == pytest.approx( + 50 * litellm.model_cost["gpt-4o-mini-tts"]["output_cost_per_audio_token"] + ) + + +@pytest.mark.asyncio +async def test_buffered_token_priced_speech_returns_audio_and_bills_the_audio_tokens_openai_reports(): + requests: Final[list[dict[str, object]]] = [] + + response: Final = await _token_priced_speech(requests, stream=False, frame_size=64, optional_params={}) + + assert response.content == b"first-half|second-half" + assert response.content_type == "audio/mpeg" + assert _audio_output_cost(response) == pytest.approx( + 50 * litellm.model_cost["gpt-4o-mini-tts"]["output_cost_per_audio_token"] + ) + + +def test_sync_streamed_token_priced_speech_returns_audio_and_the_usage_openai_reports(): + requests: Final[list[dict[str, object]]] = [] + + response: Final = OpenAIChatCompletion().audio_speech( + model="gpt-4o-mini-tts", + input="hello", + voice="alloy", + optional_params={}, + api_key="fake-key", + api_base=None, + organization=None, + project=None, + max_retries=0, + timeout=10.0, + logging_obj=Mock(), + client=OpenAI(api_key="fake-key", http_client=httpx.Client(transport=_sse_transport(requests, 7))), + stream=True, + ) + + assert b"".join(response.iter_bytes()) == b"first-half|second-half" + assert response.usage is not None + assert (response.usage.prompt_tokens, response.usage.completion_tokens) == (6, 50) + + +@pytest.mark.asyncio +async def test_streamed_speech_drops_the_cost_stamped_before_the_audio_tokens_arrived(): + response: Final = await _token_priced_speech([], stream=True, frame_size=64, optional_params={}) + response.set_response_cost(3.6e-06) + + async for _ in await response.aiter_bytes(): + pass + + assert "response_cost" not in response.hidden_params + + +@pytest.mark.asyncio +async def test_speech_sse_asked_for_by_the_caller_is_passed_through_untouched(): + requests: Final[list[dict[str, object]]] = [] + + response: Final = await _token_priced_speech( + requests, stream=False, frame_size=4096, optional_params={"stream_format": "sse"} + ) + + assert requests == [{"model": "gpt-4o-mini-tts", "voice": "alloy", "input": "hello", "stream_format": "sse"}] + assert response.content == _SPEECH_SSE + + +@pytest.mark.asyncio +async def test_speech_model_priced_per_character_is_not_asked_for_sse(): + requests: Final[list[dict[str, object]]] = [] + + def _respond(request: httpx.Request) -> httpx.Response: + requests.append(json.loads(request.content)) + return httpx.Response(200, headers={"content-type": "audio/mpeg"}, content=b"audio") + + response: Final = await OpenAIChatCompletion().async_audio_speech( + model="tts-1", + input="hello", + voice="alloy", + optional_params={}, + api_key="fake-key", + api_base=None, + organization=None, + project=None, + max_retries=0, + timeout=10.0, + logging_obj=Mock(), + client=AsyncOpenAI(api_key="fake-key", http_client=httpx.AsyncClient(transport=httpx.MockTransport(_respond))), + stream=False, + ) + + assert requests == [{"model": "tts-1", "voice": "alloy", "input": "hello"}] + assert response.content == b"audio" + + +def _sync_streamed_speech(transport: httpx.MockTransport): + return OpenAIChatCompletion().audio_speech( + model="gpt-4o-mini-tts", + input="hello", + voice="alloy", + optional_params={}, + api_key="fake-key", + api_base=None, + organization=None, + project=None, + max_retries=0, + timeout=10.0, + logging_obj=Mock(), + client=OpenAI(api_key="fake-key", http_client=httpx.Client(transport=transport)), + stream=True, + ) + + +def test_streamed_speech_keeps_the_usage_of_a_last_event_with_no_trailing_newline(): + response: Final = _sync_streamed_speech( + _sse_transport([], 64, sse=_SPEECH_SSE.removesuffix(b"\n\ndata: [DONE]\n\n")) + ) + + assert b"".join(response.iter_bytes()) == b"first-half|second-half" + assert response.usage is not None + assert response.usage.completion_tokens == 50 + + +def test_token_priced_speech_asks_the_provider_not_to_compress_the_events_it_decodes(): + encodings: Final[list[str]] = [] + + response: Final = _sync_streamed_speech(_sse_transport([], 64, encodings=encodings)) + + assert b"".join(response.iter_bytes()) == b"first-half|second-half" + assert encodings == ["identity"]