fix(openai): bill the audio output tokens OpenAI reports for gpt-4o-mini-tts speech (#52317)

* fix(openai): bill the audio output tokens OpenAI reports for token-priced speech models

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(openai): type the speech usage stream and expect stream_format on the wire test

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(openai): flush the last speech event and ask for uncompressed events

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Kerry Lu 2026-11-17 11:35:30 -08:00 • committed by GitHub
parent 52412090b9
commit 1efca39054
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 433 additions and 10 deletions

View file

@ -79,6 +79,7 @@ from .common_utils import (
model_dump_provider_response,
upstream_error_response,
)
from .speech_usage_stream import speech_response_with_usage, with_usage_stream
from .workload_identity import (
build_async_openai_client,
build_openai_client,
@ -1705,7 +1706,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
model: str,
input: str,
voice: str,
optional_params: dict,
optional_params: dict[str, object],
api_key: str | None,
api_base: str | None,
organization: str | None,
@ -1759,13 +1760,13 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
},
)
params: Final = speech_create_params(model, voice, input, optional_params)
params: Final = speech_create_params(model, voice, input, with_usage_stream(model, optional_params))
raw_response: Final = (
sync_client.audio.speech.with_streaming_response.create(**params).__enter__().http_response
if stream
else sync_client.audio.speech.create(**params).response
)
speech_response: Final = HttpxBinaryResponseContent(response=raw_response)
speech_response: Final = speech_response_with_usage(raw_response, model, optional_params)
set_provider_response_headers_in_hidden_params(speech_response, raw_response.headers)
return speech_response
@ -1774,7 +1775,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
model: str,
input: str,
voice: str,
optional_params: dict,
optional_params: dict[str, object],
api_key: str | None,
api_base: str | None,
organization: str | None,
@ -1810,13 +1811,13 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
},
)
params: Final = speech_create_params(model, voice, input, optional_params)
params: Final = speech_create_params(model, voice, input, with_usage_stream(model, optional_params))
raw_response: Final = (
(await openai_client.audio.speech.with_streaming_response.create(**params).__aenter__()).http_response
if stream
else (await openai_client.audio.speech.create(**params)).response
)
speech_response: Final = HttpxBinaryResponseContent(response=raw_response)
speech_response: Final = speech_response_with_usage(raw_response, model, optional_params)
set_provider_response_headers_in_hidden_params(speech_response, raw_response.headers)
return speech_response

View file

@ -0,0 +1,193 @@
import base64
from collections.abc import AsyncIterator, Iterator, Mapping
from typing import Final
import httpx
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
from litellm.litellm_core_utils import model_cost_table
from litellm.types.llms.openai import HttpxBinaryResponseContent
from litellm.types.utils import CompletionTokensDetailsWrapper, Usage
_SSE_MEDIA_TYPE: Final = "text/event-stream"
_STREAM_FORMAT_KEY: Final = "stream_format"
_AUDIO_MEDIA_TYPES: Final[Mapping[str, str]] = {
"mp3": "audio/mpeg",
"opus": "audio/opus",
"aac": "audio/aac",
"flac": "audio/flac",
"wav": "audio/wav",
"pcm": "audio/pcm",
}
class _SpeechUsage(BaseModel):
model_config = ConfigDict(frozen=True)
input_tokens: int
output_tokens: int
class _SpeechPricing(BaseModel):
model_config = ConfigDict(frozen=True)
litellm_provider: str | None = None
output_cost_per_audio_token: float | None = None
_EXTRA_BODY: Final = TypeAdapter(dict[str, object])
class _SpeechEvent(BaseModel):
model_config = ConfigDict(frozen=True)
type: str
audio: str | None = None
usage: _SpeechUsage | None = None
def _parse_event(line: bytes) -> _SpeechEvent | None:
if not line.startswith(b"data:"):
return None
try:
return _SpeechEvent.model_validate_json(line.removeprefix(b"data:").strip())
except ValidationError:
return None
class SpeechUsageDecoder:
"""Turns OpenAI's speech SSE events back into the audio bytes and keeps the usage the last event reports."""
def __init__(self) -> None:
self._pending = b""
self.usage: Usage | None = None
def feed(self, chunk: bytes) -> bytes:
*lines, self._pending = (self._pending + chunk).split(b"\n")
return b"".join(self._audio(event) for line in lines if (event := _parse_event(line)) is not None)
def _audio(self, event: _SpeechEvent) -> bytes:
if event.usage is not None:
self.usage = Usage(
prompt_tokens=event.usage.input_tokens,
completion_tokens=event.usage.output_tokens,
total_tokens=event.usage.input_tokens + event.usage.output_tokens,
completion_tokens_details=CompletionTokensDetailsWrapper(audio_tokens=event.usage.output_tokens),
)
return b"" if event.audio is None else base64.b64decode(event.audio)
def _decoded_audio(decoder: SpeechUsageDecoder, speech_response: HttpxBinaryResponseContent, chunk: bytes) -> bytes:
audio: Final = decoder.feed(chunk)
if decoder.usage is not None and speech_response.usage is not decoder.usage:
speech_response.usage = decoder.usage # rebind-ok: the caller already holds this response while it streams
speech_response.set_response_cost(None)
return audio
class _UsageAsyncByteStream(httpx.AsyncByteStream):
def __init__(self, stream: httpx.AsyncByteStream, speech_response: HttpxBinaryResponseContent) -> None:
self._stream: Final = stream
self._speech_response: Final = speech_response
self._decoder: Final = SpeechUsageDecoder()
async def __aiter__(self) -> AsyncIterator[bytes]:
async for chunk in self._stream:
if audio := _decoded_audio(self._decoder, self._speech_response, chunk):
yield audio
if last_audio := _decoded_audio(self._decoder, self._speech_response, b"\n"):
yield last_audio
async def aclose(self) -> None:
await self._stream.aclose()
class _UsageSyncByteStream(httpx.SyncByteStream):
def __init__(self, stream: httpx.SyncByteStream, speech_response: HttpxBinaryResponseContent) -> None:
self._stream: Final = stream
self._speech_response: Final = speech_response
self._decoder: Final = SpeechUsageDecoder()
def __iter__(self) -> Iterator[bytes]:
for chunk in self._stream:
if audio := _decoded_audio(self._decoder, self._speech_response, chunk):
yield audio
if last_audio := _decoded_audio(self._decoder, self._speech_response, b"\n"):
yield last_audio
def close(self) -> None:
self._stream.close()
def bills_audio_output_tokens(model: str) -> bool:
row: Final = model_cost_table.published().get(model)
if row is None:
return False
try:
pricing: Final = _SpeechPricing.model_validate(row)
except ValidationError:
return False
return pricing.litellm_provider == "openai" and pricing.output_cost_per_audio_token is not None
def _caller_extra_body(optional_params: Mapping[str, object]) -> Mapping[str, object]:
return _EXTRA_BODY.validate_python(optional_params.get("extra_body") or {})
def _caller_extra_headers(optional_params: Mapping[str, object]) -> Mapping[str, object]:
return _EXTRA_BODY.validate_python(optional_params.get("extra_headers") or {})
def wants_usage_stream(model: str, optional_params: Mapping[str, object]) -> bool:
caller_asked: Final = _STREAM_FORMAT_KEY in optional_params or _STREAM_FORMAT_KEY in _caller_extra_body(
optional_params
)
return bills_audio_output_tokens(model) and not caller_asked
def with_usage_stream(model: str, optional_params: Mapping[str, object]) -> Mapping[str, object]:
"""Asks OpenAI for the SSE form of the audio, the only form that reports the audio tokens it bills."""
if not wants_usage_stream(model, optional_params):
return optional_params
return {
**optional_params,
"extra_body": {**_caller_extra_body(optional_params), _STREAM_FORMAT_KEY: "sse"},
"extra_headers": {**_caller_extra_headers(optional_params), "Accept-Encoding": "identity"},
}
def _audio_headers(headers: httpx.Headers, response_format: object) -> httpx.Headers:
media_type: Final = _AUDIO_MEDIA_TYPES.get(str(response_format), _AUDIO_MEDIA_TYPES["mp3"])
kept: Final = tuple(
(name, value)
for name, value in headers.multi_items()
if name.lower() not in ("content-type", "content-length", "content-encoding")
)
return httpx.Headers((*kept, ("content-type", media_type)))
def speech_response_with_usage(
raw_response: httpx.Response, model: str, optional_params: Mapping[str, object]
) -> HttpxBinaryResponseContent:
"""The speech response the caller gets. The SSE answer we asked for is decoded back to audio with its usage."""
unchanged: Final = HttpxBinaryResponseContent(response=raw_response)
if not wants_usage_stream(model, optional_params) or not (unchanged.content_type or "").startswith(_SSE_MEDIA_TYPE):
return unchanged
headers: Final = _audio_headers(raw_response.headers, optional_params.get("response_format"))
if raw_response.is_stream_consumed:
decoder: Final = SpeechUsageDecoder()
audio: Final = decoder.feed(raw_response.content + b"\n")
return HttpxBinaryResponseContent(
response=httpx.Response(
status_code=raw_response.status_code, headers=headers, content=audio, request=raw_response.request
),
usage=decoder.usage,
)
raw_response.headers = headers # rebind-ok: the unread provider response is what streams the audio to the caller
speech_response: Final = HttpxBinaryResponseContent(response=raw_response)
raw_response.stream = ( # rebind-ok: the unread provider response is what streams the audio to the caller
_UsageAsyncByteStream(raw_response.stream, speech_response)
if isinstance(raw_response.stream, httpx.AsyncByteStream)
else _UsageSyncByteStream(raw_response.stream, speech_response)
)
return speech_response

View file

@ -40,7 +40,7 @@ def test_audio_speech_forwards_the_full_json_body_and_streams_the_audio(gateway:
assert request.target == "/v1/audio/speech"
assert request.headers["authorization"] == "Bearer synthetic-openai-key"
body: Final = json.loads(request.body)
assert body == _SPEECH_REQUEST
assert body == {**_SPEECH_REQUEST, "stream_format": "sse"}
assert type(body["speed"]) is float
return Reply(body=_SPEECH_BYTES, content_type="audio/wav")

View file

@ -12467,9 +12467,16 @@ GPT_4O_MINI_TTS_SPEECH: Final = CostTrackingTestCase(
model="gpt-4o-mini-tts",
endpoint="/v1/audio/speech",
request={"input": "The pricing harness says hello.", "voice": "alloy", "response_format": "mp3"},
response=BinaryResponse(content_type="audio/mpeg", length=2048),
response=SseResponse(
content_type="text/event-stream",
frames=(
'data: {"type":"speech.audio.delta","audio":"c2NyaXB0ZWQgYXVkaW8="}',
'data: {"type":"speech.audio.done","usage":{"input_tokens":6,"output_tokens":50,"total_tokens":56}}',
"data: [DONE]",
),
),
expected=ExactExpected(
spend=3.66e-06, input_cost=3.66e-06, output_cost=0.0, prompt_tokens=0, completion_tokens=0
spend=0.00060366, input_cost=3.66e-06, output_cost=0.0006, prompt_tokens=6, completion_tokens=50
),
)

View file

@ -2,13 +2,16 @@
from __future__ import annotations
import base64
import json
from typing import Final
from unittest.mock import Mock
import httpx
import pytest
from openai import AsyncAzureOpenAI, AsyncOpenAI
from openai import AsyncAzureOpenAI, AsyncOpenAI, OpenAI
import litellm
from litellm.llms.azure.azure import AzureChatCompletion
from litellm.llms.openai.openai import OpenAIChatCompletion
@ -103,3 +106,222 @@ async def test_azure_audio_speech_yields_chunks_before_upstream_finishes():
assert await anext(audio) == b"one"
assert produced == [b"one"]
_SPEECH_SSE: Final = (
b'data: {"type":"speech.audio.delta","audio":"%s"}\n\n'
b'data: {"type":"speech.audio.delta","audio":"%s"}\n\n'
b'data: {"type":"speech.audio.done","usage":{"input_tokens":6,"output_tokens":50,"total_tokens":56}}\n\n'
b"data: [DONE]\n\n"
) % (base64.b64encode(b"first-half|"), base64.b64encode(b"second-half"))
class _SpeechSseStream(httpx.SyncByteStream, httpx.AsyncByteStream):
def __init__(self, frames: list[bytes]):
self._frames = frames
def __iter__(self):
yield from self._frames
async def __aiter__(self):
for frame in self._frames:
yield frame
def _sse_transport(
requests: list[dict[str, object]], frame_size: int, sse: bytes = _SPEECH_SSE, encodings: list[str] | None = None
) -> httpx.MockTransport:
def _respond(request: httpx.Request) -> httpx.Response:
requests.append(json.loads(request.content))
if encodings is not None:
encodings.append(request.headers["accept-encoding"])
frames: Final = [sse[start : start + frame_size] for start in range(0, len(sse), frame_size)]
return httpx.Response(
status_code=200,
headers={"content-type": "text/event-stream; charset=utf-8"},
stream=_SpeechSseStream(frames),
)
return httpx.MockTransport(_respond)
async def _token_priced_speech(
requests: list[dict[str, object]], *, stream: bool, frame_size: int, optional_params: dict[str, object]
):
return await OpenAIChatCompletion().async_audio_speech(
model="gpt-4o-mini-tts",
input="hello",
voice="alloy",
optional_params=optional_params,
api_key="fake-key",
api_base=None,
organization=None,
project=None,
max_retries=0,
timeout=10.0,
logging_obj=Mock(),
client=AsyncOpenAI(
api_key="fake-key", http_client=httpx.AsyncClient(transport=_sse_transport(requests, frame_size))
),
stream=stream,
)
def _audio_output_cost(response: litellm.HttpxBinaryResponseContent) -> float:
_, completion_cost = litellm.cost_per_token(
model="gpt-4o-mini-tts", custom_llm_provider="openai", usage_object=response.usage, call_type="aspeech"
)
return completion_cost
@pytest.mark.asyncio
@pytest.mark.parametrize("frame_size", [7, 64, 4096])
async def test_streamed_token_priced_speech_returns_audio_and_bills_the_audio_tokens_openai_reports(frame_size: int):
requests: Final[list[dict[str, object]]] = []
response: Final = await _token_priced_speech(
requests, stream=True, frame_size=frame_size, optional_params={"response_format": "wav"}
)
audio: Final = b"".join([chunk async for chunk in await response.aiter_bytes()])
assert audio == b"first-half|second-half"
assert response.content_type == "audio/wav"
assert requests == [
{
"model": "gpt-4o-mini-tts",
"voice": "alloy",
"input": "hello",
"response_format": "wav",
"stream_format": "sse",
}
]
assert response.usage is not None
assert (response.usage.prompt_tokens, response.usage.completion_tokens) == (6, 50)
assert _audio_output_cost(response) == pytest.approx(
50 * litellm.model_cost["gpt-4o-mini-tts"]["output_cost_per_audio_token"]
)
@pytest.mark.asyncio
async def test_buffered_token_priced_speech_returns_audio_and_bills_the_audio_tokens_openai_reports():
requests: Final[list[dict[str, object]]] = []
response: Final = await _token_priced_speech(requests, stream=False, frame_size=64, optional_params={})
assert response.content == b"first-half|second-half"
assert response.content_type == "audio/mpeg"
assert _audio_output_cost(response) == pytest.approx(
50 * litellm.model_cost["gpt-4o-mini-tts"]["output_cost_per_audio_token"]
)
def test_sync_streamed_token_priced_speech_returns_audio_and_the_usage_openai_reports():
requests: Final[list[dict[str, object]]] = []
response: Final = OpenAIChatCompletion().audio_speech(
model="gpt-4o-mini-tts",
input="hello",
voice="alloy",
optional_params={},
api_key="fake-key",
api_base=None,
organization=None,
project=None,
max_retries=0,
timeout=10.0,
logging_obj=Mock(),
client=OpenAI(api_key="fake-key", http_client=httpx.Client(transport=_sse_transport(requests, 7))),
stream=True,
)
assert b"".join(response.iter_bytes()) == b"first-half|second-half"
assert response.usage is not None
assert (response.usage.prompt_tokens, response.usage.completion_tokens) == (6, 50)
@pytest.mark.asyncio
async def test_streamed_speech_drops_the_cost_stamped_before_the_audio_tokens_arrived():
response: Final = await _token_priced_speech([], stream=True, frame_size=64, optional_params={})
response.set_response_cost(3.6e-06)
async for _ in await response.aiter_bytes():
pass
assert "response_cost" not in response.hidden_params
@pytest.mark.asyncio
async def test_speech_sse_asked_for_by_the_caller_is_passed_through_untouched():
requests: Final[list[dict[str, object]]] = []
response: Final = await _token_priced_speech(
requests, stream=False, frame_size=4096, optional_params={"stream_format": "sse"}
)
assert requests == [{"model": "gpt-4o-mini-tts", "voice": "alloy", "input": "hello", "stream_format": "sse"}]
assert response.content == _SPEECH_SSE
@pytest.mark.asyncio
async def test_speech_model_priced_per_character_is_not_asked_for_sse():
requests: Final[list[dict[str, object]]] = []
def _respond(request: httpx.Request) -> httpx.Response:
requests.append(json.loads(request.content))
return httpx.Response(200, headers={"content-type": "audio/mpeg"}, content=b"audio")
response: Final = await OpenAIChatCompletion().async_audio_speech(
model="tts-1",
input="hello",
voice="alloy",
optional_params={},
api_key="fake-key",
api_base=None,
organization=None,
project=None,
max_retries=0,
timeout=10.0,
logging_obj=Mock(),
client=AsyncOpenAI(api_key="fake-key", http_client=httpx.AsyncClient(transport=httpx.MockTransport(_respond))),
stream=False,
)
assert requests == [{"model": "tts-1", "voice": "alloy", "input": "hello"}]
assert response.content == b"audio"
def _sync_streamed_speech(transport: httpx.MockTransport):
return OpenAIChatCompletion().audio_speech(
model="gpt-4o-mini-tts",
input="hello",
voice="alloy",
optional_params={},
api_key="fake-key",
api_base=None,
organization=None,
project=None,
max_retries=0,
timeout=10.0,
logging_obj=Mock(),
client=OpenAI(api_key="fake-key", http_client=httpx.Client(transport=transport)),
stream=True,
)
def test_streamed_speech_keeps_the_usage_of_a_last_event_with_no_trailing_newline():
response: Final = _sync_streamed_speech(
_sse_transport([], 64, sse=_SPEECH_SSE.removesuffix(b"\n\ndata: [DONE]\n\n"))
)
assert b"".join(response.iter_bytes()) == b"first-half|second-half"
assert response.usage is not None
assert response.usage.completion_tokens == 50
def test_token_priced_speech_asks_the_provider_not_to_compress_the_events_it_decodes():
encodings: Final[list[str]] = []
response: Final = _sync_streamed_speech(_sse_transport([], 64, encodings=encodings))
assert b"".join(response.iter_bytes()) == b"first-half|second-half"
assert encodings == ["identity"]