fix(transcription): bill uploaded seconds when a stream fails mid-transcript

This commit is contained in:
mateo-berri 2026-10-31 15:23:55 -07:00
parent 1f83922591
commit a961d773f9
2 changed files with 128 additions and 3 deletions

View file

@ -16,7 +16,10 @@ from openai.types.audio import (
from pydantic import TypeAdapter, ValidationError
from litellm.litellm_core_utils.core_helpers import process_response_headers
from litellm.types.utils import TranscriptionResponse
from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import (
TranscriptionUsageObjectTransformation,
)
from litellm.types.utils import TranscriptionResponse, TranscriptionUsageTokensObject, Usage
TRANSCRIPTION_RAN_UNSTREAMED: Final = "transcription_ran_unstreamed"
@ -125,6 +128,10 @@ class TranscriptionStreamLogging(Protocol):
end_time: datetime.datetime,
) -> None: ...
def response_cost_calculator(self, result: TranscriptionResponse) -> float | None: ...
def record_partial_usage_for_failure(self, usage: Usage, response_cost: float) -> None: ...
class _TranscriptionEventCollector:
def __init__(self, duration: float | None) -> None:
@ -150,6 +157,26 @@ class _TranscriptionEventCollector:
return response
def _billed_usage(response: TranscriptionResponse) -> Usage:
usage: Final[object] = getattr(response, "usage", None)
token_usage: Final = (
TranscriptionUsageObjectTransformation.transform_transcription_usage_object(usage)
if isinstance(usage, TranscriptionUsageTokensObject)
else None
)
return token_usage or Usage(prompt_tokens=0, completion_tokens=0, total_tokens=0)
def _record_billable_partial_spend(
logging_obj: TranscriptionStreamLogging, collector: _TranscriptionEventCollector
) -> None:
partial_response: Final = collector.response()
partial_cost: Final = logging_obj.response_cost_calculator(result=partial_response) or 0.0
if partial_cost <= 0:
return
logging_obj.record_partial_usage_for_failure(_billed_usage(partial_response), partial_cost)
class LoggingTranscriptionStream(Stream[TranscriptionStreamEvent]):
def __init__(
self,
@ -180,6 +207,7 @@ class LoggingTranscriptionStream(Stream[TranscriptionStreamEvent]):
yield event
except Exception as exception:
self._failed = True
_record_billable_partial_spend(self._logging_obj, self._collector)
mapped_exception: Final = self._map_exception(exception)
end_time: Final = datetime.datetime.now() # noqa: DTZ005 # callback timestamps use the legacy naive contract
self._logging_obj.failure_handler(mapped_exception, traceback.format_exc(), self._start_time, end_time)
@ -238,6 +266,7 @@ class LoggingAsyncTranscriptionStream(AsyncStream[TranscriptionStreamEvent]):
yield event
except Exception as exception:
self._failed = True
_record_billable_partial_spend(self._logging_obj, self._collector)
mapped_exception: Final = self._map_exception(exception)
end_time: Final = datetime.datetime.now() # noqa: DTZ005 # callback timestamps use the legacy naive contract
self._logging_obj.failure_handler(

View file

@ -1,5 +1,8 @@
import datetime
from collections.abc import AsyncIterator
import io
import math
import wave
from collections.abc import AsyncIterator, Iterator
from typing import Final
import anyio
@ -7,7 +10,7 @@ import anyio.lowlevel
import httpx
import pytest
import respx
from openai import AsyncOpenAI
from openai import AsyncOpenAI, OpenAI
import litellm
from litellm import Router
@ -221,3 +224,96 @@ async def test_an_async_stream_that_fails_mid_transcript_is_logged_as_one_failur
pass
assert counter.failures == ("async_failure",)
class _FirstDeltaThenResetSync(httpx.SyncByteStream):
def __iter__(self) -> Iterator[bytes]:
yield b'data: {"type":"transcript.text.delta","delta":"Good"}\n\n'
raise httpx.ReadError("connection reset")
class _FailureCostRecorder(CustomLogger):
def __init__(self) -> None:
super().__init__()
self.costs: tuple[object, ...] = ()
def log_failure_event(self, kwargs: object, response_obj: object, start_time: object, end_time: object) -> None:
self.costs = (*self.costs, kwargs.get("response_cost") if isinstance(kwargs, dict) else None)
async def async_log_failure_event(
self, kwargs: object, response_obj: object, start_time: object, end_time: object
) -> None:
self.costs = (*self.costs, kwargs.get("response_cost") if isinstance(kwargs, dict) else None)
def _silent_wav(seconds: int) -> bytes:
buffer: Final = io.BytesIO()
with wave.open(buffer, "wb") as wav:
wav.setnchannels(1)
wav.setsampwidth(2)
wav.setframerate(16_000)
wav.writeframes(b"\x00\x00" * 16_000 * seconds)
return buffer.getvalue()
def _reset_mid_transcript(request: httpx.Request) -> httpx.Response:
return httpx.Response(
200,
headers={"content-type": "text/event-stream"},
stream=_FirstDeltaThenFails(httpx.ReadError("connection reset")),
)
def _reset_mid_transcript_sync(request: httpx.Request) -> httpx.Response:
return httpx.Response(200, headers={"content-type": "text/event-stream"}, stream=_FirstDeltaThenResetSync())
def _per_second_rate(model: str) -> float:
rate: Final = litellm.model_cost[model]["input_cost_per_second"]
assert isinstance(rate, float)
return rate
@pytest.mark.asyncio
async def test_an_async_stream_that_fails_mid_transcript_bills_the_uploaded_seconds(
monkeypatch: pytest.MonkeyPatch,
) -> None:
recorder: Final = _FailureCostRecorder()
monkeypatch.setattr(litellm, "callbacks", [recorder])
upstream: Final = AsyncOpenAI(
api_key="sk-test",
base_url="https://upstream.example/v1",
http_client=httpx.AsyncClient(transport=httpx.MockTransport(_reset_mid_transcript)),
)
stream: Final = await litellm.atranscription(
model="openai/gpt-transcribe", file=("sample.wav", _silent_wav(4)), stream=True, client=upstream
)
with pytest.raises(litellm.APIConnectionError):
async for _ in stream:
pass
assert len(recorder.costs) == 1
assert isinstance(recorder.costs[0], float)
assert math.isclose(recorder.costs[0], 4 * _per_second_rate("gpt-transcribe"))
def test_a_sync_stream_that_fails_mid_transcript_bills_the_uploaded_seconds(monkeypatch: pytest.MonkeyPatch) -> None:
recorder: Final = _FailureCostRecorder()
monkeypatch.setattr(litellm, "callbacks", [recorder])
upstream: Final = OpenAI(
api_key="sk-test",
base_url="https://upstream.example/v1",
http_client=httpx.Client(transport=httpx.MockTransport(_reset_mid_transcript_sync)),
)
stream: Final = litellm.transcription(
model="openai/gpt-transcribe", file=("sample.wav", _silent_wav(4)), stream=True, client=upstream
)
with pytest.raises(litellm.APIConnectionError):
for _ in stream:
pass
assert len(recorder.costs) == 1
assert isinstance(recorder.costs[0], float)
assert math.isclose(recorder.costs[0], 4 * _per_second_rate("gpt-transcribe"))