diff --git a/litellm/litellm_core_utils/audio_utils/transcription_streaming.py b/litellm/litellm_core_utils/audio_utils/transcription_streaming.py index 4fc5df55273..44b1667dd20 100644 --- a/litellm/litellm_core_utils/audio_utils/transcription_streaming.py +++ b/litellm/litellm_core_utils/audio_utils/transcription_streaming.py @@ -16,7 +16,10 @@ from openai.types.audio import ( from pydantic import TypeAdapter, ValidationError from litellm.litellm_core_utils.core_helpers import process_response_headers -from litellm.types.utils import TranscriptionResponse +from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import ( + TranscriptionUsageObjectTransformation, +) +from litellm.types.utils import TranscriptionResponse, TranscriptionUsageTokensObject, Usage TRANSCRIPTION_RAN_UNSTREAMED: Final = "transcription_ran_unstreamed" @@ -125,6 +128,10 @@ class TranscriptionStreamLogging(Protocol): end_time: datetime.datetime, ) -> None: ... + def response_cost_calculator(self, result: TranscriptionResponse) -> float | None: ... + + def record_partial_usage_for_failure(self, usage: Usage, response_cost: float) -> None: ... + class _TranscriptionEventCollector: def __init__(self, duration: float | None) -> None: @@ -150,6 +157,26 @@ class _TranscriptionEventCollector: return response +def _billed_usage(response: TranscriptionResponse) -> Usage: + usage: Final[object] = getattr(response, "usage", None) + token_usage: Final = ( + TranscriptionUsageObjectTransformation.transform_transcription_usage_object(usage) + if isinstance(usage, TranscriptionUsageTokensObject) + else None + ) + return token_usage or Usage(prompt_tokens=0, completion_tokens=0, total_tokens=0) + + +def _record_billable_partial_spend( + logging_obj: TranscriptionStreamLogging, collector: _TranscriptionEventCollector +) -> None: + partial_response: Final = collector.response() + partial_cost: Final = logging_obj.response_cost_calculator(result=partial_response) or 0.0 + if partial_cost <= 0: + return + logging_obj.record_partial_usage_for_failure(_billed_usage(partial_response), partial_cost) + + class LoggingTranscriptionStream(Stream[TranscriptionStreamEvent]): def __init__( self, @@ -180,6 +207,7 @@ class LoggingTranscriptionStream(Stream[TranscriptionStreamEvent]): yield event except Exception as exception: self._failed = True + _record_billable_partial_spend(self._logging_obj, self._collector) mapped_exception: Final = self._map_exception(exception) end_time: Final = datetime.datetime.now() # noqa: DTZ005 # callback timestamps use the legacy naive contract self._logging_obj.failure_handler(mapped_exception, traceback.format_exc(), self._start_time, end_time) @@ -238,6 +266,7 @@ class LoggingAsyncTranscriptionStream(AsyncStream[TranscriptionStreamEvent]): yield event except Exception as exception: self._failed = True + _record_billable_partial_spend(self._logging_obj, self._collector) mapped_exception: Final = self._map_exception(exception) end_time: Final = datetime.datetime.now() # noqa: DTZ005 # callback timestamps use the legacy naive contract self._logging_obj.failure_handler( diff --git a/tests/unit/litellm_core_utils/audio_utils/test_transcription_streaming.py b/tests/unit/litellm_core_utils/audio_utils/test_transcription_streaming.py index 386d1b4917c..5296ea3b15a 100644 --- a/tests/unit/litellm_core_utils/audio_utils/test_transcription_streaming.py +++ b/tests/unit/litellm_core_utils/audio_utils/test_transcription_streaming.py @@ -1,5 +1,8 @@ import datetime -from collections.abc import AsyncIterator +import io +import math +import wave +from collections.abc import AsyncIterator, Iterator from typing import Final import anyio @@ -7,7 +10,7 @@ import anyio.lowlevel import httpx import pytest import respx -from openai import AsyncOpenAI +from openai import AsyncOpenAI, OpenAI import litellm from litellm import Router @@ -221,3 +224,96 @@ async def test_an_async_stream_that_fails_mid_transcript_is_logged_as_one_failur pass assert counter.failures == ("async_failure",) + + +class _FirstDeltaThenResetSync(httpx.SyncByteStream): + def __iter__(self) -> Iterator[bytes]: + yield b'data: {"type":"transcript.text.delta","delta":"Good"}\n\n' + raise httpx.ReadError("connection reset") + + +class _FailureCostRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.costs: tuple[object, ...] = () + + def log_failure_event(self, kwargs: object, response_obj: object, start_time: object, end_time: object) -> None: + self.costs = (*self.costs, kwargs.get("response_cost") if isinstance(kwargs, dict) else None) + + async def async_log_failure_event( + self, kwargs: object, response_obj: object, start_time: object, end_time: object + ) -> None: + self.costs = (*self.costs, kwargs.get("response_cost") if isinstance(kwargs, dict) else None) + + +def _silent_wav(seconds: int) -> bytes: + buffer: Final = io.BytesIO() + with wave.open(buffer, "wb") as wav: + wav.setnchannels(1) + wav.setsampwidth(2) + wav.setframerate(16_000) + wav.writeframes(b"\x00\x00" * 16_000 * seconds) + return buffer.getvalue() + + +def _reset_mid_transcript(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + stream=_FirstDeltaThenFails(httpx.ReadError("connection reset")), + ) + + +def _reset_mid_transcript_sync(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, headers={"content-type": "text/event-stream"}, stream=_FirstDeltaThenResetSync()) + + +def _per_second_rate(model: str) -> float: + rate: Final = litellm.model_cost[model]["input_cost_per_second"] + assert isinstance(rate, float) + return rate + + +@pytest.mark.asyncio +async def test_an_async_stream_that_fails_mid_transcript_bills_the_uploaded_seconds( + monkeypatch: pytest.MonkeyPatch, +) -> None: + recorder: Final = _FailureCostRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + upstream: Final = AsyncOpenAI( + api_key="sk-test", + base_url="https://upstream.example/v1", + http_client=httpx.AsyncClient(transport=httpx.MockTransport(_reset_mid_transcript)), + ) + stream: Final = await litellm.atranscription( + model="openai/gpt-transcribe", file=("sample.wav", _silent_wav(4)), stream=True, client=upstream + ) + + with pytest.raises(litellm.APIConnectionError): + async for _ in stream: + pass + + assert len(recorder.costs) == 1 + assert isinstance(recorder.costs[0], float) + assert math.isclose(recorder.costs[0], 4 * _per_second_rate("gpt-transcribe")) + + +def test_a_sync_stream_that_fails_mid_transcript_bills_the_uploaded_seconds(monkeypatch: pytest.MonkeyPatch) -> None: + recorder: Final = _FailureCostRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + upstream: Final = OpenAI( + api_key="sk-test", + base_url="https://upstream.example/v1", + http_client=httpx.Client(transport=httpx.MockTransport(_reset_mid_transcript_sync)), + ) + stream: Final = litellm.transcription( + model="openai/gpt-transcribe", file=("sample.wav", _silent_wav(4)), stream=True, client=upstream + ) + + with pytest.raises(litellm.APIConnectionError): + for _ in stream: + pass + + assert len(recorder.costs) == 1 + assert isinstance(recorder.costs[0], float) + assert math.isclose(recorder.costs[0], 4 * _per_second_rate("gpt-transcribe"))