mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(transcription): bill uploaded seconds when a stream fails mid-transcript
This commit is contained in:
parent
1f83922591
commit
a961d773f9
2 changed files with 128 additions and 3 deletions
|
|
@ -16,7 +16,10 @@ from openai.types.audio import (
|
|||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from litellm.litellm_core_utils.core_helpers import process_response_headers
|
||||
from litellm.types.utils import TranscriptionResponse
|
||||
from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import (
|
||||
TranscriptionUsageObjectTransformation,
|
||||
)
|
||||
from litellm.types.utils import TranscriptionResponse, TranscriptionUsageTokensObject, Usage
|
||||
|
||||
TRANSCRIPTION_RAN_UNSTREAMED: Final = "transcription_ran_unstreamed"
|
||||
|
||||
|
|
@ -125,6 +128,10 @@ class TranscriptionStreamLogging(Protocol):
|
|||
end_time: datetime.datetime,
|
||||
) -> None: ...
|
||||
|
||||
def response_cost_calculator(self, result: TranscriptionResponse) -> float | None: ...
|
||||
|
||||
def record_partial_usage_for_failure(self, usage: Usage, response_cost: float) -> None: ...
|
||||
|
||||
|
||||
class _TranscriptionEventCollector:
|
||||
def __init__(self, duration: float | None) -> None:
|
||||
|
|
@ -150,6 +157,26 @@ class _TranscriptionEventCollector:
|
|||
return response
|
||||
|
||||
|
||||
def _billed_usage(response: TranscriptionResponse) -> Usage:
|
||||
usage: Final[object] = getattr(response, "usage", None)
|
||||
token_usage: Final = (
|
||||
TranscriptionUsageObjectTransformation.transform_transcription_usage_object(usage)
|
||||
if isinstance(usage, TranscriptionUsageTokensObject)
|
||||
else None
|
||||
)
|
||||
return token_usage or Usage(prompt_tokens=0, completion_tokens=0, total_tokens=0)
|
||||
|
||||
|
||||
def _record_billable_partial_spend(
|
||||
logging_obj: TranscriptionStreamLogging, collector: _TranscriptionEventCollector
|
||||
) -> None:
|
||||
partial_response: Final = collector.response()
|
||||
partial_cost: Final = logging_obj.response_cost_calculator(result=partial_response) or 0.0
|
||||
if partial_cost <= 0:
|
||||
return
|
||||
logging_obj.record_partial_usage_for_failure(_billed_usage(partial_response), partial_cost)
|
||||
|
||||
|
||||
class LoggingTranscriptionStream(Stream[TranscriptionStreamEvent]):
|
||||
def __init__(
|
||||
self,
|
||||
|
|
@ -180,6 +207,7 @@ class LoggingTranscriptionStream(Stream[TranscriptionStreamEvent]):
|
|||
yield event
|
||||
except Exception as exception:
|
||||
self._failed = True
|
||||
_record_billable_partial_spend(self._logging_obj, self._collector)
|
||||
mapped_exception: Final = self._map_exception(exception)
|
||||
end_time: Final = datetime.datetime.now() # noqa: DTZ005 # callback timestamps use the legacy naive contract
|
||||
self._logging_obj.failure_handler(mapped_exception, traceback.format_exc(), self._start_time, end_time)
|
||||
|
|
@ -238,6 +266,7 @@ class LoggingAsyncTranscriptionStream(AsyncStream[TranscriptionStreamEvent]):
|
|||
yield event
|
||||
except Exception as exception:
|
||||
self._failed = True
|
||||
_record_billable_partial_spend(self._logging_obj, self._collector)
|
||||
mapped_exception: Final = self._map_exception(exception)
|
||||
end_time: Final = datetime.datetime.now() # noqa: DTZ005 # callback timestamps use the legacy naive contract
|
||||
self._logging_obj.failure_handler(
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
import datetime
|
||||
from collections.abc import AsyncIterator
|
||||
import io
|
||||
import math
|
||||
import wave
|
||||
from collections.abc import AsyncIterator, Iterator
|
||||
from typing import Final
|
||||
|
||||
import anyio
|
||||
|
|
@ -7,7 +10,7 @@ import anyio.lowlevel
|
|||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
from openai import AsyncOpenAI
|
||||
from openai import AsyncOpenAI, OpenAI
|
||||
|
||||
import litellm
|
||||
from litellm import Router
|
||||
|
|
@ -221,3 +224,96 @@ async def test_an_async_stream_that_fails_mid_transcript_is_logged_as_one_failur
|
|||
pass
|
||||
|
||||
assert counter.failures == ("async_failure",)
|
||||
|
||||
|
||||
class _FirstDeltaThenResetSync(httpx.SyncByteStream):
|
||||
def __iter__(self) -> Iterator[bytes]:
|
||||
yield b'data: {"type":"transcript.text.delta","delta":"Good"}\n\n'
|
||||
raise httpx.ReadError("connection reset")
|
||||
|
||||
|
||||
class _FailureCostRecorder(CustomLogger):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.costs: tuple[object, ...] = ()
|
||||
|
||||
def log_failure_event(self, kwargs: object, response_obj: object, start_time: object, end_time: object) -> None:
|
||||
self.costs = (*self.costs, kwargs.get("response_cost") if isinstance(kwargs, dict) else None)
|
||||
|
||||
async def async_log_failure_event(
|
||||
self, kwargs: object, response_obj: object, start_time: object, end_time: object
|
||||
) -> None:
|
||||
self.costs = (*self.costs, kwargs.get("response_cost") if isinstance(kwargs, dict) else None)
|
||||
|
||||
|
||||
def _silent_wav(seconds: int) -> bytes:
|
||||
buffer: Final = io.BytesIO()
|
||||
with wave.open(buffer, "wb") as wav:
|
||||
wav.setnchannels(1)
|
||||
wav.setsampwidth(2)
|
||||
wav.setframerate(16_000)
|
||||
wav.writeframes(b"\x00\x00" * 16_000 * seconds)
|
||||
return buffer.getvalue()
|
||||
|
||||
|
||||
def _reset_mid_transcript(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
200,
|
||||
headers={"content-type": "text/event-stream"},
|
||||
stream=_FirstDeltaThenFails(httpx.ReadError("connection reset")),
|
||||
)
|
||||
|
||||
|
||||
def _reset_mid_transcript_sync(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, headers={"content-type": "text/event-stream"}, stream=_FirstDeltaThenResetSync())
|
||||
|
||||
|
||||
def _per_second_rate(model: str) -> float:
|
||||
rate: Final = litellm.model_cost[model]["input_cost_per_second"]
|
||||
assert isinstance(rate, float)
|
||||
return rate
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_async_stream_that_fails_mid_transcript_bills_the_uploaded_seconds(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
recorder: Final = _FailureCostRecorder()
|
||||
monkeypatch.setattr(litellm, "callbacks", [recorder])
|
||||
upstream: Final = AsyncOpenAI(
|
||||
api_key="sk-test",
|
||||
base_url="https://upstream.example/v1",
|
||||
http_client=httpx.AsyncClient(transport=httpx.MockTransport(_reset_mid_transcript)),
|
||||
)
|
||||
stream: Final = await litellm.atranscription(
|
||||
model="openai/gpt-transcribe", file=("sample.wav", _silent_wav(4)), stream=True, client=upstream
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.APIConnectionError):
|
||||
async for _ in stream:
|
||||
pass
|
||||
|
||||
assert len(recorder.costs) == 1
|
||||
assert isinstance(recorder.costs[0], float)
|
||||
assert math.isclose(recorder.costs[0], 4 * _per_second_rate("gpt-transcribe"))
|
||||
|
||||
|
||||
def test_a_sync_stream_that_fails_mid_transcript_bills_the_uploaded_seconds(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
recorder: Final = _FailureCostRecorder()
|
||||
monkeypatch.setattr(litellm, "callbacks", [recorder])
|
||||
upstream: Final = OpenAI(
|
||||
api_key="sk-test",
|
||||
base_url="https://upstream.example/v1",
|
||||
http_client=httpx.Client(transport=httpx.MockTransport(_reset_mid_transcript_sync)),
|
||||
)
|
||||
stream: Final = litellm.transcription(
|
||||
model="openai/gpt-transcribe", file=("sample.wav", _silent_wav(4)), stream=True, client=upstream
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.APIConnectionError):
|
||||
for _ in stream:
|
||||
pass
|
||||
|
||||
assert len(recorder.costs) == 1
|
||||
assert isinstance(recorder.costs[0], float)
|
||||
assert math.isclose(recorder.costs[0], 4 * _per_second_rate("gpt-transcribe"))
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue