diff --git a/litellm/litellm_core_utils/audio_utils/utils.py b/litellm/litellm_core_utils/audio_utils/utils.py index dab3e48f91a..5acee9014e9 100644 --- a/litellm/litellm_core_utils/audio_utils/utils.py +++ b/litellm/litellm_core_utils/audio_utils/utils.py @@ -4,8 +4,9 @@ Utils used for litellm.transcription() and litellm.atranscription() import hashlib import os +from collections.abc import Sized from dataclasses import dataclass -from typing import Final +from typing import Final, Protocol from litellm.types.files import ( AUDIO_FILE_TYPES, @@ -16,6 +17,40 @@ from litellm.types.files import ( ) from litellm.types.utils import FileTypes +# libsndfile uses SF_COUNT_MAX (2**63 - 1) for "unknown length"; no real upload gets near 2**62 frames +_MAX_PLAUSIBLE_AUDIO_FRAMES: Final = 2**62 +_DECODE_BLOCK_FRAMES: Final = 1 << 16 + + +class _ReadableAudio(Protocol): + def read(self, frames: int, dtype: str) -> Sized: ... + + def close(self) -> None: ... + + +def _open_forward_only(file_content: bytes) -> _ReadableAudio: + import io + + import soundfile as sf + + class _ForwardOnlySoundFile(sf.SoundFile): # pyright: ignore[reportUntypedBaseClass, reportUnknownMemberType] # soundfile ships no type stubs + # soundfile sizes seekable reads from tell(), which libsndfile can't answer for an + # unknown-length stream, so read it front to back instead + def seekable(self) -> bool: + return False + + return _ForwardOnlySoundFile(io.BytesIO(file_content)) + + +def _decoded_frame_count(file_content: bytes) -> int: + """Count frames by decoding, for audio whose header carries no length""" + audio: Final = _open_forward_only(file_content) + try: + # iter() stops at the first empty block, i.e. the end of the stream + return sum(iter(lambda: len(audio.read(_DECODE_BLOCK_FRAMES, "int16")), 0)) + finally: + audio.close() + @dataclass class ProcessedAudioFile: @@ -323,13 +358,21 @@ def calculate_request_duration(file: FileTypes) -> float | None: # Extract duration using soundfile file_object: Final = io.BytesIO(file_content) with sf.SoundFile(file_object) as audio: - duration: Final = len(audio) / audio.samplerate - return duration + reported_frames: Final[int] = len(audio) + samplerate: Final[int] = audio.samplerate + # libsndfile reports an unknown length (e.g. a FLAC with no sample count + # in STREAMINFO) as SF_COUNT_MAX, which would bill as ~2^63 frames, so + # count the real frames by decoding instead of trusting the header + frames: Final = ( + reported_frames if reported_frames < _MAX_PLAUSIBLE_AUDIO_FRAMES else _decoded_frame_count(file_content) + ) except Exception: # Silently fail if duration extraction fails return None + return frames / samplerate + DEFAULT_SPEECH_MEDIA_TYPE: Final = "audio/mpeg" diff --git a/litellm/llms/mistral/audio_transcription/transformation.py b/litellm/llms/mistral/audio_transcription/transformation.py index 4fce7a95350..843ef3f89ad 100644 --- a/litellm/llms/mistral/audio_transcription/transformation.py +++ b/litellm/llms/mistral/audio_transcription/transformation.py @@ -7,6 +7,7 @@ API reference: https://docs.mistral.ai/api/#tag/audio/operation/audio_transcript from typing import Final import httpx +from pydantic import BaseModel, ConfigDict, ValidationError from litellm.litellm_core_utils.audio_utils.utils import process_audio_file from litellm.llms.base_llm.audio_transcription.transformation import ( @@ -26,6 +27,26 @@ class MistralAudioTranscriptionException(BaseLLMException): pass +class _MistralTranscriptionUsage(BaseModel): + model_config = ConfigDict(frozen=True) + + prompt_audio_seconds: float | None = None + + +class _MistralTranscriptionBody(BaseModel): + model_config = ConfigDict(frozen=True) + + usage: _MistralTranscriptionUsage | None = None + + +def _billed_audio_seconds(body: object) -> float | None: + try: + parsed: Final = _MistralTranscriptionBody.model_validate(body) + except ValidationError: + return None + return parsed.usage.prompt_audio_seconds if parsed.usage is not None else None + + class MistralAudioTranscriptionConfig(BaseAudioTranscriptionConfig): def get_supported_openai_params(self, model: str) -> list[OpenAIAudioTranscriptionOptionalParams]: return [ @@ -147,5 +168,12 @@ class MistralAudioTranscriptionConfig(BaseAudioTranscriptionConfig): if "language" in response_json: response["language"] = response_json["language"] - response.hidden_params = response_json + # Mistral bills transcription on usage.prompt_audio_seconds, so cost uses it + # instead of measuring the uploaded file locally + billed_seconds: Final = _billed_audio_seconds(response_json) + response.hidden_params = ( + response_json + if billed_seconds is None + else {**response_json, "audio_transcription_duration": billed_seconds} + ) return response diff --git a/litellm/main.py b/litellm/main.py index 3eae64d828e..192dca6c155 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -8024,7 +8024,7 @@ async def atranscription(*args, **kwargs) -> TranscriptionResponse: # a plain Transcription is a TranscriptionVerbose/Diarized type. if response is not None and not isinstance(response, Coroutine) and file is not None: existing_duration: Final = getattr(response, "duration", None) - if existing_duration is None: + if existing_duration is None and response.hidden_params.get("audio_transcription_duration") is None: calculated_duration: Final = calculate_request_duration(file) if calculated_duration is not None: response.hidden_params["audio_transcription_duration"] = calculated_duration @@ -8295,12 +8295,12 @@ def transcription( # exposing it in the response body (see sync path comment above). if response is not None and not isinstance(response, Coroutine): existing_duration: Final = getattr(response, "duration", None) - if existing_duration is None: + response_hidden_params: Final = cast( # cast-ok: preserve dynamic mapping behavior + dict[str, object], getattr(response, HIDDEN_PARAMS_ATTR) + ) + if existing_duration is None and response_hidden_params.get("audio_transcription_duration") is None: calculated_duration: Final = calculate_request_duration(file) if calculated_duration is not None: - response_hidden_params: Final = cast( # cast-ok: preserve dynamic mapping behavior - dict[str, object], getattr(response, HIDDEN_PARAMS_ATTR) - ) response_hidden_params["audio_transcription_duration"] = calculated_duration if response is None: diff --git a/tests/unit/litellm_core_utils/test_audio_utils.py b/tests/unit/litellm_core_utils/test_audio_utils.py index 155f6680416..69dfa66844d 100644 --- a/tests/unit/litellm_core_utils/test_audio_utils.py +++ b/tests/unit/litellm_core_utils/test_audio_utils.py @@ -276,6 +276,31 @@ class TestCalculateRequestDuration: wav_header ), "File position should be restored to original position" + @pytest.mark.skipif( + os.environ.get("SKIP_AUDIO_TESTS") == "true", + reason="Skipping audio tests - soundfile may not be available", + ) + def test_flac_without_sample_count_counts_decoded_frames(self): + """ + A FLAC whose STREAMINFO has no total sample count (what ffmpeg writes to a + pipe) makes libsndfile report SF_COUNT_MAX frames. The duration must come + from the decoded audio, not a ~2^63 frame count and not a free 0 or None + """ + np = pytest.importorskip("numpy") + sf = pytest.importorskip("soundfile") + + buffer = io.BytesIO() + sf.write(buffer, np.zeros(1600, dtype="int16"), 16000, format="FLAC") + known_length = buffer.getvalue() + assert calculate_request_duration(known_length) == pytest.approx(0.1) + + # STREAMINFO starts at byte 8; the low 36 bits of bytes 18..25 hold the total sample count + unknown_length = bytearray(known_length) + packed = int.from_bytes(unknown_length[18:26], "big") & ~((1 << 36) - 1) + unknown_length[18:26] = packed.to_bytes(8, "big") + + assert calculate_request_duration(bytes(unknown_length)) == pytest.approx(0.1) + class TestGetAudioFileContentHash: """Test the get_audio_file_content_hash function for cache key generation""" diff --git a/tests/unit/llms/mistral/audio_transcription/test_mistral_audio_transcription_transformation.py b/tests/unit/llms/mistral/audio_transcription/test_mistral_audio_transcription_transformation.py index b475cf7e3ff..4dff79fd97a 100644 --- a/tests/unit/llms/mistral/audio_transcription/test_mistral_audio_transcription_transformation.py +++ b/tests/unit/llms/mistral/audio_transcription/test_mistral_audio_transcription_transformation.py @@ -2,11 +2,14 @@ import os from unittest.mock import MagicMock import httpx +import pytest + import litellm from litellm.llms.base_llm.audio_transcription.transformation import ( BaseAudioTranscriptionConfig, ) +from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.llms.mistral.audio_transcription.transformation import ( MistralAudioTranscriptionConfig, ) @@ -191,3 +194,89 @@ def test_mistral_audio_transcription_response_transform_empty(): assert isinstance(response, TranscriptionResponse) assert response.text == "" + + +def test_mistral_audio_transcription_response_uses_prompt_audio_seconds_for_cost(): + """Mistral bills on usage.prompt_audio_seconds, so that is the duration cost reads""" + config = MistralAudioTranscriptionConfig() + + mock_response = MagicMock(spec=httpx.Response) + mock_response.json.return_value = { + "text": "hello", + "usage": {"prompt_audio_seconds": 600, "total_tokens": 10}, + } + + response = config.transform_audio_transcription_response(mock_response) + + assert response.hidden_params["audio_transcription_duration"] == 600.0 + assert response.hidden_params["usage"] == {"prompt_audio_seconds": 600, "total_tokens": 10} + assert getattr(response, "duration", None) is None + + +@pytest.mark.parametrize( + "usage", + [ + {"prompt_audio_seconds": None}, + {"prompt_audio_seconds": "not-a-number"}, + "not-an-object", + ], +) +def test_mistral_audio_transcription_response_without_usable_usage_has_no_duration(usage): + config = MistralAudioTranscriptionConfig() + + mock_response = MagicMock(spec=httpx.Response) + mock_response.json.return_value = {"text": "hello", "usage": usage} + + response = config.transform_audio_transcription_response(mock_response) + + assert "audio_transcription_duration" not in response.hidden_params + + +def test_mistral_transcription_cost_uses_provider_billed_seconds(): + """ + End to end through litellm.transcription: the seconds Mistral reports win over the + local file measurement (about 17s for this wav), so spend matches what Mistral bills + """ + billed_seconds = 30 + wav_path = os.path.join( + os.path.dirname(__file__), + "../../../../..", + "tests", + "llm_translation", + "gettysburg.wav", + ) + with open(wav_path, "rb") as f: + audio_bytes = f.read() + + def respond(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + json={ + "model": "voxtral-mini-2602", + "text": "a tone", + "language": None, + "segments": [], + "usage": {"prompt_audio_seconds": billed_seconds, "total_tokens": 3}, + }, + request=request, + ) + + client = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))) + + response = litellm.transcription( + model="mistral/voxtral-mini-2602", + file=("gettysburg.wav", audio_bytes, "audio/wav"), + api_key="fake-key", + client=client, + ) + + assert response.hidden_params["audio_transcription_duration"] == float(billed_seconds) + + cost_per_second = litellm.get_model_info("mistral/voxtral-mini-2602")["input_cost_per_second"] + assert cost_per_second + cost = litellm.completion_cost( + completion_response=response, + model="mistral/voxtral-mini-2602", + call_type="transcription", + ) + assert cost == pytest.approx(billed_seconds * cost_per_second) diff --git a/tests/unit/llms/soniox/audio_transcription/test_soniox_audio_transcription_handler.py b/tests/unit/llms/soniox/audio_transcription/test_soniox_audio_transcription_handler.py index 9c70ebe7ece..5886b2b06ba 100644 --- a/tests/unit/llms/soniox/audio_transcription/test_soniox_audio_transcription_handler.py +++ b/tests/unit/llms/soniox/audio_transcription/test_soniox_audio_transcription_handler.py @@ -1266,3 +1266,69 @@ class TestSpendTracking: litellm.model_cost = original_model_cost litellm.get_model_info.cache_clear() + @staticmethod + def _wav_bytes() -> bytes: + import os + + wav_path = os.path.join( + os.path.dirname(__file__), "../../../../..", "tests", "llm_translation", "gettysburg.wav" + ) + with open(wav_path, "rb") as f: + return f.read() + + @staticmethod + def _responses_without_duration() -> Dict[str, List[httpx.Response]]: + # Soniox omits audio_duration_ms, so the handler stores a None duration + return { + "POST https://api.soniox.com/v1/files": [_make_response({"id": "file_1"})], + "POST https://api.soniox.com/v1/transcriptions": [_make_response({"id": "tx_1"})], + "GET https://api.soniox.com/v1/transcriptions/tx_1": [_make_response({"status": "completed"})], + "GET https://api.soniox.com/v1/transcriptions/tx_1/transcript": [ + _make_response({"text": "four score", "tokens": []}) + ], + "DELETE https://api.soniox.com/v1/transcriptions/tx_1": [_make_response({})], + "DELETE https://api.soniox.com/v1/files/file_1": [_make_response({})], + } + + def test_should_fall_back_to_local_duration_when_soniox_omits_it(self, monkeypatch): + import litellm + from litellm.litellm_core_utils.audio_utils.utils import calculate_request_duration + + monkeypatch.setattr("time.sleep", lambda *_: None) + audio = self._wav_bytes() + local_duration = calculate_request_duration(audio) + assert local_duration is not None and local_duration > 0 + + resp = litellm.transcription( + model="soniox/stt-async-v4", + file=("gettysburg.wav", audio, "audio/wav"), + api_key="sk-test", + client=_MockSyncClient(self._responses_without_duration()), + ) + + assert resp._hidden_params["audio_transcription_duration"] == pytest.approx(local_duration) + cost = litellm.completion_cost(completion_response=resp, model="soniox/stt-async-v4", call_type="transcription") + assert cost > 0 + + def test_should_fall_back_to_local_duration_when_soniox_omits_it_async(self, monkeypatch): + import litellm + from litellm.litellm_core_utils.audio_utils.utils import calculate_request_duration + + async def _no_sleep(*_a, **_kw): + return None + + monkeypatch.setattr(asyncio, "sleep", _no_sleep) + audio = self._wav_bytes() + local_duration = calculate_request_duration(audio) + assert local_duration is not None and local_duration > 0 + + resp = asyncio.new_event_loop().run_until_complete( + litellm.atranscription( + model="soniox/stt-async-v4", + file=("gettysburg.wav", audio, "audio/wav"), + api_key="sk-test", + client=_MockAsyncClient(self._responses_without_duration()), + ) + ) + + assert resp._hidden_params["audio_transcription_duration"] == pytest.approx(local_duration)