From d6b54ae2b028653efbc8f93feaf71660b4468501 Mon Sep 17 00:00:00 2001 From: Irvallen Saragih Date: Sat, 17 Oct 2026 05:27:47 +0900 Subject: [PATCH] fix(openai)!: route gpt-transcribe to GPT audio transcription config (#44021) * fix(openai): route gpt-transcribe to GPT audio transcription config * test(openai): move gpt-transcribe tests under tests/unit * fix(openai): bill gpt-transcribe from provider usage seconds when duration is unknown Fall back to the provider-reported usage.seconds when neither a measured file duration nor a response duration is available, so json responses for undecodable uploads are no longer billed at zero. Move the routing test next to the other provider config tests and replace the structure test with a behavior test. * fix(openai): bill provider usage seconds only for gpt transcription models * fix(cost): decide provider-billed transcription seconds from the deployment model, not its pricing id * test(integration): cover gpt-transcribe default request, provider-billed spend, and health probe --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/cost_calculator.py | 18 ++ litellm/utils.py | 2 +- .../test_audio_transcriptions_wire.py | 174 +++++++++++++++++- .../management/test_model_health_check.py | 37 ++++ tests/unit/test_cost_calculator.py | 72 ++++++++ tests/unit/test_utils.py | 27 +++ 6 files changed, 328 insertions(+), 2 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 0e315e97432..05dc2900349 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -86,6 +86,9 @@ from litellm.llms.openai.cost_calculation import ( from litellm.llms.openai.cost_calculation import ( video_output_cost_per_second, ) +from litellm.llms.openai.transcriptions.gpt_transformation import ( + OpenAIGPTAudioTranscriptionConfig, +) from litellm.llms.perplexity.cost_calculator import ( cost_per_token as perplexity_cost_per_token, ) @@ -325,6 +328,15 @@ def _transcription_usage_has_token_details( return (prompt_tokens_val > 0) or (completion_tokens_val > 0) +def _bills_provider_transcription_seconds(model: str | None, custom_llm_provider: str | None) -> bool: + if model is None or custom_llm_provider != LlmProviders.OPENAI.value: + return False + transcription_config: Final = ProviderConfigManager.get_provider_audio_transcription_config( + model=model.removeprefix(f"{LlmProviders.OPENAI.value}/"), provider=LlmProviders.OPENAI + ) + return isinstance(transcription_config, OpenAIGPTAudioTranscriptionConfig) + + OCRPricingField = Literal["ocr_cost_per_page", "ocr_cost_per_credit", "annotation_cost_per_page"] @@ -1739,6 +1751,12 @@ def completion_cost( audio_transcription_file_duration = _hidden.get( "audio_transcription_duration", getattr(completion_response, "duration", 0.0), + ) or ( + completion_response.usage.seconds + if isinstance(completion_response, TranscriptionResponse) + and isinstance(completion_response.usage, TranscriptionUsageDurationObject) + and _bills_provider_transcription_seconds(requested_model, custom_llm_provider) + else 0.0 ) elif call_type in _RERANK_CALL_TYPES: if completion_response is not None and isinstance(completion_response, RerankResponse): diff --git a/litellm/utils.py b/litellm/utils.py index 8aa6eb23c68..5a933d8f2b8 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9252,7 +9252,7 @@ class ProviderConfigManager: return XAIAudioTranscriptionConfig() elif litellm.LlmProviders.OPENAI == provider: - if "gpt-4o" in model: + if "gpt-4o" in model or (model.startswith("gpt-") and "transcribe" in model): return litellm.OpenAIGPTAudioTranscriptionConfig() else: return litellm.OpenAIWhisperAudioTranscriptionConfig() diff --git a/tests/integration/compatibility/test_audio_transcriptions_wire.py b/tests/integration/compatibility/test_audio_transcriptions_wire.py index 7845e3d2423..02a09bce5fd 100644 --- a/tests/integration/compatibility/test_audio_transcriptions_wire.py +++ b/tests/integration/compatibility/test_audio_transcriptions_wire.py @@ -1,15 +1,18 @@ +import io import json import math import mimetypes import uuid +import wave from email.message import Message from email.parser import BytesParser from email.policy import HTTP +from hashlib import sha256 from typing import Final, cast import httpx import pytest -from integration._support.client import Gateway, eventually +from integration._support.client import JSON_OBJECT, Gateway, eventually, object_value from integration._support.database import read_rows from integration._support.wire import Reply, Request, wire_server from openai import OpenAI @@ -671,3 +674,172 @@ def test_audio_transcription_bills_audio_tokens_at_the_input_rate_when_no_audio_ rel_tol=1e-9, ), response.text assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/v1/audio/transcriptions")] + + +_GPT_TRANSCRIBE_VERBOSE_JSON_REFUSAL: Final = { + "error": { + "message": "response_format 'verbose_json' is not compatible with model 'gpt-transcribe'", + "type": "invalid_request_error", + "param": "response_format", + "code": "unsupported_value", + } +} +_UNDECODABLE_M4A_BYTES: Final = b"\x00\x00\x00\x20ftypM4A \x00\x00\x02\x00M4A isommp42\x00\x00\x00\x08free" +_DEPLOYMENT_COST_PER_SECOND: Final = 0.0003 + + +def _openai_gpt_transcribe_reply(request: Request, reported_seconds: float) -> Reply: + fields: Final = dict(_text_parts(_multipart_parts(request))) + if fields.get("response_format") == "verbose_json": + return Reply(status=400, body=json.dumps(_GPT_TRANSCRIBE_VERBOSE_JSON_REFUSAL).encode()) + return Reply( + body=json.dumps({"text": "hello world", "usage": {"type": "duration", "seconds": reported_seconds}}).encode() + ) + + +def _uploaded_files(request: Request) -> tuple[tuple[str, str, bytes], ...]: + return tuple((name, filename, payload) for name, filename, _, payload in _file_parts(_multipart_parts(request))) + + +def _silent_wav(seconds: float) -> bytes: + output: Final = io.BytesIO() + with wave.open(output, "wb") as wav: + wav.setnchannels(1) + wav.setsampwidth(2) + wav.setframerate(16000) + wav.writeframes(b"\x00\x00" * round(16000 * seconds)) + return output.getvalue() + + +def _advertised_cost_per_second(gateway: Gateway, model: str) -> float: + entries: Final = gateway.get("/v2/model/info", {"model": model})["data"] + assert isinstance(entries, list) and len(entries) == 1, entries + price: Final = object_value(object_value(entries[0])["model_info"])["input_cost_per_second"] + assert isinstance(price, float) and price > 0, entries + return price + + +def _response_cost(response: httpx.Response) -> float: + return float(response.headers["x-litellm-response-cost"]) if "x-litellm-response-cost" in response.headers else 0.0 + + +def _logged_spend(call_id: str) -> float: + rows: Final = eventually( + lambda: read_rows('SELECT spend FROM "LiteLLM_SpendLogs" WHERE request_id = %s', (call_id,)), + lambda values: len(values) == 1, + seconds=70, + ) + return float(str(rows[0]["spend"])) + + +def test_gpt_transcribe_without_response_format_reaches_openai_without_verbose_json(gateway: Gateway) -> None: + upload: Final = _UNDECODABLE_M4A_BYTES + uuid.uuid4().bytes + + with ( + wire_server(lambda request: _openai_gpt_transcribe_reply(request, 2)) as wire, + gateway.scenario() as scenario, + ): + model: Final = scenario.model( + model="openai/gpt-transcribe", api_base=f"{wire.url}/v1", api_key="synthetic-openai-key" + ) + with _sdk(gateway, "/v1", gateway.key) as sdk: + response: Final = sdk.audio.transcriptions.with_raw_response.create( + model=model, file=("a.m4a", upload, "audio/mp4") + ) + requests: Final = wire.drain() + + assert response.status_code == 200, response.text + assert JSON_OBJECT.validate_json(response.content)["text"] == "hello world", response.text + assert [(request.method, request.target) for request in requests] == [("POST", "/v1/audio/transcriptions")] + assert _text_parts(_multipart_parts(requests[0])) == (("model", "gpt-transcribe"),) + assert _uploaded_files(requests[0]) == (("file", "a.m4a", upload),) + + +@pytest.mark.parametrize( + "configured_cost_per_second", + (None, _DEPLOYMENT_COST_PER_SECOND), + ids=("cost_map_price", "deployment_price"), +) +def test_gpt_transcribe_unmeasurable_upload_bills_the_seconds_openai_reports( + gateway: Gateway, configured_cost_per_second: float | None +) -> None: + reported_seconds: Final = 2.0 + upload: Final = _UNDECODABLE_M4A_BYTES + uuid.uuid4().bytes + + with ( + wire_server(lambda request: _openai_gpt_transcribe_reply(request, reported_seconds)) as wire, + gateway.scenario() as scenario, + ): + model: Final = ( + scenario.model( + model="openai/gpt-transcribe", + api_base=f"{wire.url}/v1", + api_key="synthetic-openai-key", + input_cost_per_second=configured_cost_per_second, + ) + if configured_cost_per_second is not None + else scenario.model( + model="openai/gpt-transcribe", api_base=f"{wire.url}/v1", api_key="synthetic-openai-key" + ) + ) + key: Final = scenario.key(models=[model]) + cost_per_second: Final = ( + configured_cost_per_second + if configured_cost_per_second is not None + else _advertised_cost_per_second(gateway, model) + ) + response: Final = gateway.request_multipart( + "/v1/audio/transcriptions", + {"model": model, "response_format": "json"}, + {"file": ("a.m4a", upload, "audio/mp4")}, + key=key, + ) + expected_cost: Final = reported_seconds * cost_per_second + + assert response.status_code == 200, response.text + assert JSON_OBJECT.validate_json(response.content)["text"] == "hello world", response.text + assert math.isclose(_response_cost(response), expected_cost), (_response_cost(response), expected_cost) + assert math.isclose(_logged_spend(response.headers["x-litellm-call-id"]), expected_cost) + key_rows: Final = eventually( + lambda: read_rows( + 'SELECT spend FROM "LiteLLM_VerificationToken" WHERE token = %s', + (sha256(key.encode()).hexdigest(),), + ), + lambda rows: len(rows) == 1 and float(str(rows[0]["spend"])) > 0, + seconds=70, + ) + assert math.isclose(float(str(key_rows[0]["spend"])), expected_cost), key_rows + requests: Final = wire.drain() + + assert [(request.method, request.target) for request in requests] == [("POST", "/v1/audio/transcriptions")] + assert _uploaded_files(requests[0]) == (("file", "a.m4a", upload),) + + +def test_gpt_transcribe_measured_upload_bills_the_measured_length_over_reported_seconds(gateway: Gateway) -> None: + measured_seconds: Final = 1.0 + upload: Final = _silent_wav(measured_seconds) + + with ( + wire_server(lambda request: _openai_gpt_transcribe_reply(request, 7)) as wire, + gateway.scenario() as scenario, + ): + model: Final = scenario.model( + model="openai/gpt-transcribe", + api_base=f"{wire.url}/v1", + api_key="synthetic-openai-key", + input_cost_per_second=_DEPLOYMENT_COST_PER_SECOND, + ) + response: Final = gateway.request_multipart( + "/v1/audio/transcriptions", + {"model": model, "response_format": "json"}, + {"file": ("a.wav", upload, _WAV_CONTENT_TYPE)}, + ) + expected_cost: Final = measured_seconds * _DEPLOYMENT_COST_PER_SECOND + + assert response.status_code == 200, response.text + assert math.isclose(_response_cost(response), expected_cost), (_response_cost(response), expected_cost) + assert math.isclose(_logged_spend(response.headers["x-litellm-call-id"]), expected_cost) + requests: Final = wire.drain() + + assert [(request.method, request.target) for request in requests] == [("POST", "/v1/audio/transcriptions")] + assert _uploaded_files(requests[0]) == (("file", "a.wav", upload),) diff --git a/tests/integration/management/test_model_health_check.py b/tests/integration/management/test_model_health_check.py index 2b9ec658d87..9945d16af07 100644 --- a/tests/integration/management/test_model_health_check.py +++ b/tests/integration/management/test_model_health_check.py @@ -1352,3 +1352,40 @@ def test_evaluation_mode_health_check_of_databricks_ai_decide_resolves_the_mode_ }, ) ] + + +def _openai_gpt_transcribe_health_reply(request: Request) -> Reply: + if _multipart_text_fields(_multipart_parts(request)).get("response_format") == "verbose_json": + return Reply( + status=400, + body=json.dumps( + { + "error": { + "message": "response_format 'verbose_json' is not compatible with model 'gpt-transcribe'", + "type": "invalid_request_error", + "param": "response_format", + "code": "unsupported_value", + } + } + ).encode(), + ) + return Reply(body=json.dumps({"text": "healthy", "usage": {"type": "duration", "seconds": 1}}).encode()) + + +def test_health_check_of_gpt_transcribe_probes_without_verbose_json_and_reports_it_healthy(gateway: Gateway) -> None: + with wire_server(_openai_gpt_transcribe_health_reply) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model="openai/gpt-transcribe", + api_base=f"{wire.url}/v1", + api_key="synthetic-openai-key", + model_info={"mode": "audio_transcription"}, + ) + report: Final = _health_report(gateway, model) + requests: Final = wire.drain() + + assert (report["healthy_count"], report["unhealthy_count"]) == (1, 0), report + assert requests, report + assert { + (request.method, request.target, tuple(sorted(_multipart_text_fields(_multipart_parts(request)).items()))) + for request in requests + } == {("POST", "/v1/audio/transcriptions", (("model", "gpt-transcribe"),))} diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index cc2a1d62d52..c61ebb3737c 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -6426,6 +6426,78 @@ def test_whisper_azure(): assert round(cost, 5) == round(expected_cost, 5) +@pytest.mark.parametrize( + ("provider", "requested_model", "response_extras", "expected_seconds"), + [ + ("openai", "openai/gpt-fake-transcribe", {}, 2.0), + ("openai", "openai/gpt-fake-transcribe", {"_audio_transcription_duration": 1.35}, 1.35), + ("openai", "openai/gpt-fake-transcribe", {"duration": 1.34}, 1.34), + ("openai", "openai/whisper-fake", {}, 0.0), + ("azure", "gpt-fake-transcribe", {}, 0.0), + ], + ids=[ + "undecodable_upload_bills_provider_seconds", + "measured_duration_still_wins", + "provider_duration_still_wins", + "whisper_models_never_bill_provider_seconds", + "azure_never_bills_provider_seconds", + ], +) +def test_transcription_cost_falls_back_to_provider_billed_seconds( + monkeypatch: pytest.MonkeyPatch, + provider: str, + requested_model: str, + response_extras: Mapping[str, float], + expected_seconds: float, +) -> None: + model: Final = requested_model.removeprefix(f"{provider}/") + monkeypatch.setitem( + litellm.model_cost, + model, + {"input_cost_per_second": 0.01, "litellm_provider": provider, "mode": "audio_transcription"}, + ) + transcription: Final = convert_to_model_response_object( + response_object={ + "text": "What is the weather in Paris?", + "usage": {"type": "duration", "seconds": 2}, + **response_extras, + }, + model_response_object=TranscriptionResponse(), + hidden_params={"model": model, "custom_llm_provider": provider}, + response_type="audio_transcription", + ) + + cost: Final = completion_cost( + completion_response=transcription, model=requested_model, call_type="atranscription" + ) + + assert cost == pytest.approx(0.01 * expected_seconds) + + +def test_custom_priced_transcription_deployment_bills_provider_seconds(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setitem( + litellm.model_cost, + "priced-deployment", + {"input_cost_per_second": 0.000075, "litellm_provider": "openai", "mode": "audio_transcription"}, + ) + transcription: Final = convert_to_model_response_object( + response_object={"text": "What is the weather in Paris?", "usage": {"type": "duration", "seconds": 2}}, + model_response_object=TranscriptionResponse(), + hidden_params={"model": "gpt-fake-transcribe", "custom_llm_provider": "openai"}, + response_type="audio_transcription", + ) + + cost: Final = completion_cost( + completion_response=transcription, + model="openai/gpt-fake-transcribe", + call_type="atranscription", + custom_pricing=True, + router_model_id="priced-deployment", + ) + + assert cost == pytest.approx(0.000075 * 2) + + def test_gpt_image_2_azure_cost_tracking(): azure_image_generation_response: Final = { "created": 1758585600, diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 98025f3a79a..75cec231f16 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -9585,6 +9585,33 @@ def test_get_provider_audio_transcription_config_resolves_for_every_provider() - assert isinstance(configs[LlmProviders.OPENAI], litellm.OpenAIWhisperAudioTranscriptionConfig) +@pytest.mark.parametrize( + ("model", "optional_params", "expected_response_format"), + [ + ("whisper-1", {}, "verbose_json"), + ("my-transcribe-whisper", {}, "verbose_json"), + ("whisper-1", {"response_format": "srt"}, "srt"), + ("gpt-4o-mini-transcribe", {}, None), + ("gpt-transcribe", {}, None), + ("gpt-transcribe", {"response_format": "text"}, "text"), + ], +) +def test_openai_transcription_defaults_verbose_json_only_for_whisper_models( + model: str, optional_params: dict[str, str], expected_response_format: str | None +) -> None: + config: Final = ProviderConfigManager.get_provider_audio_transcription_config( + model=model, provider=LlmProviders.OPENAI + ) + assert config is not None + + request: Final = config.transform_audio_transcription_request( + model=model, audio_file=b"audio", optional_params=dict(optional_params), litellm_params={} + ) + + assert isinstance(request.data, dict) + assert request.data.get("response_format") == expected_response_format + + def test_get_valid_models_from_provider_cache_invalidation(monkeypatch): """ Test that get_valid_models returns the correct models for a given provider