scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-18 04:44:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 123

fix(openai)!: route gpt-transcribe to GPT audio transcription config (#44021)

Browse source 
* fix(openai): route gpt-transcribe to GPT audio transcription config

* test(openai): move gpt-transcribe tests under tests/unit

* fix(openai): bill gpt-transcribe from provider usage seconds when duration is unknown

Fall back to the provider-reported usage.seconds when neither a measured
file duration nor a response duration is available, so json responses
for undecodable uploads are no longer billed at zero. Move the routing
test next to the other provider config tests and replace the structure
test with a behavior test.

* fix(openai): bill provider usage seconds only for gpt transcription models

* fix(cost): decide provider-billed transcription seconds from the deployment model, not its pricing id

* test(integration): cover gpt-transcribe default request, provider-billed spend, and health probe

---------

Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
Irvallen Saragih 2026-10-17 05:27:47 +09:00 • committed by GitHub
parent 33b73207e3
commit d6b54ae2b0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 328 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
litellm/cost_calculator.py
View file

@ -86,6 +86,9 @@ from litellm.llms.openai.cost_calculation import (
from litellm.llms.openai.cost_calculation import (
video_output_cost_per_second,
)
from litellm.llms.openai.transcriptions.gpt_transformation import (
OpenAIGPTAudioTranscriptionConfig,
)
from litellm.llms.perplexity.cost_calculator import (
cost_per_token as perplexity_cost_per_token,
)
@ -325,6 +328,15 @@ def _transcription_usage_has_token_details(
return (prompt_tokens_val > 0) or (completion_tokens_val > 0)
def _bills_provider_transcription_seconds(model: str | None, custom_llm_provider: str | None) -> bool:
if model is None or custom_llm_provider != LlmProviders.OPENAI.value:
return False
transcription_config: Final = ProviderConfigManager.get_provider_audio_transcription_config(
model=model.removeprefix(f"{LlmProviders.OPENAI.value}/"), provider=LlmProviders.OPENAI
)
return isinstance(transcription_config, OpenAIGPTAudioTranscriptionConfig)
OCRPricingField = Literal["ocr_cost_per_page", "ocr_cost_per_credit", "annotation_cost_per_page"]
@ -1739,6 +1751,12 @@ def completion_cost(
audio_transcription_file_duration = _hidden.get(
"audio_transcription_duration",
getattr(completion_response, "duration", 0.0),
) or (
completion_response.usage.seconds
if isinstance(completion_response, TranscriptionResponse)
and isinstance(completion_response.usage, TranscriptionUsageDurationObject)
and _bills_provider_transcription_seconds(requested_model, custom_llm_provider)
else 0.0
)
elif call_type in _RERANK_CALL_TYPES:
if completion_response is not None and isinstance(completion_response, RerankResponse):

2
litellm/utils.py
View file

@ -9252,7 +9252,7 @@ class ProviderConfigManager:
return XAIAudioTranscriptionConfig()
elif litellm.LlmProviders.OPENAI == provider:
if "gpt-4o" in model:
if "gpt-4o" in model or (model.startswith("gpt-") and "transcribe" in model):
return litellm.OpenAIGPTAudioTranscriptionConfig()
else:
return litellm.OpenAIWhisperAudioTranscriptionConfig()

174
tests/integration/compatibility/test_audio_transcriptions_wire.py
View file

@ -1,15 +1,18 @@
import io
import json
import math
import mimetypes
import uuid
import wave
from email.message import Message
from email.parser import BytesParser
from email.policy import HTTP
from hashlib import sha256
from typing import Final, cast
import httpx
import pytest
from integration._support.client import Gateway, eventually
from integration._support.client import JSON_OBJECT, Gateway, eventually, object_value
from integration._support.database import read_rows
from integration._support.wire import Reply, Request, wire_server
from openai import OpenAI
@ -671,3 +674,172 @@ def test_audio_transcription_bills_audio_tokens_at_the_input_rate_when_no_audio_
rel_tol=1e-9,
), response.text
assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/v1/audio/transcriptions")]
_GPT_TRANSCRIBE_VERBOSE_JSON_REFUSAL: Final = {
"error": {
"message": "response_format 'verbose_json' is not compatible with model 'gpt-transcribe'",
"type": "invalid_request_error",
"param": "response_format",
"code": "unsupported_value",
}
}
_UNDECODABLE_M4A_BYTES: Final = b"\x00\x00\x00\x20ftypM4A \x00\x00\x02\x00M4A isommp42\x00\x00\x00\x08free"
_DEPLOYMENT_COST_PER_SECOND: Final = 0.0003
def _openai_gpt_transcribe_reply(request: Request, reported_seconds: float) -> Reply:
fields: Final = dict(_text_parts(_multipart_parts(request)))
if fields.get("response_format") == "verbose_json":
return Reply(status=400, body=json.dumps(_GPT_TRANSCRIBE_VERBOSE_JSON_REFUSAL).encode())
return Reply(
body=json.dumps({"text": "hello world", "usage": {"type": "duration", "seconds": reported_seconds}}).encode()
)
def _uploaded_files(request: Request) -> tuple[tuple[str, str, bytes], ...]:
return tuple((name, filename, payload) for name, filename, _, payload in _file_parts(_multipart_parts(request)))
def _silent_wav(seconds: float) -> bytes:
output: Final = io.BytesIO()
with wave.open(output, "wb") as wav:
wav.setnchannels(1)
wav.setsampwidth(2)
wav.setframerate(16000)
wav.writeframes(b"\x00\x00" * round(16000 * seconds))
return output.getvalue()
def _advertised_cost_per_second(gateway: Gateway, model: str) -> float:
entries: Final = gateway.get("/v2/model/info", {"model": model})["data"]
assert isinstance(entries, list) and len(entries) == 1, entries
price: Final = object_value(object_value(entries[0])["model_info"])["input_cost_per_second"]
assert isinstance(price, float) and price > 0, entries
return price
def _response_cost(response: httpx.Response) -> float:
return float(response.headers["x-litellm-response-cost"]) if "x-litellm-response-cost" in response.headers else 0.0
def _logged_spend(call_id: str) -> float:
rows: Final = eventually(
lambda: read_rows('SELECT spend FROM "LiteLLM_SpendLogs" WHERE request_id = %s', (call_id,)),
lambda values: len(values) == 1,
seconds=70,
)
return float(str(rows[0]["spend"]))
def test_gpt_transcribe_without_response_format_reaches_openai_without_verbose_json(gateway: Gateway) -> None:
upload: Final = _UNDECODABLE_M4A_BYTES + uuid.uuid4().bytes
with (
wire_server(lambda request: _openai_gpt_transcribe_reply(request, 2)) as wire,
gateway.scenario() as scenario,
):
model: Final = scenario.model(
model="openai/gpt-transcribe", api_base=f"{wire.url}/v1", api_key="synthetic-openai-key"
)
with _sdk(gateway, "/v1", gateway.key) as sdk:
response: Final = sdk.audio.transcriptions.with_raw_response.create(
model=model, file=("a.m4a", upload, "audio/mp4")
)
requests: Final = wire.drain()
assert response.status_code == 200, response.text
assert JSON_OBJECT.validate_json(response.content)["text"] == "hello world", response.text
assert [(request.method, request.target) for request in requests] == [("POST", "/v1/audio/transcriptions")]
assert _text_parts(_multipart_parts(requests[0])) == (("model", "gpt-transcribe"),)
assert _uploaded_files(requests[0]) == (("file", "a.m4a", upload),)
@pytest.mark.parametrize(
"configured_cost_per_second",
(None, _DEPLOYMENT_COST_PER_SECOND),
ids=("cost_map_price", "deployment_price"),
)
def test_gpt_transcribe_unmeasurable_upload_bills_the_seconds_openai_reports(
gateway: Gateway, configured_cost_per_second: float | None
) -> None:
reported_seconds: Final = 2.0
upload: Final = _UNDECODABLE_M4A_BYTES + uuid.uuid4().bytes
with (
wire_server(lambda request: _openai_gpt_transcribe_reply(request, reported_seconds)) as wire,
gateway.scenario() as scenario,
):
model: Final = (
scenario.model(
model="openai/gpt-transcribe",
api_base=f"{wire.url}/v1",
api_key="synthetic-openai-key",
input_cost_per_second=configured_cost_per_second,
)
if configured_cost_per_second is not None
else scenario.model(
model="openai/gpt-transcribe", api_base=f"{wire.url}/v1", api_key="synthetic-openai-key"
)
)
key: Final = scenario.key(models=[model])
cost_per_second: Final = (
configured_cost_per_second
if configured_cost_per_second is not None
else _advertised_cost_per_second(gateway, model)
)
response: Final = gateway.request_multipart(
"/v1/audio/transcriptions",
{"model": model, "response_format": "json"},
{"file": ("a.m4a", upload, "audio/mp4")},
key=key,
)
expected_cost: Final = reported_seconds * cost_per_second
assert response.status_code == 200, response.text
assert JSON_OBJECT.validate_json(response.content)["text"] == "hello world", response.text
assert math.isclose(_response_cost(response), expected_cost), (_response_cost(response), expected_cost)
assert math.isclose(_logged_spend(response.headers["x-litellm-call-id"]), expected_cost)
key_rows: Final = eventually(
lambda: read_rows(
'SELECT spend FROM "LiteLLM_VerificationToken" WHERE token = %s',
(sha256(key.encode()).hexdigest(),),
),
lambda rows: len(rows) == 1 and float(str(rows[0]["spend"])) > 0,
seconds=70,
)
assert math.isclose(float(str(key_rows[0]["spend"])), expected_cost), key_rows
requests: Final = wire.drain()
assert [(request.method, request.target) for request in requests] == [("POST", "/v1/audio/transcriptions")]
assert _uploaded_files(requests[0]) == (("file", "a.m4a", upload),)
def test_gpt_transcribe_measured_upload_bills_the_measured_length_over_reported_seconds(gateway: Gateway) -> None:
measured_seconds: Final = 1.0
upload: Final = _silent_wav(measured_seconds)
with (
wire_server(lambda request: _openai_gpt_transcribe_reply(request, 7)) as wire,
gateway.scenario() as scenario,
):
model: Final = scenario.model(
model="openai/gpt-transcribe",
api_base=f"{wire.url}/v1",
api_key="synthetic-openai-key",
input_cost_per_second=_DEPLOYMENT_COST_PER_SECOND,
)
response: Final = gateway.request_multipart(
"/v1/audio/transcriptions",
{"model": model, "response_format": "json"},
{"file": ("a.wav", upload, _WAV_CONTENT_TYPE)},
)
expected_cost: Final = measured_seconds * _DEPLOYMENT_COST_PER_SECOND
assert response.status_code == 200, response.text
assert math.isclose(_response_cost(response), expected_cost), (_response_cost(response), expected_cost)
assert math.isclose(_logged_spend(response.headers["x-litellm-call-id"]), expected_cost)
requests: Final = wire.drain()
assert [(request.method, request.target) for request in requests] == [("POST", "/v1/audio/transcriptions")]
assert _uploaded_files(requests[0]) == (("file", "a.wav", upload),)

37
tests/integration/management/test_model_health_check.py
View file

@ -1352,3 +1352,40 @@ def test_evaluation_mode_health_check_of_databricks_ai_decide_resolves_the_mode_
},
)
]
def _openai_gpt_transcribe_health_reply(request: Request) -> Reply:
if _multipart_text_fields(_multipart_parts(request)).get("response_format") == "verbose_json":
return Reply(
status=400,
body=json.dumps(
{
"error": {
"message": "response_format 'verbose_json' is not compatible with model 'gpt-transcribe'",
"type": "invalid_request_error",
"param": "response_format",
"code": "unsupported_value",
}
}
).encode(),
)
return Reply(body=json.dumps({"text": "healthy", "usage": {"type": "duration", "seconds": 1}}).encode())
def test_health_check_of_gpt_transcribe_probes_without_verbose_json_and_reports_it_healthy(gateway: Gateway) -> None:
with wire_server(_openai_gpt_transcribe_health_reply) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(
model="openai/gpt-transcribe",
api_base=f"{wire.url}/v1",
api_key="synthetic-openai-key",
model_info={"mode": "audio_transcription"},
)
report: Final = _health_report(gateway, model)
requests: Final = wire.drain()
assert (report["healthy_count"], report["unhealthy_count"]) == (1, 0), report
assert requests, report
assert {
(request.method, request.target, tuple(sorted(_multipart_text_fields(_multipart_parts(request)).items())))
for request in requests
} == {("POST", "/v1/audio/transcriptions", (("model", "gpt-transcribe"),))}

72
tests/unit/test_cost_calculator.py
View file

@ -6426,6 +6426,78 @@ def test_whisper_azure():
assert round(cost, 5) == round(expected_cost, 5)
@pytest.mark.parametrize(
("provider", "requested_model", "response_extras", "expected_seconds"),
[
("openai", "openai/gpt-fake-transcribe", {}, 2.0),
("openai", "openai/gpt-fake-transcribe", {"_audio_transcription_duration": 1.35}, 1.35),
("openai", "openai/gpt-fake-transcribe", {"duration": 1.34}, 1.34),
("openai", "openai/whisper-fake", {}, 0.0),
("azure", "gpt-fake-transcribe", {}, 0.0),
],
ids=[
"undecodable_upload_bills_provider_seconds",
"measured_duration_still_wins",
"provider_duration_still_wins",
"whisper_models_never_bill_provider_seconds",
"azure_never_bills_provider_seconds",
],
)
def test_transcription_cost_falls_back_to_provider_billed_seconds(
monkeypatch: pytest.MonkeyPatch,
provider: str,
requested_model: str,
response_extras: Mapping[str, float],
expected_seconds: float,
) -> None:
model: Final = requested_model.removeprefix(f"{provider}/")
monkeypatch.setitem(
litellm.model_cost,
model,
{"input_cost_per_second": 0.01, "litellm_provider": provider, "mode": "audio_transcription"},
)
transcription: Final = convert_to_model_response_object(
response_object={
"text": "What is the weather in Paris?",
"usage": {"type": "duration", "seconds": 2},
**response_extras,
},
model_response_object=TranscriptionResponse(),
hidden_params={"model": model, "custom_llm_provider": provider},
response_type="audio_transcription",
)
cost: Final = completion_cost(
completion_response=transcription, model=requested_model, call_type="atranscription"
)
assert cost == pytest.approx(0.01 * expected_seconds)
def test_custom_priced_transcription_deployment_bills_provider_seconds(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setitem(
litellm.model_cost,
"priced-deployment",
{"input_cost_per_second": 0.000075, "litellm_provider": "openai", "mode": "audio_transcription"},
)
transcription: Final = convert_to_model_response_object(
response_object={"text": "What is the weather in Paris?", "usage": {"type": "duration", "seconds": 2}},
model_response_object=TranscriptionResponse(),
hidden_params={"model": "gpt-fake-transcribe", "custom_llm_provider": "openai"},
response_type="audio_transcription",
)
cost: Final = completion_cost(
completion_response=transcription,
model="openai/gpt-fake-transcribe",
call_type="atranscription",
custom_pricing=True,
router_model_id="priced-deployment",
)
assert cost == pytest.approx(0.000075 * 2)
def test_gpt_image_2_azure_cost_tracking():
azure_image_generation_response: Final = {
"created": 1758585600,

27
tests/unit/test_utils.py
View file

@ -9585,6 +9585,33 @@ def test_get_provider_audio_transcription_config_resolves_for_every_provider() -
assert isinstance(configs[LlmProviders.OPENAI], litellm.OpenAIWhisperAudioTranscriptionConfig)
@pytest.mark.parametrize(
("model", "optional_params", "expected_response_format"),
[
("whisper-1", {}, "verbose_json"),
("my-transcribe-whisper", {}, "verbose_json"),
("whisper-1", {"response_format": "srt"}, "srt"),
("gpt-4o-mini-transcribe", {}, None),
("gpt-transcribe", {}, None),
("gpt-transcribe", {"response_format": "text"}, "text"),
],
)
def test_openai_transcription_defaults_verbose_json_only_for_whisper_models(
model: str, optional_params: dict[str, str], expected_response_format: str | None
) -> None:
config: Final = ProviderConfigManager.get_provider_audio_transcription_config(
model=model, provider=LlmProviders.OPENAI
)
assert config is not None
request: Final = config.transform_audio_transcription_request(
model=model, audio_file=b"audio", optional_params=dict(optional_params), litellm_params={}
)
assert isinstance(request.data, dict)
assert request.data.get("response_format") == expected_response_format
def test_get_valid_models_from_provider_cache_invalidation(monkeypatch):
"""
Test that get_valid_models returns the correct models for a given provider