scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-05 02:41:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 49

fix(realtime): address transcription and billing review findings

Browse source
This commit is contained in:
Emerson Gomes 2026-09-22 20:16:11 -05:00
parent 34d0773076
commit 17dbd6ae92
No known key found for this signature in database
GPG key ID: D3DF28AB5D1B5E17
23 changed files with 384 additions and 149 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
litellm/__init__.py
View file

@ -72,7 +72,6 @@ from litellm.constants import (
OPENAI_CHAT_COMPLETION_PARAMS as _openai_completion_params, # backwards compatibility
OPENAI_FINISH_REASONS,
OPENAI_FINISH_REASONS as _openai_finish_reasons, # backwards compatibility
OPENAI_REALTIME_AND_TRANSCRIPTION_MODELS,
openai_compatible_endpoints,
openai_compatible_providers,
openai_text_completion_compatible_providers,
@ -1008,7 +1007,6 @@ def add_known_models(model_cost_map: Optional[Dict] = None):
_populate_provider_model_sets(model_cost)
open_ai_chat_completion_models.update(OPENAI_REALTIME_AND_TRANSCRIPTION_MODELS)
# known openai compatible endpoints - we'll eventually move this list to the model_prices_and_context_window.json dictionary
# this is maintained for Exception Mapping

31
litellm/constants.py
View file

@ -835,37 +835,6 @@ OPENAI_TRANSCRIPTION_PARAMS: Final = [
"timestamp_granularities",
]
OPENAI_REALTIME_AND_TRANSCRIPTION_MODELS: Final = frozenset(
{
"gpt-realtime-2",
"gpt-realtime-2.1",
"gpt-realtime-2.1-mini",
"gpt-realtime-translate",
"gpt-realtime-whisper",
"gpt-transcribe",
"gpt-live-transcribe",
}
)
AZURE_GA_REALTIME_MODELS: Final = frozenset(
{
"gpt-realtime-2",
"gpt-realtime-2-2026-05-06",
"gpt-realtime-2.1",
"gpt-realtime-2.1-2026-07-07",
"gpt-realtime-2.1-mini",
"gpt-realtime-2.1-mini-2026-07-07",
"gpt-realtime-translate",
"gpt-realtime-translate-2026-05-06",
"gpt-realtime-translate-2026-05-07",
"gpt-realtime-whisper",
"gpt-realtime-whisper-2026-05-06",
"gpt-realtime-whisper-2026-05-07",
"gpt-transcribe",
"gpt-live-transcribe",
}
)
OPENAI_EMBEDDING_PARAMS: Final = ["dimensions", "encoding_format", "user"]
DEFAULT_EMBEDDING_PARAM_VALUES: Final = {

34
litellm/cost_calculator.py
View file

@ -3023,14 +3023,21 @@ def handle_realtime_translation_cost_calculation(
custom_llm_provider: str,
litellm_model_name: str,
) -> float:
output_seconds = 0.0 # rebind-ok: duration is accumulated across translation close events
for result in results:
if result.get("type") != _TRANSLATION_CLOSED_EVENT_TYPE:
continue
usage = result.get("usage")
if isinstance(usage, dict) and isinstance(usage.get("output_seconds"), (int, float)):
output_seconds += float(usage["output_seconds"])
if output_seconds <= 0:
usage_events: Final = tuple(
usage
for result in results
if result.get("type") == _TRANSLATION_CLOSED_EVENT_TYPE
if isinstance(usage := result.get("usage"), dict)
)
input_seconds: Final = sum(
float(usage["input_seconds"]) for usage in usage_events if isinstance(usage.get("input_seconds"), (int, float))
)
output_seconds: Final = sum(
float(usage["output_seconds"])
for usage in usage_events
if isinstance(usage.get("output_seconds"), (int, float))
)
if input_seconds <= 0 and output_seconds <= 0:
return 0.0
try:
model_info: Final = litellm.get_model_info(
@ -3039,10 +3046,15 @@ def handle_realtime_translation_cost_calculation(
)
except Exception: # noqa: BLE001 # unknown model metadata should yield zero translation cost
return 0.0
input_cost_per_second: Final = model_info.get("input_cost_per_second")
output_cost_per_second: Final = model_info.get("output_cost_per_second")
if not isinstance(output_cost_per_second, (int, float)):
return 0.0
return output_seconds * output_cost_per_second
input_cost: Final = (
input_seconds * input_cost_per_second if isinstance(input_cost_per_second, (int, float)) else 0.0
)
output_cost: Final = (
output_seconds * output_cost_per_second if isinstance(output_cost_per_second, (int, float)) else 0.0
)
return input_cost + output_cost
def handle_realtime_transcription_cost_calculation(

4
litellm/litellm_core_utils/audio_utils/transcription_streaming.py
View file

@ -120,6 +120,8 @@ class LoggingTranscriptionStream(Stream[TranscriptionStreamEvent]):
if self._finalized or self._failed:
return
self._finalized = True
if self._collector.done_event is None and self._collector.duration is None:
return
self._logging_obj.success_handler(
self._collector.response(),
self._start_time,
@ -174,6 +176,8 @@ class LoggingAsyncTranscriptionStream(AsyncStream[TranscriptionStreamEvent]):
if self._finalized or self._failed:
return
self._finalized = True
if self._collector.done_event is None and self._collector.duration is None:
return
response: Final = self._collector.response()
end_time: Final = datetime.datetime.now() # noqa: DTZ005 # callback timestamps use the legacy naive contract
self._logging_obj.handle_sync_success_callbacks_for_async_calls(

5
litellm/litellm_core_utils/realtime_streaming.py
View file

@ -441,12 +441,15 @@ class RealTimeStreaming:
usage: Final = event_obj.get("usage")
output_seconds: Final = usage.get("output_seconds") if isinstance(usage, dict) else None
if isinstance(output_seconds, (int, float)):
input_seconds: Final = usage.get("input_seconds") if isinstance(usage, dict) else None
if not self._should_store_message(event_obj):
self.messages.append(
OpenAIRealtimeTranslationClosedEvent(
type="session.closed",
usage=OpenAIRealtimeTranslationDurationUsage(
type="duration", output_seconds=output_seconds
type="duration",
output_seconds=output_seconds,
**({"input_seconds": input_seconds} if isinstance(input_seconds, (int, float)) else {}),
),
)
)

6
litellm/llms/azure/audio_transcriptions.py
View file

@ -6,6 +6,7 @@ from pydantic import BaseModel
from litellm._uuid import uuid
from litellm.litellm_core_utils.audio_utils.utils import get_audio_file_name
from litellm.llms.base_llm.audio_transcription.transformation import sdk_compatible_transcription_request_data
from litellm.types.utils import FileTypes
from litellm.utils import (
TranscriptionResponse,
@ -40,11 +41,12 @@ class AzureAudioTranscription(AzureChatCompletion):
custom_llm_provider: str = "azure",
) -> TranscriptionResponse | Coroutine[Any, Any, TranscriptionResponse]:
data: Final = {"model": model, "file": audio_file, **optional_params}
sdk_data: Final = sdk_compatible_transcription_request_data(data)
if atranscription is True:
return self.async_audio_transcriptions(
audio_file=audio_file,
data=data,
data=sdk_data,
model_response=model_response,
timeout=timeout,
api_key=api_key,
@ -86,7 +88,7 @@ class AzureAudioTranscription(AzureChatCompletion):
)
response: Final = azure_client.audio.transcriptions.create(
**data, # pyright: ignore[reportArgumentType] # SDK TypedDict lags accepted transcription options
**sdk_data, # pyright: ignore[reportArgumentType] # SDK TypedDict lags accepted transcription options
timeout=timeout,
)

19
litellm/llms/azure/realtime/handler.py
View file

@ -8,6 +8,7 @@ from collections.abc import Mapping
from types import MappingProxyType
from typing import Any, Final, Protocol, cast
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
from litellm.types.realtime import RealtimeQueryParams
@ -39,16 +40,30 @@ async def forward_messages(client_ws: Any, backend_ws: Any):
pass
def azure_realtime_requires_ga(model: str) -> bool:
try:
model_info: Final = litellm.get_model_info(model=model, custom_llm_provider="azure")
except Exception: # noqa: BLE001 # unmapped deployments can select a protocol explicitly
return False
return (model_info.get("provider_specific_entry") or {}).get("realtime_ga_only") == 1
def azure_realtime_protocol_for_client(
configured_protocol: object,
*,
model: str,
realtime_mode: str,
query_params: RealtimeQueryParams | None,
websocket: ScopedWebSocket,
) -> str:
if azure_realtime_requires_ga(model):
if isinstance(configured_protocol, str) and configured_protocol.upper() not in ("GA", "V1"):
raise ValueError(f"{model} requires the Azure OpenAI v1 Realtime API")
return "GA"
if realtime_mode == "translation" or (query_params or {}).get("intent") == "transcription":
return "GA"
if isinstance(configured_protocol, str) and configured_protocol:
return configured_protocol
if (query_params or {}).get("intent") == "transcription":
return "GA"
return "beta" if client_sent_openai_beta_realtime_header(websocket) else "GA"

21
litellm/llms/azure/realtime/http_transformation.py
View file

@ -1,17 +1,21 @@
"""Azure OpenAI realtime HTTP transformation config (client_secrets + realtime_calls)."""
from collections.abc import Mapping
from typing import Final
import litellm
from litellm.constants import AZURE_GA_REALTIME_MODELS
from litellm.llms.azure.common_utils import get_azure_ad_token
from litellm.llms.base_llm.realtime.http_transformation import BaseRealtimeHTTPConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.router import GenericLiteLLMParams
from .handler import azure_realtime_requires_ga
class AzureRealtimeHTTPConfig(BaseRealtimeHTTPConfig):
@staticmethod
def _uses_ga_api(model: str, api_version: str | None) -> bool:
return api_version in ("preview", "latest", "v1") or model in AZURE_GA_REALTIME_MODELS
return api_version in ("preview", "latest", "v1") or azure_realtime_requires_ga(model)
def get_api_base(self, api_base: str | None, **kwargs) -> str:
return api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") or ""
@ -19,6 +23,19 @@ class AzureRealtimeHTTPConfig(BaseRealtimeHTTPConfig):
def get_api_key(self, api_key: str | None, **kwargs) -> str:
return api_key or litellm.api_key or get_secret_str("AZURE_API_KEY") or ""
def get_extra_headers(
self,
litellm_params: GenericLiteLLMParams,
api_key: str,
extra_headers: Mapping[str, object] | None,
) -> Mapping[str, object] | None:
if api_key:
return extra_headers
azure_ad_token: Final = get_azure_ad_token(litellm_params)
if azure_ad_token is None:
return extra_headers
return {**(extra_headers or {}), "Authorization": f"Bearer {azure_ad_token}"}
def get_complete_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str:
base: Final = self.get_api_base(api_base).rstrip("/")
if self._uses_ga_api(model, api_version):

17
litellm/llms/base_llm/audio_transcription/transformation.py
View file

@ -11,6 +11,23 @@ from litellm.types.llms.openai import (
)
from litellm.types.utils import FileTypes, ModelResponse, TranscriptionResponse
def sdk_compatible_transcription_request_data(data: dict) -> dict:
extension_keys: Final = ("keywords", "languages")
extension_body: Final = {key: data[key] for key in extension_keys if key in data}
if not extension_body:
return data
existing_extra_body: Final = data.get("extra_body")
return {
**{key: value for key, value in data.items() if key not in extension_keys},
"extra_body": {
**(existing_extra_body if isinstance(existing_extra_body, dict) else {}),
**extension_body,
},
}
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer

11
litellm/llms/base_llm/realtime/http_transformation.py
View file

@ -7,10 +7,13 @@ These are HTTP (not WebSocket) endpoints used by the WebRTC flow:
"""
from abc import ABC, abstractmethod
from collections.abc import Mapping
from typing import Final
import httpx
from litellm.types.router import GenericLiteLLMParams
class BaseRealtimeHTTPConfig(ABC):
"""
@ -49,6 +52,14 @@ class BaseRealtimeHTTPConfig(ABC):
explicit api_key → litellm.api_key → env var → ""
"""
def get_extra_headers(
self,
litellm_params: GenericLiteLLMParams,
api_key: str,
extra_headers: Mapping[str, object] | None,
) -> Mapping[str, object] | None:
return extra_headers
# ------------------------------------------------------------------ #
# client_secrets endpoint #
# ------------------------------------------------------------------ #

22
litellm/llms/openai/transcriptions/handler.py
View file

@ -12,6 +12,7 @@ from litellm.litellm_core_utils.audio_utils.utils import get_audio_file_name
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.audio_transcription.transformation import (
BaseAudioTranscriptionConfig,
sdk_compatible_transcription_request_data,
)
from litellm.types.utils import FileTypes
from litellm.utils import (
@ -25,23 +26,6 @@ from ..openai import OpenAIChatCompletion
class OpenAIAudioTranscription(OpenAIChatCompletion):
# Audio Transcriptions
@staticmethod
def _sdk_compatible_request_data(data: dict) -> dict:
"""Route API fields that predate SDK support through ``extra_body``."""
extension_keys: Final = ("keywords", "languages")
extension_body: Final = {key: data[key] for key in extension_keys if key in data}
if not extension_body:
return data
existing_extra_body: Final = data.get("extra_body")
return { # mutable-ok: OpenAI SDK requires a mutable request mapping
**{key: value for key, value in data.items() if key not in extension_keys},
"extra_body": {
**(existing_extra_body if isinstance(existing_extra_body, dict) else {}),
**extension_body,
},
}
async def make_openai_audio_transcriptions_request(
self,
openai_aclient: AsyncOpenAI,
@ -54,7 +38,7 @@ class OpenAIAudioTranscription(OpenAIChatCompletion):
- call openai_aclient.audio.transcriptions.create by default
"""
try:
sdk_data: Final = self._sdk_compatible_request_data(data)
sdk_data: Final = sdk_compatible_transcription_request_data(data)
if data.get("stream") is True:
stream_response: Final = await openai_aclient.audio.transcriptions.create(**sdk_data, timeout=timeout)
return {}, stream_response # mutable-ok: response headers use the existing mutable mapping contract
@ -80,7 +64,7 @@ class OpenAIAudioTranscription(OpenAIChatCompletion):
- call openai_aclient.audio.transcriptions.create by default
"""
try:
sdk_data: Final = self._sdk_compatible_request_data(data)
sdk_data: Final = sdk_compatible_transcription_request_data(data)
if data.get("stream") is True:
response = openai_client.audio.transcriptions.create(**sdk_data, timeout=timeout)
return None, response

16
litellm/main.py
View file

@ -7879,8 +7879,14 @@ def _validate_gpt_transcription_request(
response_format: str | None,
api_version: str | None,
) -> str | None:
model_info: Final = get_model_info(model=model) if model in litellm.model_cost else {}
model_cost_key: Final = f"{custom_llm_provider}/{model}" if custom_llm_provider == "azure" else model
model_info: Final = (
get_model_info(model=model, custom_llm_provider=custom_llm_provider)
if model_cost_key in litellm.model_cost
else {}
)
supported_endpoints: Final = model_info.get("supported_endpoints")
provider_specific_entry: Final = model_info.get("provider_specific_entry") or {}
if language is not None and languages is not None:
raise litellm.UnsupportedParamsError(
message="language and languages cannot be used together",
@ -7893,14 +7899,14 @@ def _validate_gpt_transcription_request(
model=model,
llm_provider=custom_llm_provider,
)
if model == "gpt-transcribe" and response_format not in (None, "json"):
if provider_specific_entry.get("transcription_json_only") == 1 and response_format not in (None, "json"):
raise litellm.UnsupportedParamsError(
message="gpt-transcribe only supports response_format='json'",
message=f"{model} only supports response_format='json'",
model=model,
llm_provider=custom_llm_provider,
)
if custom_llm_provider == "azure" and model == "gpt-transcribe":
if api_version in (None, "v1", "latest", "preview"):
if provider_specific_entry.get("transcription_deployment_api") == 1:
if api_version in ("v1", "latest", "preview"):
return litellm.AZURE_DEFAULT_API_VERSION
return api_version
return api_version

41
litellm/model_prices_and_context_window_backup.json
View file

@ -3,6 +3,9 @@
"input_cost_per_second": 0.0002833333333333333,
"litellm_provider": "azure",
"mode": "audio_transcription",
"provider_specific_entry": {
"realtime_ga_only": 1
},
"source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-gpt-transcribe-and-gpt-live-transcribe-in-microsoft-foundry/4541740",
"supported_endpoints": [
"/v1/realtime",
@ -30,6 +33,9 @@
"max_output_tokens": 4096,
"max_tokens": 4096,
"mode": "realtime",
"provider_specific_entry": {
"realtime_ga_only": 1
},
"output_cost_per_audio_token": 6.4e-05,
"output_cost_per_token": 2.4e-05,
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure",
@ -67,6 +73,9 @@
"max_output_tokens": 4096,
"max_tokens": 4096,
"mode": "realtime",
"provider_specific_entry": {
"realtime_ga_only": 1
},
"output_cost_per_audio_token": 2e-05,
"output_cost_per_token": 2.4e-06,
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure",
@ -98,6 +107,9 @@
"max_output_tokens": 4096,
"max_tokens": 4096,
"mode": "realtime",
"provider_specific_entry": {
"realtime_ga_only": 1
},
"output_cost_per_second": 0.0005666666666666667,
"source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/a-new-chapter-for-realtime-ai-reasoning-translation-and-real-time-transcription/4517124",
"supported_endpoints": [
@ -122,6 +134,9 @@
"max_output_tokens": 4096,
"max_tokens": 4096,
"mode": "realtime",
"provider_specific_entry": {
"realtime_ga_only": 1
},
"output_cost_per_second": 0.0005666666666666667,
"source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/a-new-chapter-for-realtime-ai-reasoning-translation-and-real-time-transcription/4517124",
"supported_endpoints": [
@ -146,6 +161,9 @@
"max_output_tokens": 4096,
"max_tokens": 4096,
"mode": "realtime",
"provider_specific_entry": {
"realtime_ga_only": 1
},
"output_cost_per_second": 0.0005666666666666667,
"source": "https://ai.azure.com/catalog/models/gpt-realtime-translate",
"supported_endpoints": [
@ -168,6 +186,9 @@
"input_cost_per_second": 0.0002833333333333333,
"litellm_provider": "azure",
"mode": "audio_transcription",
"provider_specific_entry": {
"realtime_ga_only": 1
},
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper",
"supported_endpoints": [
"/v1/realtime",
@ -188,6 +209,9 @@
"input_cost_per_second": 0.0002833333333333333,
"litellm_provider": "azure",
"mode": "audio_transcription",
"provider_specific_entry": {
"realtime_ga_only": 1
},
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper",
"supported_endpoints": [
"/v1/realtime",
@ -208,6 +232,11 @@
"input_cost_per_second": 7.5e-05,
"litellm_provider": "azure",
"mode": "audio_transcription",
"provider_specific_entry": {
"realtime_ga_only": 1,
"transcription_deployment_api": 1,
"transcription_json_only": 1
},
"source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-gpt-transcribe-and-gpt-live-transcribe-in-microsoft-foundry/4541740",
"supported_endpoints": [
"/v1/audio/transcriptions",
@ -6057,6 +6086,9 @@
"max_output_tokens": 4096,
"max_tokens": 4096,
"mode": "realtime",
"provider_specific_entry": {
"realtime_ga_only": 1
},
"output_cost_per_audio_token": 6.4e-05,
"output_cost_per_token": 2.4e-05,
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure",
@ -6095,6 +6127,9 @@
"max_output_tokens": 4096,
"max_tokens": 4096,
"mode": "realtime",
"provider_specific_entry": {
"realtime_ga_only": 1
},
"output_cost_per_audio_token": 2e-05,
"output_cost_per_token": 2.4e-06,
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure",
@ -6290,6 +6325,9 @@
"input_cost_per_second": 0.0002833333333333333,
"litellm_provider": "azure",
"mode": "audio_transcription",
"provider_specific_entry": {
"realtime_ga_only": 1
},
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper",
"supported_endpoints": [
"/v1/realtime",
@ -57550,6 +57588,9 @@
"input_cost_per_second": 7.5e-05,
"litellm_provider": "openai",
"mode": "audio_transcription",
"provider_specific_entry": {
"transcription_json_only": 1
},
"source": "https://developers.openai.com/api/docs/pricing",
"supported_endpoints": [
"/v1/audio/transcriptions",

52
litellm/realtime_api/main.py
View file

@ -10,7 +10,6 @@ import httpx
import litellm
from litellm.constants import (
AZURE_GA_REALTIME_MODELS,
AZURE_OPENAI_AUDIO_PROVIDERS,
REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS,
REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES,
@ -51,8 +50,6 @@ from ..utils import client as wrapper_client
if TYPE_CHECKING:
from fastapi import WebSocket
from litellm.llms.base_llm.realtime.http_transformation import BaseRealtimeHTTPConfig
azure_realtime: Final = AzureOpenAIRealtime()
openai_realtime: Final = OpenAIRealtime()
bedrock_realtime: Final = BedrockRealtime()
@ -112,21 +109,6 @@ def _build_litellm_metadata(kwargs: dict) -> dict:
return metadata
def _resolve_azure_realtime_protocol(
model: str,
realtime_protocol: str | None,
query_params: RealtimeQueryParams | None,
realtime_mode: str,
) -> str:
if model in AZURE_GA_REALTIME_MODELS:
if realtime_protocol is not None and realtime_protocol.upper() not in ("GA", "V1"):
raise ValueError(f"{model} requires the Azure OpenAI v1 Realtime API")
return "GA"
if realtime_mode == "translation" or (query_params or {}).get("intent") == "transcription":
return "GA"
return realtime_protocol or "beta"
def _get_realtime_http_provider_config(
custom_llm_provider: str,
dynamic_api_base: str | None,
@ -164,19 +146,18 @@ def _get_realtime_http_provider_config(
def _get_realtime_http_extra_headers(
custom_llm_provider: str,
provider_config: BaseRealtimeHTTPConfig | None,
litellm_params: GenericLiteLLMParams,
resolved_api_key: str,
extra_headers: Mapping[str, object] | None,
) -> Mapping[str, object] | None:
resolved_headers: Final = { # mutable-ok: Azure authentication may extend caller-supplied headers
**(extra_headers or {})
}
if custom_llm_provider == "azure" and not resolved_api_key:
azure_ad_token: Final = get_azure_ad_token(litellm_params)
if azure_ad_token:
resolved_headers["Authorization"] = f"Bearer {azure_ad_token}"
return resolved_headers or None
if provider_config is None:
return extra_headers
return provider_config.get_extra_headers(
litellm_params=litellm_params,
api_key=resolved_api_key,
extra_headers=extra_headers,
)
@wrapper_client
@ -229,7 +210,7 @@ async def acreate_realtime_client_secret(
litellm_params=litellm_params,
)
resolved_extra_headers: Final = _get_realtime_http_extra_headers(
custom_llm_provider=custom_llm_provider,
provider_config=provider_config,
litellm_params=litellm_params,
resolved_api_key=resolved_api_key,
extra_headers=kwargs.get("extra_headers"),
@ -302,7 +283,7 @@ async def acreate_realtime_translation_client_secret(
litellm_params=litellm_params,
)
resolved_extra_headers: Final = _get_realtime_http_extra_headers(
custom_llm_provider=custom_llm_provider,
provider_config=provider_config,
litellm_params=litellm_params,
resolved_api_key=resolved_api_key,
extra_headers=kwargs.get("extra_headers"),
@ -385,7 +366,7 @@ async def acreate_realtime_transcription_session(
litellm_params=litellm_params,
)
resolved_extra_headers: Final = _get_realtime_http_extra_headers(
custom_llm_provider=custom_llm_provider,
provider_config=provider_config,
litellm_params=litellm_params,
resolved_api_key=resolved_api_key,
extra_headers=kwargs.get("extra_headers"),
@ -660,17 +641,12 @@ async def _arealtime(
or litellm_params.get("realtime_protocol")
or os.environ.get("LITELLM_AZURE_REALTIME_PROTOCOL")
)
realtime_protocol: Final = _resolve_azure_realtime_protocol(
realtime_protocol: Final = azure_realtime_protocol_for_client(
configured_realtime_protocol,
model=model,
realtime_protocol=(
configured_realtime_protocol
if model in AZURE_GA_REALTIME_MODELS or realtime_mode == "translation"
else azure_realtime_protocol_for_client(
configured_realtime_protocol, query_params=query_params, websocket=websocket
)
),
query_params=query_params,
realtime_mode=realtime_mode,
websocket=websocket,
)
await azure_realtime.async_realtime(
model=model,

1
litellm/types/llms/openai.py
View file

@ -2314,6 +2314,7 @@ class OpenAIRealtimeResponseUsage(TypedDict):
class OpenAIRealtimeTranslationDurationUsage(TypedDict):
type: ReadOnly[Literal["duration"]]
output_seconds: ReadOnly[float]
input_seconds: NotRequired[ReadOnly[float]]
class OpenAIRealtimeTranslationClosedEvent(TypedDict):

41
model_prices_and_context_window.json
View file

@ -3,6 +3,9 @@
"input_cost_per_second": 0.0002833333333333333,
"litellm_provider": "azure",
"mode": "audio_transcription",
"provider_specific_entry": {
"realtime_ga_only": 1
},
"source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-gpt-transcribe-and-gpt-live-transcribe-in-microsoft-foundry/4541740",
"supported_endpoints": [
"/v1/realtime",
@ -30,6 +33,9 @@
"max_output_tokens": 4096,
"max_tokens": 4096,
"mode": "realtime",
"provider_specific_entry": {
"realtime_ga_only": 1
},
"output_cost_per_audio_token": 6.4e-05,
"output_cost_per_token": 2.4e-05,
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure",
@ -67,6 +73,9 @@
"max_output_tokens": 4096,
"max_tokens": 4096,
"mode": "realtime",
"provider_specific_entry": {
"realtime_ga_only": 1
},
"output_cost_per_audio_token": 2e-05,
"output_cost_per_token": 2.4e-06,
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure",
@ -98,6 +107,9 @@
"max_output_tokens": 4096,
"max_tokens": 4096,
"mode": "realtime",
"provider_specific_entry": {
"realtime_ga_only": 1
},
"output_cost_per_second": 0.0005666666666666667,
"source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/a-new-chapter-for-realtime-ai-reasoning-translation-and-real-time-transcription/4517124",
"supported_endpoints": [
@ -122,6 +134,9 @@
"max_output_tokens": 4096,
"max_tokens": 4096,
"mode": "realtime",
"provider_specific_entry": {
"realtime_ga_only": 1
},
"output_cost_per_second": 0.0005666666666666667,
"source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/a-new-chapter-for-realtime-ai-reasoning-translation-and-real-time-transcription/4517124",
"supported_endpoints": [
@ -146,6 +161,9 @@
"max_output_tokens": 4096,
"max_tokens": 4096,
"mode": "realtime",
"provider_specific_entry": {
"realtime_ga_only": 1
},
"output_cost_per_second": 0.0005666666666666667,
"source": "https://ai.azure.com/catalog/models/gpt-realtime-translate",
"supported_endpoints": [
@ -168,6 +186,9 @@
"input_cost_per_second": 0.0002833333333333333,
"litellm_provider": "azure",
"mode": "audio_transcription",
"provider_specific_entry": {
"realtime_ga_only": 1
},
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper",
"supported_endpoints": [
"/v1/realtime",
@ -188,6 +209,9 @@
"input_cost_per_second": 0.0002833333333333333,
"litellm_provider": "azure",
"mode": "audio_transcription",
"provider_specific_entry": {
"realtime_ga_only": 1
},
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper",
"supported_endpoints": [
"/v1/realtime",
@ -208,6 +232,11 @@
"input_cost_per_second": 7.5e-05,
"litellm_provider": "azure",
"mode": "audio_transcription",
"provider_specific_entry": {
"realtime_ga_only": 1,
"transcription_deployment_api": 1,
"transcription_json_only": 1
},
"source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-gpt-transcribe-and-gpt-live-transcribe-in-microsoft-foundry/4541740",
"supported_endpoints": [
"/v1/audio/transcriptions",
@ -6057,6 +6086,9 @@
"max_output_tokens": 4096,
"max_tokens": 4096,
"mode": "realtime",
"provider_specific_entry": {
"realtime_ga_only": 1
},
"output_cost_per_audio_token": 6.4e-05,
"output_cost_per_token": 2.4e-05,
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure",
@ -6095,6 +6127,9 @@
"max_output_tokens": 4096,
"max_tokens": 4096,
"mode": "realtime",
"provider_specific_entry": {
"realtime_ga_only": 1
},
"output_cost_per_audio_token": 2e-05,
"output_cost_per_token": 2.4e-06,
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure",
@ -6290,6 +6325,9 @@
"input_cost_per_second": 0.0002833333333333333,
"litellm_provider": "azure",
"mode": "audio_transcription",
"provider_specific_entry": {
"realtime_ga_only": 1
},
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/gpt-realtime-whisper",
"supported_endpoints": [
"/v1/realtime",
@ -57550,6 +57588,9 @@
"input_cost_per_second": 7.5e-05,
"litellm_provider": "openai",
"mode": "audio_transcription",
"provider_specific_entry": {
"transcription_json_only": 1
},
"source": "https://developers.openai.com/api/docs/pricing",
"supported_endpoints": [
"/v1/audio/transcriptions",

4
tests/test_litellm/litellm_core_utils/test_realtime_streaming.py
View file

@ -2990,13 +2990,13 @@ def test_translation_prefers_provider_duration_over_audio_byte_estimate():
{"type": "session.output_audio.delta", "delta": base64.b64encode(bytes(48000)).decode()}
)
streaming._capture_translation_output_audio(
{"type": "session.closed", "usage": {"type": "duration", "output_seconds": 0.5}}
{"type": "session.closed", "usage": {"type": "duration", "input_seconds": 0.25, "output_seconds": 0.5}}
)
streaming._finalize_translation_usage()
closed_events = [event for event in streaming.messages if event.get("type") == "session.closed"]
assert len(closed_events) == 1
assert closed_events[0]["usage"] == {"type": "duration", "output_seconds": 0.5}
assert closed_events[0]["usage"] == {"type": "duration", "input_seconds": 0.25, "output_seconds": 0.5}
@pytest.mark.asyncio

10
tests/test_litellm/llms/openai/realtime/test_transcription_sessions.py
View file

@ -10,6 +10,7 @@ from unittest.mock import AsyncMock, MagicMock
import httpx
import pytest
import litellm
from litellm.llms.azure.realtime.http_transformation import AzureRealtimeHTTPConfig
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
@ -46,15 +47,18 @@ def test_azure_transcription_session_url_uses_deployment_and_api_version():
@pytest.mark.parametrize("api_version", [None, "v1", "latest", "preview"])
def test_azure_ga_realtime_http_urls(api_version):
def test_azure_ga_realtime_http_urls(api_version, monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
litellm.get_model_info.cache_clear()
cfg = AzureRealtimeHTTPConfig()
base = "https://my.openai.azure.com"
assert cfg.get_complete_url(base, "gpt-realtime-2", api_version) == (f"{base}/openai/v1/realtime/client_secrets")
assert cfg.get_realtime_calls_url(base, "gpt-realtime-2", api_version) == (f"{base}/openai/v1/realtime/calls")
assert cfg.get_complete_url(base, "gpt-realtime-2.1", api_version) == (f"{base}/openai/v1/realtime/client_secrets")
assert cfg.get_realtime_calls_url(base, "gpt-realtime-2.1", api_version) == (f"{base}/openai/v1/realtime/calls")
assert cfg.get_transcription_session_url(base, "gpt-live-transcribe", api_version) == (
f"{base}/openai/v1/realtime/transcription_sessions"
)
litellm.get_model_info.cache_clear()
def test_request_resolves_model_returns_none_when_both_absent():

12
tests/test_litellm/llms/openai/realtime/test_translation.py
View file

@ -11,6 +11,18 @@ from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.llms.openai.realtime.http_transformation import OpenAIRealtimeHTTPConfig
from litellm.types.realtime import RealtimeSessionConfig
from litellm.types.router import GenericLiteLLMParams
def test_azure_realtime_http_headers_use_ad_token_without_api_key():
config = AzureRealtimeHTTPConfig()
params = GenericLiteLLMParams(azure_ad_token="test-ad-token")
assert config.get_extra_headers(params, "", {"X-Trace": "trace"}) == {
"X-Trace": "trace",
"Authorization": "Bearer test-ad-token",
}
assert config.get_extra_headers(params, "test-api-key", {"X-Trace": "trace"}) == {"X-Trace": "trace"}
def test_realtime_session_config_supports_translation_and_live_transcription_fields():

35
tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py
View file

@ -5,11 +5,10 @@ from typing import Final
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
import litellm
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.public_endpoints import router
from litellm.router_strategy.complexity_router.fuse_presets import get_fuse_presets
@ -74,32 +73,20 @@ def test_get_provider_create_fields():
), "Expected at least one provider to have detailed credential fields"
def test_get_litellm_model_cost_map_returns_cost_map():
app = FastAPI()
def test_get_litellm_model_cost_map_returns_cost_map(monkeypatch: pytest.MonkeyPatch) -> None:
model_cost: Final = {
"audio-model": {"input_cost_per_second": 0.1, "mode": "audio_transcription"},
"token-model": {"input_cost_per_token": 0.01, "mode": "chat"},
}
monkeypatch.setattr(litellm, "model_cost", model_cost)
app: Final = FastAPI()
app.include_router(router)
client = TestClient(app)
client: Final = TestClient(app)
response = client.get("/public/litellm_model_cost_map")
response: Final = client.get("/public/litellm_model_cost_map")
assert response.status_code == 200
payload = response.json()
assert isinstance(payload, dict)
assert len(payload) > 0, "Expected model cost map to contain at least one model"
# Verify the structure contains expected keys for at least one model
# Check for a common model like gpt-4 or gpt-3.5-turbo
model_keys = list(payload.keys())
assert len(model_keys) > 0
# Verify at least one model has expected cost fields
sample_model = model_keys[0]
sample_model_data = payload[sample_model]
assert isinstance(sample_model_data, dict)
# Check for common cost fields that should be present
assert (
"input_cost_per_token" in sample_model_data
or "output_cost_per_token" in sample_model_data
)
assert response.json() == model_cost
def test_public_ai_hub_info_is_public_by_default(monkeypatch):

20
tests/test_litellm/test_cost_calculator.py
View file

@ -4685,3 +4685,23 @@ def test_realtime_translation_duration_cost(_local_model_cost_map):
)
assert actual == pytest.approx(2 * litellm.model_cost[model]["output_cost_per_second"])
def test_realtime_translation_duration_cost_includes_provider_input_usage(_local_model_cost_map):
from litellm.cost_calculator import handle_realtime_translation_cost_calculation
model: Final = "gpt-realtime-translate"
events: Final[OpenAIRealtimeStreamList] = [
{"type": "session.closed", "usage": {"type": "duration", "input_seconds": 3.0, "output_seconds": 2.0}}
]
actual: Final = handle_realtime_translation_cost_calculation(
results=events,
custom_llm_provider="openai",
litellm_model_name=model,
)
expected: Final = (
3 * litellm.model_cost[model]["input_cost_per_second"]
+ 2 * litellm.model_cost[model]["output_cost_per_second"]
)
assert actual == pytest.approx(expected)

106
tests/unit/llms/openai/transcriptions/test_gpt_transcribe.py
View file

@ -1,12 +1,13 @@
import io
import json
import wave
from collections.abc import Iterator
from datetime import datetime
from unittest.mock import AsyncMock, MagicMock
import httpx
import pytest
from openai import AsyncOpenAI, AsyncStream, AzureOpenAI
from openai import AsyncAzureOpenAI, AsyncOpenAI, AsyncStream, AzureOpenAI
import litellm
from litellm.litellm_core_utils.audio_utils.transcription_streaming import wrap_transcription_stream
@ -20,6 +21,14 @@ from litellm.types.utils import TranscriptionResponse
from litellm.utils import get_optional_params_transcription
@pytest.fixture
def local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
litellm.get_model_info.cache_clear()
yield
litellm.get_model_info.cache_clear()
def test_gpt_transcribe_config_uses_native_parameters_and_json():
config = OpenAIGPTTranscribeAudioTranscriptionConfig()
supported = config.get_supported_openai_params("gpt-transcribe")
@ -122,6 +131,37 @@ async def test_openai_handler_returns_native_typed_stream():
assert logged_response.languages == []
@pytest.mark.asyncio
async def test_closed_transcription_stream_without_usage_or_duration_does_not_log_success():
async def send_response(request: httpx.Request) -> httpx.Response:
events = (
{"type": "transcript.text.delta", "delta": "hello"},
{"type": "transcript.text.done", "text": "hello", "usage": {"type": "duration", "seconds": 1}},
)
content = "".join(f"data: {json.dumps(event)}\n\n" for event in events).encode()
return httpx.Response(200, content=content, headers={"content-type": "text/event-stream"})
http_client = httpx.AsyncClient(transport=httpx.MockTransport(send_response))
client = AsyncOpenAI(api_key="sk-test", base_url="https://example.com/v1", http_client=http_client)
stream = await client.audio.transcriptions.create(
model="gpt-transcribe", file=("sample.webm", b"audio"), stream=True
)
logging_obj = MagicMock()
logging_obj.async_success_handler = AsyncMock()
logging_obj.async_failure_handler = AsyncMock()
wrapped_stream = wrap_transcription_stream(stream, logging_obj, datetime.now())
async for event in wrapped_stream:
assert event.type == "transcript.text.delta"
break
await wrapped_stream.close()
await client.close()
logging_obj.handle_sync_success_callbacks_for_async_calls.assert_not_called()
logging_obj.async_success_handler.assert_not_awaited()
logging_obj.async_failure_handler.assert_not_awaited()
@pytest.mark.asyncio
async def test_atranscription_stream_preserves_duration_for_callback_cost():
async def send_response(request: httpx.Request) -> httpx.Response:
@ -188,19 +228,20 @@ def test_gpt_transcribe_rejects_conflicting_language_inputs():
)
def test_gpt_transcribe_rejects_whisper_response_formats():
@pytest.mark.parametrize("model", ["gpt-transcribe", "azure/gpt-transcribe"])
def test_gpt_transcribe_rejects_whisper_response_formats(local_model_cost_map: None, model: str) -> None:
audio_file = io.BytesIO(b"audio")
audio_file.name = "sample.wav"
with pytest.raises(litellm.UnsupportedParamsError, match="only supports response_format='json'"):
litellm.transcription(
model="gpt-transcribe",
model=model,
file=audio_file,
response_format="verbose_json",
api_key="sk-test",
)
def test_gpt_live_transcribe_rejects_file_transcription():
def test_gpt_live_transcribe_rejects_file_transcription(local_model_cost_map: None) -> None:
audio_file = io.BytesIO(b"audio")
audio_file.name = "sample.wav"
with pytest.raises(litellm.UnsupportedParamsError, match="Realtime API"):
@ -232,8 +273,8 @@ def test_azure_async_gpt_transcribe_forwards_v1_api_version():
assert handler.async_audio_transcriptions.call_args.kwargs["api_version"] == "v1"
@pytest.mark.parametrize("api_version", [None, "v1", "latest", "preview"])
def test_azure_gpt_transcribe_uses_deployment_scoped_api_version(api_version: str | None):
@pytest.mark.parametrize("api_version", ["v1", "latest", "preview"])
def test_azure_gpt_transcribe_uses_deployment_scoped_api_version(local_model_cost_map: None, api_version: str) -> None:
resolved_api_version = _validate_gpt_transcription_request(
model="gpt-transcribe",
custom_llm_provider="azure",
@ -246,6 +287,20 @@ def test_azure_gpt_transcribe_uses_deployment_scoped_api_version(api_version: st
assert resolved_api_version == litellm.AZURE_DEFAULT_API_VERSION
def test_azure_gpt_transcribe_keeps_unset_api_version_for_configured_default(local_model_cost_map: None) -> None:
assert (
_validate_gpt_transcription_request(
model="gpt-transcribe",
custom_llm_provider="azure",
language=None,
languages=None,
response_format="json",
api_version=None,
)
is None
)
def test_azure_gpt_transcribe_uses_deployment_scoped_route():
def send_response(request: httpx.Request) -> httpx.Response:
assert str(request.url) == (
@ -287,7 +342,7 @@ def test_azure_gpt_transcribe_uses_deployment_scoped_route():
client.close()
def test_azure_gpt_transcribe_preserves_dated_api_version():
def test_azure_gpt_transcribe_preserves_dated_api_version(local_model_cost_map: None) -> None:
resolved_api_version = _validate_gpt_transcription_request(
model="gpt-transcribe",
custom_llm_provider="azure",
@ -298,3 +353,40 @@ def test_azure_gpt_transcribe_preserves_dated_api_version():
)
assert resolved_api_version == "2025-04-01-preview"
@pytest.mark.asyncio
async def test_azure_gpt_transcribe_sends_language_hints_in_sdk_extra_body():
async def send_response(request: httpx.Request) -> httpx.Response:
body = await request.aread()
assert b'name="keywords[]"' in body
assert b'name="languages[]"' in body
return httpx.Response(200, json={"text": "hello"})
http_client = httpx.AsyncClient(transport=httpx.MockTransport(send_response))
client = AsyncAzureOpenAI(
api_key="azure-test-key",
azure_endpoint="https://example.openai.azure.com",
api_version="2025-04-01-preview",
http_client=http_client,
)
audio_file = io.BytesIO(b"audio")
audio_file.name = "sample.wav"
response = await AzureAudioTranscription().audio_transcriptions(
model="gpt-transcribe",
audio_file=audio_file,
optional_params={"keywords": ["LiteLLM"], "languages": ["en"]},
logging_obj=MagicMock(),
model_response=TranscriptionResponse(),
timeout=10,
max_retries=0,
api_key="azure-test-key",
api_base="https://example.openai.azure.com",
api_version="2025-04-01-preview",
client=client,
atranscription=True,
)
assert response.text == "hello"
await client.close()

23
tests/unit/realtime_api/test_main.py
View file

@ -463,6 +463,29 @@ _GA_CLIENT: Final = _ClientWebSocketWithHeaders(headers=())
_BETA_CLIENT: Final = _ClientWebSocketWithHeaders(headers=((b"openai-beta", b"realtime=v1"),))
def test_azure_ga_only_protocol_comes_from_model_metadata(local_model_cost_map) -> None:
from litellm.llms.azure.realtime.handler import azure_realtime_protocol_for_client
assert (
azure_realtime_protocol_for_client(
None,
model="gpt-realtime-2.1",
realtime_mode="realtime",
query_params=None,
websocket=_BETA_CLIENT,
)
== "GA"
)
with pytest.raises(ValueError, match="requires the Azure OpenAI v1 Realtime API"):
azure_realtime_protocol_for_client(
"beta",
model="gpt-realtime-2.1",
realtime_mode="realtime",
query_params=None,
websocket=_BETA_CLIENT,
)
async def _azure_backend_url_dialed_for(websocket: _ClientWebSocketWithHeaders, **kwargs: object) -> str | None:
connect: Final = _ConnectThatStopsAfterCapturingTheUrl()
with patch("websockets.connect", connect):