diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index 3d907daec44..b7f99992593 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -38,7 +38,7 @@ from litellm.types.realtime import ( ) from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import CallTypes, LlmProviders -from litellm.utils import ProviderConfigManager, resolve_api_key_secret_reference +from litellm.utils import ProviderConfigManager, get_base_model_from_metadata, resolve_api_key_secret_reference from ..litellm_core_utils.core_helpers import normalize_drop_params from ..litellm_core_utils.credential_accessor import CredentialAccessor @@ -462,6 +462,11 @@ def _with_upstream_transcription_model(request_data: Mapping[str, object], model } +def _deployment_base_model(kwargs: Mapping[str, object], custom_llm_provider: str) -> str | None: + base_model: Final = get_base_model_from_metadata(model_call_details={"litellm_params": kwargs}) + return None if base_model is None else base_model.removeprefix(f"{custom_llm_provider}/") + + @wrapper_client async def acreate_realtime_transcription_session( model: str | None = None, @@ -534,7 +539,7 @@ async def acreate_realtime_transcription_session( logging_obj=litellm_logging_obj, timeout=timeout or request_timeout, provider_config=provider_config, - model=model_name, + model=_deployment_base_model(kwargs, custom_llm_provider) or model_name, extra_headers=resolved_extra_headers, client=kwargs.get("client"), api_version=litellm_params.api_version, @@ -905,15 +910,10 @@ async def _arealtime( GenericLiteLLMParams(**kwargs, azure_ad_token=azure_ad_token), ) ) - configured_base_model: Final = (as_str_mapping(litellm_params_dict) or {}).get("base_model") realtime_protocol: Final = azure_realtime_protocol_for_client( configured_realtime_protocol, model=model, - base_model=( - configured_base_model.removeprefix(f"{_custom_llm_provider}/") - if isinstance(configured_base_model, str) - else None - ), + base_model=_deployment_base_model(kwargs, _custom_llm_provider), query_params=query_params, realtime_mode=realtime_mode, websocket=websocket, diff --git a/tests/unit/realtime_api/test_main.py b/tests/unit/realtime_api/test_main.py index 44c469f6ed7..071af4135ae 100644 --- a/tests/unit/realtime_api/test_main.py +++ b/tests/unit/realtime_api/test_main.py @@ -1091,14 +1091,79 @@ async def test_arealtime_azure_bridges_a_beta_client_to_ga_when_the_deployment_b assert await _azure_session_created_seen_by_a_beta_client(base_model_kwargs) == ( "wss://my-endpoint.openai.azure.com/openai/v1/realtime?model=my-realtime-deployment", - { - "type": "session.created", - "event_id": "event_1", - "session": {"id": "sess_1", "model": "my-realtime-deployment", "modalities": ["text"], "voice": "marin"}, - }, + _GA_SESSION_CREATED_BRIDGED_TO_BETA, ) +_GA_SESSION_CREATED_BRIDGED_TO_BETA: Final = { + "type": "session.created", + "event_id": "event_1", + "session": {"id": "sess_1", "model": "my-realtime-deployment", "modalities": ["text"], "voice": "marin"}, +} + +_ROUTER_REQUEST_SHAPES: Final = ( + pytest.param({}, id="sdk-router-call"), + pytest.param({"metadata": {"user_api_key_hash": "hashed-key"}}, id="proxy-request-metadata"), +) + + +def _router_with_an_azure_deployment_aliasing(deployment_model: str, base_model: str) -> litellm.Router: + return litellm.Router( + model_list=[ + { + "model_name": "team-alias", + "litellm_params": { + "model": deployment_model, + "api_base": "https://my-endpoint.openai.azure.com", + "api_key": "fake-key", + }, + "model_info": {"base_model": base_model}, + } + ] + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("request_kwargs", _ROUTER_REQUEST_SHAPES) +async def test_router_realtime_bridges_a_beta_client_to_ga_when_model_info_base_model_is_ga_only( + local_model_cost_map, monkeypatch: pytest.MonkeyPatch, request_kwargs: Mapping[str, object] +) -> None: + monkeypatch.delenv("LITELLM_AZURE_REALTIME_PROTOCOL", raising=False) + router: Final = _router_with_an_azure_deployment_aliasing( + "azure/my-realtime-deployment", "azure/gpt-realtime-2.1-mini" + ) + client: Final = _BetaClientThatStaysConnected() + connect: Final = _ConnectToABackendThatSendsOneFrame(_GA_SESSION_CREATED) + + with patch("websockets.connect", connect): + await router._arealtime(model="team-alias", websocket=client, **request_kwargs) + + assert (connect.url, json.loads(client.sent[0])) == ( + "wss://my-endpoint.openai.azure.com/openai/v1/realtime?model=my-realtime-deployment", + _GA_SESSION_CREATED_BRIDGED_TO_BETA, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("request_kwargs", _ROUTER_REQUEST_SHAPES) +async def test_router_transcription_session_uses_the_ga_route_when_model_info_base_model_is_ga_only( + local_model_cost_map, request_kwargs: Mapping[str, object] +) -> None: + router: Final = _router_with_an_azure_deployment_aliasing( + "azure/my-whisper-deployment", "azure/gpt-realtime-whisper" + ) + gateway: Final = _GatewayThatRecordsTheUrl() + + await router.acreate_realtime_transcription_session( + model="team-alias", + transcription_session={"input_audio_format": "pcm16"}, + client=AsyncHTTPHandler(transport=httpx.MockTransport(gateway)), + **request_kwargs, + ) + + assert gateway.url == "https://my-endpoint.openai.azure.com/openai/v1/realtime/transcription_sessions" + + async def _vertex_provider_config_for(monkeypatch, model: str, vertex_location: str | None): from litellm.llms.vertex_ai.audio_transcription.realtime_transformation import VertexChirpRealtimeConfig from litellm.llms.vertex_ai.realtime.transformation import VertexAIRealtimeConfig