diff --git a/litellm/proxy/realtime_endpoints/__init__.py b/litellm/proxy/realtime_endpoints/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index 1b791980af3..285de6d101a 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -54,6 +54,8 @@ ROUTE_ENDPOINT_MAPPING = { "avideo_status": "/videos/{video_id}", "avideo_content": "/videos/{video_id}/content", "avideo_remix": "/videos/{video_id}/remix", + "acreate_realtime_client_secret": "/realtime/client_secrets", + "arealtime_calls": "/realtime/calls", "acreate_container": "/containers", "alist_containers": "/containers", "aretrieve_container": "/containers/{container_id}", @@ -164,6 +166,8 @@ async def route_request( # noqa: PLR0915 - Complex routing function, refactorin "acreate_response_reply", "alist_input_items", "_arealtime", # private function for realtime API + "acreate_realtime_client_secret", + "arealtime_calls", "_aresponses_websocket", # private function for responses WebSocket mode "aimage_edit", "agenerate_content", @@ -296,6 +300,8 @@ async def route_request( # noqa: PLR0915 - Complex routing function, refactorin "aget_run", "acancel_run", "adelete_run", + "acreate_realtime_client_secret", + "arealtime_calls", ]: # If a model is provided, get its credentials from the router model = data.get("model") diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index 83ab63ef146..01b76ad805c 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -1,15 +1,16 @@ """Abstraction function for OpenAI's realtime API""" import os -from typing import Any, Optional, cast +from typing import Any, Dict, Optional, cast import litellm +from litellm.constants import request_timeout from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.secret_managers.main import get_secret_str -from litellm.types.realtime import RealtimeQueryParams +from litellm.types.realtime import RealtimeClientSecretRequest, RealtimeQueryParams from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders from litellm.utils import ProviderConfigManager @@ -42,6 +43,148 @@ def _build_litellm_metadata(kwargs: dict) -> dict: return metadata +def _get_realtime_http_provider_config( + custom_llm_provider: str, + dynamic_api_base: Optional[str], + dynamic_api_key: Optional[str], + litellm_params: GenericLiteLLMParams, +) -> tuple[Any, str, str]: + """ + Return (provider_config, resolved_api_base, resolved_api_key) for the + realtime HTTP endpoints (client_secrets / realtime_calls). + + Uses ProviderConfigManager so each provider keeps its credential-resolution + and URL-construction logic in its own transformation class. + """ + from litellm.llms.base_llm.realtime.http_transformation import BaseRealtimeHTTPConfig + + provider_config: Optional[BaseRealtimeHTTPConfig] = None + if custom_llm_provider in LlmProviders._member_map_.values(): + provider_config = ProviderConfigManager.get_provider_realtime_http_config( + model="", + provider=LlmProviders(custom_llm_provider), + ) + + raw_api_base = dynamic_api_base or litellm_params.api_base + raw_api_key = dynamic_api_key or litellm_params.api_key + + if provider_config is not None: + resolved_api_base = provider_config.get_api_base(api_base=raw_api_base) + resolved_api_key = provider_config.get_api_key(api_key=raw_api_key) + else: + # Fallback for providers without a dedicated HTTP config (treated as OpenAI-compatible). + resolved_api_base = ( + raw_api_base + or litellm.api_base + or "https://api.openai.com" + ) + resolved_api_key = ( + raw_api_key + or litellm.api_key + or litellm.openai_key + or get_secret_str("OPENAI_API_KEY") + or "" + ) + + return provider_config, resolved_api_base.rstrip("/"), resolved_api_key + + +@wrapper_client +async def acreate_realtime_client_secret( + model: Optional[str] = None, + session: Optional[Dict[str, Any]] = None, + expires_after: Optional[Dict[str, Any]] = None, + timeout: Optional[float] = None, + **kwargs, +): + req = RealtimeClientSecretRequest( + model=model, + session=session, + expires_after=expires_after, + ) + model_name = ( + (req.session.model if req.session is not None else None) + or req.model + or "gpt-4o-realtime-preview" + ) + litellm_logging_obj: LiteLLMLogging = kwargs.get("litellm_logging_obj") # type: ignore + litellm_params = GenericLiteLLMParams(**kwargs) + + model_name, custom_llm_provider, dynamic_api_key, dynamic_api_base = get_llm_provider( + model=model_name, + api_base=litellm_params.api_base, + api_key=litellm_params.api_key, + ) + provider_config, resolved_api_base, resolved_api_key = _get_realtime_http_provider_config( + custom_llm_provider=custom_llm_provider, + dynamic_api_base=dynamic_api_base, + dynamic_api_key=dynamic_api_key, + litellm_params=litellm_params, + ) + litellm_logging_obj.update_environment_variables( + model=model_name, + optional_params={"expires_after": expires_after, "session": session}, + litellm_params={"api_base": resolved_api_base}, + custom_llm_provider=custom_llm_provider, + ) + request_data = req.model_dump(exclude_none=True, exclude={"model"}) + return await base_llm_http_handler.async_realtime_client_secret_handler( + api_base=resolved_api_base, + api_key=resolved_api_key, + request_data=request_data, + logging_obj=litellm_logging_obj, + timeout=timeout or request_timeout, + provider_config=provider_config, + model=model_name, + extra_headers=kwargs.get("extra_headers"), + client=kwargs.get("client"), + ) + + +@wrapper_client +async def arealtime_calls( + openai_ephemeral_key: str, + sdp_body: bytes, + model: Optional[str] = None, + session: Optional[Dict[str, Any]] = None, + timeout: Optional[float] = None, + **kwargs, +): + model_name = model or "gpt-4o-realtime-preview" + litellm_logging_obj: LiteLLMLogging = kwargs.get("litellm_logging_obj") # type: ignore + litellm_params = GenericLiteLLMParams(**kwargs) + + model_name, custom_llm_provider, dynamic_api_key, dynamic_api_base = get_llm_provider( + model=model_name, + api_base=litellm_params.api_base, + api_key=litellm_params.api_key, + ) + provider_config, resolved_api_base, _ = _get_realtime_http_provider_config( + custom_llm_provider=custom_llm_provider, + dynamic_api_base=dynamic_api_base, + dynamic_api_key=dynamic_api_key, + litellm_params=litellm_params, + ) + litellm_logging_obj.update_environment_variables( + model=model_name, + optional_params={"realtime_calls": True, "session": session}, + litellm_params={"api_base": resolved_api_base}, + custom_llm_provider=custom_llm_provider, + ) + return await base_llm_http_handler.async_realtime_calls_handler( + api_base=resolved_api_base, + openai_ephemeral_key=openai_ephemeral_key, + sdp_body=sdp_body, + logging_obj=litellm_logging_obj, + timeout=timeout or request_timeout, + provider_config=provider_config, + model=model_name, + session_config=session, + extra_headers=kwargs.get("extra_headers"), + client=kwargs.get("client"), + ) + + @wrapper_client async def _arealtime( # noqa: PLR0915 model: str, diff --git a/litellm/types/realtime.py b/litellm/types/realtime.py index 1ec41f40b3d..d341a32654d 100644 --- a/litellm/types/realtime.py +++ b/litellm/types/realtime.py @@ -1,6 +1,7 @@ -from typing import List, Literal, Optional, Union +from typing import Any, Dict, List, Literal, Optional, Union -from typing_extensions import TypedDict +from pydantic import BaseModel +from typing_extensions import TypedDict # noqa: F401 – re-exported from .llms.openai import ( OpenAIRealtimeEvents, @@ -49,3 +50,68 @@ class RealtimeQueryParams(TypedDict, total=False): model: str intent: Optional[str] # Add more fields as needed + + +# --------------------------------------------------------------------------- +# WebRTC / client_secrets types (POST /v1/realtime/client_secrets) +# --------------------------------------------------------------------------- + + +class RealtimeExpiresAfter(BaseModel): + """Expiration config for a client secret.""" + + anchor: Optional[str] = "created_at" + seconds: Optional[int] = None + + +class RealtimeSessionConfig(BaseModel): + """ + Session configuration nested inside the client_secrets request body. + + Mirrors OpenAI's RealtimeSessionCreateRequest (type=realtime) and + RealtimeTranscriptionSessionCreateRequest (type=transcription). + Extra/unknown fields are passed through unchanged. + """ + + model_config = {"extra": "allow"} + + type: Optional[str] = None + model: Optional[str] = None + instructions: Optional[str] = None + audio: Optional[Dict[str, Any]] = None + include: Optional[List[str]] = None + max_output_tokens: Optional[Union[int, str]] = None + output_modalities: Optional[List[str]] = None + tool_choice: Optional[Any] = None + tools: Optional[List[Dict[str, Any]]] = None + tracing: Optional[Any] = None + truncation: Optional[Any] = None + prompt: Optional[Dict[str, Any]] = None + + +class RealtimeClientSecretRequest(BaseModel): + """ + Request body for POST /v1/realtime/client_secrets. + + LiteLLM also accepts a top-level `model` field for routing when + session.model is absent (LiteLLM extension, not forwarded to OpenAI). + """ + + expires_after: Optional[RealtimeExpiresAfter] = None + session: Optional[RealtimeSessionConfig] = None + # LiteLLM-only routing hint — stripped before forwarding upstream + model: Optional[str] = None + + +class RealtimeClientSecretResponse(BaseModel): + """ + Response from POST /v1/realtime/client_secrets. + + Both the top-level `value` and `session.client_secret.value` + will contain the encrypted token instead of the raw ephemeral key. + The `session` field is kept as a raw dict so unknown fields pass through. + """ + + expires_at: int + value: str + session: Optional[Dict[str, Any]] = None diff --git a/litellm/utils.py b/litellm/utils.py index 88312354bc0..17dd6f91e91 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8846,6 +8846,30 @@ class ProviderConfigManager: return GeminiRealtimeConfig() return None + @staticmethod + def get_provider_realtime_http_config( + model: str, + provider: LlmProviders, + ) -> Optional["BaseRealtimeHTTPConfig"]: + """ + Return the HTTP transformation config for realtime HTTP endpoints + (POST /realtime/client_secrets and POST /realtime/calls). + """ + + if LlmProviders.OPENAI == provider: + from litellm.llms.openai.realtime.http_transformation import ( + OpenAIRealtimeHTTPConfig, + ) + + return OpenAIRealtimeHTTPConfig() + if LlmProviders.AZURE == provider: + from litellm.llms.azure.realtime.http_transformation import ( + AzureRealtimeHTTPConfig, + ) + + return AzureRealtimeHTTPConfig() + return None + @staticmethod def get_provider_image_edit_config( model: str,