diff --git a/docs/my-website/blog/realtime_webrtc_http_endpoints/index.md b/docs/my-website/blog/realtime_webrtc_http_endpoints/index.md new file mode 100644 index 00000000000..1316b5c7308 --- /dev/null +++ b/docs/my-website/blog/realtime_webrtc_http_endpoints/index.md @@ -0,0 +1,134 @@ +--- +slug: realtime_webrtc_http_endpoints +title: "Realtime WebRTC HTTP Endpoints" +date: 2026-03-12T10:00:00 +authors: + - name: Sameer Kankute + title: SWE @ LiteLLM (LLM Translation) + url: https://www.linkedin.com/in/sameer-kankute/ + image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg + - name: Krrish Dholakia + title: "CEO, LiteLLM" + url: https://www.linkedin.com/in/krish-d/ + image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg + - name: Ishaan Jaff + title: "CTO, LiteLLM" + url: https://www.linkedin.com/in/reffajnaahsi/ + image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg +description: "Use the LiteLLM proxy to route OpenAI-style WebRTC realtime via HTTP: client_secrets and SDP exchange." +tags: [realtime, webrtc, proxy, openai] +hide_table_of_contents: false +--- + +import WebRTCTester from '@site/src/components/WebRTCTester'; + +Connect to the Realtime API via WebRTC from browser/mobile clients. LiteLLM handles auth and key management; audio streams directly to OpenAI/Azure. + +**Providers:** OpenAI · Azure OpenAI + +:::info **WebRTC vs WebSocket** +- **WebSocket** (`/v1/realtime`) — server-to-server +- **WebRTC** (`/v1/realtime/client_secrets` + `/v1/realtime/calls`) — browser/mobile, lower latency +::: + +## How it works + +LiteLLM issues tokens and relays SDP; audio never passes through the proxy. + +``` +Browser LiteLLM Proxy OpenAI/Azure + | | | + |-- POST /v1/realtime/ | | + | client_secrets -------->|-- POST sessions -------->| + | |<-- { ek_... } -----------| + |<-- { encrypted_token } ---| | + |-- POST /v1/realtime/calls |-- POST calls ----------->| + | [SDP + token] --------->| | + |<-- SDP answer ------------|<-- SDP answer -----------| + |===== audio P2P direct to OpenAI/Azure =============>| +``` + +## Proxy Setup + +```yaml +model_list: + - model_name: gpt-4o-realtime + litellm_params: + model: openai/gpt-4o-realtime-preview-2024-12-17 + api_key: os.environ/OPENAI_API_KEY + model_info: + mode: realtime +``` + +**Azure:** use `model: azure/gpt-4o-realtime-preview`, `api_key`, `api_base`. + +```bash +litellm --config /path/to/config.yaml +``` + +## Try it live + + + +## Client Usage + +**1. Get token** — `POST /v1/realtime/client_secrets` with LiteLLM API key and `{ model }`. + +**2. WebRTC handshake** — Create `RTCPeerConnection`, add mic track, create data channel `oai-events`, send SDP offer to `POST /v1/realtime/calls` with `Authorization: Bearer ` and `Content-Type: application/sdp`. + +**3. Events** — Use the data channel for `session.update` and other events. + +
+Full code example + +```javascript +// 1. Token +const r = await fetch("http://proxy:4000/v1/realtime/client_secrets", { + method: "POST", + headers: { "Authorization": "Bearer sk-litellm-key", "Content-Type": "application/json" }, + body: JSON.stringify({ model: "gpt-4o-realtime" }), +}); +const { client_secret } = await r.json(); +const token = client_secret.value; + +// 2. WebRTC +const pc = new RTCPeerConnection(); +const audio = document.createElement("audio"); +audio.autoplay = true; +pc.ontrack = (e) => (audio.srcObject = e.streams[0]); +const ms = await navigator.mediaDevices.getUserMedia({ audio: true }); +pc.addTrack(ms.getTracks()[0]); +const dc = pc.createDataChannel("oai-events"); +const offer = await pc.createOffer(); +await pc.setLocalDescription(offer); + +const sdpRes = await fetch("http://proxy:4000/v1/realtime/calls", { + method: "POST", + headers: { "Authorization": `Bearer ${token}`, "Content-Type": "application/sdp" }, + body: offer.sdp, +}); +await pc.setRemoteDescription({ type: "answer", sdp: await sdpRes.text() }); + +// 3. Events +dc.send(JSON.stringify({ type: "session.update", session: { instructions: "..." } })); +``` + +
+ +## FAQ + +**Q: What do I do if I get a 401 Token expired error?** +A: Tokens are short-lived. Get a fresh token right before creating the WebRTC offer. + +**Q: Which key should I use for `/v1/realtime/calls`?** +A: Use the **encrypted token** from `client_secrets`, not your raw API key. + +**Q: Should I pass the `model` parameter when making the call?** +A: No, the encrypted token already encodes all routing information including model. + +**Q: How do I resolve Azure `api-version` errors?** +A: Set the correct `api_version` in `litellm_params` (or via the `AZURE_API_VERSION` environment variable), along with the right `api_base` and deployment values. + +**Q: What if I get no audio?** +A: Make sure you grant microphone permission, ensure `pc.ontrack` assigns the audio element with `autoplay` enabled, check your network/firewall for WebRTC traffic, and inspect the browser console for ICE or SDP errors. + diff --git a/docs/my-website/docs/proxy/realtime_webrtc.md b/docs/my-website/docs/proxy/realtime_webrtc.md new file mode 100644 index 00000000000..694f293652b --- /dev/null +++ b/docs/my-website/docs/proxy/realtime_webrtc.md @@ -0,0 +1,84 @@ +# /realtime - WebRTC Support + +Connect to the Realtime API via WebRTC from browser/mobile clients. LiteLLM handles auth; audio streams directly to OpenAI/Azure. + +**Providers:** OpenAI · Azure + +:::info **WebRTC vs WebSocket** +- **WebSocket** (`/v1/realtime`) — server-to-server +- **WebRTC** (`/v1/realtime/client_secrets` + `/v1/realtime/calls`) — browser/mobile, lower latency +::: + +## How it works + +LiteLLM issues tokens and relays SDP; audio never passes through the proxy. + +``` +Browser LiteLLM Proxy OpenAI/Azure + | | | + |-- POST client_secrets --->|-- POST sessions -------->| + |<-- encrypted_token -------|<-- ek_... ---------------| + |-- POST calls [SDP+token] ->|-- POST calls ----------->| + |<-- SDP answer ------------|<-- SDP answer -----------| + |===== audio P2P direct ===============================>| +``` + +## Proxy Setup + +```yaml +model_list: + - model_name: gpt-4o-realtime + litellm_params: + model: openai/gpt-4o-realtime-preview-2024-12-17 + api_key: os.environ/OPENAI_API_KEY + model_info: + mode: realtime +``` + +**Azure:** `model: azure/gpt-4o-realtime-preview`, `api_key`, `api_base`. + +```bash +litellm --config /path/to/config.yaml +``` + +## Client Usage + +1. **Token** — `POST /v1/realtime/client_secrets` with LiteLLM key and `{ model }`. +2. **WebRTC** — Create `RTCPeerConnection`, add mic, data channel `oai-events`, send SDP offer to `POST /v1/realtime/calls` with `Authorization: Bearer `, `Content-Type: application/sdp`. +3. **Events** — Use data channel for `session.update` and other events. + +```javascript +const r = await fetch("http://proxy:4000/v1/realtime/client_secrets", { + method: "POST", + headers: { "Authorization": "Bearer sk-litellm-key", "Content-Type": "application/json" }, + body: JSON.stringify({ model: "gpt-4o-realtime" }), +}); +const token = (await r.json()).client_secret.value; + +const pc = new RTCPeerConnection(); +const audio = document.createElement("audio"); +audio.autoplay = true; +pc.ontrack = (e) => (audio.srcObject = e.streams[0]); +const ms = await navigator.mediaDevices.getUserMedia({ audio: true }); +pc.addTrack(ms.getTracks()[0]); +const dc = pc.createDataChannel("oai-events"); +const offer = await pc.createOffer(); +await pc.setLocalDescription(offer); + +const sdpRes = await fetch("http://proxy:4000/v1/realtime/calls", { + method: "POST", + headers: { "Authorization": `Bearer ${token}`, "Content-Type": "application/sdp" }, + body: offer.sdp, +}); +await pc.setRemoteDescription({ type: "answer", sdp: await sdpRes.text() }); + +dc.send(JSON.stringify({ type: "session.update", session: { instructions: "..." } })); +``` + +## FAQ + +- **401 Token expired** — Get a fresh token right before creating the WebRTC offer. +- **Which key for `/calls`?** — Encrypted token from `client_secrets`, not raw key. +- **Pass `model`?** — No. Token encodes routing. +- **Azure `api-version`** — Set `api_version` in `litellm_params` and correct `api_base`. +- **No audio** — Grant mic; ensure `pc.ontrack` sets autoplay audio; check firewall/WebRTC; inspect console. \ No newline at end of file diff --git a/docs/my-website/src/components/WebRTCTester.jsx b/docs/my-website/src/components/WebRTCTester.jsx new file mode 100644 index 00000000000..3ade6dd7689 --- /dev/null +++ b/docs/my-website/src/components/WebRTCTester.jsx @@ -0,0 +1,83 @@ +import DashboardWebRTCTester from "../../../../ui/litellm-dashboard/src/components/WebRTCTester.jsx"; + +const LIGHT_MODE_OVERRIDES = ` +.wrt-wrap { + background: #1f2937; + border: 1px solid #334155; +} +.wrt-toggle, +.wrt-toggle:hover { + background: #111827; +} +.wrt-toggle-title, +.we-msg { + color: #e2e8f0; +} +.wrt-toggle-sub, +.wrt-label, +.wrt-field label, +.wrt-flow-box, +.wrt-flow-arrow, +.wrt-meta-row span:first-child, +.wrt-header-title, +.wrt-tab, +.we-time { + color: #94a3b8; +} +.wrt-body, +.wrt-sidebar, +.wrt-main, +.wrt-header, +.wrt-tabs, +.wrt-sdp-box, +.wrt-sdp-hdr, +.wrt-divider { + border-color: #334155; +} +.wrt-header { + background: #111827; +} +.wrt-field input, +.wrt-mic-btn, +.wrt-status-pill { + background: #0b1220; + border-color: #334155; + color: #e2e8f0; +} +.wrt-field input:focus, +.wrt-btn-ghost:hover { + border-color: #60a5fa; +} +.wrt-btn-ghost { + background: #0b1220; + border-color: #334155; + color: #e2e8f0; +} +.wrt-log::-webkit-scrollbar-thumb { + background: #475569; +} +.wrt-tab.active { + color: #93c5fd; + border-bottom-color: #93c5fd; +} +.wrt-empty, +.wrt-audio-status, +.wrt-meta-row span:last-child { + color: #cbd5e1; +} +.wrt-sdp-dot { + background: #475569; +} +.wrt-sdp-pane textarea { + color: #e2e8f0; +} +`; + +export default function WebRTCTester() { + return ( + <> + + + + ); +} diff --git a/litellm/__init__.py b/litellm/__init__.py index 847b16d0630..439f88f205a 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1261,7 +1261,7 @@ from .containers.main import * from .ocr.main import * from .rag.main import * from .search.main import * -from .realtime_api.main import _arealtime +from .realtime_api.main import _arealtime, acreate_realtime_client_secret, arealtime_calls from .responses.main import _aresponses_websocket from .fine_tuning.main import * from .files.main import * diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index 0b4b21154ae..06ba9675ca2 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -377,6 +377,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac user_api_key_dict: UserAPIKeyAuth, response: Any, request_headers: Optional[Dict[str, str]] = None, + litellm_call_info: Optional[Dict[str, Any]] = None, ) -> Optional[Dict[str, str]]: """ Called after an LLM API call (success or failure) to allow injecting custom HTTP response headers. @@ -386,6 +387,11 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac - user_api_key_dict: UserAPIKeyAuth - The user API key dictionary. - response: Any - The response object (None for failure cases). - request_headers: Optional[Dict[str, str]] - The original request headers. + - litellm_call_info: Optional[Dict[str, Any]] - Normalized routing metadata: + - custom_llm_provider: str - The LLM provider (e.g. "openai", "azure") + - model_info: dict - The model_info from router config + - api_base: str - The API base URL used + - model_id: str - The deployment model ID Returns: - Optional[Dict[str, str]]: A dictionary of headers to inject into the HTTP response. diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index a40ff94b347..566ff3218ad 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -5345,6 +5345,20 @@ def get_standard_logging_object_payload( model_name = reconstruct_model_name( kwargs.get("model", "") or "", custom_llm_provider, metadata ) + response_model_name: Optional[str] = None + if isinstance(final_response_obj, dict): + response_model_name = final_response_obj.get("model") + + # For Azure Model Router, preserve the actual model in the top-level standard + # logging payload only when the user has opted in. + requested_model = kwargs.get("model") + if ( + isinstance(requested_model, str) + and ("model_router" in requested_model.lower() or "model-router" in requested_model.lower()) + and isinstance(response_model_name, str) + and response_model_name + ): + model_name = response_model_name payload: StandardLoggingPayload = StandardLoggingPayload( id=str(id), diff --git a/litellm/llms/azure/realtime/http_transformation.py b/litellm/llms/azure/realtime/http_transformation.py new file mode 100644 index 00000000000..ef9a2d92d48 --- /dev/null +++ b/litellm/llms/azure/realtime/http_transformation.py @@ -0,0 +1,52 @@ +"""Azure OpenAI realtime HTTP transformation config (client_secrets + realtime_calls).""" + +from typing import Optional + +import litellm +from litellm.llms.base_llm.realtime.http_transformation import BaseRealtimeHTTPConfig +from litellm.secret_managers.main import get_secret_str + + +class AzureRealtimeHTTPConfig(BaseRealtimeHTTPConfig): + def get_api_base(self, api_base: Optional[str], **kwargs) -> str: + return ( + api_base + or litellm.api_base + or get_secret_str("AZURE_API_BASE") + or "" + ) + + def get_api_key(self, api_key: Optional[str], **kwargs) -> str: + return ( + api_key + or litellm.api_key + or get_secret_str("AZURE_API_KEY") + or "" + ) + + def get_complete_url(self, api_base: Optional[str], model: str, api_version: Optional[str] = None) -> str: + base = self.get_api_base(api_base).rstrip("/") + version = api_version or get_secret_str("AZURE_API_VERSION") or "2024-12-17" + return f"{base}/openai/realtime/client_secrets?api-version={version}" + + def validate_environment( + self, + headers: dict, + model: str, + api_key: Optional[str] = None, + ) -> dict: + return { + **headers, + "api-key": api_key or "", + "Content-Type": "application/json", + } + + def get_realtime_calls_url(self, api_base: Optional[str], model: str, api_version: Optional[str] = None) -> str: + base = self.get_api_base(api_base).rstrip("/") + version = api_version or get_secret_str("AZURE_API_VERSION") or "2024-12-17" + return f"{base}/openai/realtime/calls?api-version={version}" + + def get_realtime_calls_headers(self, ephemeral_key: str) -> dict: + return { + "api-key": ephemeral_key, + } diff --git a/litellm/llms/azure_ai/azure_model_router/transformation.py b/litellm/llms/azure_ai/azure_model_router/transformation.py index c768b591827..b5eb0edba59 100644 --- a/litellm/llms/azure_ai/azure_model_router/transformation.py +++ b/litellm/llms/azure_ai/azure_model_router/transformation.py @@ -63,25 +63,18 @@ class AzureModelRouterConfig(AzureAIStudioConfig): ) -> ModelResponse: """ Transform response for Model Router. - - Preserves the original model path (including model_router/ prefix) in the response - for proper cost tracking and logging. + + Extracts the actual model used from the Azure response (e.g., gpt-5-nano-2025-08-07) + and returns it with the azure_ai/ prefix for proper display and cost tracking. """ from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo - # Preserve the original model from litellm_params (includes routing prefixes like model_router/) - # This ensures cost tracking and logging use the full model path - original_model: str = litellm_params.get("model") or model - if not original_model.startswith("azure_ai/"): - # Add provider prefix if not already present - model_response.model = f"azure_ai/{original_model}" - else: - model_response.model = original_model - # Get base model for the parent call (strips routing prefixes for API compatibility) base_model: str = AzureFoundryModelInfo.get_base_model(model) - - return super().transform_response( + + # Call parent transform_response first - this will extract the actual model + # from the raw response (e.g., "gpt-5-nano-2025-08-07") + model_response = super().transform_response( model=base_model, raw_response=raw_response, model_response=model_response, @@ -94,6 +87,7 @@ class AzureModelRouterConfig(AzureAIStudioConfig): api_key=api_key, json_mode=json_mode, ) + return model_response def calculate_additional_costs( self, model: str, prompt_tokens: int, completion_tokens: int diff --git a/litellm/llms/base_llm/realtime/http_transformation.py b/litellm/llms/base_llm/realtime/http_transformation.py new file mode 100644 index 00000000000..7aadd49ffd3 --- /dev/null +++ b/litellm/llms/base_llm/realtime/http_transformation.py @@ -0,0 +1,115 @@ +""" +Base transformation class for realtime HTTP endpoints (client_secrets, realtime_calls). + +These are HTTP (not WebSocket) endpoints used by the WebRTC flow: + POST /v1/realtime/client_secrets — obtains a short-lived ephemeral key + POST /v1/realtime/calls — exchanges an SDP offer using that key +""" + +from abc import ABC, abstractmethod +from typing import Optional, Union + +import httpx + + +class BaseRealtimeHTTPConfig(ABC): + """ + Abstract base for provider-specific realtime HTTP credential / URL logic. + + Implement one subclass per provider (OpenAI, Azure, …). + """ + + # ------------------------------------------------------------------ # + # Credential resolution # + # ------------------------------------------------------------------ # + + @abstractmethod + def get_api_base( + self, + api_base: Optional[str], + **kwargs, + ) -> str: + """ + Resolve the provider API base URL. + + Resolution order (provider-specific): + explicit api_base → litellm.api_base → env var → hard-coded default + """ + + @abstractmethod + def get_api_key( + self, + api_key: Optional[str], + **kwargs, + ) -> str: + """ + Resolve the provider API key. + + Resolution order (provider-specific): + explicit api_key → litellm.api_key → env var → "" + """ + + # ------------------------------------------------------------------ # + # client_secrets endpoint # + # ------------------------------------------------------------------ # + + @abstractmethod + def get_complete_url(self, api_base: Optional[str], model: str, api_version: Optional[str] = None) -> str: + """Return the full URL for POST /realtime/client_secrets.""" + + @abstractmethod + def validate_environment( + self, + headers: dict, + model: str, + api_key: Optional[str] = None, + ) -> dict: + """ + Build and return the request headers for the client_secrets call. + + Merge `headers` (caller-supplied extras) with auth / content-type + headers required by this provider. + """ + + # ------------------------------------------------------------------ # + # realtime_calls endpoint # + # ------------------------------------------------------------------ # + + def get_realtime_calls_url( + self, api_base: Optional[str], model: str, api_version: Optional[str] = None + ) -> str: + """Return the full URL for POST /realtime/calls (SDP exchange).""" + base = (api_base or "").rstrip("/") + return f"{base}/v1/realtime/calls" + + def get_realtime_calls_headers(self, ephemeral_key: str) -> dict: + """ + Build headers for the realtime_calls POST. + + The Bearer token here is the ephemeral key obtained from + client_secrets, not the long-lived provider key. + """ + return { + "Authorization": f"Bearer {ephemeral_key}", + } + + # ------------------------------------------------------------------ # + # Error handling # + # ------------------------------------------------------------------ # + + def get_error_class( + self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] + ): + """ + Map HTTP errors to LiteLLM exception types. + + Default: generic exception. Override in subclasses for provider-specific + error mapping (e.g., Azure uses different error codes). + """ + from litellm.llms.base_llm.chat.transformation import BaseLLMException + + return BaseLLMException( + status_code=status_code, + message=error_message, + headers=headers, + ) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 1e7b1dcffbd..1442be71c30 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -4808,6 +4808,153 @@ class BaseLLMHTTPHandler: f"Unexpected error while closing WebSocket: {close_error}" ) + async def async_realtime_client_secret_handler( + self, + api_base: str, + api_key: str, + request_data: Dict[str, Any], + logging_obj: LiteLLMLoggingObj, + timeout: Union[float, httpx.Timeout], + provider_config: Optional[Any] = None, + model: Optional[str] = None, + extra_headers: Optional[Dict[str, Any]] = None, + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + api_version: Optional[str] = None, + ) -> httpx.Response: + """ + Forward POST /v1/realtime/client_secrets to upstream provider. + + Uses provider_config (BaseRealtimeHTTPConfig) for URL construction and + header auth when available; falls back to the legacy OpenAI-style defaults. + """ + if client is None or not isinstance(client, AsyncHTTPHandler): + async_httpx_client = get_async_httpx_client( + llm_provider=litellm.LlmProviders.OPENAI, + ) + else: + async_httpx_client = client + + if provider_config is not None: + url = provider_config.get_complete_url(api_base=api_base, model=model or "", api_version=api_version) + headers: Dict[str, Any] = provider_config.validate_environment( + headers={}, model=model or "", api_key=api_key + ) + else: + url = f"{api_base.rstrip('/')}/v1/realtime/client_secrets" + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + "OpenAI-Beta": "realtime=v1", + } + + if extra_headers: + headers.update(extra_headers) + + logging_obj.pre_call( + input=request_data, + api_key="", + additional_args={ + "complete_input_dict": request_data, + "api_base": url, + "headers": headers, + }, + ) + + try: + return await async_httpx_client.post( + url=url, + headers=headers, + json=request_data, + timeout=timeout, + ) + except Exception as e: + raise self._handle_error( + e=e, + provider_config=provider_config, + ) + + async def async_realtime_calls_handler( + self, + api_base: str, + openai_ephemeral_key: str, + sdp_body: bytes, + logging_obj: LiteLLMLoggingObj, + timeout: Union[float, httpx.Timeout], + provider_config: Optional[Any] = None, + model: Optional[str] = None, + session_config: Optional[Dict[str, Any]] = None, + extra_headers: Optional[Dict[str, Any]] = None, + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + api_version: Optional[str] = None, + ) -> httpx.Response: + """ + Forward POST /v1/realtime/calls (SDP exchange) to upstream provider. + + Uses provider_config (BaseRealtimeHTTPConfig) for URL construction and + header auth when available; falls back to the legacy OpenAI-style defaults. + + OpenAI's GA realtime API expects multipart/form-data with: + - sdp: the SDP offer (text) + - session: JSON string with {"type": "realtime", "model": "...", ...} + """ + if client is None or not isinstance(client, AsyncHTTPHandler): + async_httpx_client = get_async_httpx_client( + llm_provider=litellm.LlmProviders.OPENAI, + ) + else: + async_httpx_client = client + + if provider_config is not None: + url = provider_config.get_realtime_calls_url(api_base=api_base, model=model or "", api_version=api_version) + headers: Dict[str, Any] = provider_config.get_realtime_calls_headers( + ephemeral_key=openai_ephemeral_key + ) + else: + url = f"{api_base.rstrip('/')}/v1/realtime/calls" + headers = { + "Authorization": f"Bearer {openai_ephemeral_key}", + } + + if extra_headers: + headers.update(extra_headers) + + # Build multipart form data: sdp + session JSON + session_data = session_config or {} + if "type" not in session_data: + session_data["type"] = "realtime" + if "model" not in session_data and model: + session_data["model"] = model + + sdp_text = sdp_body.decode("utf-8") if isinstance(sdp_body, bytes) else sdp_body + + files = { + "sdp": (None, sdp_text, "text/plain"), + "session": (None, json.dumps(session_data), "application/json"), + } + + logging_obj.pre_call( + input="realtime_sdp_offer", + api_key="", + additional_args={ + "api_base": url, + "headers": headers, + "session": session_data, + }, + ) + + try: + return await async_httpx_client.post( + url=url, + headers=headers, + files=files, + timeout=timeout, + ) + except Exception as e: + raise self._handle_error( + e=e, + provider_config=provider_config, + ) + async def async_responses_websocket( self, model: str, diff --git a/litellm/llms/openai/realtime/http_transformation.py b/litellm/llms/openai/realtime/http_transformation.py new file mode 100644 index 00000000000..ff69ef987db --- /dev/null +++ b/litellm/llms/openai/realtime/http_transformation.py @@ -0,0 +1,50 @@ +"""OpenAI realtime HTTP transformation config (client_secrets + realtime_calls).""" + +from typing import Optional + +import litellm +from litellm.llms.base_llm.realtime.http_transformation import BaseRealtimeHTTPConfig +from litellm.secret_managers.main import get_secret_str + + +class OpenAIRealtimeHTTPConfig(BaseRealtimeHTTPConfig): + def get_api_base(self, api_base: Optional[str], **kwargs) -> str: + return ( + api_base + or litellm.api_base + or get_secret_str("OPENAI_API_BASE") + or "https://api.openai.com" + ) + + def get_api_key(self, api_key: Optional[str], **kwargs) -> str: + return ( + api_key + or litellm.api_key + or litellm.openai_key + or get_secret_str("OPENAI_API_KEY") + or "" + ) + + def get_complete_url(self, api_base: Optional[str], model: str, api_version: Optional[str] = None) -> str: + base = self.get_api_base(api_base).rstrip("/") + if base.endswith("/v1"): + base = base[:-3] + return f"{base}/v1/realtime/client_secrets" + + def get_realtime_calls_url(self, api_base: Optional[str], model: str, api_version: Optional[str] = None) -> str: + base = self.get_api_base(api_base).rstrip("/") + if base.endswith("/v1"): + base = base[:-3] + return f"{base}/v1/realtime/calls" + + def validate_environment( + self, + headers: dict, + model: str, + api_key: Optional[str] = None, + ) -> dict: + return { + **headers, + "Authorization": f"Bearer {api_key or ''}", + "Content-Type": "application/json", + } diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index ce39ecf52dc..5a3f3a984b3 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -246,6 +246,29 @@ async def create_response( ) +def _is_azure_model_router_request(model: str) -> bool: + """ + Check if the requested model is an Azure Model Router. + + Azure Model Router models follow the pattern: + - azure_ai/model_router/ + - azure_ai/model-router + - model_router/ + - model-router + + Args: + model: The requested model name + + Returns: + bool: True if this is an Azure Model Router request + """ + model_lower = model.lower() + return ( + "model-router" in model_lower + or "model_router" in model_lower + ) + + def _override_openai_response_model( *, response_obj: Any, @@ -265,9 +288,11 @@ def _override_openai_response_model( Errors are reserved for cases where the proxy cannot read/override the response model field. - Exception: If a fallback occurred (indicated by x-litellm-attempted-fallbacks header), - we should preserve the actual model that was used (the fallback model) rather than - overriding it with the originally requested model. + Exceptions: + 1. If a fallback occurred (indicated by x-litellm-attempted-fallbacks header), + we preserve the actual model that was used (the fallback model). + 2. If the request was to an Azure Model Router, we preserve the actual model + that was used (e.g., gpt-5-nano-2025-08-07) instead of the router model. """ if not requested_model: return @@ -288,6 +313,14 @@ def _override_openai_response_model( ) return + # Check if this is an Azure Model Router request - if so, preserve the actual model used + if _is_azure_model_router_request(requested_model): + verbose_proxy_logger.debug( + "%s: Azure Model Router detected - preserving actual model used from response instead of overriding to router model.", + log_context, + ) + return + if isinstance(response_obj, dict): downstream_model = response_obj.get("model") if downstream_model != requested_model: @@ -744,6 +777,8 @@ class ProxyBaseLLMRequestProcessing: "aembedding", "aresponses", "_arealtime", + "acreate_realtime_client_secret", + "arealtime_calls", "aget_responses", "adelete_responses", "acancel_responses", @@ -920,6 +955,7 @@ class ProxyBaseLLMRequestProcessing: data=self.data, user_api_key_dict=user_api_key_dict, response=response, + request_headers=dict(request.headers), ) if callback_headers: custom_headers.update(callback_headers) @@ -1028,6 +1064,7 @@ class ProxyBaseLLMRequestProcessing: data=self.data, user_api_key_dict=user_api_key_dict, response=response, + request_headers=dict(request.headers), ) if callback_headers: fastapi_response.headers.update(callback_headers) @@ -1196,6 +1233,7 @@ class ProxyBaseLLMRequestProcessing: data=self.data, user_api_key_dict=user_api_key_dict, response=None, + request_headers=(self.data.get("proxy_server_request") or {}).get("headers", {}), ) if callback_headers: headers.update(callback_headers) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 9a502d53a89..0c24e8b6b5b 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -260,6 +260,7 @@ from litellm.proxy.anthropic_endpoints.claude_code_endpoints import ( claude_code_marketplace_router, ) from litellm.proxy.anthropic_endpoints.endpoints import router as anthropic_router +from litellm.proxy.realtime_endpoints.endpoints import router as webrtc_router from litellm.proxy.anthropic_endpoints.skills_endpoints import ( router as anthropic_skills_router, ) @@ -289,6 +290,7 @@ from litellm.proxy.batches_endpoints.endpoints import router as batches_router from litellm.proxy.caching_routes import router as caching_router from litellm.proxy.common_request_processing import ( ProxyBaseLLMRequestProcessing, + _is_azure_model_router_request, create_response, ) from litellm.proxy.common_utils.callback_utils import initialize_callbacks_on_proxy @@ -5434,6 +5436,12 @@ def _restamp_streaming_chunk_model( if not requested_model_from_client or not isinstance(chunk, (BaseModel, dict)): return chunk, model_mismatch_logged + # For Azure Model Router, preserve the actual model used in each chunk + if _is_azure_model_router_request( + requested_model_from_client + ): + return chunk, model_mismatch_logged + downstream_model = ( chunk.get("model") if isinstance(chunk, dict) else getattr(chunk, "model", None) ) @@ -7525,6 +7533,16 @@ async def audio_transcriptions( ) ) + # Call response headers hook (matches base_process_llm_request behavior) + callback_headers = await proxy_logging_obj.post_call_response_headers_hook( + data=data, + user_api_key_dict=user_api_key_dict, + response=response, + request_headers=dict(request.headers), + ) + if callback_headers: + fastapi_response.headers.update(callback_headers) + return response except Exception as e: await proxy_logging_obj.post_call_failure_hook( @@ -13187,6 +13205,7 @@ app.include_router(vector_store_management_router) app.include_router(vector_store_files_router) app.include_router(credential_router) app.include_router(llm_passthrough_router) +app.include_router(webrtc_router) app.include_router(mcp_management_router) app.include_router(mcp_byok_oauth_router) app.include_router(anthropic_router) diff --git a/litellm/proxy/realtime_endpoints/__init__.py b/litellm/proxy/realtime_endpoints/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/proxy/realtime_endpoints/endpoints.py b/litellm/proxy/realtime_endpoints/endpoints.py new file mode 100644 index 00000000000..bb286d1fd0d --- /dev/null +++ b/litellm/proxy/realtime_endpoints/endpoints.py @@ -0,0 +1,371 @@ +#### Realtime WebRTC Endpoints ##### + +import json +import time +from typing import Any, Dict, Optional + +import httpx +from fastapi import APIRouter, Depends, HTTPException, Request, Response +from fastapi import status as http_status + +from litellm._logging import verbose_proxy_logger +from litellm.proxy._types import ProxyException, UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing +from litellm.proxy.common_utils.encrypt_decrypt_utils import ( + decrypt_value_helper, + encrypt_value_helper, +) +from litellm.proxy.common_utils.http_parsing_utils import _read_request_body +from litellm.types.realtime import ( + RealtimeClientSecretRequest, + RealtimeClientSecretResponse, +) + +router = APIRouter() + +_REALTIME_TOKEN_VERSION = "realtime_v1" + + +def _encode_realtime_token_payload( + ephemeral_key: str, + model_id: str, + user_id: Optional[str], + team_id: Optional[str], + expires_at: Optional[int], +) -> str: + """ + Encode metadata with the upstream ephemeral key so /realtime/calls can + route without requiring model as a query param. + """ + payload: Dict[str, Any] = { + "v": _REALTIME_TOKEN_VERSION, + "ephemeral_key": ephemeral_key, + "model_id": model_id, + "user_id": user_id or "", + "team_id": team_id or "", + "expires_at": expires_at, + } + return json.dumps(payload, separators=(",", ":")) + + +def _decode_realtime_token_payload( + decrypted_value: str, +) -> Optional[Dict[str, Any]]: + """ + Decode realtime token payload; returns None for legacy/raw ephemeral tokens. + """ + try: + decoded = json.loads(decrypted_value) + except Exception: + return None + + if not isinstance(decoded, dict): + return None + if decoded.get("v") != _REALTIME_TOKEN_VERSION: + return None + if not isinstance(decoded.get("ephemeral_key"), str): + return None + if not isinstance(decoded.get("model_id"), str): + return None + return decoded + + +@router.post( + "/v1/realtime/client_secrets", + dependencies=[Depends(user_api_key_auth)], + tags=["realtime"], +) +@router.post( + "/realtime/client_secrets", + dependencies=[Depends(user_api_key_auth)], + tags=["realtime"], +) +@router.post( + "/openai/v1/realtime/client_secrets", + dependencies=[Depends(user_api_key_auth)], + tags=["realtime"], +) +async def create_realtime_client_secret( + request: Request, + fastapi_response: Response, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +) -> RealtimeClientSecretResponse: + from litellm.proxy.proxy_server import ( + add_litellm_data_to_request, + general_settings, + llm_router, + proxy_config, + proxy_logging_obj, + route_request, + user_model, + version, + ) + + data: dict = {} + try: + body = await _read_request_body(request=request) + req = RealtimeClientSecretRequest(**body) + + model: str = ( + (req.session.model if req.session else None) + or req.model + or "gpt-4o-realtime-preview" + ) + + data = {"model": model} + + # If session is provided, use it; otherwise create one from model + if req.session: + data["session"] = req.session.model_dump(exclude_none=True) + elif req.model: + # User provided model at root level, convert to session format + data["session"] = {"type": "realtime", "model": model} + + if req.expires_after: + data["expires_after"] = req.expires_after.model_dump(exclude_none=True) + + data = await add_litellm_data_to_request( + data=data, + request=request, + general_settings=general_settings, + user_api_key_dict=user_api_key_dict, + version=version, + proxy_config=proxy_config, + ) + + data = await proxy_logging_obj.pre_call_hook( + user_api_key_dict=user_api_key_dict, + data=data, + call_type="acreate_realtime_client_secret", + ) + + verbose_proxy_logger.debug( + "WebRTC: /v1/realtime/client_secrets (model=%s)", model + ) + + llm_call = await route_request( + data=data, + route_type="acreate_realtime_client_secret", + llm_router=llm_router, + user_model=user_model, + ) + upstream_resp: httpx.Response = await llm_call # type: ignore + + except Exception as e: + await proxy_logging_obj.post_call_failure_hook( + user_api_key_dict=user_api_key_dict, + original_exception=e, + request_data=data, + ) + verbose_proxy_logger.error( + "litellm.proxy.realtime_endpoints.webrtc.create_realtime_client_secret(): Exception - %s", + str(e), + ) + if isinstance(e, HTTPException): + raise ProxyException( + message=getattr(e, "message", str(e)), + type=getattr(e, "type", "None"), + param=getattr(e, "param", "None"), + code=getattr(e, "status_code", http_status.HTTP_400_BAD_REQUEST), + ) + raise ProxyException( + message=getattr(e, "message", str(e)), + type=getattr(e, "type", "None"), + param=getattr(e, "param", "None"), + code=getattr(e, "status_code", 500), + ) + + if upstream_resp.status_code != 200: + verbose_proxy_logger.error( + "WebRTC client_secrets upstream error %s: %s", + upstream_resp.status_code, + upstream_resp.text, + ) + return Response( + content=upstream_resp.content, + status_code=upstream_resp.status_code, + media_type="application/json", + ) + + upstream_json: dict = upstream_resp.json() + + # Encrypt upstream ephemeral key with routing metadata so /realtime/calls + # can recover model without requiring query params. + raw_value: str = upstream_json.get("value", "") + expires_at = upstream_json.get("expires_at") + token_payload = _encode_realtime_token_payload( + ephemeral_key=raw_value, + model_id=model, + user_id=getattr(user_api_key_dict, "user_id", None), + team_id=getattr(user_api_key_dict, "team_id", None), + expires_at=expires_at if isinstance(expires_at, int) else None, + ) + encrypted_token: str = encrypt_value_helper(token_payload) + upstream_json["value"] = encrypted_token + + session_obj: Optional[dict] = upstream_json.get("session") + if isinstance(session_obj, dict): + cs = session_obj.get("client_secret") + if isinstance(cs, dict) and "value" in cs: + cs["value"] = encrypted_token + upstream_json["session"] = session_obj + + return RealtimeClientSecretResponse(**upstream_json) + + +@router.post( + "/v1/realtime/calls", + tags=["realtime"], +) +@router.post( + "/realtime/calls", + tags=["realtime"], +) +@router.post( + "/openai/v1/realtime/calls", + tags=["realtime"], +) +async def proxy_realtime_calls( + request: Request, + fastapi_response: Response, +) -> Response: + from litellm.proxy.proxy_server import ( + add_litellm_data_to_request, + general_settings, + llm_router, + proxy_config, + proxy_logging_obj, + route_request, + user_model, + version, + ) + + # Auth: the Bearer token is the encrypted ephemeral key issued by + # /realtime/client_secrets, not a standard proxy API key. + auth_header: Optional[str] = request.headers.get("Authorization") + if not auth_header or not auth_header.startswith("Bearer "): + return Response( + content=json.dumps({"error": "Missing or invalid Authorization header"}), + status_code=http_status.HTTP_401_UNAUTHORIZED, + media_type="application/json", + ) + + encrypted_token = auth_header.removeprefix("Bearer ").strip() + decrypted_token_value = decrypt_value_helper( + value=encrypted_token, + key="realtime_calls_auth", + ) + if not decrypted_token_value: + return Response( + content=json.dumps({"error": "Invalid or expired token"}), + status_code=http_status.HTTP_401_UNAUTHORIZED, + media_type="application/json", + ) + + sdp_body: bytes = await request.body() + decoded_payload = _decode_realtime_token_payload(decrypted_token_value) + if decoded_payload is not None: + # Check token expiry + expires_at = decoded_payload.get("expires_at") + if expires_at is not None and isinstance(expires_at, int): + if time.time() > expires_at: + return Response( + content=json.dumps({"error": "Token has expired"}), + status_code=http_status.HTTP_401_UNAUTHORIZED, + media_type="application/json", + ) + + openai_ephemeral_key = decoded_payload.get("ephemeral_key", "") + model = ( + decoded_payload.get("model_id") + or request.query_params.get("model") + or "gpt-4o-realtime-preview" + ) + user_id = decoded_payload.get("user_id") or None + team_id = decoded_payload.get("team_id") or None + else: + # Backward compatibility: older tokens contained only encrypted upstream key. + openai_ephemeral_key = decrypted_token_value + model = request.query_params.get("model", "gpt-4o-realtime-preview") + user_id = None + team_id = None + + # Build a minimal UserAPIKeyAuth with user/team IDs from the token + # so spend tracking and budget enforcement work correctly. + minimal_auth = UserAPIKeyAuth( + user_id=user_id, + team_id=team_id, + ) + + data: dict = {} + try: + # Build session config for the multipart form data + session_config = { + "type": "realtime", + "model": model, + } + + data = { + "model": model, + "openai_ephemeral_key": openai_ephemeral_key, + "sdp_body": sdp_body, + "session": session_config, + } + + data = await add_litellm_data_to_request( + data=data, + request=request, + general_settings=general_settings, + user_api_key_dict=minimal_auth, + version=version, + proxy_config=proxy_config, + ) + + data = await proxy_logging_obj.pre_call_hook( + user_api_key_dict=minimal_auth, + data=data, + call_type="arealtime_calls", + ) + + verbose_proxy_logger.debug( + "WebRTC: /v1/realtime/calls (model=%s)", model + ) + + llm_call = await route_request( + data=data, + route_type="arealtime_calls", + llm_router=llm_router, + user_model=user_model, + ) + upstream_resp: httpx.Response = await llm_call # type: ignore + + except Exception as e: + await proxy_logging_obj.post_call_failure_hook( + user_api_key_dict=minimal_auth, + original_exception=e, + request_data=data, + ) + verbose_proxy_logger.error( + "litellm.proxy.realtime_endpoints.webrtc.proxy_realtime_calls(): Exception - %s", + str(e), + ) + if isinstance(e, HTTPException): + raise ProxyException( + message=getattr(e, "message", str(e)), + type=getattr(e, "type", "None"), + param=getattr(e, "param", "None"), + code=getattr(e, "status_code", http_status.HTTP_400_BAD_REQUEST), + ) + raise ProxyException( + message=getattr(e, "message", str(e)), + type=getattr(e, "type", "None"), + param=getattr(e, "param", "None"), + code=getattr(e, "status_code", 500), + ) + + return Response( + content=upstream_resp.content, + status_code=upstream_resp.status_code, + media_type=upstream_resp.headers.get("content-type", "application/sdp"), + ) diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index 1a4af62312e..9fb5fe9fee4 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -54,6 +54,8 @@ ROUTE_ENDPOINT_MAPPING = { "avideo_status": "/videos/{video_id}", "avideo_content": "/videos/{video_id}/content", "avideo_remix": "/videos/{video_id}/remix", + "acreate_realtime_client_secret": "/realtime/client_secrets", + "arealtime_calls": "/realtime/calls", "acreate_container": "/containers", "alist_containers": "/containers", "aretrieve_container": "/containers/{container_id}", @@ -164,6 +166,8 @@ async def route_request( # noqa: PLR0915 - Complex routing function, refactorin "acreate_response_reply", "alist_input_items", "_arealtime", # private function for realtime API + "acreate_realtime_client_secret", + "arealtime_calls", "_aresponses_websocket", # private function for responses WebSocket mode "aimage_edit", "agenerate_content", @@ -296,6 +300,8 @@ async def route_request( # noqa: PLR0915 - Complex routing function, refactorin "aget_run", "acancel_run", "adelete_run", + "acreate_realtime_client_secret", + "arealtime_calls", ]: # If a model is provided, get its credentials from the router model = data.get("model") diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 85a982f3724..3019de617f0 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1,6 +1,7 @@ import asyncio import copy import hashlib +import inspect import json import os import smtplib @@ -285,6 +286,19 @@ class InternalUsageCache: ### LOGGING ### + +# Cache for inspect.signature checks — avoids repeated introspection per request +_CALLBACK_ACCEPTS_CALL_INFO: Dict[int, bool] = {} + + +def _accepts_litellm_call_info(cb: CustomLogger) -> bool: + key = id(type(cb)) + if key not in _CALLBACK_ACCEPTS_CALL_INFO: + sig = inspect.signature(cb.async_post_call_response_headers_hook) + _CALLBACK_ACCEPTS_CALL_INFO[key] = "litellm_call_info" in sig.parameters + return _CALLBACK_ACCEPTS_CALL_INFO[key] + + class ProxyLogging: """ Logging/Custom Handlers for proxy. @@ -1977,6 +1991,9 @@ class ProxyLogging: """ merged_headers: Dict[str, str] = {} try: + # Build litellm_call_info — normalized routing metadata for callbacks + litellm_call_info = self._build_litellm_call_info(data=data, response=response) + for callback in litellm.callbacks: _callback: Optional[CustomLogger] = None if isinstance(callback, str): @@ -1987,12 +2004,22 @@ class ProxyLogging: _callback = callback # type: ignore if _callback is not None and isinstance(_callback, CustomLogger): - result = await _callback.async_post_call_response_headers_hook( - data=data, - user_api_key_dict=user_api_key_dict, - response=response, - request_headers=request_headers, - ) + if _accepts_litellm_call_info(_callback): + result = await _callback.async_post_call_response_headers_hook( + data=data, + user_api_key_dict=user_api_key_dict, + response=response, + request_headers=request_headers, + litellm_call_info=litellm_call_info, + ) + else: + # Backwards compat: callback doesn't accept litellm_call_info + result = await _callback.async_post_call_response_headers_hook( + data=data, + user_api_key_dict=user_api_key_dict, + response=response, + request_headers=request_headers, + ) if result is not None: merged_headers.update(result) except Exception as e: @@ -2001,6 +2028,30 @@ class ProxyLogging: ) return merged_headers + @staticmethod + def _build_litellm_call_info( + data: dict, response: Any + ) -> Dict[str, Any]: + """ + Build a normalized dict of routing metadata from response._hidden_params + and data, abstracting away the metadata vs litellm_metadata split. + """ + hidden_params = getattr(response, "_hidden_params", {}) or {} + + # model_info: check both metadata keys (chat uses "metadata", responses uses "litellm_metadata") + model_info = ( + (data.get("metadata") or {}).get("model_info") + or (data.get("litellm_metadata") or {}).get("model_info") + or {} + ) + + return { + "custom_llm_provider": hidden_params.get("custom_llm_provider"), + "model_info": model_info, + "api_base": hidden_params.get("api_base"), + "model_id": hidden_params.get("model_id"), + } + def is_a2a_streaming_response(self, response: dict) -> bool: expected_keys = ["jsonrpc", "id", "result"] return all(key in response for key in expected_keys) diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index 7906113213a..2e5efe1c338 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -1,15 +1,15 @@ """Abstraction function for OpenAI's realtime API""" import os -from typing import Any, Optional, cast +from typing import Any, Dict, Optional, cast import litellm -from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES +from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, request_timeout from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.secret_managers.main import get_secret_str -from litellm.types.realtime import RealtimeQueryParams +from litellm.types.realtime import RealtimeClientSecretRequest, RealtimeQueryParams from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders from litellm.utils import ProviderConfigManager @@ -46,6 +46,152 @@ def _build_litellm_metadata(kwargs: dict) -> dict: return metadata +def _get_realtime_http_provider_config( + custom_llm_provider: str, + dynamic_api_base: Optional[str], + dynamic_api_key: Optional[str], + litellm_params: GenericLiteLLMParams, +) -> tuple[Any, str, str]: + """ + Return (provider_config, resolved_api_base, resolved_api_key) for the + realtime HTTP endpoints (client_secrets / realtime_calls). + + Uses ProviderConfigManager so each provider keeps its credential-resolution + and URL-construction logic in its own transformation class. + """ + from litellm.llms.base_llm.realtime.http_transformation import ( + BaseRealtimeHTTPConfig, + ) + + provider_config: Optional[BaseRealtimeHTTPConfig] = None + if custom_llm_provider in LlmProviders._member_map_.values(): + provider_config = ProviderConfigManager.get_provider_realtime_http_config( + model="", + provider=LlmProviders(custom_llm_provider), + ) + + raw_api_base = dynamic_api_base or litellm_params.api_base + raw_api_key = dynamic_api_key or litellm_params.api_key + + if provider_config is not None: + resolved_api_base = provider_config.get_api_base(api_base=raw_api_base) + resolved_api_key = provider_config.get_api_key(api_key=raw_api_key) + else: + # Fallback for providers without a dedicated HTTP config (treated as OpenAI-compatible). + resolved_api_base = ( + raw_api_base + or litellm.api_base + or "https://api.openai.com" + ) + resolved_api_key = ( + raw_api_key + or litellm.api_key + or litellm.openai_key + or get_secret_str("OPENAI_API_KEY") + or "" + ) + + return provider_config, resolved_api_base.rstrip("/"), resolved_api_key + + +@wrapper_client +async def acreate_realtime_client_secret( + model: Optional[str] = None, + session: Optional[Dict[str, Any]] = None, + expires_after: Optional[Dict[str, Any]] = None, + timeout: Optional[float] = None, + **kwargs, +): + req = RealtimeClientSecretRequest( + model=model, + session=session, + expires_after=expires_after, + ) + model_name = ( + (req.session.model if req.session is not None else None) + or req.model + or "gpt-4o-realtime-preview" + ) + litellm_logging_obj: LiteLLMLogging = kwargs.get("litellm_logging_obj") # type: ignore + litellm_params = GenericLiteLLMParams(**kwargs) + + model_name, custom_llm_provider, dynamic_api_key, dynamic_api_base = get_llm_provider( + model=model_name, + api_base=litellm_params.api_base, + api_key=litellm_params.api_key, + ) + provider_config, resolved_api_base, resolved_api_key = _get_realtime_http_provider_config( + custom_llm_provider=custom_llm_provider, + dynamic_api_base=dynamic_api_base, + dynamic_api_key=dynamic_api_key, + litellm_params=litellm_params, + ) + litellm_logging_obj.update_environment_variables( + model=model_name, + optional_params={"expires_after": expires_after, "session": session}, + litellm_params={"api_base": resolved_api_base}, + custom_llm_provider=custom_llm_provider, + ) + request_data = req.model_dump(exclude_none=True, exclude={"model"}) + return await base_llm_http_handler.async_realtime_client_secret_handler( + api_base=resolved_api_base, + api_key=resolved_api_key, + request_data=request_data, + logging_obj=litellm_logging_obj, + timeout=timeout or request_timeout, + provider_config=provider_config, + model=model_name, + extra_headers=kwargs.get("extra_headers"), + client=kwargs.get("client"), + api_version=litellm_params.api_version, + ) + + +@wrapper_client +async def arealtime_calls( + openai_ephemeral_key: str, + sdp_body: bytes, + model: Optional[str] = None, + session: Optional[Dict[str, Any]] = None, + timeout: Optional[float] = None, + **kwargs, +): + model_name = model or "gpt-4o-realtime-preview" + litellm_logging_obj: LiteLLMLogging = kwargs.get("litellm_logging_obj") # type: ignore + litellm_params = GenericLiteLLMParams(**kwargs) + + model_name, custom_llm_provider, dynamic_api_key, dynamic_api_base = get_llm_provider( + model=model_name, + api_base=litellm_params.api_base, + api_key=litellm_params.api_key, + ) + provider_config, resolved_api_base, _ = _get_realtime_http_provider_config( + custom_llm_provider=custom_llm_provider, + dynamic_api_base=dynamic_api_base, + dynamic_api_key=dynamic_api_key, + litellm_params=litellm_params, + ) + litellm_logging_obj.update_environment_variables( + model=model_name, + optional_params={"realtime_calls": True, "session": session}, + litellm_params={"api_base": resolved_api_base}, + custom_llm_provider=custom_llm_provider, + ) + return await base_llm_http_handler.async_realtime_calls_handler( + api_base=resolved_api_base, + openai_ephemeral_key=openai_ephemeral_key, + sdp_body=sdp_body, + logging_obj=litellm_logging_obj, + timeout=timeout or request_timeout, + provider_config=provider_config, + model=model_name, + session_config=session, + extra_headers=kwargs.get("extra_headers"), + client=kwargs.get("client"), + api_version=litellm_params.api_version, + ) + + @wrapper_client async def _arealtime( # noqa: PLR0915 model: str, diff --git a/litellm/responses/main.py b/litellm/responses/main.py index e4c4df50b69..6f7e38dc8b8 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -511,6 +511,9 @@ async def aresponses( litellm_metadata=kwargs.get("litellm_metadata", {}), custom_llm_provider=custom_llm_provider, ) + # Stamp custom_llm_provider so callbacks can identify the provider + # (mirrors litellm/main.py:1371 for chat completions) + response._hidden_params["custom_llm_provider"] = custom_llm_provider if response is None: raise ValueError( @@ -785,6 +788,9 @@ def responses( litellm_metadata=kwargs.get("litellm_metadata", {}), custom_llm_provider=custom_llm_provider, ) + # Stamp custom_llm_provider so callbacks can identify the provider + # (mirrors litellm/main.py:1371 for chat completions) + response._hidden_params["custom_llm_provider"] = custom_llm_provider return response except Exception as e: diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 0d0e78d9790..073ee926063 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -83,6 +83,7 @@ class BaseResponsesAPIStreamingIterator: self._hidden_params = { "model_id": _model_info.get("id", None), "api_base": _api_base, + "custom_llm_provider": custom_llm_provider, } self._hidden_params["additional_headers"] = process_response_headers( self.response.headers or {} diff --git a/litellm/types/realtime.py b/litellm/types/realtime.py index 1ec41f40b3d..62e4044061b 100644 --- a/litellm/types/realtime.py +++ b/litellm/types/realtime.py @@ -1,6 +1,7 @@ -from typing import List, Literal, Optional, Union +from typing import Any, Dict, List, Literal, Optional, Union -from typing_extensions import TypedDict +from pydantic import BaseModel +from typing_extensions import TypedDict # noqa: F401 – re-exported from .llms.openai import ( OpenAIRealtimeEvents, @@ -49,3 +50,68 @@ class RealtimeQueryParams(TypedDict, total=False): model: str intent: Optional[str] # Add more fields as needed + + +# --------------------------------------------------------------------------- +# WebRTC / client_secrets types (POST /v1/realtime/client_secrets) +# --------------------------------------------------------------------------- + + +class RealtimeExpiresAfter(BaseModel): + """Expiration config for a client secret.""" + + anchor: Optional[str] = "created_at" + seconds: Optional[int] = None + + +class RealtimeSessionConfig(BaseModel): + """ + Session configuration nested inside the client_secrets request body. + + Mirrors OpenAI's RealtimeSessionCreateRequest (type=realtime) and + RealtimeTranscriptionSessionCreateRequest (type=transcription). + Extra/unknown fields are passed through unchanged. + """ + + model_config = {"extra": "allow"} + + type: Optional[str] = None + model: Optional[str] = None + instructions: Optional[str] = None + audio: Optional[Dict[str, Any]] = None + include: Optional[List[str]] = None + max_output_tokens: Optional[Union[int, str]] = None + output_modalities: Optional[List[str]] = None + tool_choice: Optional[Any] = None + tools: Optional[List[Dict[str, Any]]] = None + tracing: Optional[Any] = None + truncation: Optional[Any] = None + prompt: Optional[Dict[str, Any]] = None + + +class RealtimeClientSecretRequest(BaseModel): + """ + Request body for POST /v1/realtime/client_secrets. + + LiteLLM also accepts a top-level `model` field for routing when + session.model is absent (LiteLLM extension, not forwarded to OpenAI). + """ + + expires_after: Optional[RealtimeExpiresAfter] = None + session: Optional[RealtimeSessionConfig] = None + # LiteLLM-only routing hint — stripped before forwarding upstream + model: Optional[str] = None + + +class RealtimeClientSecretResponse(BaseModel): + """ + Response from POST /v1/realtime/client_secrets. + + Both the top-level `value` and `session.client_secret.value` + will contain the encrypted token instead of the raw ephemeral key. + The `session` field is kept as a raw dict so unknown fields pass through. + """ + + expires_at: Optional[int] = None + value: str + session: Optional[Dict[str, Any]] = None diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 70fb164c97d..957905935be 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -492,6 +492,8 @@ CallTypesLiteral = Literal[ "aresponses", "responses", "acreate_skill", + "acreate_realtime_client_secret", + "arealtime_calls", ] # Mapping of API routes to their corresponding call types diff --git a/litellm/utils.py b/litellm/utils.py index 14c71c89eb5..67c7fb3e82a 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8854,6 +8854,30 @@ class ProviderConfigManager: return GeminiRealtimeConfig() return None + @staticmethod + def get_provider_realtime_http_config( + model: str, + provider: LlmProviders, + ) -> Optional["BaseRealtimeHTTPConfig"]: + """ + Return the HTTP transformation config for realtime HTTP endpoints + (POST /realtime/client_secrets and POST /realtime/calls). + """ + + if LlmProviders.OPENAI == provider: + from litellm.llms.openai.realtime.http_transformation import ( + OpenAIRealtimeHTTPConfig, + ) + + return OpenAIRealtimeHTTPConfig() + if LlmProviders.AZURE == provider: + from litellm.llms.azure.realtime.http_transformation import ( + AzureRealtimeHTTPConfig, + ) + + return AzureRealtimeHTTPConfig() + return None + @staticmethod def get_provider_image_edit_config( model: str, diff --git a/tests/logging_callback_tests/test_standard_logging_payload.py b/tests/logging_callback_tests/test_standard_logging_payload.py index b725d077e68..163e2d94353 100644 --- a/tests/logging_callback_tests/test_standard_logging_payload.py +++ b/tests/logging_callback_tests/test_standard_logging_payload.py @@ -809,6 +809,102 @@ def test_usage_dict_roundtrip_in_payload(use_combined_usage_object): assert usage_obj["total_tokens"] == 100 +def test_standard_logging_payload_uses_actual_model_for_azure_router(): + from litellm.litellm_core_utils.litellm_logging import ( + Logging, + get_standard_logging_object_payload, + ) + + logging_obj = Logging( + model="azure_ai/model-router", + messages=[{"role": "user", "content": "Hello"}], + stream=False, + call_type="completion", + start_time=datetime.now(), + litellm_call_id="test-azure-router-opt-in", + function_id="test-fn", + ) + + kwargs = { + "model": "azure_ai/model-router", + "messages": [{"role": "user", "content": "Hello"}], + "response_cost": 0.00001, + "custom_llm_provider": "azure_ai", + } + mock_response = { + "id": "chatcmpl-azure-router-opt-in", + "object": "chat.completion", + "model": "azure_ai/gpt-5-nano-2025-08-07", + "usage": {"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "hello"}, + "finish_reason": "stop", + } + ], + } + + payload = get_standard_logging_object_payload( + kwargs=kwargs, + init_response_obj=mock_response, + start_time=datetime.now(), + end_time=datetime.now(), + logging_obj=logging_obj, + status="success", + ) + assert payload is not None + assert payload["model"] == "azure_ai/gpt-5-nano-2025-08-07" + + +def test_standard_logging_payload_uses_actual_model_for_azure_router_with_underscore(): + from litellm.litellm_core_utils.litellm_logging import ( + Logging, + get_standard_logging_object_payload, + ) + + logging_obj = Logging( + model="azure_ai/model_router", + messages=[{"role": "user", "content": "Hello"}], + stream=False, + call_type="completion", + start_time=datetime.now(), + litellm_call_id="test-azure-router-underscore", + function_id="test-fn", + ) + + kwargs = { + "model": "azure_ai/model_router", + "messages": [{"role": "user", "content": "Hello"}], + "response_cost": 0.00001, + "custom_llm_provider": "azure_ai", + } + mock_response = { + "id": "chatcmpl-azure-router-underscore", + "object": "chat.completion", + "model": "azure_ai/gpt-5-nano-2025-08-07", + "usage": {"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "hello"}, + "finish_reason": "stop", + } + ], + } + + payload = get_standard_logging_object_payload( + kwargs=kwargs, + init_response_obj=mock_response, + start_time=datetime.now(), + end_time=datetime.now(), + logging_obj=logging_obj, + status="success", + ) + assert payload is not None + assert payload["model"] == "azure_ai/gpt-5-nano-2025-08-07" + + def test_merge_litellm_metadata_basic(): """ Test that merge_litellm_metadata correctly merges metadata and litellm_metadata. diff --git a/tests/test_litellm/llms/azure_ai/chat/test_azure_ai_transformation.py b/tests/test_litellm/llms/azure_ai/chat/test_azure_ai_transformation.py index d903d7c85f1..a26f7e7021d 100644 --- a/tests/test_litellm/llms/azure_ai/chat/test_azure_ai_transformation.py +++ b/tests/test_litellm/llms/azure_ai/chat/test_azure_ai_transformation.py @@ -8,6 +8,9 @@ import pytest sys.path.insert( 0, os.path.abspath("../../../../..") ) # Adds the parent directory to the system path +from litellm.llms.azure_ai.azure_model_router.transformation import ( + AzureModelRouterConfig, +) from litellm.llms.azure_ai.chat.transformation import AzureAIStudioConfig @@ -117,3 +120,80 @@ def test_azure_ai_grok_stop_parameter_handling(): # Test supported parameters for non-Grok models gpt_params = config.get_supported_openai_params("gpt-4") assert "stop" in gpt_params, "GPT models should support stop parameter" + + +def test_azure_model_router_response_shows_actual_model(): + """ + Test that Azure Model Router returns the actual model used in the response, + not the router model. + + According to the documentation, when using Azure Model Router, the response + should show the actual model that handled the request (e.g., gpt-5-nano-2025-08-07) + rather than the router model (e.g., model-router). + + Regression test for: Azure Model Router should show actual model in response + """ + from httpx import Response + + from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj + from litellm.types.utils import ModelResponse + + config = AzureModelRouterConfig() + + # Mock raw response from Azure that includes the actual model used + raw_response_json = { + "id": "chatcmpl-test123", + "object": "chat.completion", + "created": 1234567890, + "model": "gpt-5-nano-2025-08-07", # Actual model used by the router + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Hello!", + }, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 10, + "completion_tokens": 5, + "total_tokens": 15, + }, + } + + # Create mock Response object + mock_response = MagicMock(spec=Response) + mock_response.json.return_value = raw_response_json + mock_response.text = json.dumps(raw_response_json) + mock_response.headers = {} + + # Create ModelResponse object + model_response = ModelResponse() + + # Create mock logging object with required methods + logging_obj = MagicMock(spec=LiteLLMLoggingObj) + logging_obj.post_call = MagicMock() + logging_obj.model_call_details = {} + + # Call transform_response with router model + result = config.transform_response( + model="model-router", # This is the router model (without prefix) + raw_response=mock_response, + model_response=model_response, + logging_obj=logging_obj, + request_data={}, + messages=[{"role": "user", "content": "Hello"}], + optional_params={}, + litellm_params={"model": "azure_ai/model-router"}, # Original request model + encoding=None, + api_key="test-key", + json_mode=False, + ) + + # Verify that the response contains the actual model used, not the router model + assert result.model == "azure_ai/gpt-5-nano-2025-08-07", ( + f"Expected model to be 'azure_ai/gpt-5-nano-2025-08-07' (actual model used), " + f"but got '{result.model}'" + ) diff --git a/tests/test_litellm/proxy/hooks/test_post_call_response_headers_hook.py b/tests/test_litellm/proxy/hooks/test_post_call_response_headers_hook.py index 6a12366fdd3..3399a34e075 100644 --- a/tests/test_litellm/proxy/hooks/test_post_call_response_headers_hook.py +++ b/tests/test_litellm/proxy/hooks/test_post_call_response_headers_hook.py @@ -195,3 +195,134 @@ async def test_default_hook_returns_none(): response=None, ) assert result is None + + +# --- Tests for litellm_call_info parameter --- + + +class CallInfoInspectorLogger(CustomLogger): + """Logger that captures litellm_call_info for inspection.""" + + def __init__(self): + self.called = False + self.received_call_info = None + + async def async_post_call_response_headers_hook( + self, + data: dict, + user_api_key_dict: UserAPIKeyAuth, + response: Any, + request_headers: Optional[Dict[str, str]] = None, + litellm_call_info: Optional[Dict[str, Any]] = None, + ) -> Optional[Dict[str, str]]: + self.called = True + self.received_call_info = litellm_call_info + return None + + +@pytest.mark.asyncio +async def test_litellm_call_info_from_hidden_params(): + """Test that litellm_call_info is built from response._hidden_params.""" + inspector = CallInfoInspectorLogger() + + class MockResponse: + _hidden_params = { + "custom_llm_provider": "openai", + "api_base": "https://api.openai.com", + "model_id": "model-abc", + } + + with patch("litellm.callbacks", [inspector]): + from litellm.proxy.utils import ProxyLogging + from litellm.caching.caching import DualCache + + proxy_logging = ProxyLogging(user_api_key_cache=DualCache()) + + await proxy_logging.post_call_response_headers_hook( + data={"model": "gpt-4", "metadata": {"model_info": {"id": "model-abc", "provider": "HubSpot"}}}, + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), + response=MockResponse(), + ) + + assert inspector.called is True + assert inspector.received_call_info is not None + assert inspector.received_call_info["custom_llm_provider"] == "openai" + assert inspector.received_call_info["api_base"] == "https://api.openai.com" + assert inspector.received_call_info["model_id"] == "model-abc" + assert inspector.received_call_info["model_info"]["provider"] == "HubSpot" + + +@pytest.mark.asyncio +async def test_litellm_call_info_from_litellm_metadata(): + """Test that litellm_call_info finds model_info under litellm_metadata (responses API path).""" + inspector = CallInfoInspectorLogger() + + class MockResponse: + _hidden_params = { + "custom_llm_provider": "azure", + "api_base": "https://east.openai.azure.com", + "model_id": "deploy-xyz", + } + + with patch("litellm.callbacks", [inspector]): + from litellm.proxy.utils import ProxyLogging + from litellm.caching.caching import DualCache + + proxy_logging = ProxyLogging(user_api_key_cache=DualCache()) + + await proxy_logging.post_call_response_headers_hook( + data={"model": "gpt-4", "litellm_metadata": {"model_info": {"id": "deploy-xyz"}}}, + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), + response=MockResponse(), + ) + + assert inspector.received_call_info["model_info"]["id"] == "deploy-xyz" + assert inspector.received_call_info["custom_llm_provider"] == "azure" + + +@pytest.mark.asyncio +async def test_litellm_call_info_with_none_response(): + """Test that litellm_call_info handles None response (failure path).""" + inspector = CallInfoInspectorLogger() + + with patch("litellm.callbacks", [inspector]): + from litellm.proxy.utils import ProxyLogging + from litellm.caching.caching import DualCache + + proxy_logging = ProxyLogging(user_api_key_cache=DualCache()) + + await proxy_logging.post_call_response_headers_hook( + data={"model": "gpt-4", "metadata": {}}, + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), + response=None, + ) + + assert inspector.called is True + assert inspector.received_call_info is not None + assert inspector.received_call_info["custom_llm_provider"] is None + assert inspector.received_call_info["model_info"] == {} + + +@pytest.mark.asyncio +async def test_litellm_call_info_backwards_compatible(): + """Test that existing callbacks without litellm_call_info parameter still work.""" + # HeaderInjectorLogger doesn't accept litellm_call_info — must not crash + injector = HeaderInjectorLogger(headers={"x-test": "1"}) + + class MockResponse: + _hidden_params = {"custom_llm_provider": "openai", "api_base": "https://api.openai.com", "model_id": "m1"} + + with patch("litellm.callbacks", [injector]): + from litellm.proxy.utils import ProxyLogging + from litellm.caching.caching import DualCache + + proxy_logging = ProxyLogging(user_api_key_cache=DualCache()) + + result = await proxy_logging.post_call_response_headers_hook( + data={"model": "gpt-4", "metadata": {}}, + user_api_key_dict=UserAPIKeyAuth(api_key="test-key"), + response=MockResponse(), + ) + + assert result == {"x-test": "1"} + assert injector.called is True diff --git a/tests/test_litellm/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py b/tests/test_litellm/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py new file mode 100644 index 00000000000..3d82e4177a5 --- /dev/null +++ b/tests/test_litellm/proxy/realtime_endpoints/test_realtime_webrtc_endpoints.py @@ -0,0 +1,301 @@ +""" +Tests for LiteLLM proxy realtime WebRTC HTTP endpoints: +- POST /v1/realtime/client_secrets +- POST /v1/realtime/calls +""" + +import json +import os +import sys +import time +from unittest.mock import AsyncMock, MagicMock, patch + +import httpx +import pytest +from fastapi.testclient import TestClient + +sys.path.insert(0, os.path.abspath("../../../..")) + +from litellm.proxy.common_utils.encrypt_decrypt_utils import ( + decrypt_value_helper, + encrypt_value_helper, +) +from litellm.proxy.realtime_endpoints.endpoints import ( + _decode_realtime_token_payload, + _encode_realtime_token_payload, +) + +# --- Unit tests: token encode/decode helpers --- + + +def test_encode_realtime_token_payload(): + payload = _encode_realtime_token_payload( + ephemeral_key="epk_abc123", + model_id="gpt-4o-realtime-preview", + user_id="user-1", + team_id="team-1", + expires_at=1234567890, + ) + decoded = json.loads(payload) + assert decoded["v"] == "realtime_v1" + assert decoded["ephemeral_key"] == "epk_abc123" + assert decoded["model_id"] == "gpt-4o-realtime-preview" + assert decoded["user_id"] == "user-1" + assert decoded["team_id"] == "team-1" + assert decoded["expires_at"] == 1234567890 + + +def test_encode_realtime_token_payload_none_optional_fields(): + payload = _encode_realtime_token_payload( + ephemeral_key="epk_xyz", + model_id="gpt-4o-realtime", + user_id=None, + team_id=None, + expires_at=None, + ) + decoded = json.loads(payload) + assert decoded["user_id"] == "" + assert decoded["team_id"] == "" + assert decoded["expires_at"] is None + + +def test_decode_realtime_token_payload_valid(): + future_expires_at = int(time.time()) + 3600 + payload = _encode_realtime_token_payload( + ephemeral_key="epk_abc", + model_id="gpt-4o", + user_id=None, + team_id=None, + expires_at=future_expires_at, + ) + decrypted = json.loads(payload) # simulate decrypted value + result = _decode_realtime_token_payload(json.dumps(decrypted)) + assert result is not None + assert result["ephemeral_key"] == "epk_abc" + assert result["model_id"] == "gpt-4o" + assert result["expires_at"] == future_expires_at + + +def test_decode_realtime_token_payload_invalid_version(): + payload = json.dumps({ + "v": "realtime_v2", + "ephemeral_key": "epk", + "model_id": "gpt-4o", + }) + assert _decode_realtime_token_payload(payload) is None + + +def test_decode_realtime_token_payload_invalid_json(): + assert _decode_realtime_token_payload("not-json") is None + + +def test_decode_realtime_token_payload_missing_ephemeral_key(): + payload = json.dumps({"v": "realtime_v1", "model_id": "gpt-4o"}) + assert _decode_realtime_token_payload(payload) is None + + +def test_decode_realtime_token_payload_ephemeral_key_not_string(): + payload = json.dumps({ + "v": "realtime_v1", + "ephemeral_key": 123, + "model_id": "gpt-4o", + }) + assert _decode_realtime_token_payload(payload) is None + + +# --- Integration tests: proxy endpoints (mocked upstream) --- + + +@pytest.fixture +def proxy_app(): + from litellm.proxy import proxy_server + + proxy_server.master_key = "sk-test-master-key" + return proxy_server.app + + +@pytest.fixture +def mock_route_request_client_secrets(): + """Mock route_request to return a fake upstream client_secrets response.""" + future_expires_at = int(time.time()) + 3600 + mock_resp = MagicMock(spec=httpx.Response) + mock_resp.status_code = 200 + mock_resp.text = f'{{"value":"upstream_ephemeral_key","expires_at":{future_expires_at}}}' + mock_resp.content = f'{{"value":"upstream_ephemeral_key","expires_at":{future_expires_at}}}'.encode() + mock_resp.headers = {} + mock_resp.json.return_value = { + "value": "upstream_ephemeral_key", + "expires_at": future_expires_at, + } + + async def _mock_route(*args, **kwargs): + async def _inner(): + return mock_resp + + return _inner() + + return _mock_route + + +@pytest.fixture +def mock_route_request_realtime_calls(): + """Mock route_request to return a fake SDP answer.""" + mock_resp = MagicMock(spec=httpx.Response) + mock_resp.status_code = 201 + mock_resp.content = b"v=0\r\no=- 0 0 IN IP4 0.0.0.0\r\ns=-\r\n" + mock_resp.headers = {"content-type": "application/sdp"} + + async def _mock_route(*args, **kwargs): + async def _inner(): + return mock_resp + + return _inner() + + return _mock_route + + +@pytest.fixture +def mock_add_litellm_data(): + async def _mock(data, **kwargs): + return data + + return _mock + + +@pytest.fixture +def mock_pre_call_hook(): + async def _mock(user_api_key_dict, data, call_type): + return data + + return _mock + + +def test_client_secrets_requires_auth(proxy_app): + """POST /v1/realtime/client_secrets returns 401 without Authorization.""" + client = TestClient(proxy_app) + with patch( + "litellm.proxy.proxy_server.route_request", + new_callable=AsyncMock, + ): + response = client.post( + "/v1/realtime/client_secrets", + json={"model": "gpt-4o-realtime-preview"}, + ) + assert response.status_code == 401 + + +@pytest.mark.asyncio +async def test_client_secrets_success_with_mock( + proxy_app, + mock_route_request_client_secrets, + mock_add_litellm_data, + mock_pre_call_hook, +): + """POST /v1/realtime/client_secrets returns 200 with valid auth and mocked upstream.""" + client = TestClient(proxy_app) + with ( + patch( + "litellm.proxy.proxy_server.route_request", + side_effect=mock_route_request_client_secrets, + ), + patch( + "litellm.proxy.proxy_server.add_litellm_data_to_request", + side_effect=mock_add_litellm_data, + ), + patch( + "litellm.proxy.proxy_server.proxy_logging_obj" + ) as mock_logging, + ): + mock_logging.pre_call_hook = AsyncMock(side_effect=mock_pre_call_hook) + mock_logging.post_call_failure_hook = AsyncMock() + + response = client.post( + "/v1/realtime/client_secrets", + headers={"Authorization": "Bearer sk-test-master-key"}, + json={"model": "gpt-4o-realtime-preview"}, + ) + + assert response.status_code == 200 + data = response.json() + assert "value" in data + assert data["expires_at"] is not None + assert data["expires_at"] > int(time.time()) # Should be in the future + # Proxy encrypts the upstream value, so returned value should differ + assert data["value"] != "upstream_ephemeral_key" + + +def test_realtime_calls_requires_auth(proxy_app): + """POST /v1/realtime/calls returns 401 without Authorization.""" + client = TestClient(proxy_app) + with patch( + "litellm.proxy.proxy_server.route_request", + new_callable=AsyncMock, + ): + response = client.post( + "/v1/realtime/calls", + content=b"v=0\r\no=- 0 0 IN IP4 0.0.0.0\r\n", + ) + assert response.status_code == 401 + + +def test_realtime_calls_invalid_token_returns_401(proxy_app): + """POST /v1/realtime/calls returns 401 with invalid Bearer token.""" + client = TestClient(proxy_app) + response = client.post( + "/v1/realtime/calls", + headers={"Authorization": "Bearer invalid-token-not-encrypted"}, + content=b"v=0\r\n", + ) + assert response.status_code == 401 + assert "Invalid or expired token" in response.json().get("error", "") + + +@pytest.mark.asyncio +async def test_realtime_calls_success_with_valid_encrypted_token( + proxy_app, + mock_route_request_realtime_calls, + mock_add_litellm_data, + mock_pre_call_hook, +): + """POST /v1/realtime/calls returns 201 with valid encrypted token from client_secrets.""" + from litellm.proxy import proxy_server + + proxy_server.master_key = "sk-test-master-key" + + # Build a valid encrypted token (same format as client_secrets returns) + future_expires_at = int(time.time()) + 3600 + token_payload = _encode_realtime_token_payload( + ephemeral_key="fake_upstream_epk", + model_id="gpt-4o-realtime-preview", + user_id=None, + team_id=None, + expires_at=future_expires_at, + ) + encrypted_token = encrypt_value_helper(token_payload) + + client = TestClient(proxy_app) + with ( + patch( + "litellm.proxy.proxy_server.route_request", + side_effect=mock_route_request_realtime_calls, + ), + patch( + "litellm.proxy.proxy_server.add_litellm_data_to_request", + side_effect=mock_add_litellm_data, + ), + patch( + "litellm.proxy.proxy_server.proxy_logging_obj" + ) as mock_logging, + ): + mock_logging.pre_call_hook = AsyncMock(side_effect=mock_pre_call_hook) + mock_logging.post_call_failure_hook = AsyncMock() + + response = client.post( + "/v1/realtime/calls", + headers={"Authorization": f"Bearer {encrypted_token}"}, + content=b"v=0\r\no=- 0 0 IN IP4 0.0.0.0\r\ns=-\r\n", + ) + + assert response.status_code == 201 + assert response.content.startswith(b"v=0") + assert b"application/sdp" in response.headers.get("content-type", "").encode() diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index ba1084eafe0..3869a24d356 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -15,6 +15,7 @@ from litellm.proxy.common_request_processing import ( ProxyConfig, _extract_error_from_sse_chunk, _get_cost_breakdown_from_logging_obj, + _is_azure_model_router_request, _override_openai_response_model, _parse_event_data_for_error, create_response, @@ -1368,6 +1369,84 @@ class TestOverrideOpenAIResponseModel: # Verify the model was not changed assert response_obj.model == fallback_model + def test_override_model_preserves_azure_model_router_actual_model(self): + """ + Test that when the requested model is an Azure Model Router, the actual + model used (returned in the response) is preserved instead of being + overridden. + """ + requested_model = "azure_ai/model_router" + actual_model_used = "azure_ai/gpt-5-nano-2025-08-07" + + response_obj = MagicMock() + response_obj.model = actual_model_used + response_obj._hidden_params = {"additional_headers": {}} + + _override_openai_response_model( + response_obj=response_obj, + requested_model=requested_model, + log_context="test_context", + ) + assert response_obj.model == actual_model_used + assert response_obj.model != requested_model + + def test_override_model_preserves_azure_model_router_with_deployment_name(self): + """ + Test that Azure Model Router with deployment name pattern also preserves + the actual model used. + """ + requested_model = "azure_ai/model_router/my-deployment" + actual_model_used = "azure_ai/gpt-4.1-nano-2025-04-14" + + response_obj = MagicMock() + response_obj.model = actual_model_used + response_obj._hidden_params = {"additional_headers": {}} + + _override_openai_response_model( + response_obj=response_obj, + requested_model=requested_model, + log_context="test_context", + ) + assert response_obj.model == actual_model_used + assert response_obj.model != requested_model + + def test_override_model_preserves_azure_model_router_with_hyphen(self): + """ + Test that Azure Model Router with hyphen pattern (model-router) also preserves + the actual model used. + """ + requested_model = "azure_ai/model-router" + actual_model_used = "azure_ai/gpt-5-nano-2025-08-07" + + response_obj = MagicMock() + response_obj.model = actual_model_used + response_obj._hidden_params = {"additional_headers": {}} + + _override_openai_response_model( + response_obj=response_obj, + requested_model=requested_model, + log_context="test_context", + ) + assert response_obj.model == actual_model_used + assert response_obj.model != requested_model + + +class TestIsAzureModelRouterRequest: + """Tests for _is_azure_model_router_request helper""" + + def test_detects_model_router_with_underscore(self): + assert _is_azure_model_router_request("azure_ai/model_router") is True + assert _is_azure_model_router_request("azure_ai/model_router/my-deployment") is True + + def test_detects_model_router_with_hyphen(self): + assert _is_azure_model_router_request("azure_ai/model-router") is True + assert _is_azure_model_router_request("model-router") is True + + def test_rejects_regular_models(self): + assert _is_azure_model_router_request("azure_ai/gpt-4") is False + assert _is_azure_model_router_request("gpt-4") is False + assert _is_azure_model_router_request("openai/gpt-3.5-turbo") is False + class TestStreamingOverheadHeader: """ diff --git a/tests/test_litellm/proxy/test_response_model_sanitization.py b/tests/test_litellm/proxy/test_response_model_sanitization.py index b1bb8d0ed39..22785bbcb9e 100644 --- a/tests/test_litellm/proxy/test_response_model_sanitization.py +++ b/tests/test_litellm/proxy/test_response_model_sanitization.py @@ -23,7 +23,11 @@ def _initialize_proxy_with_config(config: dict, tmp_path) -> TestClient: IMPORTANT: proxy_server.initialize() mutates module-level globals. We must call cleanup_router_config_variables() before initializing to prevent cross-test bleed. """ - from litellm.proxy.proxy_server import app, cleanup_router_config_variables, initialize + from litellm.proxy.proxy_server import ( + app, + cleanup_router_config_variables, + initialize, + ) cleanup_router_config_variables() @@ -123,8 +127,8 @@ async def test_proxy_streaming_chunks_do_not_return_provider_prefixed_model(monk client_model = "vllm-model" internal_model = f"hosted_vllm/{client_model}" - from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy import proxy_server + from litellm.proxy._types import UserAPIKeyAuth # Patch proxy_logging_obj hooks so async_data_generator yields exactly our chunk. async def _iterator_hook( @@ -176,8 +180,8 @@ async def test_proxy_streaming_chunks_use_client_requested_model_before_alias_ma canonical_model = "vllm-model" internal_model = f"hosted_vllm/{canonical_model}" - from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy import proxy_server + from litellm.proxy._types import UserAPIKeyAuth async def _iterator_hook( user_api_key_dict: UserAPIKeyAuth, @@ -215,3 +219,57 @@ async def test_proxy_streaming_chunks_use_client_requested_model_before_alias_ma payload = json.loads(first[len("data: ") :].strip()) assert payload["model"] == client_model_alias assert not payload["model"].startswith("hosted_vllm/") + + +@pytest.mark.asyncio +async def test_proxy_streaming_azure_model_router_preserves_actual_model(monkeypatch): + """ + Regression test for Azure Model Router streaming: + + When the client requests azure_ai/model_router, the streaming chunks should + preserve the actual model used (e.g., azure_ai/gpt-5-nano-2025-08-07) from + the downstream response, NOT override to the router model. + """ + router_model = "azure_ai/model_router" + actual_model_used = "azure_ai/gpt-5-nano-2025-08-07" + + from litellm.proxy import proxy_server + from litellm.proxy._types import UserAPIKeyAuth + + async def _iterator_hook( + user_api_key_dict: UserAPIKeyAuth, + response: AsyncGenerator, + request_data: dict, + ): + yield _make_model_response_stream_chunk(model=actual_model_used) + + monkeypatch.setattr(proxy_server.proxy_logging_obj, "async_post_call_streaming_iterator_hook", _iterator_hook) + monkeypatch.setattr( + proxy_server.proxy_logging_obj, + "async_post_call_streaming_hook", + AsyncMock(side_effect=lambda **kwargs: kwargs["response"]), + ) + + user_api_key_dict = UserAPIKeyAuth(api_key="sk-1234") + + gen = proxy_server.async_data_generator( + response=MagicMock(), + user_api_key_dict=user_api_key_dict, + request_data={ + "model": router_model, + "_litellm_client_requested_model": router_model, + }, + ) + + chunks = [] + async for item in gen: + chunks.append(item) + + assert len(chunks) >= 2 + first = chunks[0] + assert first.startswith("data: ") + + payload = json.loads(first[len("data: ") :].strip()) + # Azure Model Router: preserve actual model used, not the router model + assert payload["model"] == actual_model_used + assert payload["model"] != router_model diff --git a/ui/litellm-dashboard/src/components/WebRTCTester.jsx b/ui/litellm-dashboard/src/components/WebRTCTester.jsx new file mode 100644 index 00000000000..b26cf2a2dba --- /dev/null +++ b/ui/litellm-dashboard/src/components/WebRTCTester.jsx @@ -0,0 +1,571 @@ +import { useState, useRef, useEffect, useCallback } from 'react'; + +const STYLES = ` +.wrt-wrap { + font-family: 'JetBrains Mono', 'Fira Code', monospace; + background: #0d0d14; + border: 1px solid #1e1e2e; + border-radius: 10px; + overflow: hidden; + margin: 24px 0; +} + +.wrt-toggle { + display: flex; + align-items: center; + justify-content: space-between; + padding: 14px 20px; + cursor: pointer; + user-select: none; + background: #0d0d14; + transition: background 0.15s; +} +.wrt-toggle:hover { background: #111120; } + +.wrt-toggle-left { display: flex; align-items: center; gap: 10px; } + +.wrt-live-dot { + width: 8px; height: 8px; border-radius: 50%; + background: #00ff88; + box-shadow: 0 0 8px #00ff88; + animation: wrt-blink 2s infinite; +} +@keyframes wrt-blink { 0%,100%{opacity:1} 50%{opacity:0.4} } + +.wrt-toggle-title { font-size: 12px; font-weight: 600; color: #e2e8f0; letter-spacing: 0.06em; } +.wrt-toggle-sub { font-size: 10px; color: #4a5568; margin-top: 1px; } +.wrt-chevron { font-size: 11px; color: #4a5568; transition: transform 0.2s; } +.wrt-chevron.open { transform: rotate(180deg); } + +.wrt-body { + border-top: 1px solid #1e1e2e; + display: grid; + grid-template-columns: 280px 1fr; + height: 460px; +} + +.wrt-sidebar { + border-right: 1px solid #1e1e2e; + padding: 14px; + display: flex; + flex-direction: column; + gap: 12px; + overflow-y: auto; +} + +.wrt-label { + font-size: 9px; + letter-spacing: 0.15em; + color: #4a5568; + text-transform: uppercase; + margin-bottom: 5px; +} + +.wrt-field { display: flex; flex-direction: column; gap: 4px; margin-bottom: 6px; } +.wrt-field label { font-size: 10px; color: #4a5568; } +.wrt-field input { + background: #0a0a0f; + border: 1px solid #1e1e2e; + border-radius: 5px; + color: #e2e8f0; + font-family: inherit; + font-size: 11px; + padding: 7px 9px; + outline: none; + width: 100%; + transition: border-color 0.2s; +} +.wrt-field input:focus { border-color: #7c3aed; } + +.wrt-divider { height: 1px; background: #1e1e2e; } + +.wrt-btn { + display: flex; align-items: center; justify-content: center; + border: none; border-radius: 5px; cursor: pointer; + font-family: inherit; font-size: 11px; font-weight: 600; + padding: 8px; width: 100%; + transition: all 0.15s; letter-spacing: 0.04em; +} +.wrt-btn + .wrt-btn { margin-top: 5px; } +.wrt-btn-primary { background: #00ff88; color: #000; } +.wrt-btn-primary:hover:not(:disabled) { filter: brightness(1.1); } +.wrt-btn-primary:disabled { opacity: 0.35; cursor: not-allowed; } +.wrt-btn-danger { background: transparent; color: #ff4466; border: 1px solid #ff4466; } +.wrt-btn-danger:hover:not(:disabled) { background: rgba(255,68,102,0.08); } +.wrt-btn-danger:disabled { opacity: 0.3; cursor: not-allowed; } +.wrt-btn-ghost { background: #111118; color: #e2e8f0; border: 1px solid #1e1e2e; } +.wrt-btn-ghost:hover { border-color: #7c3aed; } + +.wrt-flow { display: flex; align-items: center; padding: 4px 0; gap: 0; } +.wrt-flow-box { + padding: 4px 7px; border-radius: 4px; font-size: 9px; + border: 1px solid #1e1e2e; color: #4a5568; + transition: all 0.3s; white-space: nowrap; +} +.wrt-flow-box.active { border-color: #00ff88; color: #00ff88; box-shadow: 0 0 8px rgba(0,255,136,0.15); } +.wrt-flow-arrow { font-size: 10px; color: #4a5568; padding: 0 4px; transition: color 0.3s; } +.wrt-flow-arrow.active { color: #00ff88; } + +.wrt-meta { display: flex; flex-direction: column; gap: 4px; } +.wrt-meta-row { display: flex; justify-content: space-between; font-size: 10px; } +.wrt-meta-row span:first-child { color: #4a5568; } +.wrt-meta-row span:last-child { color: #e2e8f0; } + +.wrt-status-pill { + display: flex; align-items: center; gap: 6px; + font-size: 10px; color: #4a5568; + background: #111118; border: 1px solid #1e1e2e; + border-radius: 100px; padding: 3px 10px; +} +.wrt-status-dot { + width: 6px; height: 6px; border-radius: 50%; + background: #4a5568; transition: all 0.3s; +} +.wrt-status-dot.connected { background: #00ff88; box-shadow: 0 0 6px #00ff88; } +.wrt-status-dot.connecting { background: #ffaa00; animation: wrt-blink 1s infinite; } +.wrt-status-dot.error { background: #ff4466; } + +.wrt-main { display: flex; flex-direction: column; overflow: hidden; } + +.wrt-header { + display: flex; align-items: center; justify-content: space-between; + padding: 8px 14px; border-bottom: 1px solid #1e1e2e; background: #111118; +} +.wrt-header-title { font-size: 10px; color: #4a5568; letter-spacing: 0.08em; } + +.wrt-tabs { display: flex; padding: 0 14px; border-bottom: 1px solid #1e1e2e; } +.wrt-tab { + font-size: 9px; letter-spacing: 0.08em; padding: 10px 12px; cursor: pointer; + color: #4a5568; border-bottom: 2px solid transparent; transition: all 0.15s; + user-select: none; +} +.wrt-tab.active { color: #00ff88; border-bottom-color: #00ff88; } +.wrt-tab:hover:not(.active) { color: #e2e8f0; } + +.wrt-tab-content { flex: 1; overflow: hidden; display: none; flex-direction: column; } +.wrt-tab-content.active { display: flex; } + +.wrt-log { + flex: 1; overflow-y: auto; padding: 8px 12px; + display: flex; flex-direction: column; gap: 2px; +} +.wrt-log::-webkit-scrollbar { width: 3px; } +.wrt-log::-webkit-scrollbar-thumb { background: #1e1e2e; border-radius: 2px; } + +.wrt-entry { + display: grid; grid-template-columns: 58px 56px 1fr; gap: 8px; + padding: 3px 7px; border-radius: 3px; + border-left: 2px solid transparent; + font-size: 10px; line-height: 1.5; + animation: wrt-fadein 0.15s ease; +} +@keyframes wrt-fadein { from { opacity:0; transform:translateY(2px); } to { opacity:1; transform:none; } } + +.wrt-entry.info { border-left-color: #7c3aed; } +.wrt-entry.info .we-tag { color: #7c3aed; } +.wrt-entry.success { border-left-color: #00ff88; } +.wrt-entry.success .we-tag { color: #00ff88; } +.wrt-entry.error { border-left-color: #ff4466; } +.wrt-entry.error .we-tag { color: #ff4466; } +.wrt-entry.warn { border-left-color: #ffaa00; } +.wrt-entry.warn .we-tag { color: #ffaa00; } +.wrt-entry.step { border-left-color: #60a5fa; } +.wrt-entry.step .we-tag { color: #60a5fa; } + +.we-time { color: #4a5568; font-size: 9px; padding-top: 1px; } +.we-tag { font-size: 9px; font-weight: 700; padding-top: 1px; } +.we-msg { color: #e2e8f0; word-break: break-all; white-space: pre-wrap; } + +.wrt-empty { + display: flex; flex-direction: column; align-items: center; justify-content: center; + flex: 1; gap: 6px; color: #4a5568; font-size: 11px; +} + +.wrt-sdp-pane { flex: 1; display: grid; grid-template-columns: 1fr 1fr; overflow: hidden; } +.wrt-sdp-box { display: flex; flex-direction: column; border-right: 1px solid #1e1e2e; overflow: hidden; } +.wrt-sdp-box:last-child { border-right: none; } +.wrt-sdp-hdr { + padding: 7px 12px; border-bottom: 1px solid #1e1e2e; + font-size: 9px; color: #4a5568; letter-spacing: 0.08em; + display: flex; align-items: center; gap: 6px; +} +.wrt-sdp-dot { width: 5px; height: 5px; border-radius: 50%; background: #1e1e2e; } +.wrt-sdp-dot.active { background: #00ff88; } +.wrt-sdp-pane textarea { + flex: 1; background: transparent; border: none; color: #e2e8f0; + font-family: inherit; font-size: 10px; padding: 10px 12px; + resize: none; outline: none; line-height: 1.5; +} + +.wrt-audio-pane { + flex: 1; display: flex; flex-direction: column; + align-items: center; justify-content: center; gap: 14px; +} +.wrt-viz { display: flex; align-items: center; gap: 2px; height: 44px; } +.wrt-bar { width: 3px; border-radius: 2px; min-height: 2px; background: #00ff88; transition: height 0.05s; } +.wrt-mic-btn { + width: 52px; height: 52px; border-radius: 50%; + background: #111118; border: 1.5px solid #1e1e2e; + font-size: 18px; cursor: pointer; + display: flex; align-items: center; justify-content: center; transition: all 0.2s; +} +.wrt-mic-btn.active { border-color: #00ff88; box-shadow: 0 0 16px rgba(0,255,136,0.2); } +.wrt-audio-status { font-size: 10px; color: #4a5568; text-align: center; } +`; + +function useLog() { + const [entries, setEntries] = useState([]); + const add = useCallback((level, tag, msg) => { + const time = new Date().toTimeString().slice(0, 8); + setEntries(prev => [...prev, { level, tag, msg, time, id: Date.now() + Math.random() }]); + }, []); + const clear = useCallback(() => setEntries([]), []); + return { entries, add, clear }; +} + +export default function WebRTCTester() { + const [open, setOpen] = useState(false); + const [activeTab, setActiveTab] = useState('logs'); + const [proxyUrl, setProxyUrl] = useState('http://localhost:4000'); + const [apiKey, setApiKey] = useState('sk-1234'); + const [model, setModel] = useState('gpt-4o-realtime-preview'); + const [status, setStatus] = useState('idle'); + const [flowStep, setFlowStep] = useState(0); + const [tokenPreview, setTokenPreview] = useState('—'); + const [iceState, setIceState] = useState('—'); + const [connState, setConnState] = useState('—'); + const [dcState, setDcState] = useState('—'); + const [sdpOffer, setSdpOffer] = useState(''); + const [sdpAnswer, setSdpAnswer] = useState(''); + const [offerActive, setOfferActive] = useState(false); + const [answerActive, setAnswerActive] = useState(false); + const [audioStatus, setAudioStatus] = useState('Start a session first'); + const [micActive, setMicActive] = useState(false); + const [bars, setBars] = useState(Array(28).fill(2)); + const [connected, setConnected] = useState(false); + + const { entries, add: log, clear: clearLogs } = useLog(); + const logRef = useRef(null); + + const pcRef = useRef(null); + const dcRef = useRef(null); + const streamRef = useRef(null); + const audioCtxRef = useRef(null); + const analyserRef = useRef(null); + const animRef = useRef(null); + const tokenRef = useRef(null); + const micRef = useRef(false); + const remoteAudioRef = useRef(null); + + useEffect(() => { + if (logRef.current) logRef.current.scrollTop = logRef.current.scrollHeight; + }, [entries]); + + function drawBars() { + animRef.current = requestAnimationFrame(drawBars); + if (!analyserRef.current) return; + const data = new Uint8Array(analyserRef.current.frequencyBinCount); + analyserRef.current.getByteFrequencyData(data); + setBars(Array.from({ length: 28 }, (_, i) => Math.max(2, ((data[i] || 0) / 255) * 42))); + } + + function setupAnalyser(stream) { + audioCtxRef.current = new AudioContext(); + const src = audioCtxRef.current.createMediaStreamSource(stream); + analyserRef.current = audioCtxRef.current.createAnalyser(); + analyserRef.current.fftSize = 64; + src.connect(analyserRef.current); + drawBars(); + } + + async function startSession() { + const url = proxyUrl.trim().replace(/\/$/, ''); + const key = apiKey.trim(); + const mdl = model.trim(); + + setConnected(true); + setStatus('connecting'); + setFlowStep(1); + + // Step 1: ephemeral token + log('step', 'STEP 1', `POST ${url}/v1/realtime/client_secrets`); + let tokenResp; + try { + const r = await fetch(`${url}/v1/realtime/client_secrets`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'Authorization': `Bearer ${key}` }, + body: JSON.stringify({ model: mdl }), + }); + log('info', 'HTTP', `${r.status} ${r.statusText}`); + const raw = await r.text(); + if (!r.ok) { log('error', 'ERR', raw); stopSession(); return; } + tokenResp = JSON.parse(raw); + log('success', 'TOKEN', 'Received encrypted ephemeral token'); + } catch (e) { + log('error', 'ERR', `client_secrets failed: ${e.message}`); + stopSession(); return; + } + + const token = tokenResp?.client_secret?.value ?? tokenResp?.value; + if (!token) { log('error', 'ERR', `Cannot extract token: ${JSON.stringify(tokenResp)}`); stopSession(); return; } + tokenRef.current = token; + setTokenPreview(token.slice(0, 10) + '…'); + log('info', 'TOKEN', `Preview: ${token.slice(0, 10)}…`); + + // Step 2: PeerConnection + log('step', 'STEP 2', 'Creating RTCPeerConnection'); + const pc = new RTCPeerConnection(); + pcRef.current = pc; + + pc.oniceconnectionstatechange = () => { + setIceState(pc.iceConnectionState); + log('info', 'ICE', pc.iceConnectionState); + if (pc.iceConnectionState === 'connected' || pc.iceConnectionState === 'completed') { + setStatus('connected'); setFlowStep(3); + } + if (pc.iceConnectionState === 'failed' || pc.iceConnectionState === 'disconnected') { + setStatus('error'); + } + }; + + pc.onconnectionstatechange = () => { + setConnState(pc.connectionState); + log('info', 'CONN', pc.connectionState); + }; + + pc.ontrack = (e) => { + log('success', 'AUDIO', 'Remote audio track received from OpenAI'); + if (remoteAudioRef.current) remoteAudioRef.current.srcObject = e.streams[0]; + setupAnalyser(e.streams[0]); + setAudioStatus('Receiving audio from OpenAI ✓'); + }; + + const dc = pc.createDataChannel('oai-events'); + dcRef.current = dc; + dc.onopen = () => { setDcState('open'); log('success', 'DC', 'Data channel open — ready!'); setStatus('connected'); }; + dc.onclose = () => { setDcState('closed'); log('warn', 'DC', 'Closed'); }; + dc.onmessage = (e) => { + try { log('info', 'EVENT', JSON.parse(e.data).type ?? 'unknown'); } + catch { log('info', 'EVENT', e.data.slice(0, 100)); } + }; + + // Mic + try { + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + streamRef.current = stream; + stream.getTracks().forEach(t => pc.addTrack(t, stream)); + log('success', 'MIC', 'Microphone access granted'); + setAudioStatus('Mic active — waiting for remote audio'); + micRef.current = true; + setMicActive(true); + } catch (e) { + log('warn', 'MIC', `Mic denied: ${e.message}`); + const ctx = new AudioContext(); + const dest = ctx.createMediaStreamDestination(); + dest.stream.getTracks().forEach(t => pc.addTrack(t, dest.stream)); + } + + // Step 3: SDP offer + log('step', 'STEP 3', 'Creating SDP offer'); + const offer = await pc.createOffer(); + await pc.setLocalDescription(offer); + setSdpOffer(offer.sdp); + setOfferActive(true); + log('info', 'SDP', `Offer created (${offer.sdp.split('\n').length} lines)`); + + // Step 4: SDP exchange + setFlowStep(2); + log('step', 'STEP 4', `POST ${url}/v1/realtime/calls`); + try { + const r = await fetch(`${url}/v1/realtime/calls`, { + method: 'POST', + headers: { 'Authorization': `Bearer ${token}`, 'Content-Type': 'application/sdp' }, + body: offer.sdp, + }); + log('info', 'HTTP', `${r.status} ${r.statusText}`); + if (!r.ok) { log('error', 'ERR', await r.text()); stopSession(); return; } + const ans = await r.text(); + log('success', 'SDP', `Answer received (${ans.split('\n').length} lines)`); + + // Step 5: remote description + log('step', 'STEP 5', 'Setting remote description'); + await pc.setRemoteDescription({ type: 'answer', sdp: ans }); + setSdpAnswer(ans); + setAnswerActive(true); + log('success', 'CONN', '✓ Session established — Browser ↔ LiteLLM ↔ OpenAI'); + } catch (e) { + log('error', 'ERR', `calls failed: ${e.message}`); + stopSession(); + } + } + + function stopSession() { + if (pcRef.current) { pcRef.current.close(); pcRef.current = null; } + if (streamRef.current) { streamRef.current.getTracks().forEach(t => t.stop()); streamRef.current = null; } + if (animRef.current) { cancelAnimationFrame(animRef.current); animRef.current = null; } + tokenRef.current = null; + micRef.current = false; + setConnected(false); + setStatus('idle'); + setFlowStep(0); + setTokenPreview('—'); + setIceState('—'); + setConnState('—'); + setDcState('—'); + setMicActive(false); + setOfferActive(false); + setAnswerActive(false); + setBars(Array(28).fill(2)); + setAudioStatus('Start a session first'); + log('warn', 'SESSION', 'Session stopped'); + } + + function toggleMic() { + if (!streamRef.current) { log('warn', 'MIC', 'No active session'); return; } + const next = !micRef.current; + micRef.current = next; + streamRef.current.getAudioTracks().forEach(t => { t.enabled = next; }); + setMicActive(next); + log('info', 'MIC', next ? 'Unmuted' : 'Muted'); + } + + const f = (n) => flowStep >= n; + + return ( + <> + +
+ {/* Toggle header */} +
setOpen(o => !o)}> +
+
+
+
INTERACTIVE TESTER
+
Browser → LiteLLM → OpenAI · WebRTC
+
+
+ ▼ +
+ + {open && ( +
+ {/* Sidebar */} +
+
+
Proxy Config
+
+ + setProxyUrl(e.target.value)} placeholder="http://localhost:4000" /> +
+
+ + setApiKey(e.target.value)} placeholder="sk-1234" /> +
+
+ + setModel(e.target.value)} /> +
+
+ +
+ +
+
Flow
+
+
Browser
+
→
+
LiteLLM
+
→
+
OpenAI
+
+
+ +
+ +
+
Controls
+ + + +
+ +
+ +
+
Session Info
+
+ {[['token', tokenPreview], ['ice', iceState], ['conn', connState], ['data ch.', dcState]].map(([k, v]) => ( +
{k}{v}
+ ))} +
+
+
+ + {/* Right panel */} +
+
+ WEBRTC REALTIME TESTER +
+
+ {status} +
+
+ +
+ {['logs','sdp','audio'].map(t => ( +
setActiveTab(t)}> + {t.toUpperCase()} +
+ ))} +
+ + {/* Logs */} +
+
+ {entries.length === 0 + ?
📡
Hit "Start Session" to begin
+ : entries.map(e => ( +
+ {e.time} + [{e.tag}] + {e.msg} +
+ )) + } +
+
+ + {/* SDP */} +
+
+
+
SDP OFFER
+