From 3ec07745036897f52bacf24351dd71f999902034 Mon Sep 17 00:00:00 2001 From: Timothy Jaeryang Baek Date: Wed, 14 Oct 2026 17:49:31 +0400 Subject: [PATCH] refac --- backend/open_webui/config.py | 4 + backend/open_webui/routers/audio/__init__.py | 6 +- backend/open_webui/routers/audio/realtime.py | 104 +++++++++++++----- src/lib/apis/audio/index.ts | 14 ++- .../components/admin/Settings/Audio.svelte | 79 ++++++++++--- src/lib/i18n/locales/en-US/translation.json | 9 +- 6 files changed, 168 insertions(+), 48 deletions(-) diff --git a/backend/open_webui/config.py b/backend/open_webui/config.py index a5e2205149..16b4436226 100644 --- a/backend/open_webui/config.py +++ b/backend/open_webui/config.py @@ -1650,6 +1650,8 @@ AUDIO_TTS_VOICE = os.getenv('AUDIO_TTS_VOICE', 'alloy') REALTIME_TTS_PROMPT_TEMPLATE = os.getenv('REALTIME_TTS_PROMPT_TEMPLATE') AUDIO_REALTIME_ENABLED = os.getenv('AUDIO_REALTIME_ENABLED', 'False').lower() == 'true' +AUDIO_REALTIME_PROVIDER = os.getenv('AUDIO_REALTIME_PROVIDER', 'openai') +AUDIO_REALTIME_AUTH_TYPE = os.getenv('AUDIO_REALTIME_AUTH_TYPE', 'api_key') AUDIO_REALTIME_OPENAI_API_BASE_URL = os.getenv('AUDIO_REALTIME_OPENAI_API_BASE_URL', 'https://api.openai.com/v1') AUDIO_REALTIME_OPENAI_API_KEY = os.getenv('AUDIO_REALTIME_OPENAI_API_KEY', '') AUDIO_REALTIME_MODEL = os.getenv('AUDIO_REALTIME_MODEL', 'gpt-realtime-2.1-mini') @@ -3127,6 +3129,8 @@ DEFAULT_CONFIG = { 'audio.tts.openai.params': AUDIO_TTS_OPENAI_PARAMS, 'audio.tts.api_key': AUDIO_TTS_API_KEY, 'audio.realtime.enabled': AUDIO_REALTIME_ENABLED, + 'audio.realtime.provider': AUDIO_REALTIME_PROVIDER, + 'audio.realtime.auth_type': AUDIO_REALTIME_AUTH_TYPE, 'audio.realtime.openai.api_base_url': AUDIO_REALTIME_OPENAI_API_BASE_URL, 'audio.realtime.openai.api_key': AUDIO_REALTIME_OPENAI_API_KEY, 'audio.realtime.model': AUDIO_REALTIME_MODEL, diff --git a/backend/open_webui/routers/audio/__init__.py b/backend/open_webui/routers/audio/__init__.py index eb0d0456e8..e2b9b3b3fc 100644 --- a/backend/open_webui/routers/audio/__init__.py +++ b/backend/open_webui/routers/audio/__init__.py @@ -12,7 +12,7 @@ import uuid import wave from fnmatch import fnmatch from pathlib import Path -from typing import Optional +from typing import Literal, Optional from urllib.parse import urlencode, urlsplit, urlunsplit import aiofiles @@ -101,6 +101,8 @@ TTS_CONFIG_KEYS = { REALTIME_CONFIG_KEYS = { 'ENABLED': 'audio.realtime.enabled', + 'PROVIDER': 'audio.realtime.provider', + 'AUTH_TYPE': 'audio.realtime.auth_type', 'OPENAI_API_BASE_URL': 'audio.realtime.openai.api_base_url', 'OPENAI_API_KEY': 'audio.realtime.openai.api_key', 'MODEL': 'audio.realtime.model', @@ -293,6 +295,8 @@ class STTConfigForm(BaseModel): class RealtimeConfigForm(BaseModel): ENABLED: bool = False + PROVIDER: Literal['openai', 'azure'] = 'openai' + AUTH_TYPE: Literal['api_key', 'microsoft_entra_id'] = 'api_key' OPENAI_API_BASE_URL: str = 'https://api.openai.com/v1' OPENAI_API_KEY: str = '' MODEL: str = Field(default='gpt-realtime-2.1-mini', min_length=1, max_length=200) diff --git a/backend/open_webui/routers/audio/realtime.py b/backend/open_webui/routers/audio/realtime.py index 5d1c5b0320..c8a9d5a744 100644 --- a/backend/open_webui/routers/audio/realtime.py +++ b/backend/open_webui/routers/audio/realtime.py @@ -15,6 +15,7 @@ from open_webui.models.config import Config from open_webui.models.models import Models, ModelVoice from open_webui.utils.access_control import has_permission from open_webui.utils.auth import get_verified_user_by_token +from open_webui.utils.headers import get_microsoft_entra_id_access_token from open_webui.utils.json_codec import JSONCodec from open_webui.utils.models import check_model_access, get_all_models from open_webui.utils.session_pool import get_session @@ -300,6 +301,55 @@ class CallProtocol: raise ValueError('Unsupported call command') +async def _get_realtime_connection(config: dict) -> tuple[str, dict[str, str]]: + provider = config.get('audio.realtime.provider', 'openai') + auth_type = config.get('audio.realtime.auth_type', 'api_key') + if provider not in {'openai', 'azure'} or auth_type not in {'api_key', 'microsoft_entra_id'}: + raise ValueError('Unsupported Realtime provider or authentication method') + if provider != 'azure' and auth_type == 'microsoft_entra_id': + raise ValueError('Microsoft Entra ID requires the Azure OpenAI Realtime provider') + + try: + url = urlsplit(config.get('audio.realtime.openai.api_base_url') or '') + valid_url = ( + url.scheme in {'http', 'https'} + and url.hostname + and not (url.username or url.password or url.query or url.fragment) + ) + url.port # Validate a supplied port before opening the connection. + except ValueError: + valid_url = False + if not valid_url: + raise ValueError('Invalid Realtime provider URL') + model = config.get('audio.realtime.model') + if not model: + raise ValueError('Configure the Realtime model') + ws_url = urlunsplit( + ( + 'wss' if url.scheme == 'https' else 'ws', + url.netloc, + url.path.rstrip('/') + '/realtime', + urlencode({'model': model}), + '', + ) + ) + + if auth_type == 'microsoft_entra_id': + try: + token = await asyncio.to_thread(get_microsoft_entra_id_access_token) + except Exception: + raise ValueError('Unable to obtain a Microsoft Entra ID token for Realtime calls') from None + if not token: + raise ValueError('Unable to obtain a Microsoft Entra ID token for Realtime calls') + headers = {'Authorization': f'Bearer {token}'} + else: + key = config.get('audio.realtime.openai.api_key') + if not key: + raise ValueError('Configure the Realtime API key') + headers = {'api-key': key} if provider == 'azure' else {'Authorization': f'Bearer {key}'} + return ws_url, headers + + @router.websocket('/realtime') async def realtime_call(ws: WebSocket): await ws.accept() @@ -321,6 +371,8 @@ async def realtime_call(ws: WebSocket): raise ValueError('Authentication expired or invalid') config = await Config.get_many( 'audio.realtime.enabled', + 'audio.realtime.provider', + 'audio.realtime.auth_type', 'audio.realtime.openai.api_base_url', 'audio.realtime.openai.api_key', 'audio.realtime.model', @@ -360,38 +412,30 @@ async def realtime_call(ws: WebSocket): override = ModelVoice.model_validate((model_info.meta.model_dump().get('voice') if model_info else None) or {}) voice_model = config.get('audio.realtime.model') voice = override.voice or config.get('audio.realtime.voice') - key = config.get('audio.realtime.openai.api_key') - url = urlsplit(config.get('audio.realtime.openai.api_base_url') or '') - if ( - url.scheme not in {'http', 'https'} - or not url.netloc - or url.username - or url.password - or url.query - or url.fragment - ): - raise ValueError('Invalid Realtime provider URL') - if not key or not voice_model or not voice or not config.get('audio.realtime.transcription_model'): - raise ValueError('Configure the Realtime API key, model, voice, and transcription model') - ws_url = urlunsplit( - ( - 'wss' if url.scheme == 'https' else 'ws', - url.netloc, - url.path.rstrip('/') + '/realtime', - urlencode({'model': voice_model}), - '', - ) - ) - session = await get_session() + if not voice_model or not voice or not config.get('audio.realtime.transcription_model'): + raise ValueError('Configure the Realtime model, voice, and transcription model') async with asyncio.timeout(30): async with asyncio.timeout(15): - upstream = await session.ws_connect( - ws_url, - headers={'Authorization': f'Bearer {key}'}, - ssl=AIOHTTP_CLIENT_SESSION_SSL, - heartbeat=20, - max_msg_size=MAX_EVENT_BYTES, - ) + ws_url, headers = await _get_realtime_connection(config) + session = await get_session() + try: + upstream = await session.ws_connect( + ws_url, + headers=headers, + ssl=AIOHTTP_CLIENT_SESSION_SSL, + heartbeat=20, + max_msg_size=MAX_EVENT_BYTES, + ) + except aiohttp.WSServerHandshakeError as exc: + if exc.status in {401, 403}: + raise ValueError( + 'Realtime provider authentication rejected. Check the credentials and access.' + ) from None + if exc.status == 404: + raise ValueError( + 'Realtime endpoint not found. Check the API base URL and deployment name.' + ) from None + raise event = await upstream.receive_json() if event.get('type') != 'session.created': raise ValueError('Provider did not create a voice session') diff --git a/src/lib/apis/audio/index.ts b/src/lib/apis/audio/index.ts index a21abbbaaf..959a13b97a 100644 --- a/src/lib/apis/audio/index.ts +++ b/src/lib/apis/audio/index.ts @@ -27,10 +27,22 @@ export const getAudioConfig = async (token: string) => { return res; }; +export type RealtimeConfig = { + ENABLED: boolean; + PROVIDER: 'openai' | 'azure'; + AUTH_TYPE: 'api_key' | 'microsoft_entra_id'; + OPENAI_API_BASE_URL: string; + OPENAI_API_KEY: string; + MODEL: string; + VOICE: string; + TRANSCRIPTION_MODEL: string; + REALTIME_CALL_PROMPT_TEMPLATE: string | null; +}; + type AudioConfigForm = { tts: Record; stt: Record; - realtime?: Record; + realtime?: Partial; }; export const updateAudioConfig = async (token: string, payload: AudioConfigForm) => { diff --git a/src/lib/components/admin/Settings/Audio.svelte b/src/lib/components/admin/Settings/Audio.svelte index 96c4950034..f035693fc5 100644 --- a/src/lib/components/admin/Settings/Audio.svelte +++ b/src/lib/components/admin/Settings/Audio.svelte @@ -8,9 +8,10 @@ getAudioConfig, updateAudioConfig, getModels as _getModels, - getVoices as _getVoices + getVoices as _getVoices, + type RealtimeConfig } from '$lib/apis/audio'; - import { config, settings } from '$lib/stores'; + import { config } from '$lib/stores'; import Spinner from '$lib/components/common/Spinner.svelte'; import SensitiveInput from '$lib/components/common/SensitiveInput.svelte'; @@ -31,8 +32,10 @@ export let saveHandler: () => void; - let realtime = { + let realtime: RealtimeConfig & { REALTIME_CALL_PROMPT_TEMPLATE: string } = { ENABLED: false, + PROVIDER: 'openai', + AUTH_TYPE: 'api_key', OPENAI_API_BASE_URL: 'https://api.openai.com/v1', OPENAI_API_KEY: '', MODEL: 'gpt-realtime-2.1-mini', @@ -146,7 +149,7 @@ try { openaiParams = TTS_OPENAI_PARAMS ? JSON.parse(TTS_OPENAI_PARAMS) : {}; TTS_OPENAI_PARAMS = JSON.stringify(openaiParams, null, 2); - } catch (e) { + } catch { toast.error($i18n.t('Invalid JSON format for Parameters')); return; } @@ -154,7 +157,7 @@ const res = await updateAudioConfig(localStorage.token, { realtime: { ...realtime, - REALTIME_CALL_PROMPT_TEMPLATE: realtime.REALTIME_CALL_PROMPT_TEMPLATE.trim() || null + REALTIME_CALL_PROMPT_TEMPLATE: (realtime.REALTIME_CALL_PROMPT_TEMPLATE ?? '').trim() || null }, tts: { OPENAI_API_BASE_URL: TTS_OPENAI_API_BASE_URL, @@ -283,21 +286,67 @@ {#if realtime.ENABLED} -
- - - - - + (realtime.AUTH_TYPE = 'api_key')} + > + + + + + {#if realtime.PROVIDER === 'azure'} + + + + + + + {/if} +
+ + + {#if realtime.AUTH_TYPE === 'api_key'} + + + + {/if}
- + .openai.azure.com/openai/v1.", + "settings.audio.realtime.entraHelp": "Uses DefaultAzureCredential on the backend server, not the signed-in browser user. Configure a managed identity or Azure credentials on the server.", + "settings.audio.realtime.deployment": "Realtime Deployment Name", + "settings.audio.realtime.deploymentHelp": "Enter your Azure deployment name, which can differ from the model name. Choose a voice and transcription model supported by that deployment." }