From 62370596543f3c5a758ab2e98764bf17649222cc Mon Sep 17 00:00:00 2001 From: Dan Lemon Date: Tue, 6 Oct 2026 09:47:34 +0200 Subject: [PATCH 01/13] feat(soniox): add realtime speech-to-text on /v1/realtime Maps OpenAI realtime transcription sessions onto the Soniox websocket API (stt-rt-v5): server VAD uses Soniox endpoint detection, commit sends a manual finalize, deployment litellm_params pass Soniox options through, translation sessions transcribe the translated tokens, idle streams get keepalives, and cost is tracked from processed audio duration. --- .../llms/base_llm/realtime/transformation.py | 3 + litellm/llms/custom_httpx/llm_http_handler.py | 2 +- litellm/llms/soniox/realtime/__init__.py | 0 .../llms/soniox/realtime/transformation.py | 381 ++++++++++++++++ ...odel_prices_and_context_window_backup.json | 20 + .../provider_create_fields.json | 2 +- litellm/realtime_api/README.md | 2 +- litellm/realtime_api/main.py | 1 + litellm/utils.py | 5 + model_prices_and_context_window.json | 20 + provider_endpoints_support.json | 3 +- .../custom_httpx/test_llm_http_handler.py | 101 +++++ tests/unit/llms/soniox/realtime/__init__.py | 1 + .../test_soniox_realtime_transformation.py | 408 ++++++++++++++++++ tests/unit/realtime_api/test_main.py | 30 ++ 15 files changed, 975 insertions(+), 4 deletions(-) create mode 100644 litellm/llms/soniox/realtime/__init__.py create mode 100644 litellm/llms/soniox/realtime/transformation.py create mode 100644 tests/unit/llms/soniox/realtime/__init__.py create mode 100644 tests/unit/llms/soniox/realtime/test_soniox_realtime_transformation.py diff --git a/litellm/llms/base_llm/realtime/transformation.py b/litellm/llms/base_llm/realtime/transformation.py index 1f4ad29fa74..60d0f723ee0 100644 --- a/litellm/llms/base_llm/realtime/transformation.py +++ b/litellm/llms/base_llm/realtime/transformation.py @@ -100,6 +100,9 @@ class BaseRealtimeConfig(ABC): async def open_backend(self, url: str, headers: Mapping[str, str]) -> RealtimeBackend | None: return None + def wrap_backend(self, backend: RealtimeBackend) -> RealtimeBackend: + return backend + def transform_session_created_event( self, model: str, diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 86c36a7b413..9b22d1347ae 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -6309,7 +6309,7 @@ class BaseLLMHTTPHandler: ssl_context.check_hostname = False ssl_context.verify_mode = ssl.CERT_NONE provider_backend: Final = await provider_config.open_backend(url, headers) - backend_ws: Final = ( + backend_ws: Final = provider_config.wrap_backend( provider_backend if provider_backend is not None else await self._open_realtime_backend_ws(websockets, url, headers, ssl_context) diff --git a/litellm/llms/soniox/realtime/__init__.py b/litellm/llms/soniox/realtime/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/llms/soniox/realtime/transformation.py b/litellm/llms/soniox/realtime/transformation.py new file mode 100644 index 00000000000..4fe9f170cb4 --- /dev/null +++ b/litellm/llms/soniox/realtime/transformation.py @@ -0,0 +1,381 @@ +import asyncio +import time +from collections.abc import Awaitable, Callable +from types import TracebackType +from typing import Final +from urllib.parse import urlparse + +from pydantic import BaseModel, ConfigDict, JsonValue, ValidationError +from typing_extensions import Self + +from litellm import verbose_logger +from litellm._uuid import uuid +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.base_llm.realtime.transcription_protocol import ( + RealtimeTranscriptionProtocolError, + TranscriptionSessionUpdate, + completed_event, + decode_pcm16_append, + delta_event, + duration_usage, + error_event, + json_object, + parse_transcription_session_update, + speech_event, + transcription_session, + transcription_session_created_event, +) +from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeBackend +from litellm.llms.soniox.common_utils import get_soniox_api_key +from litellm.types.llms.openai import OpenAIRealtimeEvents, OpenAIRealtimeTranscriptionSessionCreated +from litellm.types.realtime import ( + RealtimeInputAudioTranscriptionUsage, + RealtimeResponseTransformInput, + RealtimeResponseTypedDict, +) + +DEFAULT_SONIOX_REALTIME_URL: Final = "wss://stt-rt.soniox.com/transcribe-websocket" +DEFAULT_SAMPLE_RATE: Final = 24_000 +KEEPALIVE_INTERVAL_SECONDS: Final = 5.0 +_FINALIZE: Final = '{"type":"finalize"}' +_KEEPALIVE: Final = '{"type":"keepalive"}' +_END_STREAM: Final = "" +_CONTROL_FRAMES: Final = frozenset((_FINALIZE, _KEEPALIVE, _END_STREAM)) +SESSION_STARTED_FRAME: Final = '{"type":"litellm.soniox.session_started"}' +_ENDPOINT_TOKEN: Final = "" +_FINALIZED_TOKEN: Final = "" + + +class SonioxProtocolError(RealtimeTranscriptionProtocolError): + pass + + +class SonioxRealtimeOptions(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + language_hints: tuple[str, ...] | None = None + language_hints_strict: bool | None = None + enable_language_identification: bool | None = None + enable_speaker_diarization: bool | None = None + context: JsonValue | None = None + translation: JsonValue | None = None + client_reference_id: str | None = None + max_endpoint_delay_ms: int | None = None + endpoint_sensitivity: float | None = None + endpoint_latency_adjustment_level: int | None = None + + +class SonioxStartRequest(SonioxRealtimeOptions): + model: str + audio_format: str + sample_rate: int + num_channels: int + enable_endpoint_detection: bool + + +class _SonioxToken(BaseModel): + text: str + is_final: bool = False + translation_status: str | None = None + + +class _SonioxResponse(BaseModel): + tokens: tuple[_SonioxToken, ...] = () + total_audio_proc_ms: int | None = None + error_code: int | None = None + error_message: str | None = None + + +def _sample_rate(update: TranscriptionSessionUpdate | None) -> int: + audio_format: Final = None if update is None else update.audio_format + if audio_format is None: + return DEFAULT_SAMPLE_RATE + if not audio_format.is_pcm16: + raise SonioxProtocolError("Soniox realtime requires pcm16 input audio") + if audio_format.channels not in (None, 1): + raise SonioxProtocolError("Soniox realtime requires mono input audio") + if audio_format.rate is None: + return DEFAULT_SAMPLE_RATE + if audio_format.rate <= 0: + raise SonioxProtocolError("sample rate must be positive") + return audio_format.rate + + +def _language_hints(language: str | None, configured: tuple[str, ...] | None) -> tuple[str, ...] | None: + if language is None: + return configured + return (language, *(hint for hint in configured or () if hint != language)) + + +def build_start_request( + model: str, update: TranscriptionSessionUpdate | None, options: SonioxRealtimeOptions +) -> SonioxStartRequest: + return SonioxStartRequest( + **options.model_dump(exclude={"language_hints"}), + language_hints=_language_hints(None if update is None else update.language, options.language_hints), + model=model, + audio_format="pcm_s16le", + sample_rate=_sample_rate(update), + num_channels=1, + enable_endpoint_detection=update is None or not update.turn_detection_disabled, + ) + + +def _session_created(session_id: str, start_request: SonioxStartRequest) -> OpenAIRealtimeTranscriptionSessionCreated: + session: Final = transcription_session( + session_id=session_id, + model=start_request.model, + sample_rate=start_request.sample_rate, + language=start_request.language_hints[0] if start_request.language_hints else None, + server_vad=start_request.enable_endpoint_detection, + ) + return transcription_session_created_event(session) + + +def build_soniox_realtime_url(api_base: str | None) -> str: + if api_base is None: + return DEFAULT_SONIOX_REALTIME_URL + if urlparse(api_base).scheme not in ("ws", "wss"): + raise ValueError("Soniox realtime api_base must be a ws:// or wss:// URL") + return api_base + + +class SonioxEventTransformer: + def __init__(self, *, translated_only: bool) -> None: + self._translated_only: Final = translated_only + self._item_id: str | None = None + self._final_text: str = "" + self._processed_ms: int = 0 + self._unbilled_seconds: float = 0.0 + + def transform(self, payload: str) -> tuple[OpenAIRealtimeEvents, ...]: + try: + response: Final = _SonioxResponse.model_validate_json(payload) + except ValidationError: + raise SonioxProtocolError("Soniox returned an invalid realtime response") from None + if response.error_code is not None or response.error_message is not None: + return (error_event(f"Soniox realtime error {response.error_code}: {response.error_message}"),) + self._bill(response.total_audio_proc_ms) + return tuple(event for token in response.tokens for event in self._token_events(token)) + + def take_unbilled_usage(self) -> RealtimeInputAudioTranscriptionUsage | None: + seconds: Final = self._unbilled_seconds + if seconds <= 0: + return None + self._unbilled_seconds = 0.0 + return duration_usage(seconds) + + def _bill(self, total_audio_proc_ms: int | None) -> None: + if total_audio_proc_ms is None or total_audio_proc_ms <= self._processed_ms: + return + self._unbilled_seconds += (total_audio_proc_ms - self._processed_ms) / 1000 + self._processed_ms = total_audio_proc_ms + + def _token_events(self, token: _SonioxToken) -> tuple[OpenAIRealtimeEvents, ...]: + if token.text in (_ENDPOINT_TOKEN, _FINALIZED_TOKEN): + return self._complete(speech_stopped=token.text == _ENDPOINT_TOKEN) + if self._translated_only and token.translation_status != "translation": + return () + started: Final = self._start() + if not token.is_final: + return started + self._final_text += token.text + return (*started, delta_event(self._current_item_id(), token.text)) + + def _start(self) -> tuple[OpenAIRealtimeEvents, ...]: + if self._item_id is not None: + return () + item_id: Final = f"item_{uuid.uuid4().hex}" + self._item_id = item_id + return (speech_event("input_audio_buffer.speech_started", item_id),) + + def _current_item_id(self) -> str: + assert self._item_id is not None # _start always runs before a delta is emitted + return self._item_id + + def _complete(self, *, speech_stopped: bool) -> tuple[OpenAIRealtimeEvents, ...]: + item_id: Final = self._item_id + if item_id is None: + return () + transcript: Final = self._final_text.strip() + self._item_id = None + self._final_text = "" + stopped: Final[tuple[OpenAIRealtimeEvents, ...]] = ( + (speech_event("input_audio_buffer.speech_stopped", item_id),) if speech_stopped else () + ) + return (*stopped, completed_event(item_id, transcript, self.take_unbilled_usage())) + + +class SonioxRealtimeBackend: + def __init__( + self, + inner: RealtimeBackend, + *, + keepalive_interval: float = KEEPALIVE_INTERVAL_SECONDS, + clock: Callable[[], float] = time.monotonic, + sleep: Callable[[float], Awaitable[None]] = asyncio.sleep, + ) -> None: + self._inner: Final = inner + self._interval: Final = keepalive_interval + self._clock: Final = clock + self._sleep: Final = sleep + self._local_frames: Final[asyncio.Queue[str]] = asyncio.Queue() + self._last_sent: float = clock() + self._started: bool = False + self._ended: bool = False + self._keepalive_task: asyncio.Task[None] | None = None + self._upstream_recv: asyncio.Task[str | bytes] | None = None + + async def __aenter__(self) -> Self: + await self._inner.__aenter__() + self._keepalive_task = asyncio.create_task(self._keepalive()) + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: + self._stop() + await self._inner.__aexit__(exc_type, exc_value, traceback) + + async def send(self, message: str | bytes) -> None: + self._last_sent = self._clock() + self._ended = self._ended or message == _END_STREAM + await self._inner.send(message) + if not self._started and isinstance(message, str) and message not in _CONTROL_FRAMES: + self._started = True + self._local_frames.put_nowait(SESSION_STARTED_FRAME) + + async def recv(self, decode: bool | None = None) -> str | bytes: + if not self._local_frames.empty(): + return self._local_frames.get_nowait() + upstream: Final = self._upstream_recv or asyncio.create_task(self._inner.recv(decode)) + local: Final = asyncio.create_task(self._local_frames.get()) + done, _ = await asyncio.wait((upstream, local), return_when=asyncio.FIRST_COMPLETED) + if local in done: + self._upstream_recv = upstream + return local.result() + local.cancel() + self._upstream_recv = None + return upstream.result() + + async def close(self) -> None: + self._stop() + await self._inner.close() + + def _stop(self) -> None: + for task in (self._keepalive_task, self._upstream_recv): + if task is not None: + task.cancel() + + async def _keepalive(self) -> None: + while not self._ended: + await self._sleep(self._interval) + if self._ended or self._clock() - self._last_sent < self._interval: + continue + try: + await self.send(_KEEPALIVE) + except Exception as e: + verbose_logger.debug("Soniox realtime: stopping keepalive after send failed: %s", e) + return + + +class SonioxRealtimeConfig(BaseRealtimeConfig): + def __init__( + self, + *, + options: SonioxRealtimeOptions | None = None, + wrap: Callable[[RealtimeBackend], RealtimeBackend] = SonioxRealtimeBackend, + ) -> None: + self._options: Final = options or SonioxRealtimeOptions() + self._wrap: Final = wrap + self._transformer: Final = SonioxEventTransformer(translated_only=self._options.translation is not None) + self._start_request: SonioxStartRequest | None = None + self._session_id: str = f"sess_{uuid.uuid4().hex}" + + def validate_environment( + self, + headers: dict[str, str], # mutable-ok: BaseRealtimeConfig contract + model: str, + api_key: str | None = None, + ) -> dict[str, str]: # mutable-ok: BaseRealtimeConfig contract + resolved_key: Final = get_soniox_api_key(api_key) + if not resolved_key: + raise ValueError("Missing Soniox API key. Set SONIOX_API_KEY or api_key in the deployment litellm_params") + return {**headers, "Authorization": f"Bearer {resolved_key}"} + + def get_complete_url(self, api_base: str | None, model: str, api_key: str | None = None) -> str: + return build_soniox_realtime_url(api_base) + + def wrap_backend(self, backend: RealtimeBackend) -> RealtimeBackend: + return self._wrap(backend) + + def transform_session_created_event( + self, + model: str, + logging_session_id: str, + session_configuration_request: str | None = None, + ) -> OpenAIRealtimeTranscriptionSessionCreated: + self._session_id = logging_session_id + return _session_created(logging_session_id, build_start_request(model, None, self._options)) + + def transform_realtime_request( + self, + message: str, + model: str, + session_configuration_request: str | None = None, + ) -> tuple[str | bytes, ...]: + request: Final = json_object(message, SonioxProtocolError) + match request.get("type"): + case "session.update" | "transcription_session.update": + return self._start(model, parse_transcription_session_update(message, SonioxProtocolError)) + case "input_audio_buffer.append": + audio: Final = decode_pcm16_append(request.get("audio"), error=SonioxProtocolError) + return (*self._start(model, None), audio) + case "input_audio_buffer.commit": + return (*self._start(model, None), _FINALIZE) + case "input_audio_buffer.end": + return (*self._start(model, None), _END_STREAM) + case event_type: + verbose_logger.debug("Soniox realtime: dropping unsupported client event %s", event_type) + return () + + def unbilled_usage_on_session_close(self, model: str) -> RealtimeInputAudioTranscriptionUsage | None: + return self._transformer.take_unbilled_usage() + + def transform_realtime_response( + self, + message: str | bytes, + model: str, + logging_obj: LiteLLMLoggingObj, + realtime_response_transform_input: RealtimeResponseTransformInput, + ) -> RealtimeResponseTypedDict: + payload: Final = message.decode("utf-8") if isinstance(message, bytes) else message + result: Final[RealtimeResponseTypedDict] = { + "response": list(self._backend_events(payload)), + "current_output_item_id": realtime_response_transform_input.get("current_output_item_id"), + "current_response_id": realtime_response_transform_input.get("current_response_id"), + "current_delta_chunks": realtime_response_transform_input.get("current_delta_chunks"), + "current_conversation_id": realtime_response_transform_input.get("current_conversation_id"), + "current_item_chunks": realtime_response_transform_input.get("current_item_chunks"), + "current_delta_type": realtime_response_transform_input.get("current_delta_type"), + "session_configuration_request": realtime_response_transform_input.get("session_configuration_request"), + } + return result + + def _backend_events(self, payload: str) -> tuple[OpenAIRealtimeEvents, ...]: + if payload != SESSION_STARTED_FRAME: + return self._transformer.transform(payload) + assert self._start_request is not None # the backend signals session start only after the start request + return (_session_created(self._session_id, self._start_request),) + + def _start(self, model: str, update: TranscriptionSessionUpdate | None) -> tuple[str, ...]: + if self._start_request is not None: + if update is not None: + verbose_logger.debug("Soniox realtime: ignoring session.update after the stream was configured") + return () + start_request: Final = build_start_request(model, update, self._options) + self._start_request = start_request + return (start_request.model_dump_json(exclude_none=True),) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 5f4b42bc923..b7d118862b1 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -59836,6 +59836,26 @@ ], "supports_audio_input": true }, + "soniox/stt-rt-v5": { + "input_cost_per_second": 3.333e-05, + "litellm_provider": "soniox", + "metadata": { + "calculation": "$0.12/3600 seconds = $0.00003333 per second", + "original_pricing_per_hour": 0.12 + }, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, "tensormesh/Qwen/Qwen3.5-397B-A17B-FP8": { "litellm_provider": "tensormesh", "mode": "chat", diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index 2dd9f50bec8..e3e7a3ebad2 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -3180,7 +3180,7 @@ "key": "api_key", "label": "Soniox API Key", "placeholder": null, - "tooltip": "Currently only the async Speech-to-Text REST API (api.soniox.com) is supported. Realtime STT (stt-rt.soniox.com) and TTS (tts-rt.soniox.com) are not yet available.", + "tooltip": "Supports async Speech-to-Text (api.soniox.com) on /v1/audio/transcriptions and realtime Speech-to-Text (stt-rt.soniox.com) on /v1/realtime. TTS (tts-rt.soniox.com) is not yet available.", "required": true, "field_type": "password", "options": null, diff --git a/litellm/realtime_api/README.md b/litellm/realtime_api/README.md index d810de2f24f..987e8be1ed1 100644 --- a/litellm/realtime_api/README.md +++ b/litellm/realtime_api/README.md @@ -4,6 +4,6 @@ Supported endpoints: - WebSocket: `/v1/realtime` (with `intent=transcription` for transcription-only sessions) - HTTP: `/v1/realtime/client_secrets`, `/v1/realtime/transcription_sessions` -Supported providers: OpenAI, Azure OpenAI, Bedrock, Vertex AI, xAI. +Supported providers: OpenAI, Azure OpenAI, Bedrock, Vertex AI, xAI, Gemini, Meta, Soniox. For user-facing documentation and usage examples, see the litellm-docs repo. \ No newline at end of file diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index 28814741852..e219b09212b 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -403,6 +403,7 @@ async def _arealtime( provider_config = ProviderConfigManager.get_provider_realtime_config( model=model, provider=LlmProviders(_custom_llm_provider), + litellm_params=kwargs, ) if provider_config is not None: await base_llm_http_handler.async_realtime( diff --git a/litellm/utils.py b/litellm/utils.py index a2dac2fcf8d..599170f68cc 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9694,6 +9694,7 @@ class ProviderConfigManager: def get_provider_realtime_config( model: str, provider: LlmProviders, + litellm_params: Mapping[str, object] = MappingProxyType({}), ) -> BaseRealtimeConfig | None: if LlmProviders.GEMINI == provider: from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig @@ -9703,6 +9704,10 @@ class ProviderConfigManager: from litellm.llms.meta.realtime.transformation import MetaRealtimeConfig return MetaRealtimeConfig() + if LlmProviders.SONIOX == provider: + from litellm.llms.soniox.realtime.transformation import SonioxRealtimeConfig, SonioxRealtimeOptions + + return SonioxRealtimeConfig(options=SonioxRealtimeOptions.model_validate(dict(litellm_params))) return None @staticmethod diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 5f4b42bc923..b7d118862b1 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -59836,6 +59836,26 @@ ], "supports_audio_input": true }, + "soniox/stt-rt-v5": { + "input_cost_per_second": 3.333e-05, + "litellm_provider": "soniox", + "metadata": { + "calculation": "$0.12/3600 seconds = $0.00003333 per second", + "original_pricing_per_hour": 0.12 + }, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, "tensormesh/Qwen/Qwen3.5-397B-A17B-FP8": { "litellm_provider": "tensormesh", "mode": "chat", diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 00904219b81..d485ebeff90 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -2450,7 +2450,8 @@ "audio_speech": false, "moderations": false, "batches": false, - "rerank": false + "rerank": false, + "realtime": true } }, "synthetic": { diff --git a/tests/unit/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py index 93d18727f22..ea9dc5a73d7 100644 --- a/tests/unit/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py @@ -4377,6 +4377,107 @@ async def test_async_realtime_bridges_a_transcription_session_through_the_provid assert [bytes(request.audio) for request in speech_client.requests[1:]] == [b"\x00\x01" * 800, b"\x00\x01" * 800] +@pytest.mark.asyncio +async def test_async_realtime_bridges_a_soniox_transcription_through_the_wrapped_websocket(): + import websockets.exceptions # noqa: F401 # binds the submodule so async_realtime's except clause resolves, as in the proxy process + + from litellm.llms.soniox.realtime.transformation import SonioxRealtimeBackend, SonioxRealtimeConfig + + class FakeSonioxSocket: + def __init__(self) -> None: + self.sent: Final[list[str | bytes]] = [] + self._responses: Final[asyncio.Queue[str]] = asyncio.Queue() + + async def __aenter__(self) -> "FakeSonioxSocket": + return self + + async def __aexit__(self, exc_type, exc_value, traceback) -> None: + return None + + async def send(self, message: str | bytes) -> None: + self.sent.append(message) + if message == '{"type":"finalize"}': + tokens = [{"text": "Hello", "is_final": True}, {"text": "", "is_final": True}] + await self._responses.put(json.dumps({"tokens": tokens, "total_audio_proc_ms": 1200})) + + async def recv(self, decode: bool | None = None) -> str: + return await self._responses.get() + + async def close(self) -> None: + return None + + upstream = FakeSonioxSocket() + wrapped: list[object] = [] + provider_config = SonioxRealtimeConfig( + wrap=lambda backend: wrapped.append(backend) or SonioxRealtimeBackend(backend) + ) + audio = b"\x00\x01" * 800 + client_ws = _ScriptedClientWebSocket( + [ + json.dumps( + { + "type": "session.update", + "session": { + "type": "transcription", + "audio": { + "input": { + "format": {"type": "audio/pcm", "rate": 16000}, + "transcription": {"model": "stt-rt-v5", "language": "en"}, + "turn_detection": None, + } + }, + }, + } + ), + json.dumps({"type": "input_audio_buffer.append", "audio": base64.b64encode(audio).decode()}), + json.dumps({"type": "input_audio_buffer.commit"}), + ], + last_event_type="conversation.item.input_audio_transcription.completed", + ) + logging_obj = Mock() + logging_obj.litellm_trace_id = "trace_1" + logging_obj.model_call_details = {} + logging_obj.dispatch_success_handlers = AsyncMock() + logging_obj.dispatch_failure_handlers = AsyncMock() + handler = BaseLLMHTTPHandler() + + with patch.object(handler, "_open_realtime_backend_ws", AsyncMock(return_value=upstream)) as dial: + await handler.async_realtime( + model="stt-rt-v5", + websocket=client_ws, + logging_obj=logging_obj, + provider_config=provider_config, + headers={}, + api_key="soniox-key", + query_params={"model": "stt-rt-v5", "intent": "transcription"}, + ) + + assert dial.await_args.args[1:3] == ( + "wss://stt-rt.soniox.com/transcribe-websocket", + {"Authorization": "Bearer soniox-key"}, + ) + assert wrapped == [upstream] + assert json.loads(upstream.sent[0]) == { + "model": "stt-rt-v5", + "audio_format": "pcm_s16le", + "sample_rate": 16000, + "num_channels": 1, + "enable_endpoint_detection": False, + "language_hints": ["en"], + } + assert upstream.sent[1:] == [audio, '{"type":"finalize"}'] + events = client_ws.sent_events() + assert [event["type"] for event in events] == [ + "session.created", + "session.updated", + "input_audio_buffer.speech_started", + "conversation.item.input_audio_transcription.delta", + "conversation.item.input_audio_transcription.completed", + ] + assert events[-1]["transcript"] == "Hello" + assert events[-1]["usage"] == {"type": "duration", "seconds": 1.2} + + @pytest.mark.asyncio async def test_responses_agentic_followup_does_not_repeat_request_params_from_plan_kwargs(monkeypatch): """A plan whose kwargs repeat a request param must not crash the Responses follow-up with a duplicate keyword""" diff --git a/tests/unit/llms/soniox/realtime/__init__.py b/tests/unit/llms/soniox/realtime/__init__.py new file mode 100644 index 00000000000..b503dff241f --- /dev/null +++ b/tests/unit/llms/soniox/realtime/__init__.py @@ -0,0 +1 @@ +"""Soniox realtime tests.""" diff --git a/tests/unit/llms/soniox/realtime/test_soniox_realtime_transformation.py b/tests/unit/llms/soniox/realtime/test_soniox_realtime_transformation.py new file mode 100644 index 00000000000..214119e7ade --- /dev/null +++ b/tests/unit/llms/soniox/realtime/test_soniox_realtime_transformation.py @@ -0,0 +1,408 @@ +import asyncio +import base64 +import json +from typing import Final +from unittest.mock import MagicMock + +import pytest + +from litellm.llms.soniox.realtime.transformation import ( + DEFAULT_SONIOX_REALTIME_URL, + SESSION_STARTED_FRAME, + SonioxEventTransformer, + SonioxRealtimeBackend, + SonioxProtocolError, + SonioxRealtimeConfig, + SonioxRealtimeOptions, +) +from litellm.types.llms.openai import OpenAIRealtimeEvents +from litellm.types.realtime import RealtimeResponseTransformInput +from litellm.types.utils import LlmProviders +from litellm.utils import ProviderConfigManager + +MODEL: Final = "stt-rt-v5" +EMPTY_TRANSFORM_INPUT: Final[RealtimeResponseTransformInput] = { + "session_configuration_request": None, + "current_output_item_id": None, + "current_response_id": None, + "current_delta_chunks": None, + "current_item_chunks": None, + "current_conversation_id": None, + "current_delta_type": None, +} +PCM: Final = b"\x01\x02" * 160 + + +def _event(event_type: str, **fields: object) -> str: + return json.dumps({"type": event_type, **fields}) + + +def _session_update(audio_input: dict[str, object]) -> str: + return _event("session.update", session={"type": "transcription", "audio": {"input": audio_input}}) + + +def _append() -> str: + return _event("input_audio_buffer.append", audio=base64.b64encode(PCM).decode()) + + +def _start_request(frames: tuple[str | bytes, ...]) -> dict[str, object]: + assert isinstance(frames[0], str) + return json.loads(frames[0]) + + +def _response(tokens: list[dict[str, object]], total_audio_proc_ms: int | None = None) -> str: + return json.dumps({"tokens": tokens, "total_audio_proc_ms": total_audio_proc_ms}) + + +def _token(text: str, is_final: bool = True, **fields: object) -> dict[str, object]: + return {"text": text, "is_final": is_final, **fields} + + +def _types(events: tuple[OpenAIRealtimeEvents, ...]) -> list[object]: + return [event["type"] for event in events] + + +def test_session_update_starts_the_stream_with_client_audio_and_deployment_options(): + config = SonioxRealtimeConfig( + options=SonioxRealtimeOptions(language_hints=("en", "fr"), enable_speaker_diarization=True) + ) + + frames = config.transform_realtime_request( + _session_update( + { + "format": {"type": "audio/pcm", "rate": 16_000}, + "transcription": {"model": "client-chosen-model", "language": "fr"}, + "turn_detection": None, + } + ), + MODEL, + ) + + assert _start_request(frames) == { + "model": MODEL, + "audio_format": "pcm_s16le", + "sample_rate": 16_000, + "num_channels": 1, + "enable_endpoint_detection": False, + "language_hints": ["fr", "en"], + "enable_speaker_diarization": True, + } + + +def test_audio_before_session_update_starts_a_default_stream_once(): + config = SonioxRealtimeConfig() + + first = config.transform_realtime_request(_append(), MODEL) + second = config.transform_realtime_request(_append(), MODEL) + late_update = config.transform_realtime_request(_session_update({"turn_detection": None}), MODEL) + + assert _start_request(first) == { + "model": MODEL, + "audio_format": "pcm_s16le", + "sample_rate": 24_000, + "num_channels": 1, + "enable_endpoint_detection": True, + } + assert first[1:] == (PCM,) + assert second == (PCM,) + assert late_update == () + + +@pytest.mark.parametrize( + ("event_type", "control_frame"), + [("input_audio_buffer.commit", '{"type":"finalize"}'), ("input_audio_buffer.end", "")], +) +def test_buffer_control_events_map_to_soniox_control_frames(event_type: str, control_frame: str): + config = SonioxRealtimeConfig() + config.transform_realtime_request(_append(), MODEL) + + assert config.transform_realtime_request(_event(event_type), MODEL) == (control_frame,) + + +def test_unsupported_client_events_are_dropped(): + config = SonioxRealtimeConfig() + + assert config.transform_realtime_request(_event("input_audio_buffer.clear"), MODEL) == () + + +@pytest.mark.parametrize( + "audio_input", + [ + {"format": {"type": "audio/pcmu"}}, + {"format": {"type": "audio/pcm", "rate": 24_000, "channels": 2}}, + {"format": {"type": "audio/pcm", "rate": 0}}, + ], +) +def test_unsupported_audio_formats_are_rejected_without_starting_the_stream(audio_input: dict[str, object]): + config = SonioxRealtimeConfig() + + with pytest.raises(SonioxProtocolError): + config.transform_realtime_request(_session_update(audio_input), MODEL) + assert _start_request(config.transform_realtime_request(_append(), MODEL))["sample_rate"] == 24_000 + + +def test_validate_environment_sends_the_key_as_a_bearer_header(monkeypatch): + monkeypatch.setenv("SONIOX_API_KEY", "env-key") + config = SonioxRealtimeConfig() + + assert config.validate_environment({"x": "y"}, MODEL, "deployment-key") == { + "x": "y", + "Authorization": "Bearer deployment-key", + } + assert config.validate_environment({}, MODEL) == {"Authorization": "Bearer env-key"} + + +def test_validate_environment_rejects_a_missing_key(monkeypatch): + monkeypatch.delenv("SONIOX_API_KEY", raising=False) + + with pytest.raises(ValueError, match="SONIOX_API_KEY"): + SonioxRealtimeConfig().validate_environment({}, MODEL) + + +def test_get_complete_url_defaults_to_soniox_and_only_accepts_websocket_bases(): + config = SonioxRealtimeConfig() + + assert config.get_complete_url(None, MODEL) == DEFAULT_SONIOX_REALTIME_URL + assert config.get_complete_url("wss://proxy.example/ws", MODEL) == "wss://proxy.example/ws" + with pytest.raises(ValueError, match="ws://"): + config.get_complete_url("https://api.soniox.com", MODEL) + + +def test_endpoint_detection_emits_one_openai_turn_billed_by_processed_audio(): + transformer = SonioxEventTransformer(translated_only=False) + + provisional = transformer.transform(_response([_token("Hel", is_final=False)], total_audio_proc_ms=500)) + finals = transformer.transform( + _response([_token("Hello"), _token(" world"), _token(" next", is_final=False)], total_audio_proc_ms=1500) + ) + endpoint = transformer.transform(_response([_token("")], total_audio_proc_ms=2000)) + + assert _types(provisional) == ["input_audio_buffer.speech_started"] + item_id = provisional[0]["item_id"] + assert [(event["type"], event.get("delta")) for event in finals] == [ + ("conversation.item.input_audio_transcription.delta", "Hello"), + ("conversation.item.input_audio_transcription.delta", " world"), + ] + assert _types(endpoint) == [ + "input_audio_buffer.speech_stopped", + "conversation.item.input_audio_transcription.completed", + ] + assert {event["item_id"] for event in (*finals, *endpoint)} == {item_id} + assert endpoint[1]["transcript"] == "Hello world" + assert endpoint[1]["usage"] == {"type": "duration", "seconds": 2.0} + assert transformer.take_unbilled_usage() is None + + +def test_manual_finalization_completes_the_turn_without_speech_stopped_and_starts_a_new_item(): + transformer = SonioxEventTransformer(translated_only=False) + + first = transformer.transform(_response([_token("One"), _token("")])) + second = transformer.transform(_response([_token("Two"), _token("")])) + + assert _types(first) == [ + "input_audio_buffer.speech_started", + "conversation.item.input_audio_transcription.delta", + "conversation.item.input_audio_transcription.completed", + ] + assert first[-1]["transcript"] == "One" + assert "usage" not in first[-1] + assert second[-1]["transcript"] == "Two" + assert second[0]["item_id"] != first[0]["item_id"] + + +def test_finalization_without_speech_emits_nothing(): + transformer = SonioxEventTransformer(translated_only=False) + + assert transformer.transform(_response([_token("")])) == () + + +def test_translation_sessions_transcribe_only_the_translated_tokens(): + transformer = SonioxEventTransformer(translated_only=True) + + events = transformer.transform( + _response( + [ + _token("Hola", translation_status="original"), + _token("Hello", translation_status="translation"), + _token(""), + ] + ) + ) + + assert [event.get("delta") for event in events if "delta" in event] == ["Hello"] + assert events[-1]["transcript"] == "Hello" + + +def test_provider_errors_surface_as_openai_error_events(): + transformer = SonioxEventTransformer(translated_only=False) + + events = transformer.transform(json.dumps({"error_code": 401, "error_message": "Invalid API key."})) + + assert events == ( + {"type": "error", "error": {"type": "server_error", "message": "Soniox realtime error 401: Invalid API key."}}, + ) + + +def test_malformed_provider_frames_are_rejected(): + with pytest.raises(SonioxProtocolError): + SonioxEventTransformer(translated_only=False).transform('{"tokens": "nope"}') + + +def test_audio_processed_after_the_last_turn_is_billed_on_session_close(): + config = SonioxRealtimeConfig() + config.transform_realtime_response( + _response([_token("Hi"), _token("")], total_audio_proc_ms=1000), MODEL, MagicMock(), EMPTY_TRANSFORM_INPUT + ) + config.transform_realtime_response( + _response([], total_audio_proc_ms=3500), MODEL, MagicMock(), EMPTY_TRANSFORM_INPUT + ) + + assert config.unbilled_usage_on_session_close(MODEL) == {"type": "duration", "seconds": 2.5} + assert config.unbilled_usage_on_session_close(MODEL) is None + + +def test_translation_option_switches_the_config_to_translated_transcripts(): + config = SonioxRealtimeConfig( + options=SonioxRealtimeOptions(translation={"type": "one_way", "target_language": "en"}) + ) + + result = config.transform_realtime_response( + _response([_token("Hola", translation_status="original"), _token("Hello", translation_status="translation")]), + MODEL, + MagicMock(), + EMPTY_TRANSFORM_INPUT, + ) + + assert [event.get("delta") for event in result["response"] if "delta" in event] == ["Hello"] + + +def test_session_created_event_advertises_the_deployment_model_for_cost_tracking(): + config = SonioxRealtimeConfig(options=SonioxRealtimeOptions(language_hints=("de",))) + + event = config.transform_session_created_event(MODEL, "session_1") + + assert event["session"]["id"] == "session_1" + assert event["session"]["audio"]["input"]["transcription"] == {"model": MODEL, "language": "de"} + + +def test_backend_session_start_is_announced_with_the_negotiated_session(): + config = SonioxRealtimeConfig() + config.transform_session_created_event(MODEL, "session_1") + config.transform_realtime_request( + _session_update( + { + "format": {"type": "audio/pcm", "rate": 16_000}, + "transcription": {"language": "it"}, + "turn_detection": None, + } + ), + MODEL, + ) + + result = config.transform_realtime_response(SESSION_STARTED_FRAME, MODEL, MagicMock(), EMPTY_TRANSFORM_INPUT) + + assert _types(tuple(result["response"])) == ["session.created"] + assert result["response"][0]["session"]["id"] == "session_1" + assert result["response"][0]["session"]["audio"]["input"] == { + "format": {"type": "audio/pcm", "rate": 16_000}, + "transcription": {"model": MODEL, "language": "it"}, + "turn_detection": None, + } + + +def test_provider_config_manager_passes_only_soniox_options_from_deployment_params(): + config = ProviderConfigManager.get_provider_realtime_config( + MODEL, + LlmProviders.SONIOX, + { + "api_key": "secret", + "model": "soniox/stt-rt-v5", + "audio_format": "mulaw", + "language_hints": ["pl"], + "context": {"terms": ["LiteLLM"]}, + "max_endpoint_delay_ms": 800, + }, + ) + + assert isinstance(config, SonioxRealtimeConfig) + assert _start_request(config.transform_realtime_request(_append(), MODEL)) == { + "model": MODEL, + "audio_format": "pcm_s16le", + "sample_rate": 24_000, + "num_channels": 1, + "enable_endpoint_detection": True, + "language_hints": ["pl"], + "context": {"terms": ["LiteLLM"]}, + "max_endpoint_delay_ms": 800, + } + + +class _RecordingBackend: + def __init__(self) -> None: + self.sent: Final[list[str | bytes]] = [] + self.upstream: Final[asyncio.Queue[str]] = asyncio.Queue() + self.entered = False + self.exited = False + + async def __aenter__(self) -> "_RecordingBackend": + self.entered = True + return self + + async def __aexit__(self, exc_type, exc_value, traceback) -> None: + self.exited = True + + async def send(self, message: str | bytes) -> None: + self.sent.append(message) + + async def recv(self, decode: bool | None = None) -> str | bytes: + return await self.upstream.get() + + async def close(self) -> None: + return None + + +@pytest.mark.asyncio +async def test_keepalive_is_sent_only_while_the_client_is_idle_and_stops_after_end_of_stream(): + now = [0.0] + ticks: asyncio.Queue[float] = asyncio.Queue() + + async def sleep(_: float) -> None: + now[0] = await ticks.get() + + async def tick(at: float) -> None: + await ticks.put(at) + for _ in range(5): + await asyncio.sleep(0) + + inner = _RecordingBackend() + backend = SonioxRealtimeBackend(inner, keepalive_interval=5.0, clock=lambda: now[0], sleep=sleep) + + async with backend: + await tick(5.0) + assert inner.sent == ['{"type":"keepalive"}'] + now[0] = 7.0 + await backend.send(PCM) + await tick(10.0) + assert inner.sent == ['{"type":"keepalive"}', PCM] + await backend.send("") + await tick(30.0) + assert inner.sent == ['{"type":"keepalive"}', PCM, ""] + + assert inner.entered and inner.exited + + +@pytest.mark.asyncio +async def test_backend_announces_the_session_once_the_start_request_is_sent_while_relaying_upstream_frames(): + inner = _RecordingBackend() + backend = SonioxRealtimeBackend(inner) + + async with backend: + pending = asyncio.create_task(backend.recv()) + await backend.send('{"type":"keepalive"}') + await asyncio.sleep(0) + assert not pending.done() + await backend.send('{"model":"stt-rt-v5"}') + assert await pending == SESSION_STARTED_FRAME + await backend.send('{"model":"ignored-second-config"}') + await inner.upstream.put('{"tokens":[]}') + assert await backend.recv() == '{"tokens":[]}' diff --git a/tests/unit/realtime_api/test_main.py b/tests/unit/realtime_api/test_main.py index 26ed1050cea..245d97c8b01 100644 --- a/tests/unit/realtime_api/test_main.py +++ b/tests/unit/realtime_api/test_main.py @@ -1,4 +1,5 @@ import asyncio +import json import time from types import TracebackType from typing import Final @@ -651,3 +652,32 @@ async def test_arealtime_drops_model_from_the_upstream_url_only_for_transcriptio query_params=client_query_params, ) assert connect.url == expected_backend_url + + +@pytest.mark.asyncio +async def test_arealtime_routes_soniox_with_the_deployment_options(monkeypatch): + from litellm.llms.soniox.realtime.transformation import SonioxRealtimeConfig + + captured: dict[str, object] = {} + + async def mock_async_realtime(**kwargs): + captured.update(kwargs) + + monkeypatch.setattr(realtime_main.base_llm_http_handler, "async_realtime", mock_async_realtime) + await realtime_main._arealtime.__wrapped__( + model="soniox/stt-rt-v5", + websocket=MagicMock(), + litellm_logging_obj=FakeLogging(), + query_params={"model": "soniox/stt-rt-v5", "intent": "transcription"}, + api_key="deployment-key", + language_hints=["de"], + ) + + provider_config = captured["provider_config"] + assert isinstance(provider_config, SonioxRealtimeConfig) + assert captured["model"] == "stt-rt-v5" + assert captured["api_key"] == "deployment-key" + start_request = provider_config.transform_realtime_request( + json.dumps({"type": "input_audio_buffer.commit"}), "stt-rt-v5" + )[0] + assert json.loads(start_request)["language_hints"] == ["de"] From 24f8f8268a7f11b8d6d75813ac03284be5c0b39c Mon Sep 17 00:00:00 2001 From: Dan Lemon Date: Tue, 6 Oct 2026 10:07:53 +0200 Subject: [PATCH 02/13] fix(soniox): bill realtime usage from integer milliseconds --- litellm/llms/soniox/realtime/transformation.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/litellm/llms/soniox/realtime/transformation.py b/litellm/llms/soniox/realtime/transformation.py index 4fe9f170cb4..95dd963a53e 100644 --- a/litellm/llms/soniox/realtime/transformation.py +++ b/litellm/llms/soniox/realtime/transformation.py @@ -146,7 +146,7 @@ class SonioxEventTransformer: self._item_id: str | None = None self._final_text: str = "" self._processed_ms: int = 0 - self._unbilled_seconds: float = 0.0 + self._billed_ms: int = 0 def transform(self, payload: str) -> tuple[OpenAIRealtimeEvents, ...]: try: @@ -159,17 +159,15 @@ class SonioxEventTransformer: return tuple(event for token in response.tokens for event in self._token_events(token)) def take_unbilled_usage(self) -> RealtimeInputAudioTranscriptionUsage | None: - seconds: Final = self._unbilled_seconds - if seconds <= 0: + unbilled_ms: Final = self._processed_ms - self._billed_ms + if unbilled_ms <= 0: return None - self._unbilled_seconds = 0.0 - return duration_usage(seconds) + self._billed_ms = self._processed_ms + return duration_usage(unbilled_ms / 1000) def _bill(self, total_audio_proc_ms: int | None) -> None: - if total_audio_proc_ms is None or total_audio_proc_ms <= self._processed_ms: - return - self._unbilled_seconds += (total_audio_proc_ms - self._processed_ms) / 1000 - self._processed_ms = total_audio_proc_ms + if total_audio_proc_ms is not None and total_audio_proc_ms > self._processed_ms: + self._processed_ms = total_audio_proc_ms def _token_events(self, token: _SonioxToken) -> tuple[OpenAIRealtimeEvents, ...]: if token.text in (_ENDPOINT_TOKEN, _FINALIZED_TOKEN): From c260655ef7510028267953c429fb9ed22c13dc31 Mon Sep 17 00:00:00 2001 From: Dan Lemon Date: Tue, 6 Oct 2026 10:40:59 +0200 Subject: [PATCH 03/13] fix(soniox): bound session-start queue and cover realtime backend edge paths --- .../llms/soniox/realtime/transformation.py | 34 +++++----- .../test_soniox_realtime_transformation.py | 62 ++++++++++++++++++- 2 files changed, 78 insertions(+), 18 deletions(-) diff --git a/litellm/llms/soniox/realtime/transformation.py b/litellm/llms/soniox/realtime/transformation.py index 95dd963a53e..38648ad8c57 100644 --- a/litellm/llms/soniox/realtime/transformation.py +++ b/litellm/llms/soniox/realtime/transformation.py @@ -1,7 +1,7 @@ import asyncio import time from collections.abc import Awaitable, Callable -from types import TracebackType +from types import MappingProxyType, TracebackType from typing import Final from urllib.parse import urlparse @@ -12,6 +12,7 @@ from litellm import verbose_logger from litellm._uuid import uuid from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.realtime.transcription_protocol import ( + SESSION_UPDATE_EVENT_TYPES, RealtimeTranscriptionProtocolError, TranscriptionSessionUpdate, completed_event, @@ -41,6 +42,9 @@ _FINALIZE: Final = '{"type":"finalize"}' _KEEPALIVE: Final = '{"type":"keepalive"}' _END_STREAM: Final = "" _CONTROL_FRAMES: Final = frozenset((_FINALIZE, _KEEPALIVE, _END_STREAM)) +_CLIENT_CONTROL_FRAMES: Final = MappingProxyType( + {"input_audio_buffer.commit": _FINALIZE, "input_audio_buffer.end": _END_STREAM} +) SESSION_STARTED_FRAME: Final = '{"type":"litellm.soniox.session_started"}' _ENDPOINT_TOKEN: Final = "" _FINALIZED_TOKEN: Final = "" @@ -217,7 +221,7 @@ class SonioxRealtimeBackend: self._interval: Final = keepalive_interval self._clock: Final = clock self._sleep: Final = sleep - self._local_frames: Final[asyncio.Queue[str]] = asyncio.Queue() + self._local_frames: Final[asyncio.Queue[str]] = asyncio.Queue(maxsize=1) self._last_sent: float = clock() self._started: bool = False self._ended: bool = False @@ -247,8 +251,6 @@ class SonioxRealtimeBackend: self._local_frames.put_nowait(SESSION_STARTED_FRAME) async def recv(self, decode: bool | None = None) -> str | bytes: - if not self._local_frames.empty(): - return self._local_frames.get_nowait() upstream: Final = self._upstream_recv or asyncio.create_task(self._inner.recv(decode)) local: Final = asyncio.create_task(self._local_frames.get()) done, _ = await asyncio.wait((upstream, local), return_when=asyncio.FIRST_COMPLETED) @@ -326,19 +328,17 @@ class SonioxRealtimeConfig(BaseRealtimeConfig): session_configuration_request: str | None = None, ) -> tuple[str | bytes, ...]: request: Final = json_object(message, SonioxProtocolError) - match request.get("type"): - case "session.update" | "transcription_session.update": - return self._start(model, parse_transcription_session_update(message, SonioxProtocolError)) - case "input_audio_buffer.append": - audio: Final = decode_pcm16_append(request.get("audio"), error=SonioxProtocolError) - return (*self._start(model, None), audio) - case "input_audio_buffer.commit": - return (*self._start(model, None), _FINALIZE) - case "input_audio_buffer.end": - return (*self._start(model, None), _END_STREAM) - case event_type: - verbose_logger.debug("Soniox realtime: dropping unsupported client event %s", event_type) - return () + event_type: Final = request.get("type") + if event_type in SESSION_UPDATE_EVENT_TYPES: + return self._start(model, parse_transcription_session_update(message, SonioxProtocolError)) + if event_type == "input_audio_buffer.append": + audio: Final = decode_pcm16_append(request.get("audio"), error=SonioxProtocolError) + return (*self._start(model, None), audio) + control_frame: Final = _CLIENT_CONTROL_FRAMES.get(event_type) if isinstance(event_type, str) else None + if control_frame is None: + verbose_logger.debug("Soniox realtime: dropping unsupported client event %s", event_type) + return () + return (*self._start(model, None), control_frame) def unbilled_usage_on_session_close(self, model: str) -> RealtimeInputAudioTranscriptionUsage | None: return self._transformer.take_unbilled_usage() diff --git a/tests/unit/llms/soniox/realtime/test_soniox_realtime_transformation.py b/tests/unit/llms/soniox/realtime/test_soniox_realtime_transformation.py index 214119e7ade..2c98dd6a332 100644 --- a/tests/unit/llms/soniox/realtime/test_soniox_realtime_transformation.py +++ b/tests/unit/llms/soniox/realtime/test_soniox_realtime_transformation.py @@ -141,6 +141,18 @@ def test_unsupported_audio_formats_are_rejected_without_starting_the_stream(audi assert _start_request(config.transform_realtime_request(_append(), MODEL))["sample_rate"] == 24_000 +@pytest.mark.parametrize( + ("audio_input", "sample_rate"), + [({"format": {"type": "audio/pcm"}}, 24_000), ({"input_audio_format": "pcm16"}, 24_000)], +) +def test_pcm16_without_a_rate_streams_at_the_openai_default_rate(audio_input: dict[str, object], sample_rate: int): + config = SonioxRealtimeConfig() + + frames = config.transform_realtime_request(_session_update(audio_input), MODEL) + + assert _start_request(frames)["sample_rate"] == sample_rate + + def test_validate_environment_sends_the_key_as_a_bearer_header(monkeypatch): monkeypatch.setenv("SONIOX_API_KEY", "env-key") config = SonioxRealtimeConfig() @@ -343,6 +355,7 @@ class _RecordingBackend: self.upstream: Final[asyncio.Queue[str]] = asyncio.Queue() self.entered = False self.exited = False + self.closed = False async def __aenter__(self) -> "_RecordingBackend": self.entered = True @@ -358,7 +371,12 @@ class _RecordingBackend: return await self.upstream.get() async def close(self) -> None: - return None + self.closed = True + + +class _BrokenBackend(_RecordingBackend): + async def send(self, message: str | bytes) -> None: + raise ConnectionError("upstream gone") @pytest.mark.asyncio @@ -406,3 +424,45 @@ async def test_backend_announces_the_session_once_the_start_request_is_sent_whil await backend.send('{"model":"ignored-second-config"}') await inner.upstream.put('{"tokens":[]}') assert await backend.recv() == '{"tokens":[]}' + + +@pytest.mark.asyncio +async def test_backend_relays_upstream_frames_that_arrive_before_the_session_starts(): + inner = _RecordingBackend() + + async with SonioxRealtimeBackend(inner) as backend: + await inner.upstream.put('{"error_code":401,"error_message":"Invalid API key."}') + + assert await backend.recv() == '{"error_code":401,"error_message":"Invalid API key."}' + + +@pytest.mark.asyncio +async def test_closing_the_backend_cancels_pending_reads_and_closes_upstream(): + inner = _RecordingBackend() + backend = SonioxRealtimeBackend(inner) + await backend.__aenter__() + pending = asyncio.create_task(backend.recv()) + await backend.send('{"model":"stt-rt-v5"}') + assert await pending == SESSION_STARTED_FRAME + + await backend.close() + await inner.upstream.put('{"tokens":[]}') + await asyncio.sleep(0) + + assert inner.closed + assert inner.upstream.qsize() == 1 + + +@pytest.mark.asyncio +async def test_keepalive_stops_after_the_upstream_rejects_a_send(): + sleeps: list[float] = [] + + async def sleep(seconds: float) -> None: + sleeps.append(seconds) + await asyncio.sleep(0) + + async with SonioxRealtimeBackend(_BrokenBackend(), keepalive_interval=0.0, sleep=sleep): + for _ in range(5): + await asyncio.sleep(0) + + assert sleeps == [0.0] From 65207292bece55ec1fa082b4a3f23b2cb9ca46c9 Mon Sep 17 00:00:00 2001 From: Dan Lemon Date: Tue, 6 Oct 2026 17:24:23 +0200 Subject: [PATCH 04/13] fix(soniox): bill realtime stream time, start sessions on connect and surface client protocol errors Soniox charges for the whole open stream, so session close now bills the connection's open time. The backend announces the session on connect, which flushes audio sent before any session.update, and a single upstream pump makes recv safe to cancel. Invalid client frames now get an OpenAI error event instead of a silent disconnect --- .../litellm_core_utils/realtime_streaming.py | 7 +- .../llms/soniox/realtime/transformation.py | 77 +++++++------ litellm/utils.py | 4 +- .../custom_httpx/test_llm_http_handler.py | 101 +++++++++++++----- .../test_soniox_realtime_transformation.py | 54 ++++++++-- 5 files changed, 174 insertions(+), 69 deletions(-) diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index d2fbb26bb02..5e004476532 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -12,6 +12,7 @@ import litellm from litellm._logging import redact_internal_details_from_client_message, verbose_logger from litellm.constants import REALTIME_SESSION_FAILURE_LOGGED_KEY, REALTIME_SESSION_SUCCESS_LOGGED_KEY from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER +from litellm.llms.base_llm.realtime.transcription_protocol import RealtimeTranscriptionProtocolError from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeBackend from litellm.types.llms.openai import ( OpenAIRealtimeEvents, @@ -1551,7 +1552,11 @@ class RealTimeStreaming: # would permanently disable the injection if ``_send_to_backend`` # raised — neither this loop nor # ``_maybe_send_guardrail_turn_detection_update`` would retry. - sent = await self._send_to_backend(message) + try: + sent = await self._send_to_backend(message) + except RealtimeTranscriptionProtocolError as e: + await self.websocket.send_text(realtime_error_event(str(e), error_type="invalid_request_error")) + continue if guardrail_turn_detection_injected and sent: self._guardrail_turn_detection_update_sent = True diff --git a/litellm/llms/soniox/realtime/transformation.py b/litellm/llms/soniox/realtime/transformation.py index 38648ad8c57..e1b4516b7f5 100644 --- a/litellm/llms/soniox/realtime/transformation.py +++ b/litellm/llms/soniox/realtime/transformation.py @@ -1,6 +1,6 @@ import asyncio import time -from collections.abc import Awaitable, Callable +from collections.abc import Awaitable, Callable, Mapping from types import MappingProxyType, TracebackType from typing import Final from urllib.parse import urlparse @@ -38,6 +38,7 @@ from litellm.types.realtime import ( DEFAULT_SONIOX_REALTIME_URL: Final = "wss://stt-rt.soniox.com/transcribe-websocket" DEFAULT_SAMPLE_RATE: Final = 24_000 KEEPALIVE_INTERVAL_SECONDS: Final = 5.0 +_MAX_PENDING_FRAMES: Final = 64 _FINALIZE: Final = '{"type":"finalize"}' _KEEPALIVE: Final = '{"type":"keepalive"}' _END_STREAM: Final = "" @@ -45,6 +46,7 @@ _CONTROL_FRAMES: Final = frozenset((_FINALIZE, _KEEPALIVE, _END_STREAM)) _CLIENT_CONTROL_FRAMES: Final = MappingProxyType( {"input_audio_buffer.commit": _FINALIZE, "input_audio_buffer.end": _END_STREAM} ) +CONNECTED_FRAME: Final = '{"type":"litellm.soniox.connected"}' SESSION_STARTED_FRAME: Final = '{"type":"litellm.soniox.session_started"}' _ENDPOINT_TOKEN: Final = "" _FINALIZED_TOKEN: Final = "" @@ -162,11 +164,12 @@ class SonioxEventTransformer: self._bill(response.total_audio_proc_ms) return tuple(event for token in response.tokens for event in self._token_events(token)) - def take_unbilled_usage(self) -> RealtimeInputAudioTranscriptionUsage | None: - unbilled_ms: Final = self._processed_ms - self._billed_ms + def take_unbilled_usage(self, stream_ms: int = 0) -> RealtimeInputAudioTranscriptionUsage | None: + billable_ms: Final = max(self._processed_ms, stream_ms) + unbilled_ms: Final = billable_ms - self._billed_ms if unbilled_ms <= 0: return None - self._billed_ms = self._processed_ms + self._billed_ms = billable_ms return duration_usage(unbilled_ms / 1000) def _bill(self, total_audio_proc_ms: int | None) -> None: @@ -221,16 +224,16 @@ class SonioxRealtimeBackend: self._interval: Final = keepalive_interval self._clock: Final = clock self._sleep: Final = sleep - self._local_frames: Final[asyncio.Queue[str]] = asyncio.Queue(maxsize=1) + self._frames: Final[asyncio.Queue[str | bytes | Exception]] = asyncio.Queue(maxsize=_MAX_PENDING_FRAMES) self._last_sent: float = clock() self._started: bool = False self._ended: bool = False - self._keepalive_task: asyncio.Task[None] | None = None - self._upstream_recv: asyncio.Task[str | bytes] | None = None + self._tasks: tuple[asyncio.Task[None], ...] = () async def __aenter__(self) -> Self: await self._inner.__aenter__() - self._keepalive_task = asyncio.create_task(self._keepalive()) + self._frames.put_nowait(CONNECTED_FRAME) + self._tasks = (asyncio.create_task(self._pump()), asyncio.create_task(self._keepalive())) return self async def __aexit__( @@ -248,27 +251,30 @@ class SonioxRealtimeBackend: await self._inner.send(message) if not self._started and isinstance(message, str) and message not in _CONTROL_FRAMES: self._started = True - self._local_frames.put_nowait(SESSION_STARTED_FRAME) + await self._frames.put(SESSION_STARTED_FRAME) async def recv(self, decode: bool | None = None) -> str | bytes: - upstream: Final = self._upstream_recv or asyncio.create_task(self._inner.recv(decode)) - local: Final = asyncio.create_task(self._local_frames.get()) - done, _ = await asyncio.wait((upstream, local), return_when=asyncio.FIRST_COMPLETED) - if local in done: - self._upstream_recv = upstream - return local.result() - local.cancel() - self._upstream_recv = None - return upstream.result() + frame: Final = await self._frames.get() + if isinstance(frame, Exception): + raise frame + return frame async def close(self) -> None: self._stop() await self._inner.close() def _stop(self) -> None: - for task in (self._keepalive_task, self._upstream_recv): - if task is not None: - task.cancel() + for task in self._tasks: + task.cancel() + + async def _pump(self) -> None: + while True: + try: + frame = await self._inner.recv() + except Exception as e: + await self._frames.put(e) + return + await self._frames.put(frame) async def _keepalive(self) -> None: while not self._ended: @@ -288,12 +294,19 @@ class SonioxRealtimeConfig(BaseRealtimeConfig): *, options: SonioxRealtimeOptions | None = None, wrap: Callable[[RealtimeBackend], RealtimeBackend] = SonioxRealtimeBackend, + clock: Callable[[], float] = time.monotonic, ) -> None: self._options: Final = options or SonioxRealtimeOptions() self._wrap: Final = wrap + self._clock: Final = clock self._transformer: Final = SonioxEventTransformer(translated_only=self._options.translation is not None) self._start_request: SonioxStartRequest | None = None - self._session_id: str = f"sess_{uuid.uuid4().hex}" + self._session_id: Final = f"sess_{uuid.uuid4().hex}" + self._opened_at: float | None = None + + @classmethod + def from_litellm_params(cls, litellm_params: Mapping[str, object]) -> Self: + return cls(options=SonioxRealtimeOptions.model_validate(dict(litellm_params))) def validate_environment( self, @@ -310,17 +323,9 @@ class SonioxRealtimeConfig(BaseRealtimeConfig): return build_soniox_realtime_url(api_base) def wrap_backend(self, backend: RealtimeBackend) -> RealtimeBackend: + self._opened_at = self._clock() return self._wrap(backend) - def transform_session_created_event( - self, - model: str, - logging_session_id: str, - session_configuration_request: str | None = None, - ) -> OpenAIRealtimeTranscriptionSessionCreated: - self._session_id = logging_session_id - return _session_created(logging_session_id, build_start_request(model, None, self._options)) - def transform_realtime_request( self, message: str, @@ -341,7 +346,9 @@ class SonioxRealtimeConfig(BaseRealtimeConfig): return (*self._start(model, None), control_frame) def unbilled_usage_on_session_close(self, model: str) -> RealtimeInputAudioTranscriptionUsage | None: - return self._transformer.take_unbilled_usage() + opened_at: Final = self._opened_at + stream_ms: Final = 0 if opened_at is None else int((self._clock() - opened_at) * 1000) + return self._transformer.take_unbilled_usage(stream_ms) def transform_realtime_response( self, @@ -352,7 +359,7 @@ class SonioxRealtimeConfig(BaseRealtimeConfig): ) -> RealtimeResponseTypedDict: payload: Final = message.decode("utf-8") if isinstance(message, bytes) else message result: Final[RealtimeResponseTypedDict] = { - "response": list(self._backend_events(payload)), + "response": list(self._backend_events(payload, model)), "current_output_item_id": realtime_response_transform_input.get("current_output_item_id"), "current_response_id": realtime_response_transform_input.get("current_response_id"), "current_delta_chunks": realtime_response_transform_input.get("current_delta_chunks"), @@ -363,7 +370,9 @@ class SonioxRealtimeConfig(BaseRealtimeConfig): } return result - def _backend_events(self, payload: str) -> tuple[OpenAIRealtimeEvents, ...]: + def _backend_events(self, payload: str, model: str) -> tuple[OpenAIRealtimeEvents, ...]: + if payload == CONNECTED_FRAME: + return (_session_created(self._session_id, build_start_request(model, None, self._options)),) if payload != SESSION_STARTED_FRAME: return self._transformer.transform(payload) assert self._start_request is not None # the backend signals session start only after the start request diff --git a/litellm/utils.py b/litellm/utils.py index 599170f68cc..6bb1d6f1694 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9705,9 +9705,9 @@ class ProviderConfigManager: return MetaRealtimeConfig() if LlmProviders.SONIOX == provider: - from litellm.llms.soniox.realtime.transformation import SonioxRealtimeConfig, SonioxRealtimeOptions + from litellm.llms.soniox.realtime.transformation import SonioxRealtimeConfig - return SonioxRealtimeConfig(options=SonioxRealtimeOptions.model_validate(dict(litellm_params))) + return SonioxRealtimeConfig.from_litellm_params(litellm_params) return None @staticmethod diff --git a/tests/unit/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py index ea9dc5a73d7..84c0031014a 100644 --- a/tests/unit/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py @@ -4377,36 +4377,37 @@ async def test_async_realtime_bridges_a_transcription_session_through_the_provid assert [bytes(request.audio) for request in speech_client.requests[1:]] == [b"\x00\x01" * 800, b"\x00\x01" * 800] +class _FakeSonioxSocket: + def __init__(self) -> None: + self.sent: Final[list[str | bytes]] = [] + self._responses: Final[asyncio.Queue[str]] = asyncio.Queue() + + async def __aenter__(self) -> "_FakeSonioxSocket": + return self + + async def __aexit__(self, exc_type, exc_value, traceback) -> None: + return None + + async def send(self, message: str | bytes) -> None: + self.sent.append(message) + if message == '{"type":"finalize"}': + tokens = [{"text": "Hello", "is_final": True}, {"text": "", "is_final": True}] + await self._responses.put(json.dumps({"tokens": tokens, "total_audio_proc_ms": 1200})) + + async def recv(self, decode: bool | None = None) -> str: + return await self._responses.get() + + async def close(self) -> None: + return None + + @pytest.mark.asyncio async def test_async_realtime_bridges_a_soniox_transcription_through_the_wrapped_websocket(): import websockets.exceptions # noqa: F401 # binds the submodule so async_realtime's except clause resolves, as in the proxy process from litellm.llms.soniox.realtime.transformation import SonioxRealtimeBackend, SonioxRealtimeConfig - class FakeSonioxSocket: - def __init__(self) -> None: - self.sent: Final[list[str | bytes]] = [] - self._responses: Final[asyncio.Queue[str]] = asyncio.Queue() - - async def __aenter__(self) -> "FakeSonioxSocket": - return self - - async def __aexit__(self, exc_type, exc_value, traceback) -> None: - return None - - async def send(self, message: str | bytes) -> None: - self.sent.append(message) - if message == '{"type":"finalize"}': - tokens = [{"text": "Hello", "is_final": True}, {"text": "", "is_final": True}] - await self._responses.put(json.dumps({"tokens": tokens, "total_audio_proc_ms": 1200})) - - async def recv(self, decode: bool | None = None) -> str: - return await self._responses.get() - - async def close(self) -> None: - return None - - upstream = FakeSonioxSocket() + upstream = _FakeSonioxSocket() wrapped: list[object] = [] provider_config = SonioxRealtimeConfig( wrap=lambda backend: wrapped.append(backend) or SonioxRealtimeBackend(backend) @@ -4478,6 +4479,58 @@ async def test_async_realtime_bridges_a_soniox_transcription_through_the_wrapped assert events[-1]["usage"] == {"type": "duration", "seconds": 1.2} +@pytest.mark.asyncio +async def test_async_realtime_soniox_rejects_a_bad_session_update_and_still_transcribes_audio_sent_without_one(): + import websockets.exceptions # noqa: F401 # binds the submodule so async_realtime's except clause resolves, as in the proxy process + + from litellm.llms.soniox.realtime.transformation import SonioxRealtimeConfig + + upstream = _FakeSonioxSocket() + audio = b"\x00\x01" * 800 + client_ws = _ScriptedClientWebSocket( + [ + json.dumps( + { + "type": "session.update", + "session": {"type": "transcription", "audio": {"input": {"format": {"type": "audio/pcmu"}}}}, + } + ), + json.dumps({"type": "input_audio_buffer.append", "audio": base64.b64encode(audio).decode()}), + json.dumps({"type": "input_audio_buffer.commit"}), + ], + last_event_type="conversation.item.input_audio_transcription.completed", + ) + logging_obj = Mock() + logging_obj.litellm_trace_id = "trace_1" + logging_obj.model_call_details = {} + logging_obj.dispatch_success_handlers = AsyncMock() + logging_obj.dispatch_failure_handlers = AsyncMock() + handler = BaseLLMHTTPHandler() + + with patch.object(handler, "_open_realtime_backend_ws", AsyncMock(return_value=upstream)): + await handler.async_realtime( + model="stt-rt-v5", + websocket=client_ws, + logging_obj=logging_obj, + provider_config=SonioxRealtimeConfig(), + headers={}, + api_key="soniox-key", + query_params={"model": "stt-rt-v5", "intent": "transcription"}, + ) + + assert json.loads(upstream.sent[0])["sample_rate"] == 24_000 + assert upstream.sent[1:] == [audio, '{"type":"finalize"}'] + events = client_ws.sent_events() + errors = [event for event in events if event["type"] == "error"] + assert errors == [ + { + "type": "error", + "error": {"type": "invalid_request_error", "message": "Soniox realtime requires pcm16 input audio"}, + } + ] + assert events[-1]["transcript"] == "Hello" + + @pytest.mark.asyncio async def test_responses_agentic_followup_does_not_repeat_request_params_from_plan_kwargs(monkeypatch): """A plan whose kwargs repeat a request param must not crash the Responses follow-up with a duplicate keyword""" diff --git a/tests/unit/llms/soniox/realtime/test_soniox_realtime_transformation.py b/tests/unit/llms/soniox/realtime/test_soniox_realtime_transformation.py index 2c98dd6a332..e565f42a4ba 100644 --- a/tests/unit/llms/soniox/realtime/test_soniox_realtime_transformation.py +++ b/tests/unit/llms/soniox/realtime/test_soniox_realtime_transformation.py @@ -7,6 +7,7 @@ from unittest.mock import MagicMock import pytest from litellm.llms.soniox.realtime.transformation import ( + CONNECTED_FRAME, DEFAULT_SONIOX_REALTIME_URL, SESSION_STARTED_FRAME, SonioxEventTransformer, @@ -288,18 +289,31 @@ def test_translation_option_switches_the_config_to_translated_transcripts(): assert [event.get("delta") for event in result["response"] if "delta" in event] == ["Hello"] -def test_session_created_event_advertises_the_deployment_model_for_cost_tracking(): +def test_stream_open_time_is_billed_on_close_because_soniox_charges_the_whole_stream(): + now = [100.0] + config = SonioxRealtimeConfig(clock=lambda: now[0]) + config.wrap_backend(_RecordingBackend()) + config.transform_realtime_response( + _response([_token("Hi"), _token("")], total_audio_proc_ms=5000), MODEL, MagicMock(), EMPTY_TRANSFORM_INPUT + ) + now[0] = 130.5 + + assert config.unbilled_usage_on_session_close(MODEL) == {"type": "duration", "seconds": 25.5} + assert config.unbilled_usage_on_session_close(MODEL) is None + + +def test_connection_is_announced_with_the_deployment_session_for_cost_tracking(): config = SonioxRealtimeConfig(options=SonioxRealtimeOptions(language_hints=("de",))) - event = config.transform_session_created_event(MODEL, "session_1") + result = config.transform_realtime_response(CONNECTED_FRAME, MODEL, MagicMock(), EMPTY_TRANSFORM_INPUT) - assert event["session"]["id"] == "session_1" - assert event["session"]["audio"]["input"]["transcription"] == {"model": MODEL, "language": "de"} + assert _types(tuple(result["response"])) == ["session.created"] + assert result["response"][0]["session"]["audio"]["input"]["transcription"] == {"model": MODEL, "language": "de"} def test_backend_session_start_is_announced_with_the_negotiated_session(): config = SonioxRealtimeConfig() - config.transform_session_created_event(MODEL, "session_1") + connected = config.transform_realtime_response(CONNECTED_FRAME, MODEL, MagicMock(), EMPTY_TRANSFORM_INPUT) config.transform_realtime_request( _session_update( { @@ -314,7 +328,7 @@ def test_backend_session_start_is_announced_with_the_negotiated_session(): result = config.transform_realtime_response(SESSION_STARTED_FRAME, MODEL, MagicMock(), EMPTY_TRANSFORM_INPUT) assert _types(tuple(result["response"])) == ["session.created"] - assert result["response"][0]["session"]["id"] == "session_1" + assert result["response"][0]["session"]["id"] == connected["response"][0]["session"]["id"] assert result["response"][0]["session"]["audio"]["input"] == { "format": {"type": "audio/pcm", "rate": 16_000}, "transcription": {"model": MODEL, "language": "it"}, @@ -352,7 +366,7 @@ def test_provider_config_manager_passes_only_soniox_options_from_deployment_para class _RecordingBackend: def __init__(self) -> None: self.sent: Final[list[str | bytes]] = [] - self.upstream: Final[asyncio.Queue[str]] = asyncio.Queue() + self.upstream: Final[asyncio.Queue[str | Exception]] = asyncio.Queue() self.entered = False self.exited = False self.closed = False @@ -368,7 +382,10 @@ class _RecordingBackend: self.sent.append(message) async def recv(self, decode: bool | None = None) -> str | bytes: - return await self.upstream.get() + frame = await self.upstream.get() + if isinstance(frame, Exception): + raise frame + return frame async def close(self) -> None: self.closed = True @@ -415,6 +432,7 @@ async def test_backend_announces_the_session_once_the_start_request_is_sent_whil backend = SonioxRealtimeBackend(inner) async with backend: + assert await backend.recv() == CONNECTED_FRAME pending = asyncio.create_task(backend.recv()) await backend.send('{"type":"keepalive"}') await asyncio.sleep(0) @@ -433,14 +451,34 @@ async def test_backend_relays_upstream_frames_that_arrive_before_the_session_sta async with SonioxRealtimeBackend(inner) as backend: await inner.upstream.put('{"error_code":401,"error_message":"Invalid API key."}') + assert await backend.recv() == CONNECTED_FRAME assert await backend.recv() == '{"error_code":401,"error_message":"Invalid API key."}' +@pytest.mark.asyncio +async def test_a_cancelled_read_loses_no_frames_and_upstream_errors_reach_the_reader(): + inner = _RecordingBackend() + + async with SonioxRealtimeBackend(inner) as backend: + assert await backend.recv() == CONNECTED_FRAME + cancelled = asyncio.create_task(backend.recv()) + await asyncio.sleep(0) + cancelled.cancel() + await asyncio.gather(cancelled, return_exceptions=True) + await inner.upstream.put('{"tokens":[]}') + await inner.upstream.put(ConnectionError("upstream closed")) + + assert await backend.recv() == '{"tokens":[]}' + with pytest.raises(ConnectionError, match="upstream closed"): + await backend.recv() + + @pytest.mark.asyncio async def test_closing_the_backend_cancels_pending_reads_and_closes_upstream(): inner = _RecordingBackend() backend = SonioxRealtimeBackend(inner) await backend.__aenter__() + assert await backend.recv() == CONNECTED_FRAME pending = asyncio.create_task(backend.recv()) await backend.send('{"model":"stt-rt-v5"}') assert await pending == SESSION_STARTED_FRAME From 96d4cc58ebfb55e8f9b6315dbe6402bc46f47649 Mon Sep 17 00:00:00 2001 From: Dan Lemon Date: Tue, 6 Oct 2026 17:49:44 +0200 Subject: [PATCH 05/13] fix(soniox): skip deferred setup so client frames reach Soniox in order Drops the connect marker frame, which counted as an upstream frame and hid refusals, and could start a buffered stream at the default rate before the client's session.update. Soniox now opts out of deferred setup, the handler announces a synthetic session.created whenever no setup went upstream, a session.update that would change a started stream gets an error event, and stream time is billed only once a stream started --- litellm/llms/custom_httpx/llm_http_handler.py | 6 +-- .../llms/soniox/realtime/transformation.py | 36 ++++++++++----- .../test_soniox_realtime_transformation.py | 45 +++++++++++++------ 3 files changed, 58 insertions(+), 29 deletions(-) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 9b22d1347ae..ca82652b5da 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -6347,9 +6347,9 @@ class BaseLLMHTTPHandler: await backend_ws.send(_session_config) realtime_streaming.session_configuration_request = _session_config - # For providers that defer setup until client session.update, optionally - # send synthetic session.created to unblock clients waiting on connect. - if not provider_config.requires_session_configuration(): + # When no setup went upstream, optionally send a synthetic + # session.created to unblock clients waiting on connect. + if not _session_config: synthetic_session: Final = provider_config.transform_session_created_event( model=model, logging_session_id=logging_obj.litellm_trace_id, diff --git a/litellm/llms/soniox/realtime/transformation.py b/litellm/llms/soniox/realtime/transformation.py index e1b4516b7f5..099f2675bd0 100644 --- a/litellm/llms/soniox/realtime/transformation.py +++ b/litellm/llms/soniox/realtime/transformation.py @@ -46,7 +46,6 @@ _CONTROL_FRAMES: Final = frozenset((_FINALIZE, _KEEPALIVE, _END_STREAM)) _CLIENT_CONTROL_FRAMES: Final = MappingProxyType( {"input_audio_buffer.commit": _FINALIZE, "input_audio_buffer.end": _END_STREAM} ) -CONNECTED_FRAME: Final = '{"type":"litellm.soniox.connected"}' SESSION_STARTED_FRAME: Final = '{"type":"litellm.soniox.session_started"}' _ENDPOINT_TOKEN: Final = "" _FINALIZED_TOKEN: Final = "" @@ -232,7 +231,6 @@ class SonioxRealtimeBackend: async def __aenter__(self) -> Self: await self._inner.__aenter__() - self._frames.put_nowait(CONNECTED_FRAME) self._tasks = (asyncio.create_task(self._pump()), asyncio.create_task(self._keepalive())) return self @@ -301,13 +299,16 @@ class SonioxRealtimeConfig(BaseRealtimeConfig): self._clock: Final = clock self._transformer: Final = SonioxEventTransformer(translated_only=self._options.translation is not None) self._start_request: SonioxStartRequest | None = None - self._session_id: Final = f"sess_{uuid.uuid4().hex}" + self._session_id: str = f"sess_{uuid.uuid4().hex}" self._opened_at: float | None = None @classmethod def from_litellm_params(cls, litellm_params: Mapping[str, object]) -> Self: return cls(options=SonioxRealtimeOptions.model_validate(dict(litellm_params))) + def requires_session_configuration(self) -> bool: + return True + def validate_environment( self, headers: dict[str, str], # mutable-ok: BaseRealtimeConfig contract @@ -326,6 +327,15 @@ class SonioxRealtimeConfig(BaseRealtimeConfig): self._opened_at = self._clock() return self._wrap(backend) + def transform_session_created_event( + self, + model: str, + logging_session_id: str, + session_configuration_request: str | None = None, + ) -> OpenAIRealtimeTranscriptionSessionCreated: + self._session_id = logging_session_id + return _session_created(logging_session_id, build_start_request(model, None, self._options)) + def transform_realtime_request( self, message: str, @@ -347,8 +357,9 @@ class SonioxRealtimeConfig(BaseRealtimeConfig): def unbilled_usage_on_session_close(self, model: str) -> RealtimeInputAudioTranscriptionUsage | None: opened_at: Final = self._opened_at - stream_ms: Final = 0 if opened_at is None else int((self._clock() - opened_at) * 1000) - return self._transformer.take_unbilled_usage(stream_ms) + if opened_at is None or self._start_request is None: + return self._transformer.take_unbilled_usage() + return self._transformer.take_unbilled_usage(int((self._clock() - opened_at) * 1000)) def transform_realtime_response( self, @@ -359,7 +370,7 @@ class SonioxRealtimeConfig(BaseRealtimeConfig): ) -> RealtimeResponseTypedDict: payload: Final = message.decode("utf-8") if isinstance(message, bytes) else message result: Final[RealtimeResponseTypedDict] = { - "response": list(self._backend_events(payload, model)), + "response": list(self._backend_events(payload)), "current_output_item_id": realtime_response_transform_input.get("current_output_item_id"), "current_response_id": realtime_response_transform_input.get("current_response_id"), "current_delta_chunks": realtime_response_transform_input.get("current_delta_chunks"), @@ -370,19 +381,20 @@ class SonioxRealtimeConfig(BaseRealtimeConfig): } return result - def _backend_events(self, payload: str, model: str) -> tuple[OpenAIRealtimeEvents, ...]: - if payload == CONNECTED_FRAME: - return (_session_created(self._session_id, build_start_request(model, None, self._options)),) + def _backend_events(self, payload: str) -> tuple[OpenAIRealtimeEvents, ...]: if payload != SESSION_STARTED_FRAME: return self._transformer.transform(payload) assert self._start_request is not None # the backend signals session start only after the start request return (_session_created(self._session_id, self._start_request),) def _start(self, model: str, update: TranscriptionSessionUpdate | None) -> tuple[str, ...]: - if self._start_request is not None: - if update is not None: - verbose_logger.debug("Soniox realtime: ignoring session.update after the stream was configured") + started: Final = self._start_request + if started is not None and update is None: return () start_request: Final = build_start_request(model, update, self._options) + if started is not None: + if start_request != started: + raise SonioxProtocolError("Soniox realtime can't change the session once the stream has started") + return () self._start_request = start_request return (start_request.model_dump_json(exclude_none=True),) diff --git a/tests/unit/llms/soniox/realtime/test_soniox_realtime_transformation.py b/tests/unit/llms/soniox/realtime/test_soniox_realtime_transformation.py index e565f42a4ba..4a208d2743f 100644 --- a/tests/unit/llms/soniox/realtime/test_soniox_realtime_transformation.py +++ b/tests/unit/llms/soniox/realtime/test_soniox_realtime_transformation.py @@ -7,7 +7,6 @@ from unittest.mock import MagicMock import pytest from litellm.llms.soniox.realtime.transformation import ( - CONNECTED_FRAME, DEFAULT_SONIOX_REALTIME_URL, SESSION_STARTED_FRAME, SonioxEventTransformer, @@ -90,12 +89,13 @@ def test_session_update_starts_the_stream_with_client_audio_and_deployment_optio } -def test_audio_before_session_update_starts_a_default_stream_once(): +def test_audio_before_session_update_starts_a_default_stream_once_and_rejects_a_later_change(): config = SonioxRealtimeConfig() first = config.transform_realtime_request(_append(), MODEL) second = config.transform_realtime_request(_append(), MODEL) - late_update = config.transform_realtime_request(_session_update({"turn_detection": None}), MODEL) + with pytest.raises(SonioxProtocolError, match="once the stream has started"): + config.transform_realtime_request(_session_update({"turn_detection": None}), MODEL) assert _start_request(first) == { "model": MODEL, @@ -106,7 +106,7 @@ def test_audio_before_session_update_starts_a_default_stream_once(): } assert first[1:] == (PCM,) assert second == (PCM,) - assert late_update == () + assert config.transform_realtime_request(_append(), MODEL) == (PCM,) @pytest.mark.parametrize( @@ -293,6 +293,7 @@ def test_stream_open_time_is_billed_on_close_because_soniox_charges_the_whole_st now = [100.0] config = SonioxRealtimeConfig(clock=lambda: now[0]) config.wrap_backend(_RecordingBackend()) + config.transform_realtime_request(_append(), MODEL) config.transform_realtime_response( _response([_token("Hi"), _token("")], total_audio_proc_ms=5000), MODEL, MagicMock(), EMPTY_TRANSFORM_INPUT ) @@ -302,18 +303,38 @@ def test_stream_open_time_is_billed_on_close_because_soniox_charges_the_whole_st assert config.unbilled_usage_on_session_close(MODEL) is None -def test_connection_is_announced_with_the_deployment_session_for_cost_tracking(): +def test_a_connection_that_never_started_a_stream_bills_nothing(): + now = [100.0] + config = SonioxRealtimeConfig(clock=lambda: now[0]) + config.wrap_backend(_RecordingBackend()) + now[0] = 160.0 + + assert config.unbilled_usage_on_session_close(MODEL) is None + + +def test_soniox_opts_out_of_deferred_setup_so_client_frames_reach_it_in_order(): + assert SonioxRealtimeConfig().requires_session_configuration() + + +def test_a_session_update_matching_the_started_stream_is_a_no_op(): + config = SonioxRealtimeConfig() + config.transform_realtime_request(_append(), MODEL) + + assert config.transform_realtime_request(_session_update({"format": {"type": "audio/pcm", "rate": 24_000}}), MODEL) == () + + +def test_session_created_event_advertises_the_deployment_model_for_cost_tracking(): config = SonioxRealtimeConfig(options=SonioxRealtimeOptions(language_hints=("de",))) - result = config.transform_realtime_response(CONNECTED_FRAME, MODEL, MagicMock(), EMPTY_TRANSFORM_INPUT) + event = config.transform_session_created_event(MODEL, "session_1") - assert _types(tuple(result["response"])) == ["session.created"] - assert result["response"][0]["session"]["audio"]["input"]["transcription"] == {"model": MODEL, "language": "de"} + assert event["session"]["id"] == "session_1" + assert event["session"]["audio"]["input"]["transcription"] == {"model": MODEL, "language": "de"} def test_backend_session_start_is_announced_with_the_negotiated_session(): config = SonioxRealtimeConfig() - connected = config.transform_realtime_response(CONNECTED_FRAME, MODEL, MagicMock(), EMPTY_TRANSFORM_INPUT) + config.transform_session_created_event(MODEL, "session_1") config.transform_realtime_request( _session_update( { @@ -328,7 +349,7 @@ def test_backend_session_start_is_announced_with_the_negotiated_session(): result = config.transform_realtime_response(SESSION_STARTED_FRAME, MODEL, MagicMock(), EMPTY_TRANSFORM_INPUT) assert _types(tuple(result["response"])) == ["session.created"] - assert result["response"][0]["session"]["id"] == connected["response"][0]["session"]["id"] + assert result["response"][0]["session"]["id"] == "session_1" assert result["response"][0]["session"]["audio"]["input"] == { "format": {"type": "audio/pcm", "rate": 16_000}, "transcription": {"model": MODEL, "language": "it"}, @@ -432,7 +453,6 @@ async def test_backend_announces_the_session_once_the_start_request_is_sent_whil backend = SonioxRealtimeBackend(inner) async with backend: - assert await backend.recv() == CONNECTED_FRAME pending = asyncio.create_task(backend.recv()) await backend.send('{"type":"keepalive"}') await asyncio.sleep(0) @@ -451,7 +471,6 @@ async def test_backend_relays_upstream_frames_that_arrive_before_the_session_sta async with SonioxRealtimeBackend(inner) as backend: await inner.upstream.put('{"error_code":401,"error_message":"Invalid API key."}') - assert await backend.recv() == CONNECTED_FRAME assert await backend.recv() == '{"error_code":401,"error_message":"Invalid API key."}' @@ -460,7 +479,6 @@ async def test_a_cancelled_read_loses_no_frames_and_upstream_errors_reach_the_re inner = _RecordingBackend() async with SonioxRealtimeBackend(inner) as backend: - assert await backend.recv() == CONNECTED_FRAME cancelled = asyncio.create_task(backend.recv()) await asyncio.sleep(0) cancelled.cancel() @@ -478,7 +496,6 @@ async def test_closing_the_backend_cancels_pending_reads_and_closes_upstream(): inner = _RecordingBackend() backend = SonioxRealtimeBackend(inner) await backend.__aenter__() - assert await backend.recv() == CONNECTED_FRAME pending = asyncio.create_task(backend.recv()) await backend.send('{"model":"stt-rt-v5"}') assert await pending == SESSION_STARTED_FRAME From d4e8b8ec105e065a73c8fc9c0387d5c216899a1e Mon Sep 17 00:00:00 2001 From: Dan Lemon Date: Tue, 6 Oct 2026 18:02:24 +0200 Subject: [PATCH 06/13] fix(soniox): judge a follow-up session.update only on the settings it sends A partial follow-up session.update was rebuilt with defaults for the settings it left out, so restating part of a 16 kHz or VAD-off stream was rejected as a change --- .../llms/soniox/realtime/transformation.py | 18 +++++++-- .../test_soniox_realtime_transformation.py | 40 +++++++++++++++++++ 2 files changed, 54 insertions(+), 4 deletions(-) diff --git a/litellm/llms/soniox/realtime/transformation.py b/litellm/llms/soniox/realtime/transformation.py index 099f2675bd0..23c7cb54245 100644 --- a/litellm/llms/soniox/realtime/transformation.py +++ b/litellm/llms/soniox/realtime/transformation.py @@ -137,6 +137,18 @@ def _session_created(session_id: str, start_request: SonioxStartRequest) -> Open return transcription_session_created_event(session) +def _changes_started_stream(update: TranscriptionSessionUpdate, started: SonioxStartRequest) -> bool: + turn_detection_set: Final = update.turn_detection is not None or update.turn_detection_disabled + return ( + (update.audio_format is not None and _sample_rate(update) != started.sample_rate) + or ( + update.language is not None + and _language_hints(update.language, started.language_hints) != started.language_hints + ) + or (turn_detection_set and update.turn_detection_disabled == started.enable_endpoint_detection) + ) + + def build_soniox_realtime_url(api_base: str | None) -> str: if api_base is None: return DEFAULT_SONIOX_REALTIME_URL @@ -389,12 +401,10 @@ class SonioxRealtimeConfig(BaseRealtimeConfig): def _start(self, model: str, update: TranscriptionSessionUpdate | None) -> tuple[str, ...]: started: Final = self._start_request - if started is not None and update is None: - return () - start_request: Final = build_start_request(model, update, self._options) if started is not None: - if start_request != started: + if update is not None and _changes_started_stream(update, started): raise SonioxProtocolError("Soniox realtime can't change the session once the stream has started") return () + start_request: Final = build_start_request(model, update, self._options) self._start_request = start_request return (start_request.model_dump_json(exclude_none=True),) diff --git a/tests/unit/llms/soniox/realtime/test_soniox_realtime_transformation.py b/tests/unit/llms/soniox/realtime/test_soniox_realtime_transformation.py index 4a208d2743f..8c6a837fa04 100644 --- a/tests/unit/llms/soniox/realtime/test_soniox_realtime_transformation.py +++ b/tests/unit/llms/soniox/realtime/test_soniox_realtime_transformation.py @@ -323,6 +323,46 @@ def test_a_session_update_matching_the_started_stream_is_a_no_op(): assert config.transform_realtime_request(_session_update({"format": {"type": "audio/pcm", "rate": 24_000}}), MODEL) == () +def _stream_started_at_16khz_in_french_without_vad() -> SonioxRealtimeConfig: + config = SonioxRealtimeConfig() + config.transform_realtime_request( + _session_update( + {"format": {"type": "audio/pcm", "rate": 16_000}, "transcription": {"language": "fr"}, "turn_detection": None} + ), + MODEL, + ) + return config + + +@pytest.mark.parametrize( + "follow_up", + [ + {"transcription": {"language": "fr"}}, + {"format": {"type": "audio/pcm", "rate": 16_000}}, + {"turn_detection": None}, + ], +) +def test_a_partial_follow_up_update_that_restates_the_stream_is_a_no_op(follow_up: dict[str, object]): + config = _stream_started_at_16khz_in_french_without_vad() + + assert config.transform_realtime_request(_session_update(follow_up), MODEL) == () + + +@pytest.mark.parametrize( + "follow_up", + [ + {"transcription": {"language": "de"}}, + {"format": {"type": "audio/pcm", "rate": 24_000}}, + {"turn_detection": {"type": "server_vad"}}, + ], +) +def test_a_partial_follow_up_update_that_changes_a_sent_setting_is_rejected(follow_up: dict[str, object]): + config = _stream_started_at_16khz_in_french_without_vad() + + with pytest.raises(SonioxProtocolError, match="once the stream has started"): + config.transform_realtime_request(_session_update(follow_up), MODEL) + + def test_session_created_event_advertises_the_deployment_model_for_cost_tracking(): config = SonioxRealtimeConfig(options=SonioxRealtimeOptions(language_hints=("de",))) From b28e76fa203ee79cb8b783b22ee6187fdecac103 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 6 Oct 2026 13:50:40 -0700 Subject: [PATCH 07/13] fix(soniox): treat an omitted rate in a follow-up session.update as unchanged --- litellm/llms/soniox/realtime/transformation.py | 10 +++++++++- .../realtime/test_soniox_realtime_transformation.py | 8 ++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/litellm/llms/soniox/realtime/transformation.py b/litellm/llms/soniox/realtime/transformation.py index 23c7cb54245..f8ce7e242a6 100644 --- a/litellm/llms/soniox/realtime/transformation.py +++ b/litellm/llms/soniox/realtime/transformation.py @@ -137,10 +137,18 @@ def _session_created(session_id: str, start_request: SonioxStartRequest) -> Open return transcription_session_created_event(session) +def _changes_started_rate(update: TranscriptionSessionUpdate, started: SonioxStartRequest) -> bool: + audio_format: Final = update.audio_format + if audio_format is None: + return False + requested_rate: Final = _sample_rate(update) + return audio_format.rate is not None and requested_rate != started.sample_rate + + def _changes_started_stream(update: TranscriptionSessionUpdate, started: SonioxStartRequest) -> bool: turn_detection_set: Final = update.turn_detection is not None or update.turn_detection_disabled return ( - (update.audio_format is not None and _sample_rate(update) != started.sample_rate) + _changes_started_rate(update, started) or ( update.language is not None and _language_hints(update.language, started.language_hints) != started.language_hints diff --git a/tests/unit/llms/soniox/realtime/test_soniox_realtime_transformation.py b/tests/unit/llms/soniox/realtime/test_soniox_realtime_transformation.py index 8c6a837fa04..257692d07a1 100644 --- a/tests/unit/llms/soniox/realtime/test_soniox_realtime_transformation.py +++ b/tests/unit/llms/soniox/realtime/test_soniox_realtime_transformation.py @@ -339,6 +339,7 @@ def _stream_started_at_16khz_in_french_without_vad() -> SonioxRealtimeConfig: [ {"transcription": {"language": "fr"}}, {"format": {"type": "audio/pcm", "rate": 16_000}}, + {"format": {"type": "audio/pcm"}}, {"turn_detection": None}, ], ) @@ -348,6 +349,13 @@ def test_a_partial_follow_up_update_that_restates_the_stream_is_a_no_op(follow_u assert config.transform_realtime_request(_session_update(follow_up), MODEL) == () +def test_a_follow_up_update_with_an_unsupported_encoding_is_still_rejected(): + config = _stream_started_at_16khz_in_french_without_vad() + + with pytest.raises(SonioxProtocolError, match="requires pcm16"): + config.transform_realtime_request(_session_update({"format": {"type": "audio/pcmu"}}), MODEL) + + @pytest.mark.parametrize( "follow_up", [ From 36ef783c45b3e1f215b5565d1fdb69c544262641 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:35:58 -0700 Subject: [PATCH 08/13] chore(soniox): drop comments that restate the code --- litellm/llms/custom_httpx/llm_http_handler.py | 2 -- litellm/llms/soniox/realtime/transformation.py | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index ca82652b5da..ca6657ef8d2 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -6347,8 +6347,6 @@ class BaseLLMHTTPHandler: await backend_ws.send(_session_config) realtime_streaming.session_configuration_request = _session_config - # When no setup went upstream, optionally send a synthetic - # session.created to unblock clients waiting on connect. if not _session_config: synthetic_session: Final = provider_config.transform_session_created_event( model=model, diff --git a/litellm/llms/soniox/realtime/transformation.py b/litellm/llms/soniox/realtime/transformation.py index f8ce7e242a6..6e9c1f9a515 100644 --- a/litellm/llms/soniox/realtime/transformation.py +++ b/litellm/llms/soniox/realtime/transformation.py @@ -214,7 +214,7 @@ class SonioxEventTransformer: return (speech_event("input_audio_buffer.speech_started", item_id),) def _current_item_id(self) -> str: - assert self._item_id is not None # _start always runs before a delta is emitted + assert self._item_id is not None return self._item_id def _complete(self, *, speech_stopped: bool) -> tuple[OpenAIRealtimeEvents, ...]: @@ -404,7 +404,7 @@ class SonioxRealtimeConfig(BaseRealtimeConfig): def _backend_events(self, payload: str) -> tuple[OpenAIRealtimeEvents, ...]: if payload != SESSION_STARTED_FRAME: return self._transformer.transform(payload) - assert self._start_request is not None # the backend signals session start only after the start request + assert self._start_request is not None return (_session_created(self._session_id, self._start_request),) def _start(self, model: str, update: TranscriptionSessionUpdate | None) -> tuple[str, ...]: From 92f6c0097877b20bef6ef1ad15d23004a0aa86f3 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 19:44:13 -0700 Subject: [PATCH 09/13] fix(soniox): return speaker labels, log refused sessions as failures, and probe realtime health Deltas carry speaker and language when diarization or language identification is on, and the completed event lists segments per speaker and language. A session Soniox refuses before any audio is processed is logged as a failure with no spend. Billing starts at the start request. Deferred-setup providers (Meta, Vertex Chirp) send the error event and still end the session. Soniox realtime models get a websocket /health probe and cost rows for the stt-rt-v5 aliases. --- .../health_check_helpers.py | 8 +- .../litellm_core_utils/realtime_streaming.py | 19 ++- .../llms/base_llm/realtime/transformation.py | 3 + litellm/llms/soniox/common_utils.py | 8 +- .../llms/soniox/realtime/transformation.py | 143 +++++++++++++++--- ...odel_prices_and_context_window_backup.json | 100 ++++++++++++ litellm/realtime_api/main.py | 3 + model_prices_and_context_window.json | 100 ++++++++++++ .../test_health_check_helpers.py | 5 +- .../test_realtime_streaming.py | 125 +++++++++++++-- .../test_soniox_realtime_transformation.py | 91 ++++++++++- tests/unit/realtime_api/test_main.py | 50 +++++- 12 files changed, 608 insertions(+), 47 deletions(-) diff --git a/litellm/litellm_core_utils/health_check_helpers.py b/litellm/litellm_core_utils/health_check_helpers.py index b19bcd941fa..940730eeb9f 100644 --- a/litellm/litellm_core_utils/health_check_helpers.py +++ b/litellm/litellm_core_utils/health_check_helpers.py @@ -31,7 +31,13 @@ EXTRACTION_HEALTH_CHECK_FIELDS: Final[Mapping[str, Mapping[str, str]]] = Mapping ) -def native_health_check_mode(model: str, custom_llm_provider: str | None) -> Literal["anthropic_messages"] | None: +def native_health_check_mode( + model: str, custom_llm_provider: str | None +) -> Literal["anthropic_messages", "realtime"] | None: + if custom_llm_provider == "soniox": + from litellm.llms.soniox.common_utils import soniox_health_check_mode + + return soniox_health_check_mode(model) if custom_llm_provider != "bedrock_mantle": return None from litellm.llms.bedrock_mantle.common_utils import mantle_health_check_mode diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index f80a60135a2..484bacdc72c 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -1282,24 +1282,29 @@ class RealTimeStreaming: verbose_logger.exception("Connection closed in backend to client send messages - %s", e) close: Final = backend_close_from(e) self._flush_unbilled_transcription_usage() - if self._backend_refused_session(close): - await self.log_backend_refusal(e) - else: - await self.log_messages() + await self._log_session_end(e if self._backend_refused_session(close) else None) return close except asyncio.CancelledError: self._flush_unbilled_transcription_usage() - await self.log_messages() + await self._log_session_end(None) raise except Exception as e: verbose_logger.exception("Error in backend to client send messages: %s", e) self._flush_unbilled_transcription_usage() - await self.log_messages() + await self._log_session_end(None) return BackendClose(code=1011, reason="proxy failed while relaying the upstream websocket") def _backend_refused_session(self, close: BackendClose) -> bool: return close.code != 1000 and not self._backend_sent_frames + async def _log_session_end(self, transport_refusal: Exception | None) -> None: + provider_refusal: Final = None if self.provider_config is None else self.provider_config.session_refusal() + refusal: Final = provider_refusal or transport_refusal + if refusal is None: + await self.log_messages() + return + await self.log_backend_refusal(refusal) + async def log_backend_refusal(self, error: Exception) -> None: if not self.logging_obj: return @@ -1696,6 +1701,8 @@ class RealTimeStreaming: sent = await self._send_to_backend(message) except RealtimeTranscriptionProtocolError as e: await self.websocket.send_text(realtime_error_event(str(e), error_type="invalid_request_error")) + if self._uses_deferred_backend_setup(): + raise continue if guardrail_turn_detection_injected and sent: self._guardrail_turn_detection_update_sent = True diff --git a/litellm/llms/base_llm/realtime/transformation.py b/litellm/llms/base_llm/realtime/transformation.py index e07e7be0d72..f1539d16b30 100644 --- a/litellm/llms/base_llm/realtime/transformation.py +++ b/litellm/llms/base_llm/realtime/transformation.py @@ -96,6 +96,9 @@ class BaseRealtimeConfig(ABC): def unbilled_usage_on_session_close(self, model: str) -> RealtimeInputAudioTranscriptionUsage | None: return None + def session_refusal(self) -> Exception | None: + return None + async def open_backend(self, url: str, headers: Mapping[str, str]) -> RealtimeBackend | None: return None diff --git a/litellm/llms/soniox/common_utils.py b/litellm/llms/soniox/common_utils.py index 8d83b4c9218..796f6ca12e4 100644 --- a/litellm/llms/soniox/common_utils.py +++ b/litellm/llms/soniox/common_utils.py @@ -3,7 +3,7 @@ Shared utilities for the Soniox provider (https://soniox.com). """ from collections.abc import Mapping, Sequence -from typing import Final, TypeAlias +from typing import Final, Literal, TypeAlias from litellm.litellm_core_utils.audio_utils.subtitle_utils import ( SubtitleToken, @@ -17,6 +17,8 @@ SonioxToken: TypeAlias = Mapping[str, object] # Soniox API base URL. SONIOX_API_BASE: Final[str] = "https://api.soniox.com" +SONIOX_REALTIME_MODEL_PREFIX: Final = "stt-rt-" + # Default polling interval in seconds when waiting for an async transcription # to finish. Mirrors the Soniox SDK default. SONIOX_DEFAULT_POLL_INTERVAL: Final[float] = 1.0 @@ -54,6 +56,10 @@ class SonioxException(BaseLLMException): """Provider-specific exception class for Soniox.""" +def soniox_health_check_mode(model: str) -> Literal["realtime"] | None: + return "realtime" if model.startswith(SONIOX_REALTIME_MODEL_PREFIX) else None + + def get_soniox_api_key(api_key: str | None = None) -> str | None: """Resolve the Soniox API key from arg or env var.""" # Local import to avoid a circular import: litellm.secret_managers.main diff --git a/litellm/llms/soniox/realtime/transformation.py b/litellm/llms/soniox/realtime/transformation.py index 1080a7bea4c..0d678375239 100644 --- a/litellm/llms/soniox/realtime/transformation.py +++ b/litellm/llms/soniox/realtime/transformation.py @@ -1,16 +1,17 @@ import asyncio import time from collections.abc import Awaitable, Callable, Mapping -from itertools import chain +from itertools import chain, groupby from types import MappingProxyType, TracebackType from typing import Final from urllib.parse import urlparse, urlunparse from pydantic import BaseModel, ConfigDict, JsonValue, ValidationError -from typing_extensions import Self +from typing_extensions import NotRequired, ReadOnly, Required, Self, TypedDict from litellm import verbose_logger from litellm._uuid import uuid +from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.realtime.transcription_protocol import ( SESSION_UPDATE_EVENT_TYPES, @@ -29,8 +30,15 @@ from litellm.llms.base_llm.realtime.transcription_protocol import ( uses_server_vad, ) from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeBackend -from litellm.llms.soniox.common_utils import get_soniox_api_key -from litellm.types.llms.openai import OpenAIRealtimeEvents, OpenAIRealtimeTranscriptionSessionCreated +from litellm.llms.custom_httpx.http_handler import realtime_ssl_for_url +from litellm.llms.soniox.common_utils import SonioxException, get_soniox_api_key +from litellm.types.llms.openai import ( + OpenAIRealtimeErrorEvent, + OpenAIRealtimeEvents, + OpenAIRealtimeInputAudioTranscriptionCompleted, + OpenAIRealtimeInputAudioTranscriptionDelta, + OpenAIRealtimeTranscriptionSessionCreated, +) from litellm.types.realtime import ( RealtimeInputAudioTranscriptionUsage, RealtimeResponseTransformInput, @@ -81,12 +89,35 @@ class SonioxStartRequest(SonioxRealtimeOptions): enable_endpoint_detection: bool +class SonioxSpeechLabels(TypedDict, total=False): + speaker: ReadOnly[str] + language: ReadOnly[str] + + +class SonioxTranscriptSegment(SonioxSpeechLabels): + text: Required[ReadOnly[str]] + + +class SonioxTranscriptionDelta(OpenAIRealtimeInputAudioTranscriptionDelta, SonioxSpeechLabels): + pass + + +class SonioxTranscriptionCompleted(OpenAIRealtimeInputAudioTranscriptionCompleted): + segments: NotRequired[ReadOnly[tuple[SonioxTranscriptSegment, ...]]] + + class _SonioxToken(BaseModel): model_config = ConfigDict(frozen=True) text: str is_final: bool = False translation_status: str | None = None + speaker: str | None = None + language: str | None = None + + @property + def labelled(self) -> bool: + return self.speaker is not None or self.language is not None class _SonioxResponse(BaseModel): @@ -178,22 +209,65 @@ def build_soniox_realtime_url(api_base: str | None) -> str: return urlunparse(parsed._replace(scheme=scheme)) +def _labels(token: _SonioxToken) -> SonioxSpeechLabels: + speaker: Final[SonioxSpeechLabels] = {} if token.speaker is None else {"speaker": token.speaker} + language: Final[SonioxSpeechLabels] = {} if token.language is None else {"language": token.language} + return {**speaker, **language} + + +def _delta(item_id: str, token: _SonioxToken) -> SonioxTranscriptionDelta: + return {**delta_event(item_id, token.text), **_labels(token)} + + +def _segment(run: tuple[_SonioxToken, ...]) -> SonioxTranscriptSegment: + return {**_labels(run[0]), "text": "".join(token.text for token in run).strip()} + + +def _segments(tokens: tuple[_SonioxToken, ...]) -> tuple[SonioxTranscriptSegment, ...]: + return tuple(_segment(tuple(run)) for _, run in groupby(tokens, key=lambda token: (token.speaker, token.language))) + + +def _completed( + item_id: str, tokens: tuple[_SonioxToken, ...], usage: RealtimeInputAudioTranscriptionUsage | None +) -> OpenAIRealtimeInputAudioTranscriptionCompleted: + completed: Final = completed_event(item_id, "".join(token.text for token in tokens).strip(), usage) + if not any(token.labelled for token in tokens): + return completed + labelled: Final[SonioxTranscriptionCompleted] = {**completed, "segments": _segments(tokens)} + return labelled + + +def _soniox_error(response: _SonioxResponse) -> SonioxException | None: + if response.error_code is None and response.error_message is None: + return None + return SonioxException( + status_code=response.error_code or 500, + message=f"Soniox realtime error {response.error_code}: {response.error_message}", + ) + + class SonioxEventTransformer: def __init__(self, *, translated_only: bool) -> None: self._translated_only: Final = translated_only self._item_id: str | None = None - self._final_text: str = "" + self._final_tokens: tuple[_SonioxToken, ...] = () self._processed_ms: int = 0 self._billed_ms: int = 0 + self._refusal: SonioxException | None = None + + @property + def refusal(self) -> SonioxException | None: + return self._refusal def transform(self, payload: str) -> tuple[OpenAIRealtimeEvents, ...]: try: response: Final = _SonioxResponse.model_validate_json(payload) except ValidationError: raise SonioxProtocolError("Soniox returned an invalid realtime response") from None - if response.error_code is not None or response.error_message is not None: - return (error_event(f"Soniox realtime error {response.error_code}: {response.error_message}"),) self._bill(response.total_audio_proc_ms) + error: Final = _soniox_error(response) + if error is not None: + return (self._report(error),) return tuple(chain.from_iterable(self._token_events(token) for token in response.tokens)) def take_unbilled_usage(self, stream_ms: int = 0) -> RealtimeInputAudioTranscriptionUsage | None: @@ -204,6 +278,11 @@ class SonioxEventTransformer: self._billed_ms = billable_ms return duration_usage(unbilled_ms / 1000) + def _report(self, error: SonioxException) -> OpenAIRealtimeErrorEvent: + if self._processed_ms == 0: + self._refusal = error + return error_event(error.message) + def _bill(self, total_audio_proc_ms: int | None) -> None: if total_audio_proc_ms is not None and total_audio_proc_ms > self._processed_ms: self._processed_ms = total_audio_proc_ms @@ -211,13 +290,11 @@ class SonioxEventTransformer: def _token_events(self, token: _SonioxToken) -> tuple[OpenAIRealtimeEvents, ...]: if token.text in (_ENDPOINT_TOKEN, _FINALIZED_TOKEN): return self._complete(speech_stopped=token.text == _ENDPOINT_TOKEN) - if self._translated_only and token.translation_status != "translation": - return () started: Final = self._start() - if not token.is_final: + if not token.is_final or (self._translated_only and token.translation_status != "translation"): return started - self._final_text += token.text - return (*started, delta_event(self._current_item_id(), token.text)) + self._final_tokens = (*self._final_tokens, token) + return (*started, _delta(self._current_item_id(), token)) def _start(self) -> tuple[OpenAIRealtimeEvents, ...]: if self._item_id is not None: @@ -234,13 +311,13 @@ class SonioxEventTransformer: item_id: Final = self._item_id if item_id is None: return () - transcript: Final = self._final_text.strip() + tokens: Final = self._final_tokens self._item_id = None - self._final_text = "" + self._final_tokens = () stopped: Final[tuple[OpenAIRealtimeEvents, ...]] = ( (speech_event("input_audio_buffer.speech_stopped", item_id),) if speech_stopped else () ) - return (*stopped, completed_event(item_id, transcript, self.take_unbilled_usage())) + return (*stopped, _completed(item_id, tokens, self.take_unbilled_usage())) class SonioxRealtimeBackend: @@ -335,7 +412,7 @@ class SonioxRealtimeConfig(BaseRealtimeConfig): self._transformer: Final = SonioxEventTransformer(translated_only=self._options.translation is not None) self._start_request: SonioxStartRequest | None = None self._session_id: str = f"sess_{uuid.uuid4().hex}" - self._opened_at: float | None = None + self._started_at: float | None = None @classmethod def from_litellm_params(cls, litellm_params: Mapping[str, object], *, drop_params: bool = False) -> Self: @@ -359,7 +436,6 @@ class SonioxRealtimeConfig(BaseRealtimeConfig): return build_soniox_realtime_url(api_base) def wrap_backend(self, backend: RealtimeBackend) -> RealtimeBackend: - self._opened_at = self._clock() return self._wrap(backend) def transform_session_created_event( @@ -390,11 +466,16 @@ class SonioxRealtimeConfig(BaseRealtimeConfig): return () return (*self._start(model, None), control_frame) + def session_refusal(self) -> Exception | None: + return self._transformer.refusal + def unbilled_usage_on_session_close(self, model: str) -> RealtimeInputAudioTranscriptionUsage | None: - opened_at: Final = self._opened_at - if opened_at is None or self._start_request is None: + started_at: Final = self._started_at + if self._transformer.refusal is not None: + return None + if started_at is None: return self._transformer.take_unbilled_usage() - return self._transformer.take_unbilled_usage(int((self._clock() - opened_at) * 1000)) + return self._transformer.take_unbilled_usage(int((self._clock() - started_at) * 1000)) def transform_realtime_response( self, @@ -430,4 +511,26 @@ class SonioxRealtimeConfig(BaseRealtimeConfig): return () start_request: Final = build_start_request(model, update, self._options, self._drop_params) self._start_request = start_request + self._started_at = self._clock() return (start_request.model_dump_json(exclude_none=True),) + + +async def soniox_realtime_health_check(model: str, api_base: str | None, api_key: str | None) -> bool: + import websockets + + config: Final = SonioxRealtimeConfig() + url: Final = config.get_complete_url(api_base, model) + headers: Final = config.validate_environment({}, model, api_key) + start_request: Final = build_start_request(model, None, SonioxRealtimeOptions()) + async with websockets.connect( + url, + additional_headers=headers, + max_size=REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, + ssl=realtime_ssl_for_url(url), + ) as backend: + await backend.send(start_request.model_dump_json(exclude_none=True)) + await backend.send(_END_STREAM) + async for frame in backend: + if (error := _soniox_error(_SonioxResponse.model_validate_json(frame))) is not None: + raise error + return True diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 3e1a8a1bc8d..1913ee3d4ef 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -61059,6 +61059,106 @@ ], "supports_audio_input": true }, + "soniox/stt-rt-preview": { + "input_cost_per_second": 3.333e-05, + "litellm_provider": "soniox", + "metadata": { + "calculation": "$0.12/3600 seconds = $0.00003333 per second", + "original_pricing_per_hour": 0.12 + }, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "soniox/stt-rt-preview-v2": { + "input_cost_per_second": 3.333e-05, + "litellm_provider": "soniox", + "metadata": { + "calculation": "$0.12/3600 seconds = $0.00003333 per second", + "original_pricing_per_hour": 0.12 + }, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "soniox/stt-rt-v3": { + "input_cost_per_second": 3.333e-05, + "litellm_provider": "soniox", + "metadata": { + "calculation": "$0.12/3600 seconds = $0.00003333 per second", + "original_pricing_per_hour": 0.12 + }, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "soniox/stt-rt-v3-preview": { + "input_cost_per_second": 3.333e-05, + "litellm_provider": "soniox", + "metadata": { + "calculation": "$0.12/3600 seconds = $0.00003333 per second", + "original_pricing_per_hour": 0.12 + }, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "soniox/stt-rt-v4": { + "input_cost_per_second": 3.333e-05, + "litellm_provider": "soniox", + "metadata": { + "calculation": "$0.12/3600 seconds = $0.00003333 per second", + "original_pricing_per_hour": 0.12 + }, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, "tensormesh/Qwen/Qwen3.5-397B-A17B-FP8": { "litellm_provider": "tensormesh", "mode": "chat", diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index 2b45916bd4f..4f0417a1191 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -46,6 +46,7 @@ from ..llms.custom_httpx.http_handler import realtime_ssl_for_url from ..llms.openai.live.handler import OpenAILiveSessions from ..llms.openai.realtime.handler import OpenAIRealtime from ..llms.openai.workload_identity import openai_static_api_key, resolve_openai_bearer_token_async +from ..llms.soniox.realtime.transformation import soniox_realtime_health_check from ..llms.vertex_ai.audio_transcription.realtime_transformation import is_vertex_speech_to_text_model from ..llms.vertex_ai.realtime.transformation import VertexAIRealtimeConfig, vertex_realtime_config from ..llms.vertex_ai.vertex_llm_base import VertexBase @@ -858,6 +859,8 @@ async def realtime_health_check( api_base=resolved_api_base or "https://api.openai.com/", query_params={"model": model}, ) + elif custom_llm_provider == "soniox": + return await soniox_realtime_health_check(model=model, api_base=resolved_api_base, api_key=health_check_api_key) elif custom_llm_provider == "xai": url = xai_realtime.construct_url( api_base=resolved_api_base or "https://api.x.ai/v1", query_params={"model": model} diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 3e1a8a1bc8d..1913ee3d4ef 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -61059,6 +61059,106 @@ ], "supports_audio_input": true }, + "soniox/stt-rt-preview": { + "input_cost_per_second": 3.333e-05, + "litellm_provider": "soniox", + "metadata": { + "calculation": "$0.12/3600 seconds = $0.00003333 per second", + "original_pricing_per_hour": 0.12 + }, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "soniox/stt-rt-preview-v2": { + "input_cost_per_second": 3.333e-05, + "litellm_provider": "soniox", + "metadata": { + "calculation": "$0.12/3600 seconds = $0.00003333 per second", + "original_pricing_per_hour": 0.12 + }, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "soniox/stt-rt-v3": { + "input_cost_per_second": 3.333e-05, + "litellm_provider": "soniox", + "metadata": { + "calculation": "$0.12/3600 seconds = $0.00003333 per second", + "original_pricing_per_hour": 0.12 + }, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "soniox/stt-rt-v3-preview": { + "input_cost_per_second": 3.333e-05, + "litellm_provider": "soniox", + "metadata": { + "calculation": "$0.12/3600 seconds = $0.00003333 per second", + "original_pricing_per_hour": 0.12 + }, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "soniox/stt-rt-v4": { + "input_cost_per_second": 3.333e-05, + "litellm_provider": "soniox", + "metadata": { + "calculation": "$0.12/3600 seconds = $0.00003333 per second", + "original_pricing_per_hour": 0.12 + }, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, "tensormesh/Qwen/Qwen3.5-397B-A17B-FP8": { "litellm_provider": "tensormesh", "mode": "chat", diff --git a/tests/unit/litellm_core_utils/test_health_check_helpers.py b/tests/unit/litellm_core_utils/test_health_check_helpers.py index 88e2bc4b1fd..d5c64beef85 100644 --- a/tests/unit/litellm_core_utils/test_health_check_helpers.py +++ b/tests/unit/litellm_core_utils/test_health_check_helpers.py @@ -794,9 +794,12 @@ async def test_ahealth_check_keeps_the_noul_probe_for_other_decisions_providers( ("us.anthropic.claude-haiku-4-5-20251001-v1:0", "bedrock", None), ("claude-haiku-4-5", "anthropic", None), ("anthropic.claude-haiku-4-5", None, None), + ("stt-rt-v5", "soniox", "realtime"), + ("stt-rt-preview", "soniox", "realtime"), + ("stt-async-v5", "soniox", None), ), ) -def test_native_health_check_mode_is_messages_only_for_mantle_claude( +def test_native_health_check_mode_routes_mantle_claude_to_messages_and_soniox_realtime_models_to_realtime( model: str, custom_llm_provider: str | None, expected: str | None ) -> None: assert native_health_check_mode(model=model, custom_llm_provider=custom_llm_provider) == expected diff --git a/tests/unit/litellm_core_utils/test_realtime_streaming.py b/tests/unit/litellm_core_utils/test_realtime_streaming.py index 7d85da3f756..a8543ebcfc2 100644 --- a/tests/unit/litellm_core_utils/test_realtime_streaming.py +++ b/tests/unit/litellm_core_utils/test_realtime_streaming.py @@ -24,6 +24,7 @@ from litellm.litellm_core_utils.realtime_streaming import ( client_sent_openai_beta_realtime_header, ) from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.base_llm.realtime.transcription_protocol import RealtimeTranscriptionProtocolError from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig from litellm.llms.xai.realtime.transformation import XAIRealtimeNormalizer from litellm.llms.xai.realtime.usage_meter import XAIRealtimeUsageMeter @@ -827,9 +828,7 @@ async def test_usage_meter_snapshot_stored_in_model_call_details(): backend_ws.send = AsyncMock() backend_ws.recv = AsyncMock( side_effect=[ - json.dumps( - {"type": "session.created", "session": {"turn_detection": {"type": None}}} - ).encode(), + json.dumps({"type": "session.created", "session": {"turn_detection": {"type": None}}}).encode(), json.dumps( {"type": "response.output_audio.delta", "delta": base64.b64encode(b"x" * 24000).decode()} ).encode(), @@ -3187,6 +3186,7 @@ async def test_session_close_flushes_unbilled_transcription_usage(): provider_config: Final = MagicMock() provider_config.transform_realtime_response = MagicMock(return_value=transcript_frame) provider_config.unbilled_usage_on_session_close = MagicMock(return_value=usage) + provider_config.session_refusal = MagicMock(return_value=None) streaming: Final = RealTimeStreaming( client_ws, @@ -3320,10 +3320,14 @@ def _backend_ws_closing_with(*frames: bytes | Exception) -> MagicMock: return backend_ws -def _relay_session(client_ws: MagicMock, backend_ws: MagicMock) -> _RelaySession: +def _relay_session( + client_ws: MagicMock, backend_ws: MagicMock, provider_config: BaseRealtimeConfig | None = None +) -> _RelaySession: logging: Final = _RecordingLogging() worker: Final = _InlineLoggingWorker() - streaming: Final = RealTimeStreaming(client_ws, backend_ws, logging, model="gpt-realtime", logging_worker=worker) + streaming: Final = RealTimeStreaming( + client_ws, backend_ws, logging, provider_config=provider_config, model="gpt-realtime", logging_worker=worker + ) return _RelaySession(streaming=streaming, logging=logging, worker=worker) @@ -3408,6 +3412,103 @@ async def test_upstream_refusal_before_any_frame_logs_a_failure_not_a_success(): assert session.logging.logged_sessions == () +class _RefusingRealtimeConfig(_PassthroughRealtimeConfig): + def __init__(self, refusal: Exception | None) -> None: + self._refusal: Final = refusal + + def requires_session_configuration(self) -> bool: + return True + + def session_refusal(self) -> Exception | None: + return self._refusal + + +@pytest.mark.asyncio +async def test_a_session_the_provider_refused_in_band_logs_a_failure_even_after_a_normal_close(): + refusal: Final = ValueError("Soniox realtime error 401: Invalid API key.") + session: Final = _relay_session( + _client_ws_that_never_sends(), + _backend_ws_closing_with( + json.dumps({"type": "error", "error": {"message": str(refusal)}}).encode(), + ConnectionClosed(Close(1000, ""), None), + ), + _RefusingRealtimeConfig(refusal), + ) + + await session.run() + + assert session.logging.logged_failures == (refusal,) + assert session.logging.logged_sessions == () + + +@pytest.mark.asyncio +async def test_a_provider_session_without_a_refusal_logs_a_success_after_a_normal_close(): + session: Final = _relay_session( + _client_ws_that_never_sends(), + _backend_ws_closing_with( + json.dumps({"type": "session.created", "session": {"id": "sess_1"}}).encode(), + ConnectionClosed(Close(1000, ""), None), + ), + _RefusingRealtimeConfig(None), + ) + + await session.run() + + assert session.logging.logged_failures == () + assert len(session.logging.logged_sessions) == 1 + + +class _TranscriptionConfigRejectingSessionUpdates(_PassthroughRealtimeConfig): + def __init__(self, *, defers_setup: bool) -> None: + self._defers_setup: Final = defers_setup + + def requires_session_configuration(self) -> bool: + return not self._defers_setup + + def transform_realtime_request( + self, message: str, model: str, session_configuration_request: str | None = None + ) -> Sequence[str | bytes]: + if json.loads(message).get("type") == "session.update": + raise RealtimeTranscriptionProtocolError("requires pcm16 input audio") + return (message,) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "defers_setup,client_frames_read,audio_reaches_backend", + ((True, 1, False), (False, 3, True)), + ids=["deferred_setup_session_ends", "eager_setup_session_continues"], +) +async def test_a_rejected_transcription_session_update_sends_an_error_and_ends_only_a_deferred_setup_session( + defers_setup: bool, client_frames_read: int, audio_reaches_backend: bool +): + append: Final = json.dumps({"type": "input_audio_buffer.append", "audio": "AAE="}) + client_ws: Final = MagicMock() + client_ws.headers = {} + client_ws.receive_text = AsyncMock( + side_effect=[json.dumps({"type": "session.update", "session": {"type": "transcription"}}), append, RuntimeError] + ) + client_ws.send_text = AsyncMock() + backend_ws: Final = MagicMock() + backend_ws.send = AsyncMock() + streaming: Final = RealTimeStreaming( + client_ws, + backend_ws, + _RecordingLogging(), + provider_config=_TranscriptionConfigRejectingSessionUpdates(defers_setup=defers_setup), + model="chirp_3", + logging_worker=_InlineLoggingWorker(), + ) + + await asyncio.wait_for(streaming.client_ack_messages(), timeout=2) + + assert _error_events_sent_to(client_ws) == [ + {"type": "error", "error": {"type": "invalid_request_error", "message": "requires pcm16 input audio"}} + ] + assert client_ws.receive_text.await_count == client_frames_read + assert (append in [call.args[0] for call in backend_ws.send.await_args_list]) is audio_reaches_backend + + @pytest.mark.asyncio async def test_upstream_refusal_after_a_synthetic_session_created_still_logs_a_failure(): """LIT-6973: deferred Gemini Live setup stores a synthetic ``session.created`` before @@ -4056,9 +4157,7 @@ _XAI_TYPED_VAD_SESSION: Final = { pytest.param(({"type": "realtime", "instructions": "hi"},), id="no-turn-detection"), pytest.param(({"type": "realtime", "audio": {"input": {"turn_detection": None}}},), id="push-to-talk"), pytest.param((_XAI_TYPED_VAD_SESSION,), id="typed-server-vad"), - pytest.param( - ({"type": "realtime", "instructions": "hi"}, _XAI_TYPED_VAD_SESSION), id="later-typed-server-vad" - ), + pytest.param(({"type": "realtime", "instructions": "hi"}, _XAI_TYPED_VAD_SESSION), id="later-typed-server-vad"), ], ) async def test_xai_session_update_gate_leaves_out_the_vad_type( @@ -4070,9 +4169,9 @@ async def test_xai_session_update_gate_leaves_out_the_vad_type( monkeypatch, *client_sessions, event_normalizer=XAIRealtimeNormalizer() ) - assert [session["audio"]["input"]["turn_detection"] for session in forwarded] == [ - {"create_response": False} - ] * (len(client_sessions) + 1) + assert [session["audio"]["input"]["turn_detection"] for session in forwarded] == [{"create_response": False}] * ( + len(client_sessions) + 1 + ) @pytest.mark.asyncio @@ -4206,7 +4305,9 @@ async def test_client_never_sees_the_error_caused_by_the_guardrail_cancel(monkey await RealTimeStreaming(client_ws, backend_ws, logging_obj).backend_to_client_send_messages() client_errors: Final = [ - event["error"] for call in client_ws.send_text.await_args_list if (event := json.loads(call.args[0]))["type"] == "error" + event["error"] + for call in client_ws.send_text.await_args_list + if (event := json.loads(call.args[0]))["type"] == "error" ] assert [(error["code"], error.get("event_id")) for error in client_errors] == [ ("content_policy_violation", None), diff --git a/tests/unit/llms/soniox/realtime/test_soniox_realtime_transformation.py b/tests/unit/llms/soniox/realtime/test_soniox_realtime_transformation.py index fc7acc49092..b43d0e48983 100644 --- a/tests/unit/llms/soniox/realtime/test_soniox_realtime_transformation.py +++ b/tests/unit/llms/soniox/realtime/test_soniox_realtime_transformation.py @@ -6,6 +6,7 @@ from unittest.mock import MagicMock import pytest +from litellm.llms.soniox.common_utils import SonioxException from litellm.llms.soniox.realtime.transformation import ( DEFAULT_SONIOX_REALTIME_URL, SESSION_STARTED_FRAME, @@ -203,7 +204,9 @@ def test_endpoint_detection_emits_one_openai_turn_billed_by_processed_audio(): "conversation.item.input_audio_transcription.completed", ] assert {event["item_id"] for event in (*finals, *endpoint)} == {item_id} + assert all("speaker" not in event and "language" not in event for event in finals) assert endpoint[1]["transcript"] == "Hello world" + assert "segments" not in endpoint[1] assert endpoint[1]["usage"] == {"type": "duration", "seconds": 2.0} assert transformer.take_unbilled_usage() is None @@ -248,6 +251,43 @@ def test_translation_sessions_transcribe_only_the_translated_tokens(): assert events[-1]["transcript"] == "Hello" +def test_diarized_tokens_label_each_delta_and_split_the_completed_turn_into_speaker_segments(): + transformer = SonioxEventTransformer(translated_only=False) + + events = transformer.transform( + _response( + [ + _token("Hi", speaker="1", language="en"), + _token(" doctor", speaker="1", language="en"), + _token(" Salam", speaker="2", language="fa"), + _token(" Hi", speaker="1", language="en"), + _token(""), + ] + ) + ) + + assert [(event["delta"], event["speaker"], event["language"]) for event in events if "delta" in event] == [ + ("Hi", "1", "en"), + (" doctor", "1", "en"), + (" Salam", "2", "fa"), + (" Hi", "1", "en"), + ] + assert events[-1]["transcript"] == "Hi doctor Salam Hi" + assert events[-1]["segments"] == ( + {"speaker": "1", "language": "en", "text": "Hi doctor"}, + {"speaker": "2", "language": "fa", "text": "Salam"}, + {"speaker": "1", "language": "en", "text": "Hi"}, + ) + + +def test_translation_sessions_announce_speech_on_the_first_spoken_token_before_any_translation(): + transformer = SonioxEventTransformer(translated_only=True) + + events = transformer.transform(_response([_token("Hol", is_final=False, translation_status="original")])) + + assert _types(events) == ["input_audio_buffer.speech_started"] + + def test_provider_errors_surface_as_openai_error_events(): transformer = SonioxEventTransformer(translated_only=False) @@ -291,20 +331,55 @@ def test_translation_option_switches_the_config_to_translated_transcripts(): assert [event.get("delta") for event in result["response"] if "delta" in event] == ["Hello"] -def test_stream_open_time_is_billed_on_close_because_soniox_charges_the_whole_stream(): +def test_stream_time_since_the_start_request_is_billed_on_close_because_soniox_charges_the_whole_stream(): now = [100.0] config = SonioxRealtimeConfig(clock=lambda: now[0]) config.wrap_backend(_RecordingBackend()) + now[0] = 110.0 config.transform_realtime_request(_append(), MODEL) config.transform_realtime_response( _response([_token("Hi"), _token("")], total_audio_proc_ms=5000), MODEL, MagicMock(), EMPTY_TRANSFORM_INPUT ) now[0] = 130.5 - assert config.unbilled_usage_on_session_close(MODEL) == {"type": "duration", "seconds": 25.5} + assert config.unbilled_usage_on_session_close(MODEL) == {"type": "duration", "seconds": 15.5} assert config.unbilled_usage_on_session_close(MODEL) is None +def _refusal_after(processed_ms: int) -> SonioxRealtimeConfig: + now = [100.0] + config = SonioxRealtimeConfig(clock=lambda: now[0]) + config.transform_realtime_request(_append(), MODEL) + if processed_ms: + config.transform_realtime_response( + _response([_token("Hi")], total_audio_proc_ms=processed_ms), MODEL, MagicMock(), EMPTY_TRANSFORM_INPUT + ) + config.transform_realtime_response( + json.dumps({"error_code": 401, "error_message": "Invalid API key.", "total_audio_proc_ms": processed_ms}), + MODEL, + MagicMock(), + EMPTY_TRANSFORM_INPUT, + ) + now[0] = 103.0 + return config + + +def test_an_error_before_any_audio_is_processed_is_a_refused_session_that_bills_nothing(): + config = _refusal_after(processed_ms=0) + + refusal = config.session_refusal() + assert isinstance(refusal, SonioxException) + assert refusal.status_code == 401 + assert config.unbilled_usage_on_session_close(MODEL) is None + + +def test_an_error_after_audio_was_processed_keeps_the_session_a_billed_success(): + config = _refusal_after(processed_ms=1500) + + assert config.session_refusal() is None + assert config.unbilled_usage_on_session_close(MODEL) == {"type": "duration", "seconds": 3.0} + + def test_a_connection_that_never_started_a_stream_bills_nothing(): now = [100.0] config = SonioxRealtimeConfig(clock=lambda: now[0]) @@ -322,14 +397,21 @@ def test_a_session_update_matching_the_started_stream_is_a_no_op(): config = SonioxRealtimeConfig() config.transform_realtime_request(_append(), MODEL) - assert config.transform_realtime_request(_session_update({"format": {"type": "audio/pcm", "rate": 24_000}}), MODEL) == () + assert ( + config.transform_realtime_request(_session_update({"format": {"type": "audio/pcm", "rate": 24_000}}), MODEL) + == () + ) def _stream_started_at_16khz_in_french_without_vad() -> SonioxRealtimeConfig: config = SonioxRealtimeConfig() config.transform_realtime_request( _session_update( - {"format": {"type": "audio/pcm", "rate": 16_000}, "transcription": {"language": "fr"}, "turn_detection": None} + { + "format": {"type": "audio/pcm", "rate": 16_000}, + "transcription": {"language": "fr"}, + "turn_detection": None, + } ), MODEL, ) @@ -434,7 +516,6 @@ def test_provider_config_manager_passes_only_soniox_options_from_deployment_para } - SEMANTIC_VAD: Final = {"turn_detection": {"type": "semantic_vad", "eagerness": "high"}} diff --git a/tests/unit/realtime_api/test_main.py b/tests/unit/realtime_api/test_main.py index f0dea30d132..ee7f8a863d8 100644 --- a/tests/unit/realtime_api/test_main.py +++ b/tests/unit/realtime_api/test_main.py @@ -537,7 +537,11 @@ async def test_bearer_health_check_sends_the_secret_behind_an_oidc_credential( monkeypatch.setattr( litellm, "credential_list", - [CredentialItem(credential_name="voice-sa", credential_values={"api_key": f"oidc/file/{token}"}, credential_info={})], + [ + CredentialItem( + credential_name="voice-sa", credential_values={"api_key": f"oidc/file/{token}"}, credential_info={} + ) + ], ) connect: Final = _CapturingConnect() with patch("websockets.connect", connect): @@ -1228,3 +1232,47 @@ async def test_arealtime_routes_soniox_with_the_deployment_options(monkeypatch): json.dumps({"type": "input_audio_buffer.commit"}), "stt-rt-v5" )[0] assert json.loads(start_request)["language_hints"] == ["de"] + + +async def _serve_soniox(reply: str) -> tuple[object, list[object]]: + import websockets + + received: list[object] = [] + + async def handler(connection) -> None: + received.append(connection.request.headers["Authorization"]) + received.append(json.loads(await connection.recv())) + received.append(await connection.recv()) + await connection.send(reply) + + return await websockets.serve(handler, "127.0.0.1", 0), received + + +@pytest.mark.asyncio +async def test_soniox_realtime_health_check_starts_and_ends_a_stream_on_the_deployment_websocket(): + server, received = await _serve_soniox(json.dumps({"tokens": [], "finished": True})) + port = next(iter(server.sockets)).getsockname()[1] + async with server: + healthy = await realtime_main.realtime_health_check( + model="stt-rt-v5", custom_llm_provider="soniox", api_key="soniox-key", api_base=f"http://127.0.0.1:{port}" + ) + + assert healthy is True + assert received[0] == "Bearer soniox-key" + assert received[1]["model"] == "stt-rt-v5" + assert received[2] == "" + + +@pytest.mark.asyncio +async def test_soniox_realtime_health_check_fails_when_soniox_refuses_the_stream(): + from litellm.llms.soniox.common_utils import SonioxException + + server, _ = await _serve_soniox(json.dumps({"error_code": 401, "error_message": "Invalid API key."})) + port = next(iter(server.sockets)).getsockname()[1] + async with server: + with pytest.raises(SonioxException) as refused: + await realtime_main.realtime_health_check( + model="stt-rt-v5", custom_llm_provider="soniox", api_key="bad-key", api_base=f"ws://127.0.0.1:{port}" + ) + + assert refused.value.status_code == 401 From d6d55c6153b26be1a422a8c173d68dbe3462b4f8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 20:47:28 -0700 Subject: [PATCH 10/13] fix(soniox): bill failed streams, probe health with deployment options, and keep token accumulation linear Only request rejections (400, 401, 402, 403, 429) before any processed audio count as a refused session, so a 408 or 5xx still bills the open stream. The /health probe starts Soniox with the deployment's options, sends 100 ms of silence, and passes only on Soniox's finished frame. A turn's final tokens append in constant time instead of copying the tuple per token. --- .../llms/soniox/realtime/transformation.py | 34 +++++++++++++------ litellm/realtime_api/main.py | 9 +++-- .../test_soniox_realtime_transformation.py | 12 +++++-- tests/unit/realtime_api/test_main.py | 30 +++++++++++++--- 4 files changed, 67 insertions(+), 18 deletions(-) diff --git a/litellm/llms/soniox/realtime/transformation.py b/litellm/llms/soniox/realtime/transformation.py index 0d678375239..45b71fb7411 100644 --- a/litellm/llms/soniox/realtime/transformation.py +++ b/litellm/llms/soniox/realtime/transformation.py @@ -53,6 +53,7 @@ _MAX_PENDING_FRAMES: Final = 64 _FINALIZE: Final = '{"type":"finalize"}' _KEEPALIVE: Final = '{"type":"keepalive"}' _END_STREAM: Final = "" +_HEALTH_CHECK_SILENCE: Final = bytes(DEFAULT_SAMPLE_RATE // 5) _CONTROL_FRAMES: Final = frozenset((_FINALIZE, _KEEPALIVE, _END_STREAM)) _CLIENT_CONTROL_FRAMES: Final = MappingProxyType( {"input_audio_buffer.commit": _FINALIZE, "input_audio_buffer.end": _END_STREAM} @@ -60,6 +61,7 @@ _CLIENT_CONTROL_FRAMES: Final = MappingProxyType( SESSION_STARTED_FRAME: Final = '{"type":"litellm.soniox.session_started"}' _ENDPOINT_TOKEN: Final = "" _FINALIZED_TOKEN: Final = "" +_REFUSED_STREAM_CODES: Final = frozenset({400, 401, 402, 403, 429}) class SonioxProtocolError(RealtimeTranscriptionProtocolError): @@ -127,6 +129,7 @@ class _SonioxResponse(BaseModel): total_audio_proc_ms: int | None = None error_code: int | None = None error_message: str | None = None + finished: bool = False def _sample_rate(update: TranscriptionSessionUpdate | None) -> int: @@ -250,7 +253,7 @@ class SonioxEventTransformer: def __init__(self, *, translated_only: bool) -> None: self._translated_only: Final = translated_only self._item_id: str | None = None - self._final_tokens: tuple[_SonioxToken, ...] = () + self._final_tokens: Final[list[_SonioxToken]] = [] # mutable-ok: a turn's tokens append in O(1) each self._processed_ms: int = 0 self._billed_ms: int = 0 self._refusal: SonioxException | None = None @@ -279,7 +282,7 @@ class SonioxEventTransformer: return duration_usage(unbilled_ms / 1000) def _report(self, error: SonioxException) -> OpenAIRealtimeErrorEvent: - if self._processed_ms == 0: + if self._processed_ms == 0 and error.status_code in _REFUSED_STREAM_CODES: self._refusal = error return error_event(error.message) @@ -293,7 +296,7 @@ class SonioxEventTransformer: started: Final = self._start() if not token.is_final or (self._translated_only and token.translation_status != "translation"): return started - self._final_tokens = (*self._final_tokens, token) + self._final_tokens.append(token) return (*started, _delta(self._current_item_id(), token)) def _start(self) -> tuple[OpenAIRealtimeEvents, ...]: @@ -311,9 +314,9 @@ class SonioxEventTransformer: item_id: Final = self._item_id if item_id is None: return () - tokens: Final = self._final_tokens + tokens: Final = tuple(self._final_tokens) self._item_id = None - self._final_tokens = () + self._final_tokens.clear() stopped: Final[tuple[OpenAIRealtimeEvents, ...]] = ( (speech_event("input_audio_buffer.speech_stopped", item_id),) if speech_stopped else () ) @@ -515,13 +518,23 @@ class SonioxRealtimeConfig(BaseRealtimeConfig): return (start_request.model_dump_json(exclude_none=True),) -async def soniox_realtime_health_check(model: str, api_base: str | None, api_key: str | None) -> bool: +def _finishes_stream(frame: str | bytes) -> bool: + response: Final = _SonioxResponse.model_validate_json(frame) + error: Final = _soniox_error(response) + if error is not None: + raise error + return response.finished + + +async def soniox_realtime_health_check( + model: str, api_base: str | None, api_key: str | None, options: SonioxRealtimeOptions +) -> bool: import websockets config: Final = SonioxRealtimeConfig() url: Final = config.get_complete_url(api_base, model) headers: Final = config.validate_environment({}, model, api_key) - start_request: Final = build_start_request(model, None, SonioxRealtimeOptions()) + start_request: Final = build_start_request(model, None, options) async with websockets.connect( url, additional_headers=headers, @@ -529,8 +542,9 @@ async def soniox_realtime_health_check(model: str, api_base: str | None, api_key ssl=realtime_ssl_for_url(url), ) as backend: await backend.send(start_request.model_dump_json(exclude_none=True)) + await backend.send(_HEALTH_CHECK_SILENCE) await backend.send(_END_STREAM) async for frame in backend: - if (error := _soniox_error(_SonioxResponse.model_validate_json(frame))) is not None: - raise error - return True + if _finishes_stream(frame): + return True + raise SonioxException(status_code=502, message="Soniox closed the realtime stream without finishing it") diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index 4f0417a1191..0418d7d3ab5 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -46,7 +46,7 @@ from ..llms.custom_httpx.http_handler import realtime_ssl_for_url from ..llms.openai.live.handler import OpenAILiveSessions from ..llms.openai.realtime.handler import OpenAIRealtime from ..llms.openai.workload_identity import openai_static_api_key, resolve_openai_bearer_token_async -from ..llms.soniox.realtime.transformation import soniox_realtime_health_check +from ..llms.soniox.realtime.transformation import SonioxRealtimeOptions, soniox_realtime_health_check from ..llms.vertex_ai.audio_transcription.realtime_transformation import is_vertex_speech_to_text_model from ..llms.vertex_ai.realtime.transformation import VertexAIRealtimeConfig, vertex_realtime_config from ..llms.vertex_ai.vertex_llm_base import VertexBase @@ -860,7 +860,12 @@ async def realtime_health_check( query_params={"model": model}, ) elif custom_llm_provider == "soniox": - return await soniox_realtime_health_check(model=model, api_base=resolved_api_base, api_key=health_check_api_key) + return await soniox_realtime_health_check( + model=model, + api_base=resolved_api_base, + api_key=health_check_api_key, + options=SonioxRealtimeOptions.model_validate(dict(resolved_params)), + ) elif custom_llm_provider == "xai": url = xai_realtime.construct_url( api_base=resolved_api_base or "https://api.x.ai/v1", query_params={"model": model} diff --git a/tests/unit/llms/soniox/realtime/test_soniox_realtime_transformation.py b/tests/unit/llms/soniox/realtime/test_soniox_realtime_transformation.py index b43d0e48983..6d17e0dabb2 100644 --- a/tests/unit/llms/soniox/realtime/test_soniox_realtime_transformation.py +++ b/tests/unit/llms/soniox/realtime/test_soniox_realtime_transformation.py @@ -346,7 +346,7 @@ def test_stream_time_since_the_start_request_is_billed_on_close_because_soniox_c assert config.unbilled_usage_on_session_close(MODEL) is None -def _refusal_after(processed_ms: int) -> SonioxRealtimeConfig: +def _refusal_after(processed_ms: int, error_code: int = 401) -> SonioxRealtimeConfig: now = [100.0] config = SonioxRealtimeConfig(clock=lambda: now[0]) config.transform_realtime_request(_append(), MODEL) @@ -355,7 +355,7 @@ def _refusal_after(processed_ms: int) -> SonioxRealtimeConfig: _response([_token("Hi")], total_audio_proc_ms=processed_ms), MODEL, MagicMock(), EMPTY_TRANSFORM_INPUT ) config.transform_realtime_response( - json.dumps({"error_code": 401, "error_message": "Invalid API key.", "total_audio_proc_ms": processed_ms}), + json.dumps({"error_code": error_code, "error_message": "Soniox error.", "total_audio_proc_ms": processed_ms}), MODEL, MagicMock(), EMPTY_TRANSFORM_INPUT, @@ -380,6 +380,14 @@ def test_an_error_after_audio_was_processed_keeps_the_session_a_billed_success() assert config.unbilled_usage_on_session_close(MODEL) == {"type": "duration", "seconds": 3.0} +@pytest.mark.parametrize("error_code", [408, 500, 503]) +def test_a_stream_failure_before_any_audio_is_processed_still_bills_the_open_stream(error_code: int): + config = _refusal_after(processed_ms=0, error_code=error_code) + + assert config.session_refusal() is None + assert config.unbilled_usage_on_session_close(MODEL) == {"type": "duration", "seconds": 3.0} + + def test_a_connection_that_never_started_a_stream_bills_nothing(): now = [100.0] config = SonioxRealtimeConfig(clock=lambda: now[0]) diff --git a/tests/unit/realtime_api/test_main.py b/tests/unit/realtime_api/test_main.py index ee7f8a863d8..078672b9e74 100644 --- a/tests/unit/realtime_api/test_main.py +++ b/tests/unit/realtime_api/test_main.py @@ -1234,7 +1234,7 @@ async def test_arealtime_routes_soniox_with_the_deployment_options(monkeypatch): assert json.loads(start_request)["language_hints"] == ["de"] -async def _serve_soniox(reply: str) -> tuple[object, list[object]]: +async def _serve_soniox(reply: str | None) -> tuple[object, list[object]]: import websockets received: list[object] = [] @@ -1243,7 +1243,9 @@ async def _serve_soniox(reply: str) -> tuple[object, list[object]]: received.append(connection.request.headers["Authorization"]) received.append(json.loads(await connection.recv())) received.append(await connection.recv()) - await connection.send(reply) + received.append(await connection.recv()) + if reply is not None: + await connection.send(reply) return await websockets.serve(handler, "127.0.0.1", 0), received @@ -1254,13 +1256,20 @@ async def test_soniox_realtime_health_check_starts_and_ends_a_stream_on_the_depl port = next(iter(server.sockets)).getsockname()[1] async with server: healthy = await realtime_main.realtime_health_check( - model="stt-rt-v5", custom_llm_provider="soniox", api_key="soniox-key", api_base=f"http://127.0.0.1:{port}" + model="stt-rt-v5", + custom_llm_provider="soniox", + api_key="soniox-key", + api_base=f"http://127.0.0.1:{port}", + model_params={"model": "soniox/stt-rt-v5", "language_hints": ["fa", "en"]}, ) assert healthy is True assert received[0] == "Bearer soniox-key" assert received[1]["model"] == "stt-rt-v5" - assert received[2] == "" + assert received[1]["language_hints"] == ["fa", "en"] + assert received[2] == bytes(len(received[2])) + assert len(received[2]) > 0 + assert received[3] == "" @pytest.mark.asyncio @@ -1276,3 +1285,16 @@ async def test_soniox_realtime_health_check_fails_when_soniox_refuses_the_stream ) assert refused.value.status_code == 401 + + +@pytest.mark.asyncio +async def test_soniox_realtime_health_check_fails_when_soniox_closes_without_finishing_the_stream(): + from litellm.llms.soniox.common_utils import SonioxException + + server, _ = await _serve_soniox(None) + port = next(iter(server.sockets)).getsockname()[1] + async with server: + with pytest.raises(SonioxException): + await realtime_main.realtime_health_check( + model="stt-rt-v5", custom_llm_provider="soniox", api_key="soniox-key", api_base=f"ws://127.0.0.1:{port}" + ) From 9306679865719a86d01d01d056f75398a3267235 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 21:25:53 -0700 Subject: [PATCH 11/13] fix(soniox): log every Soniox error before any processed audio as a failed session --- litellm/llms/soniox/realtime/transformation.py | 3 +-- .../test_soniox_realtime_transformation.py | 15 ++++----------- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/litellm/llms/soniox/realtime/transformation.py b/litellm/llms/soniox/realtime/transformation.py index 45b71fb7411..054e6f30c75 100644 --- a/litellm/llms/soniox/realtime/transformation.py +++ b/litellm/llms/soniox/realtime/transformation.py @@ -61,7 +61,6 @@ _CLIENT_CONTROL_FRAMES: Final = MappingProxyType( SESSION_STARTED_FRAME: Final = '{"type":"litellm.soniox.session_started"}' _ENDPOINT_TOKEN: Final = "" _FINALIZED_TOKEN: Final = "" -_REFUSED_STREAM_CODES: Final = frozenset({400, 401, 402, 403, 429}) class SonioxProtocolError(RealtimeTranscriptionProtocolError): @@ -282,7 +281,7 @@ class SonioxEventTransformer: return duration_usage(unbilled_ms / 1000) def _report(self, error: SonioxException) -> OpenAIRealtimeErrorEvent: - if self._processed_ms == 0 and error.status_code in _REFUSED_STREAM_CODES: + if self._processed_ms == 0: self._refusal = error return error_event(error.message) diff --git a/tests/unit/llms/soniox/realtime/test_soniox_realtime_transformation.py b/tests/unit/llms/soniox/realtime/test_soniox_realtime_transformation.py index 6d17e0dabb2..d83fd425103 100644 --- a/tests/unit/llms/soniox/realtime/test_soniox_realtime_transformation.py +++ b/tests/unit/llms/soniox/realtime/test_soniox_realtime_transformation.py @@ -364,12 +364,13 @@ def _refusal_after(processed_ms: int, error_code: int = 401) -> SonioxRealtimeCo return config -def test_an_error_before_any_audio_is_processed_is_a_refused_session_that_bills_nothing(): - config = _refusal_after(processed_ms=0) +@pytest.mark.parametrize("error_code", [400, 401, 402, 403, 408, 429, 500, 503]) +def test_an_error_before_any_audio_is_processed_is_a_refused_session_that_bills_nothing(error_code: int): + config = _refusal_after(processed_ms=0, error_code=error_code) refusal = config.session_refusal() assert isinstance(refusal, SonioxException) - assert refusal.status_code == 401 + assert refusal.status_code == error_code assert config.unbilled_usage_on_session_close(MODEL) is None @@ -380,14 +381,6 @@ def test_an_error_after_audio_was_processed_keeps_the_session_a_billed_success() assert config.unbilled_usage_on_session_close(MODEL) == {"type": "duration", "seconds": 3.0} -@pytest.mark.parametrize("error_code", [408, 500, 503]) -def test_a_stream_failure_before_any_audio_is_processed_still_bills_the_open_stream(error_code: int): - config = _refusal_after(processed_ms=0, error_code=error_code) - - assert config.session_refusal() is None - assert config.unbilled_usage_on_session_close(MODEL) == {"type": "duration", "seconds": 3.0} - - def test_a_connection_that_never_started_a_stream_bills_nothing(): now = [100.0] config = SonioxRealtimeConfig(clock=lambda: now[0]) From 9b49cfe7357b579dd63137a471f24b88760cb452 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 22:26:14 -0700 Subject: [PATCH 12/13] fix(soniox): keep untranslated speech, support input_audio_buffer.clear, ignore an empty language --- .../llms/soniox/realtime/transformation.py | 66 ++++++++++++---- .../test_soniox_realtime_transformation.py | 78 ++++++++++++++++--- 2 files changed, 118 insertions(+), 26 deletions(-) diff --git a/litellm/llms/soniox/realtime/transformation.py b/litellm/llms/soniox/realtime/transformation.py index 054e6f30c75..2a80371e7e5 100644 --- a/litellm/llms/soniox/realtime/transformation.py +++ b/litellm/llms/soniox/realtime/transformation.py @@ -55,9 +55,6 @@ _KEEPALIVE: Final = '{"type":"keepalive"}' _END_STREAM: Final = "" _HEALTH_CHECK_SILENCE: Final = bytes(DEFAULT_SAMPLE_RATE // 5) _CONTROL_FRAMES: Final = frozenset((_FINALIZE, _KEEPALIVE, _END_STREAM)) -_CLIENT_CONTROL_FRAMES: Final = MappingProxyType( - {"input_audio_buffer.commit": _FINALIZE, "input_audio_buffer.end": _END_STREAM} -) SESSION_STARTED_FRAME: Final = '{"type":"litellm.soniox.session_started"}' _ENDPOINT_TOKEN: Final = "" _FINALIZED_TOKEN: Final = "" @@ -147,7 +144,7 @@ def _sample_rate(update: TranscriptionSessionUpdate | None) -> int: def _language_hints(language: str | None, configured: tuple[str, ...] | None) -> tuple[str, ...] | None: - if language is None: + if not language: return configured return (language, *(hint for hint in configured or () if hint != language)) @@ -156,6 +153,11 @@ def _uses_endpoint_detection(update: TranscriptionSessionUpdate, drop_params: bo return uses_server_vad(update, "Soniox realtime", drop_params, SonioxProtocolError) +def _ignored_settings(update: TranscriptionSessionUpdate) -> tuple[str, ...]: + vad_tuning: Final = (f"turn_detection.{key}" for key in sorted(update.turn_detection or {}) if key != "type") + return (*update.unsupported_transcription_keys, *vad_tuning) + + def build_start_request( model: str, update: TranscriptionSessionUpdate | None, options: SonioxRealtimeOptions, drop_params: bool = False ) -> SonioxStartRequest: @@ -249,10 +251,11 @@ def _soniox_error(response: _SonioxResponse) -> SonioxException | None: class SonioxEventTransformer: - def __init__(self, *, translated_only: bool) -> None: - self._translated_only: Final = translated_only + def __init__(self, *, drops_originals: bool) -> None: + self._drops_originals: Final = drops_originals self._item_id: str | None = None self._final_tokens: Final[list[_SonioxToken]] = [] # mutable-ok: a turn's tokens append in O(1) each + self._finalizes: tuple[bool, ...] = () self._processed_ms: int = 0 self._billed_ms: int = 0 self._refusal: SonioxException | None = None @@ -272,6 +275,12 @@ class SonioxEventTransformer: return (self._report(error),) return tuple(chain.from_iterable(self._token_events(token) for token in response.tokens)) + def queue_finalize(self, *, discards_turn: bool) -> None: + if discards_turn and not self._finalizes: + self._item_id = None + self._final_tokens.clear() + self._finalizes = (*self._finalizes, discards_turn) + def take_unbilled_usage(self, stream_ms: int = 0) -> RealtimeInputAudioTranscriptionUsage | None: billable_ms: Final = max(self._processed_ms, stream_ms) unbilled_ms: Final = billable_ms - self._billed_ms @@ -290,14 +299,27 @@ class SonioxEventTransformer: self._processed_ms = total_audio_proc_ms def _token_events(self, token: _SonioxToken) -> tuple[OpenAIRealtimeEvents, ...]: - if token.text in (_ENDPOINT_TOKEN, _FINALIZED_TOKEN): - return self._complete(speech_stopped=token.text == _ENDPOINT_TOKEN) + if token.text == _FINALIZED_TOKEN: + return self._finalized() + if self._discarding: + return () + if token.text == _ENDPOINT_TOKEN: + return self._complete(speech_stopped=True) started: Final = self._start() - if not token.is_final or (self._translated_only and token.translation_status != "translation"): + if not token.is_final or (self._drops_originals and token.translation_status == "original"): return started self._final_tokens.append(token) return (*started, _delta(self._current_item_id(), token)) + @property + def _discarding(self) -> bool: + return self._finalizes[:1] == (True,) + + def _finalized(self) -> tuple[OpenAIRealtimeEvents, ...]: + discarded: Final = self._discarding + self._finalizes = self._finalizes[1:] + return () if discarded else self._complete(speech_stopped=False) + def _start(self) -> tuple[OpenAIRealtimeEvents, ...]: if self._item_id is not None: return () @@ -411,7 +433,7 @@ class SonioxRealtimeConfig(BaseRealtimeConfig): self._drop_params: Final = drop_params self._wrap: Final = wrap self._clock: Final = clock - self._transformer: Final = SonioxEventTransformer(translated_only=self._options.translation is not None) + self._transformer: Final = SonioxEventTransformer(drops_originals=self._options.translation is not None) self._start_request: SonioxStartRequest | None = None self._session_id: str = f"sess_{uuid.uuid4().hex}" self._started_at: float | None = None @@ -458,15 +480,27 @@ class SonioxRealtimeConfig(BaseRealtimeConfig): request: Final = json_object(message, SonioxProtocolError) event_type: Final = request.get("type") if event_type in SESSION_UPDATE_EVENT_TYPES: - return self._start(model, parse_transcription_session_update(message, SonioxProtocolError)) + update: Final = parse_transcription_session_update(message, SonioxProtocolError) + ignored: Final = _ignored_settings(update) + if ignored: + verbose_logger.warning("Soniox realtime: ignoring unsupported session settings %s", ignored) + return self._start(model, update) if event_type == "input_audio_buffer.append": audio: Final = decode_pcm16_append(request.get("audio"), error=SonioxProtocolError) return (*self._start(model, None), audio) - control_frame: Final = _CLIENT_CONTROL_FRAMES.get(event_type) if isinstance(event_type, str) else None - if control_frame is None: - verbose_logger.debug("Soniox realtime: dropping unsupported client event %s", event_type) - return () - return (*self._start(model, None), control_frame) + if event_type == "input_audio_buffer.commit": + return self._finalize(model, discards_turn=False) + if event_type == "input_audio_buffer.clear": + return () if self._start_request is None else self._finalize(model, discards_turn=True) + if event_type == "input_audio_buffer.end": + return (*self._start(model, None), _END_STREAM) + verbose_logger.debug("Soniox realtime: dropping unsupported client event %s", event_type) + return () + + def _finalize(self, model: str, *, discards_turn: bool) -> tuple[str, ...]: + start: Final = self._start(model, None) + self._transformer.queue_finalize(discards_turn=discards_turn) + return (*start, _FINALIZE) def session_refusal(self) -> Exception | None: return self._transformer.refusal diff --git a/tests/unit/llms/soniox/realtime/test_soniox_realtime_transformation.py b/tests/unit/llms/soniox/realtime/test_soniox_realtime_transformation.py index d83fd425103..d52a2e718df 100644 --- a/tests/unit/llms/soniox/realtime/test_soniox_realtime_transformation.py +++ b/tests/unit/llms/soniox/realtime/test_soniox_realtime_transformation.py @@ -90,6 +90,14 @@ def test_session_update_starts_the_stream_with_client_audio_and_deployment_optio } +def test_an_empty_transcription_language_keeps_the_deployment_language_hints(): + config = SonioxRealtimeConfig(options=SonioxRealtimeOptions(language_hints=("fa",))) + + frames = config.transform_realtime_request(_session_update({"transcription": {"language": ""}}), MODEL) + + assert _start_request(frames)["language_hints"] == ["fa"] + + def test_audio_before_session_update_starts_a_default_stream_once_and_rejects_a_later_change(): config = SonioxRealtimeConfig() @@ -121,10 +129,48 @@ def test_buffer_control_events_map_to_soniox_control_frames(event_type: str, con assert config.transform_realtime_request(_event(event_type), MODEL) == (control_frame,) -def test_unsupported_client_events_are_dropped(): +@pytest.mark.parametrize("event_type", ["response.create", "input_audio_buffer.clear"]) +def test_events_with_nothing_to_send_upstream_are_dropped_without_starting_the_stream(event_type: str): config = SonioxRealtimeConfig() - assert config.transform_realtime_request(_event("input_audio_buffer.clear"), MODEL) == () + assert config.transform_realtime_request(_event(event_type), MODEL) == () + assert _start_request(config.transform_realtime_request(_append(), MODEL))["model"] == MODEL + + +def _events(config: SonioxRealtimeConfig, tokens: list[dict[str, object]]) -> tuple[OpenAIRealtimeEvents, ...]: + return tuple( + config.transform_realtime_response(_response(tokens), MODEL, MagicMock(), EMPTY_TRANSFORM_INPUT)["response"] + ) + + +def test_clear_discards_the_open_turn_and_everything_soniox_finalizes_for_it(): + config = SonioxRealtimeConfig() + config.transform_realtime_request(_append(), MODEL) + _events(config, [_token("Hel")]) + + assert config.transform_realtime_request(_event("input_audio_buffer.clear"), MODEL) == ('{"type":"finalize"}',) + cleared = _events(config, [_token("lo"), _token(""), _token("")]) + config.transform_realtime_request(_event("input_audio_buffer.commit"), MODEL) + kept = _events(config, [_token("Bye"), _token("")]) + + assert cleared == () + assert kept[-1]["transcript"] == "Bye" + + +def test_clear_after_a_commit_keeps_the_committed_turn_and_drops_only_audio_sent_after_it(): + config = SonioxRealtimeConfig() + config.transform_realtime_request(_append(), MODEL) + _events(config, [_token("One")]) + config.transform_realtime_request(_event("input_audio_buffer.commit"), MODEL) + config.transform_realtime_request(_event("input_audio_buffer.clear"), MODEL) + + committed = _events(config, [_token("")]) + cleared = _events(config, [_token(" two"), _token("")]) + later = _events(config, [_token("Three"), _token("")]) + + assert committed[-1]["transcript"] == "One" + assert cleared == () + assert later[-1]["transcript"] == "Three" @pytest.mark.parametrize( @@ -185,7 +231,7 @@ def test_get_complete_url_defaults_to_soniox_and_opens_a_websocket_for_http_base def test_endpoint_detection_emits_one_openai_turn_billed_by_processed_audio(): - transformer = SonioxEventTransformer(translated_only=False) + transformer = SonioxEventTransformer(drops_originals=False) provisional = transformer.transform(_response([_token("Hel", is_final=False)], total_audio_proc_ms=500)) finals = transformer.transform( @@ -212,7 +258,7 @@ def test_endpoint_detection_emits_one_openai_turn_billed_by_processed_audio(): def test_manual_finalization_completes_the_turn_without_speech_stopped_and_starts_a_new_item(): - transformer = SonioxEventTransformer(translated_only=False) + transformer = SonioxEventTransformer(drops_originals=False) first = transformer.transform(_response([_token("One"), _token("")])) second = transformer.transform(_response([_token("Two"), _token("")])) @@ -229,13 +275,13 @@ def test_manual_finalization_completes_the_turn_without_speech_stopped_and_start def test_finalization_without_speech_emits_nothing(): - transformer = SonioxEventTransformer(translated_only=False) + transformer = SonioxEventTransformer(drops_originals=False) assert transformer.transform(_response([_token("")])) == () def test_translation_sessions_transcribe_only_the_translated_tokens(): - transformer = SonioxEventTransformer(translated_only=True) + transformer = SonioxEventTransformer(drops_originals=True) events = transformer.transform( _response( @@ -251,8 +297,20 @@ def test_translation_sessions_transcribe_only_the_translated_tokens(): assert events[-1]["transcript"] == "Hello" +def test_translation_sessions_keep_speech_soniox_leaves_untranslated(): + transformer = SonioxEventTransformer(drops_originals=True) + + events = transformer.transform( + _response( + [_token("Hola", translation_status="none"), _token(" amigos", translation_status="none"), _token("")] + ) + ) + + assert events[-1]["transcript"] == "Hola amigos" + + def test_diarized_tokens_label_each_delta_and_split_the_completed_turn_into_speaker_segments(): - transformer = SonioxEventTransformer(translated_only=False) + transformer = SonioxEventTransformer(drops_originals=False) events = transformer.transform( _response( @@ -281,7 +339,7 @@ def test_diarized_tokens_label_each_delta_and_split_the_completed_turn_into_spea def test_translation_sessions_announce_speech_on_the_first_spoken_token_before_any_translation(): - transformer = SonioxEventTransformer(translated_only=True) + transformer = SonioxEventTransformer(drops_originals=True) events = transformer.transform(_response([_token("Hol", is_final=False, translation_status="original")])) @@ -289,7 +347,7 @@ def test_translation_sessions_announce_speech_on_the_first_spoken_token_before_a def test_provider_errors_surface_as_openai_error_events(): - transformer = SonioxEventTransformer(translated_only=False) + transformer = SonioxEventTransformer(drops_originals=False) events = transformer.transform(json.dumps({"error_code": 401, "error_message": "Invalid API key."})) @@ -300,7 +358,7 @@ def test_provider_errors_surface_as_openai_error_events(): def test_malformed_provider_frames_are_rejected(): with pytest.raises(SonioxProtocolError): - SonioxEventTransformer(translated_only=False).transform('{"tokens": "nope"}') + SonioxEventTransformer(drops_originals=False).transform('{"tokens": "nope"}') def test_audio_processed_after_the_last_turn_is_billed_on_session_close(): From 7f6c46a3563fe3417c04aec7e20ae0122f85f52e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 22:57:48 -0700 Subject: [PATCH 13/13] fix(soniox): reject frames without a type, keep strict language hints, queue finalizes in a deque --- .../llms/soniox/realtime/transformation.py | 25 ++++++++++++------- .../test_soniox_realtime_transformation.py | 17 +++++++++++++ 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/litellm/llms/soniox/realtime/transformation.py b/litellm/llms/soniox/realtime/transformation.py index 2a80371e7e5..805e24f3026 100644 --- a/litellm/llms/soniox/realtime/transformation.py +++ b/litellm/llms/soniox/realtime/transformation.py @@ -1,5 +1,6 @@ import asyncio import time +from collections import deque from collections.abc import Awaitable, Callable, Mapping from itertools import chain, groupby from types import MappingProxyType, TracebackType @@ -143,8 +144,10 @@ def _sample_rate(update: TranscriptionSessionUpdate | None) -> int: return audio_format.rate -def _language_hints(language: str | None, configured: tuple[str, ...] | None) -> tuple[str, ...] | None: - if not language: +def _language_hints( + language: str | None, configured: tuple[str, ...] | None, strict: bool | None +) -> tuple[str, ...] | None: + if not language or (strict and configured): return configured return (language, *(hint for hint in configured or () if hint != language)) @@ -163,7 +166,9 @@ def build_start_request( ) -> SonioxStartRequest: return SonioxStartRequest( **options.model_dump(exclude={"language_hints"}), - language_hints=_language_hints(None if update is None else update.language, options.language_hints), + language_hints=_language_hints( + None if update is None else update.language, options.language_hints, options.language_hints_strict + ), model=model, audio_format="pcm_s16le", sample_rate=_sample_rate(update), @@ -197,7 +202,8 @@ def _changes_started_stream(update: TranscriptionSessionUpdate, started: SonioxS _changes_started_rate(update, started) or ( update.language is not None - and _language_hints(update.language, started.language_hints) != started.language_hints + and _language_hints(update.language, started.language_hints, started.language_hints_strict) + != started.language_hints ) or (turn_detection_set and _uses_endpoint_detection(update, drop_params) != started.enable_endpoint_detection) ) @@ -255,7 +261,7 @@ class SonioxEventTransformer: self._drops_originals: Final = drops_originals self._item_id: str | None = None self._final_tokens: Final[list[_SonioxToken]] = [] # mutable-ok: a turn's tokens append in O(1) each - self._finalizes: tuple[bool, ...] = () + self._finalizes: Final[deque[bool]] = deque() # mutable-ok: pending finalizes queue and pop in O(1) each self._processed_ms: int = 0 self._billed_ms: int = 0 self._refusal: SonioxException | None = None @@ -279,7 +285,7 @@ class SonioxEventTransformer: if discards_turn and not self._finalizes: self._item_id = None self._final_tokens.clear() - self._finalizes = (*self._finalizes, discards_turn) + self._finalizes.append(discards_turn) def take_unbilled_usage(self, stream_ms: int = 0) -> RealtimeInputAudioTranscriptionUsage | None: billable_ms: Final = max(self._processed_ms, stream_ms) @@ -313,11 +319,10 @@ class SonioxEventTransformer: @property def _discarding(self) -> bool: - return self._finalizes[:1] == (True,) + return bool(self._finalizes) and self._finalizes[0] def _finalized(self) -> tuple[OpenAIRealtimeEvents, ...]: - discarded: Final = self._discarding - self._finalizes = self._finalizes[1:] + discarded: Final = self._finalizes.popleft() if self._finalizes else False return () if discarded else self._complete(speech_stopped=False) def _start(self) -> tuple[OpenAIRealtimeEvents, ...]: @@ -479,6 +484,8 @@ class SonioxRealtimeConfig(BaseRealtimeConfig): ) -> tuple[str | bytes, ...]: request: Final = json_object(message, SonioxProtocolError) event_type: Final = request.get("type") + if event_type is None: + raise SonioxProtocolError("Soniox realtime takes OpenAI realtime events, and this message has no type") if event_type in SESSION_UPDATE_EVENT_TYPES: update: Final = parse_transcription_session_update(message, SonioxProtocolError) ignored: Final = _ignored_settings(update) diff --git a/tests/unit/llms/soniox/realtime/test_soniox_realtime_transformation.py b/tests/unit/llms/soniox/realtime/test_soniox_realtime_transformation.py index d52a2e718df..3a6ef57d1ae 100644 --- a/tests/unit/llms/soniox/realtime/test_soniox_realtime_transformation.py +++ b/tests/unit/llms/soniox/realtime/test_soniox_realtime_transformation.py @@ -98,6 +98,23 @@ def test_an_empty_transcription_language_keeps_the_deployment_language_hints(): assert _start_request(frames)["language_hints"] == ["fa"] +def test_a_client_language_cannot_widen_strict_deployment_language_hints(): + config = SonioxRealtimeConfig(options=SonioxRealtimeOptions(language_hints=("fa",), language_hints_strict=True)) + + frames = config.transform_realtime_request(_session_update({"transcription": {"language": "en"}}), MODEL) + + assert _start_request(frames)["language_hints"] == ["fa"] + assert config.transform_realtime_request(_session_update({"transcription": {"language": "en"}}), MODEL) == () + + +def test_a_soniox_native_config_frame_is_rejected_without_starting_the_stream(): + config = SonioxRealtimeConfig() + + with pytest.raises(SonioxProtocolError): + config.transform_realtime_request(json.dumps({"model": "stt-rt-v5", "audio_format": "auto"}), MODEL) + assert _start_request(config.transform_realtime_request(_append(), MODEL))["sample_rate"] == 24_000 + + def test_audio_before_session_update_starts_a_default_stream_once_and_rejects_a_later_change(): config = SonioxRealtimeConfig()