diff --git a/litellm/llms/base_llm/realtime/transcription_protocol.py b/litellm/llms/base_llm/realtime/transcription_protocol.py index c3264911d46..e67fd2c2dd0 100644 --- a/litellm/llms/base_llm/realtime/transcription_protocol.py +++ b/litellm/llms/base_llm/realtime/transcription_protocol.py @@ -16,6 +16,7 @@ from litellm.types.llms.openai import ( OpenAIRealtimeServerVadTurnDetection, OpenAIRealtimeTranscriptionSession, OpenAIRealtimeTranscriptionSessionCreated, + OpenAIRealtimeTranscriptionSessionUpdated, OpenAIRealtimeTranscriptionSettings, ) from litellm.types.realtime import RealtimeInputAudioTranscriptionDurationUsage, RealtimeInputAudioTranscriptionUsage @@ -224,6 +225,17 @@ def transcription_session_created_event( return event +def transcription_session_updated_event( + session: OpenAIRealtimeTranscriptionSession, +) -> OpenAIRealtimeTranscriptionSessionUpdated: + event: Final[OpenAIRealtimeTranscriptionSessionUpdated] = { + "type": "session.updated", + "event_id": new_event_id(), + "session": session, + } + return event + + def error_event(message: str) -> OpenAIRealtimeErrorEvent: event: Final[OpenAIRealtimeErrorEvent] = { "type": "error", diff --git a/litellm/llms/sarvam/realtime/transformation.py b/litellm/llms/sarvam/realtime/transformation.py index 3a5c91e5785..39b458d7a5f 100644 --- a/litellm/llms/sarvam/realtime/transformation.py +++ b/litellm/llms/sarvam/realtime/transformation.py @@ -3,12 +3,14 @@ import json import math from collections.abc import Mapping from dataclasses import dataclass +from ipaddress import ip_address from types import MappingProxyType from typing import Final from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse from pydantic import JsonValue +import litellm from litellm import verbose_logger from litellm._uuid import uuid from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -26,6 +28,7 @@ from litellm.llms.base_llm.realtime.transcription_protocol import ( speech_event, transcription_session, transcription_session_created_event, + transcription_session_updated_event, ) from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig from litellm.secret_managers.main import get_secret_str @@ -46,12 +49,13 @@ from litellm.types.realtime import ( RealtimeResponseTransformInput, RealtimeResponseTypedDict, ) +from litellm.types.utils import LlmProviders DEFAULT_SARVAM_REALTIME_API_BASE: Final = "wss://api.sarvam.ai" SARVAM_REALTIME_PATH: Final = "/speech-to-text-realtime/ws" SARVAM_REALTIME_ENCODING: Final = "linear16" DEFAULT_SARVAM_REALTIME_MODEL: Final = "saaras:v3-realtime" -SUPPORTED_MODELS: Final = frozenset({"saaras:v3-realtime", "saaras:v4"}) +REALTIME_ENDPOINT: Final = "/v1/realtime" SUPPORTED_SAMPLE_RATES: Final = frozenset({8_000, 16_000}) AUTO_LANGUAGE: Final = "auto" SUPPORTED_LANGUAGE_CODES: Final = ( @@ -89,10 +93,28 @@ class SarvamProtocolError(RealtimeTranscriptionProtocolError): pass +def is_realtime_transcription_model(model: str) -> bool: + """Support follows the model registry rather than a list in this file, so a new Sarvam realtime model + only needs a cost map entry. Only an exact hit counts: the registry resolves unknown names to a family.""" + registry_key: Final = f"{LlmProviders.SARVAM.value}/{model}" + try: + model_info: Final = litellm.get_model_info(model=registry_key, custom_llm_provider=LlmProviders.SARVAM.value) + except Exception: # noqa: BLE001 # get_model_info raises bare Exception for unmapped models + return False + return ( + model_info["key"] == registry_key + and model_info.get("mode") == "audio_transcription" + and REALTIME_ENDPOINT in (model_info.get("supported_endpoints") or ()) + ) + + def normalize_model(model: str) -> str: - normalized: Final = model.removeprefix("sarvam/").strip() - if normalized not in SUPPORTED_MODELS: - raise SarvamProtocolError(f"unsupported Sarvam realtime model: {model}") + normalized: Final = model.removeprefix(f"{LlmProviders.SARVAM.value}/").strip() + if not is_realtime_transcription_model(normalized): + raise SarvamProtocolError( + f"unsupported Sarvam realtime model: {model}. Add it to the model cost map with " + f'"mode": "audio_transcription" and "supported_endpoints": ["{REALTIME_ENDPOINT}"]' + ) return normalized @@ -129,6 +151,20 @@ def _connection_sample_rate(query: str) -> SarvamSampleRate: return 8_000 if int(declared[0]) == 8_000 else 16_000 +def _is_loopback(hostname: str) -> bool: + if hostname == "localhost": + return True + try: + return ip_address(hostname).is_loopback + except ValueError: + return False + + +def _netloc(hostname: str, port: int | None) -> str: + host: Final = f"[{hostname}]" if ":" in hostname else hostname + return host if port is None else f"{host}:{port}" + + def build_sarvam_connection(api_base: str | None, model: str) -> SarvamConnection: """Sarvam fixes ``encoding`` and ``sample_rate`` at connect time, so the rate is read from ``api_base``. @@ -140,15 +176,18 @@ def build_sarvam_connection(api_base: str | None, model: str) -> SarvamConnectio scheme: Final = _WEBSOCKET_SCHEMES.get(parsed.scheme) if scheme is None or not parsed.hostname or parsed.username is not None or parsed.password is not None: raise ValueError("Sarvam api_base must be an absolute wss:// or https:// URL without credentials") + if scheme == "ws" and not _is_loopback(parsed.hostname): + # The subscription key travels as a connection header, so cleartext is only ever for a local test double. + raise ValueError("Sarvam api_base must use wss:// or https:// unless it points at a loopback address") sample_rate: Final = _connection_sample_rate(parsed.query) - netloc: Final = parsed.hostname if parsed.port is None else f"{parsed.hostname}:{parsed.port}" + netloc: Final = _netloc(parsed.hostname, parsed.port) query: Final = urlencode( - { - "model": normalized_model, - "language_code": AUTO_LANGUAGE, - "encoding": SARVAM_REALTIME_ENCODING, - "sample_rate": str(sample_rate), - } + ( + ("model", normalized_model), + ("language_code", AUTO_LANGUAGE), + ("encoding", SARVAM_REALTIME_ENCODING), + ("sample_rate", str(sample_rate)), + ) ) return SarvamConnection( model=normalized_model, @@ -177,15 +216,21 @@ def _validate_audio_format(update: TranscriptionSessionUpdate, sample_rate: Sarv ) -def _parse_endpointing(update: TranscriptionSessionUpdate) -> SarvamEndpointing: +def _parse_endpointing(update: TranscriptionSessionUpdate, current: SarvamEndpointing) -> SarvamEndpointing: + """A later ``session.update`` that changes only the language leaves turn detection out entirely, which must + keep the endpointing the session already runs on rather than reverting it to VAD.""" if update.turn_detection_disabled: return "manual" - if update.turn_detection_type not in (None, "server_vad"): + if update.turn_detection is None: + return current + if update.turn_detection_type != "server_vad": raise SarvamProtocolError("Sarvam realtime supports server_vad turn detection or null") return "vad" -def parse_session_update(payload: str, connection: SarvamConnection) -> SarvamSessionUpdate: +def parse_session_update( + payload: str, connection: SarvamConnection, endpointing: SarvamEndpointing +) -> SarvamSessionUpdate: update: Final = parse_transcription_session_update(payload, SarvamProtocolError) if update.session_type not in (None, "transcription", "realtime"): raise SarvamProtocolError("Sarvam realtime supports transcription sessions only") @@ -198,7 +243,7 @@ def parse_session_update(payload: str, connection: SarvamConnection) -> SarvamSe _validate_audio_format(update, connection.sample_rate) return SarvamSessionUpdate( language=None if update.language is None else normalize_language_code(update.language), - endpointing=_parse_endpointing(update), + endpointing=_parse_endpointing(update, endpointing), ) @@ -284,6 +329,7 @@ class SarvamRealtimeConfig(BaseRealtimeConfig): self._connection: SarvamConnection | None = None self._language: str = AUTO_LANGUAGE self._endpointing: SarvamEndpointing = "vad" + self._session_id: str | None = None self._speech_open: bool = False self._forwarded_audio_bytes: int = 0 self._reported_seconds: float | None = None @@ -297,7 +343,7 @@ class SarvamRealtimeConfig(BaseRealtimeConfig): key: Final = (api_key or get_secret_str("SARVAM_API_KEY") or "").strip() if not key: raise ValueError("api_key is required for Sarvam API calls") - return {**headers, "api-subscription-key": key} + return {**headers, "api-subscription-key": key} # mutable-ok: BaseRealtimeConfig returns a plain header dict def get_complete_url(self, api_base: str | None, model: str, api_key: str | None = None) -> str: connection: Final = build_sarvam_connection(api_base, model) @@ -325,7 +371,7 @@ class SarvamRealtimeConfig(BaseRealtimeConfig): return () case unsupported: verbose_logger.debug("Sarvam realtime: dropping unsupported client event %s", unsupported) - return () + return () def unbilled_usage_on_session_close(self, model: str) -> RealtimeInputAudioTranscriptionUsage | None: """Sarvam's own ``audio_duration_s`` is the billed quantity; the forwarded audio only covers a session @@ -368,20 +414,29 @@ class SarvamRealtimeConfig(BaseRealtimeConfig): return connection def _configure(self, message: str) -> tuple[str, ...]: - update: Final = parse_session_update(message, self._require_connection()) + update: Final = parse_session_update(message, self._require_connection(), self._endpointing) language: Final = update.language or self._language language_changed: Final = language != self._language endpointing_changed: Final = update.endpointing != self._endpointing self._language = language self._endpointing = update.endpointing - if not language_changed and not endpointing_changed: - return () - empty: Final[SarvamConfigUpdate] = {"event": "config.update"} - with_language: Final[SarvamConfigUpdate] = {**empty, "language_code": language} if language_changed else empty - changes: Final[SarvamConfigUpdate] = ( - {**with_language, "endpointing": update.endpointing} if endpointing_changed else with_language - ) - return (json.dumps(changes, separators=(",", ":")),) + if language_changed and endpointing_changed: + both: Final[SarvamConfigUpdate] = { + "event": "config.update", + "language_code": language, + "endpointing": update.endpointing, + } + return (json.dumps(both, separators=(",", ":")),) + if language_changed: + language_only: Final[SarvamConfigUpdate] = {"event": "config.update", "language_code": language} + return (json.dumps(language_only, separators=(",", ":")),) + if endpointing_changed: + endpointing_only: Final[SarvamConfigUpdate] = { + "event": "config.update", + "endpointing": update.endpointing, + } + return (json.dumps(endpointing_only, separators=(",", ":")),) + return () def _append_audio(self, request: Mapping[str, JsonValue]) -> tuple[str, ...]: audio: Final = decode_pcm16_append(request.get("audio"), None, SarvamProtocolError) @@ -421,6 +476,8 @@ class SarvamRealtimeConfig(BaseRealtimeConfig): return self._partial(frame) case "transcript.final": return self._final(frame) + case "config.updated": + return self._session_updated() case "session.end": self._reported_seconds = _audio_seconds(frame) return () @@ -428,15 +485,25 @@ class SarvamRealtimeConfig(BaseRealtimeConfig): return (error_event(self._error_message(frame)),) case ignored: verbose_logger.debug("Sarvam realtime: no client event for backend event %s", ignored) - return () + return () def _session_created(self, frame: Mapping[str, JsonValue]) -> OpenAIRealtimeEvents: request_id: Final = json_string(frame.get("request_id"), "session.begin request_id", SarvamProtocolError) if not request_id: raise SarvamProtocolError("session.begin event has an invalid request_id") - return transcription_session_created_event( - openai_session(self._require_connection(), request_id, self._language, self._endpointing == "vad") - ) + self._session_id = request_id + return transcription_session_created_event(self._openai_session(request_id)) + + def _session_updated(self) -> tuple[OpenAIRealtimeEvents, ...]: + """Sarvam opens the socket on its own defaults and acknowledges each ``config.update`` separately, so a + client that configures the session after ``session.created`` only learns what took effect from here.""" + session_id: Final = self._session_id + if session_id is None: + return () + return (transcription_session_updated_event(self._openai_session(session_id)),) + + def _openai_session(self, session_id: str) -> OpenAIRealtimeTranscriptionSession: + return openai_session(self._require_connection(), session_id, self._language, self._endpointing == "vad") def _speech_started(self, utterance_idx: int) -> tuple[OpenAIRealtimeEvents, ...]: state: Final = self._utterances.state(utterance_idx) diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 599b1a76249..fc2b69a0534 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -2270,6 +2270,12 @@ class OpenAIRealtimeTranscriptionSessionCreated(TypedDict): session: ReadOnly[OpenAIRealtimeTranscriptionSession] +class OpenAIRealtimeTranscriptionSessionUpdated(TypedDict): + type: ReadOnly[Literal["session.updated"]] + event_id: ReadOnly[str] + session: ReadOnly[OpenAIRealtimeTranscriptionSession] + + class OpenAIRealtimeInputAudioTranscriptionDelta(TypedDict): type: ReadOnly[Literal["conversation.item.input_audio_transcription.delta"]] event_id: ReadOnly[str] @@ -2349,6 +2355,7 @@ OpenAIRealtimeEvents = ( | OpenAIRealtimeInputAudioTranscriptionDelta | OpenAIRealtimeInputAudioTranscriptionCompleted | OpenAIRealtimeTranscriptionSessionCreated + | OpenAIRealtimeTranscriptionSessionUpdated | OpenAIRealtimeErrorEvent ) diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 38aad3ace1c..36414118328 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -1257,6 +1257,19 @@ def test_get_provider_rerank_config(): assert isinstance(config, HostedVLLMRerankConfig) +def test_get_provider_realtime_config_serves_sarvam_speech_to_text(): + """Sarvam realtime transcription only reaches /v1/realtime when the provider registry hands back its + config; without this the endpoint rejects every sarvam deployment as an unsupported model.""" + from litellm.llms.sarvam.realtime.transformation import SarvamRealtimeConfig + from litellm.utils import LlmProviders + + config = ProviderConfigManager.get_provider_realtime_config( + model="saaras:v3-realtime", provider=LlmProviders.SARVAM + ) + + assert isinstance(config, SarvamRealtimeConfig) + + def test_get_provider_text_to_speech_config_vertex_gemini_skips_cloud_tts(): """Regression for LIT-6501: mapping vertex Gemini TTS params through Google Cloud TTS dropped response_format before the speech_to_completion bridge could honor it.""" diff --git a/tests/unit/llms/sarvam/realtime/test_sarvam_realtime_transformation.py b/tests/unit/llms/sarvam/realtime/test_sarvam_realtime_transformation.py index 0cedb183e87..0183ecbad95 100644 --- a/tests/unit/llms/sarvam/realtime/test_sarvam_realtime_transformation.py +++ b/tests/unit/llms/sarvam/realtime/test_sarvam_realtime_transformation.py @@ -27,6 +27,8 @@ EMPTY_TRANSFORM_INPUT: Final[RealtimeResponseTransformInput] = { "current_delta_type": None, } MODEL: Final = f"sarvam/{DEFAULT_SARVAM_REALTIME_MODEL}" + +pytestmark: Final = pytest.mark.usefixtures("local_model_cost_map") ONE_SECOND_OF_SILENCE: Final = b"\x00\x00" * 16_000 @@ -36,6 +38,8 @@ def _session_update( language: str | None = None, model: str | None = None, ) -> str: + """``turn_detection="omit"`` leaves the key out entirely, the way a client that only changes its language + sends a partial update; ``None`` sends an explicit null, which disables turn detection.""" transcription: Final[dict[str, str]] = { **({"model": model} if model is not None else {}), **({"language": language} if language is not None else {}), @@ -48,7 +52,11 @@ def _session_update( "audio": { "input": { **({"format": {"type": "audio/pcm", "rate": rate}} if rate is not None else {}), - "turn_detection": None if turn_detection is None else {"type": turn_detection}, + **( + {} + if turn_detection == "omit" + else {"turn_detection": None if turn_detection is None else {"type": turn_detection}} + ), "transcription": transcription, } }, @@ -133,11 +141,54 @@ def test_unusable_api_base_is_rejected(api_base: str): build_sarvam_connection(api_base, MODEL) -def test_unsupported_model_is_rejected(): +@pytest.mark.parametrize("api_base", ["ws://stt.internal.example", "http://198.51.100.7:8123"]) +def test_cleartext_to_a_remote_host_is_rejected_because_the_key_is_a_connection_header(api_base: str): + with pytest.raises(ValueError, match="loopback"): + build_sarvam_connection(api_base, MODEL) + + +@pytest.mark.parametrize( + ("api_base", "expected_netloc"), + [ + ("ws://localhost:8123", "localhost:8123"), + ("ws://127.0.0.1:8123", "127.0.0.1:8123"), + ("ws://[::1]:8123", "[::1]:8123"), + ], +) +def test_a_loopback_test_double_stays_reachable_over_cleartext(api_base: str, expected_netloc: str): + url = build_sarvam_connection(api_base, MODEL).url + + assert url.startswith(f"ws://{expected_netloc}/speech-to-text-realtime/ws?") + + +def test_a_model_the_registry_does_not_serve_on_the_realtime_endpoint_is_rejected(): with pytest.raises(SarvamProtocolError, match="model"): build_sarvam_connection(None, "sarvam/whisper-large") +def test_a_model_registered_for_the_realtime_endpoint_is_accepted_without_a_code_change(): + """Support reads the model registry, so a new Sarvam realtime model ships as a cost map entry.""" + import litellm + + litellm.register_model( + { + "sarvam/saaras:v9-realtime": { + "litellm_provider": "sarvam", + "mode": "audio_transcription", + "supported_endpoints": ["/v1/realtime"], + } + } + ) + + assert build_sarvam_connection(None, "sarvam/saaras:v9-realtime").model == "saaras:v9-realtime" + + +def test_a_sarvam_chat_model_is_not_served_on_the_realtime_endpoint(): + """The registry entry has to say audio_transcription on /v1/realtime, not merely exist under sarvam/.""" + with pytest.raises(SarvamProtocolError, match="model"): + build_sarvam_connection(None, "sarvam/sarvam-m") + + @pytest.mark.parametrize( ("requested", "expected"), [("hi", "hi-IN"), ("hi-IN", "hi-IN"), ("HI-in", "hi-IN"), ("en_IN", "en-IN"), ("auto", "auto")], @@ -272,6 +323,41 @@ def test_session_begin_becomes_an_openai_transcription_session(): assert session["audio"]["input"]["turn_detection"] == {"type": "server_vad"} +def test_a_later_update_that_only_changes_the_language_keeps_manual_endpointing(): + """Turn detection left out of a partial update means unchanged. Reverting to VAD here would silently turn + the client's audio buffer commits into no-ops.""" + config = _configured(turn_detection=None) + + language_only = _sent(config, _session_update(turn_detection="omit", language="ta")) + config.transform_realtime_request(_append(ONE_SECOND_OF_SILENCE), MODEL) + committed = _sent(config, _client_event("input_audio_buffer.commit")) + + assert language_only == [{"event": "config.update", "language_code": "ta-IN"}] + assert [frame["event"] for frame in committed] == ["speech_end", "flush"] + + +def test_applied_settings_reach_the_client_as_a_session_updated_event(): + """Sarvam opens on its own defaults, so a client that configures the session after session.created only + learns the language and turn detection that took effect from the config.updated acknowledgement.""" + config = _configured() + _backend(config, event="session.begin", request_id="req-42", config={}) + config.transform_realtime_request(_session_update(language="hi", turn_detection=None), MODEL) + + events = _backend(config, event="config.updated", applied=["language_code", "endpointing"]) + + assert [event["type"] for event in events] == ["session.updated"] + session = events[0]["session"] + assert session["id"] == "req-42" + assert session["audio"]["input"]["transcription"]["language"] == "hi-IN" + assert session["audio"]["input"]["turn_detection"] is None + + +def test_an_acknowledgement_before_the_session_exists_is_dropped(): + config = _configured() + + assert _backend(config, event="config.updated", applied=["language_code"]) == [] + + def test_session_begin_reports_manual_endpointing_as_disabled_turn_detection(): config = _configured(turn_detection=None) @@ -411,7 +497,7 @@ def test_provider_errors_reach_the_client_as_openai_error_events(): assert "language not supported" in events[0]["error"]["message"] -@pytest.mark.parametrize("event", ["config.updated", "pong", "something.new"]) +@pytest.mark.parametrize("event", ["pong", "something.new"]) def test_backend_events_without_an_openai_equivalent_are_dropped(event: str): config = _configured()