fix(sarvam): address review on the realtime speech to text config

Model support now reads the model registry instead of a list in this
file, so a new Sarvam realtime model ships as a cost map entry with
mode audio_transcription on /v1/realtime.

Sarvam opens the socket on its own defaults and acknowledges each
config.update separately, so its config.updated becomes an OpenAI
session.updated. Without it a client that configures the session after
session.created never learns which language and turn detection took
effect.

A session.update that leaves turn_detection out is a partial update and
keeps the endpointing the session already runs on. Reverting to VAD
there silently turned a manual client's audio buffer commits into
no-ops.

The subscription key travels as a connection header, so cleartext is
refused unless the host is a loopback address, and an IPv6 host keeps
the brackets the URL needs to stay parseable.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Srihari Unnikrishnan 2026-09-22 20:31:08 +05:30
parent 99089d1e70
commit 93341699e4
5 changed files with 218 additions and 33 deletions

View file

@ -16,6 +16,7 @@ from litellm.types.llms.openai import (
OpenAIRealtimeServerVadTurnDetection,
OpenAIRealtimeTranscriptionSession,
OpenAIRealtimeTranscriptionSessionCreated,
OpenAIRealtimeTranscriptionSessionUpdated,
OpenAIRealtimeTranscriptionSettings,
)
from litellm.types.realtime import RealtimeInputAudioTranscriptionDurationUsage, RealtimeInputAudioTranscriptionUsage
@ -224,6 +225,17 @@ def transcription_session_created_event(
return event
def transcription_session_updated_event(
session: OpenAIRealtimeTranscriptionSession,
) -> OpenAIRealtimeTranscriptionSessionUpdated:
event: Final[OpenAIRealtimeTranscriptionSessionUpdated] = {
"type": "session.updated",
"event_id": new_event_id(),
"session": session,
}
return event
def error_event(message: str) -> OpenAIRealtimeErrorEvent:
event: Final[OpenAIRealtimeErrorEvent] = {
"type": "error",

View file

@ -3,12 +3,14 @@ import json
import math
from collections.abc import Mapping
from dataclasses import dataclass
from ipaddress import ip_address
from types import MappingProxyType
from typing import Final
from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
from pydantic import JsonValue
import litellm
from litellm import verbose_logger
from litellm._uuid import uuid
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@ -26,6 +28,7 @@ from litellm.llms.base_llm.realtime.transcription_protocol import (
speech_event,
transcription_session,
transcription_session_created_event,
transcription_session_updated_event,
)
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig
from litellm.secret_managers.main import get_secret_str
@ -46,12 +49,13 @@ from litellm.types.realtime import (
RealtimeResponseTransformInput,
RealtimeResponseTypedDict,
)
from litellm.types.utils import LlmProviders
DEFAULT_SARVAM_REALTIME_API_BASE: Final = "wss://api.sarvam.ai"
SARVAM_REALTIME_PATH: Final = "/speech-to-text-realtime/ws"
SARVAM_REALTIME_ENCODING: Final = "linear16"
DEFAULT_SARVAM_REALTIME_MODEL: Final = "saaras:v3-realtime"
SUPPORTED_MODELS: Final = frozenset({"saaras:v3-realtime", "saaras:v4"})
REALTIME_ENDPOINT: Final = "/v1/realtime"
SUPPORTED_SAMPLE_RATES: Final = frozenset({8_000, 16_000})
AUTO_LANGUAGE: Final = "auto"
SUPPORTED_LANGUAGE_CODES: Final = (
@ -89,10 +93,28 @@ class SarvamProtocolError(RealtimeTranscriptionProtocolError):
pass
def is_realtime_transcription_model(model: str) -> bool:
"""Support follows the model registry rather than a list in this file, so a new Sarvam realtime model
only needs a cost map entry. Only an exact hit counts: the registry resolves unknown names to a family."""
registry_key: Final = f"{LlmProviders.SARVAM.value}/{model}"
try:
model_info: Final = litellm.get_model_info(model=registry_key, custom_llm_provider=LlmProviders.SARVAM.value)
except Exception: # noqa: BLE001 # get_model_info raises bare Exception for unmapped models
return False
return (
model_info["key"] == registry_key
and model_info.get("mode") == "audio_transcription"
and REALTIME_ENDPOINT in (model_info.get("supported_endpoints") or ())
)
def normalize_model(model: str) -> str:
normalized: Final = model.removeprefix("sarvam/").strip()
if normalized not in SUPPORTED_MODELS:
raise SarvamProtocolError(f"unsupported Sarvam realtime model: {model}")
normalized: Final = model.removeprefix(f"{LlmProviders.SARVAM.value}/").strip()
if not is_realtime_transcription_model(normalized):
raise SarvamProtocolError(
f"unsupported Sarvam realtime model: {model}. Add it to the model cost map with "
f'"mode": "audio_transcription" and "supported_endpoints": ["{REALTIME_ENDPOINT}"]'
)
return normalized
@ -129,6 +151,20 @@ def _connection_sample_rate(query: str) -> SarvamSampleRate:
return 8_000 if int(declared[0]) == 8_000 else 16_000
def _is_loopback(hostname: str) -> bool:
if hostname == "localhost":
return True
try:
return ip_address(hostname).is_loopback
except ValueError:
return False
def _netloc(hostname: str, port: int | None) -> str:
host: Final = f"[{hostname}]" if ":" in hostname else hostname
return host if port is None else f"{host}:{port}"
def build_sarvam_connection(api_base: str | None, model: str) -> SarvamConnection:
"""Sarvam fixes ``encoding`` and ``sample_rate`` at connect time, so the rate is read from ``api_base``.
@ -140,15 +176,18 @@ def build_sarvam_connection(api_base: str | None, model: str) -> SarvamConnectio
scheme: Final = _WEBSOCKET_SCHEMES.get(parsed.scheme)
if scheme is None or not parsed.hostname or parsed.username is not None or parsed.password is not None:
raise ValueError("Sarvam api_base must be an absolute wss:// or https:// URL without credentials")
if scheme == "ws" and not _is_loopback(parsed.hostname):
# The subscription key travels as a connection header, so cleartext is only ever for a local test double.
raise ValueError("Sarvam api_base must use wss:// or https:// unless it points at a loopback address")
sample_rate: Final = _connection_sample_rate(parsed.query)
netloc: Final = parsed.hostname if parsed.port is None else f"{parsed.hostname}:{parsed.port}"
netloc: Final = _netloc(parsed.hostname, parsed.port)
query: Final = urlencode(
{
"model": normalized_model,
"language_code": AUTO_LANGUAGE,
"encoding": SARVAM_REALTIME_ENCODING,
"sample_rate": str(sample_rate),
}
(
("model", normalized_model),
("language_code", AUTO_LANGUAGE),
("encoding", SARVAM_REALTIME_ENCODING),
("sample_rate", str(sample_rate)),
)
)
return SarvamConnection(
model=normalized_model,
@ -177,15 +216,21 @@ def _validate_audio_format(update: TranscriptionSessionUpdate, sample_rate: Sarv
)
def _parse_endpointing(update: TranscriptionSessionUpdate) -> SarvamEndpointing:
def _parse_endpointing(update: TranscriptionSessionUpdate, current: SarvamEndpointing) -> SarvamEndpointing:
"""A later ``session.update`` that changes only the language leaves turn detection out entirely, which must
keep the endpointing the session already runs on rather than reverting it to VAD."""
if update.turn_detection_disabled:
return "manual"
if update.turn_detection_type not in (None, "server_vad"):
if update.turn_detection is None:
return current
if update.turn_detection_type != "server_vad":
raise SarvamProtocolError("Sarvam realtime supports server_vad turn detection or null")
return "vad"
def parse_session_update(payload: str, connection: SarvamConnection) -> SarvamSessionUpdate:
def parse_session_update(
payload: str, connection: SarvamConnection, endpointing: SarvamEndpointing
) -> SarvamSessionUpdate:
update: Final = parse_transcription_session_update(payload, SarvamProtocolError)
if update.session_type not in (None, "transcription", "realtime"):
raise SarvamProtocolError("Sarvam realtime supports transcription sessions only")
@ -198,7 +243,7 @@ def parse_session_update(payload: str, connection: SarvamConnection) -> SarvamSe
_validate_audio_format(update, connection.sample_rate)
return SarvamSessionUpdate(
language=None if update.language is None else normalize_language_code(update.language),
endpointing=_parse_endpointing(update),
endpointing=_parse_endpointing(update, endpointing),
)
@ -284,6 +329,7 @@ class SarvamRealtimeConfig(BaseRealtimeConfig):
self._connection: SarvamConnection | None = None
self._language: str = AUTO_LANGUAGE
self._endpointing: SarvamEndpointing = "vad"
self._session_id: str | None = None
self._speech_open: bool = False
self._forwarded_audio_bytes: int = 0
self._reported_seconds: float | None = None
@ -297,7 +343,7 @@ class SarvamRealtimeConfig(BaseRealtimeConfig):
key: Final = (api_key or get_secret_str("SARVAM_API_KEY") or "").strip()
if not key:
raise ValueError("api_key is required for Sarvam API calls")
return {**headers, "api-subscription-key": key}
return {**headers, "api-subscription-key": key} # mutable-ok: BaseRealtimeConfig returns a plain header dict
def get_complete_url(self, api_base: str | None, model: str, api_key: str | None = None) -> str:
connection: Final = build_sarvam_connection(api_base, model)
@ -325,7 +371,7 @@ class SarvamRealtimeConfig(BaseRealtimeConfig):
return ()
case unsupported:
verbose_logger.debug("Sarvam realtime: dropping unsupported client event %s", unsupported)
return ()
return ()
def unbilled_usage_on_session_close(self, model: str) -> RealtimeInputAudioTranscriptionUsage | None:
"""Sarvam's own ``audio_duration_s`` is the billed quantity; the forwarded audio only covers a session
@ -368,20 +414,29 @@ class SarvamRealtimeConfig(BaseRealtimeConfig):
return connection
def _configure(self, message: str) -> tuple[str, ...]:
update: Final = parse_session_update(message, self._require_connection())
update: Final = parse_session_update(message, self._require_connection(), self._endpointing)
language: Final = update.language or self._language
language_changed: Final = language != self._language
endpointing_changed: Final = update.endpointing != self._endpointing
self._language = language
self._endpointing = update.endpointing
if not language_changed and not endpointing_changed:
return ()
empty: Final[SarvamConfigUpdate] = {"event": "config.update"}
with_language: Final[SarvamConfigUpdate] = {**empty, "language_code": language} if language_changed else empty
changes: Final[SarvamConfigUpdate] = (
{**with_language, "endpointing": update.endpointing} if endpointing_changed else with_language
)
return (json.dumps(changes, separators=(",", ":")),)
if language_changed and endpointing_changed:
both: Final[SarvamConfigUpdate] = {
"event": "config.update",
"language_code": language,
"endpointing": update.endpointing,
}
return (json.dumps(both, separators=(",", ":")),)
if language_changed:
language_only: Final[SarvamConfigUpdate] = {"event": "config.update", "language_code": language}
return (json.dumps(language_only, separators=(",", ":")),)
if endpointing_changed:
endpointing_only: Final[SarvamConfigUpdate] = {
"event": "config.update",
"endpointing": update.endpointing,
}
return (json.dumps(endpointing_only, separators=(",", ":")),)
return ()
def _append_audio(self, request: Mapping[str, JsonValue]) -> tuple[str, ...]:
audio: Final = decode_pcm16_append(request.get("audio"), None, SarvamProtocolError)
@ -421,6 +476,8 @@ class SarvamRealtimeConfig(BaseRealtimeConfig):
return self._partial(frame)
case "transcript.final":
return self._final(frame)
case "config.updated":
return self._session_updated()
case "session.end":
self._reported_seconds = _audio_seconds(frame)
return ()
@ -428,15 +485,25 @@ class SarvamRealtimeConfig(BaseRealtimeConfig):
return (error_event(self._error_message(frame)),)
case ignored:
verbose_logger.debug("Sarvam realtime: no client event for backend event %s", ignored)
return ()
return ()
def _session_created(self, frame: Mapping[str, JsonValue]) -> OpenAIRealtimeEvents:
request_id: Final = json_string(frame.get("request_id"), "session.begin request_id", SarvamProtocolError)
if not request_id:
raise SarvamProtocolError("session.begin event has an invalid request_id")
return transcription_session_created_event(
openai_session(self._require_connection(), request_id, self._language, self._endpointing == "vad")
)
self._session_id = request_id
return transcription_session_created_event(self._openai_session(request_id))
def _session_updated(self) -> tuple[OpenAIRealtimeEvents, ...]:
"""Sarvam opens the socket on its own defaults and acknowledges each ``config.update`` separately, so a
client that configures the session after ``session.created`` only learns what took effect from here."""
session_id: Final = self._session_id
if session_id is None:
return ()
return (transcription_session_updated_event(self._openai_session(session_id)),)
def _openai_session(self, session_id: str) -> OpenAIRealtimeTranscriptionSession:
return openai_session(self._require_connection(), session_id, self._language, self._endpointing == "vad")
def _speech_started(self, utterance_idx: int) -> tuple[OpenAIRealtimeEvents, ...]:
state: Final = self._utterances.state(utterance_idx)

View file

@ -2270,6 +2270,12 @@ class OpenAIRealtimeTranscriptionSessionCreated(TypedDict):
session: ReadOnly[OpenAIRealtimeTranscriptionSession]
class OpenAIRealtimeTranscriptionSessionUpdated(TypedDict):
type: ReadOnly[Literal["session.updated"]]
event_id: ReadOnly[str]
session: ReadOnly[OpenAIRealtimeTranscriptionSession]
class OpenAIRealtimeInputAudioTranscriptionDelta(TypedDict):
type: ReadOnly[Literal["conversation.item.input_audio_transcription.delta"]]
event_id: ReadOnly[str]
@ -2349,6 +2355,7 @@ OpenAIRealtimeEvents = (
| OpenAIRealtimeInputAudioTranscriptionDelta
| OpenAIRealtimeInputAudioTranscriptionCompleted
| OpenAIRealtimeTranscriptionSessionCreated
| OpenAIRealtimeTranscriptionSessionUpdated
| OpenAIRealtimeErrorEvent
)

View file

@ -1257,6 +1257,19 @@ def test_get_provider_rerank_config():
assert isinstance(config, HostedVLLMRerankConfig)
def test_get_provider_realtime_config_serves_sarvam_speech_to_text():
"""Sarvam realtime transcription only reaches /v1/realtime when the provider registry hands back its
config; without this the endpoint rejects every sarvam deployment as an unsupported model."""
from litellm.llms.sarvam.realtime.transformation import SarvamRealtimeConfig
from litellm.utils import LlmProviders
config = ProviderConfigManager.get_provider_realtime_config(
model="saaras:v3-realtime", provider=LlmProviders.SARVAM
)
assert isinstance(config, SarvamRealtimeConfig)
def test_get_provider_text_to_speech_config_vertex_gemini_skips_cloud_tts():
"""Regression for LIT-6501: mapping vertex Gemini TTS params through Google Cloud TTS
dropped response_format before the speech_to_completion bridge could honor it."""

View file

@ -27,6 +27,8 @@ EMPTY_TRANSFORM_INPUT: Final[RealtimeResponseTransformInput] = {
"current_delta_type": None,
}
MODEL: Final = f"sarvam/{DEFAULT_SARVAM_REALTIME_MODEL}"
pytestmark: Final = pytest.mark.usefixtures("local_model_cost_map")
ONE_SECOND_OF_SILENCE: Final = b"\x00\x00" * 16_000
@ -36,6 +38,8 @@ def _session_update(
language: str | None = None,
model: str | None = None,
) -> str:
"""``turn_detection="omit"`` leaves the key out entirely, the way a client that only changes its language
sends a partial update; ``None`` sends an explicit null, which disables turn detection."""
transcription: Final[dict[str, str]] = {
**({"model": model} if model is not None else {}),
**({"language": language} if language is not None else {}),
@ -48,7 +52,11 @@ def _session_update(
"audio": {
"input": {
**({"format": {"type": "audio/pcm", "rate": rate}} if rate is not None else {}),
"turn_detection": None if turn_detection is None else {"type": turn_detection},
**(
{}
if turn_detection == "omit"
else {"turn_detection": None if turn_detection is None else {"type": turn_detection}}
),
"transcription": transcription,
}
},
@ -133,11 +141,54 @@ def test_unusable_api_base_is_rejected(api_base: str):
build_sarvam_connection(api_base, MODEL)
def test_unsupported_model_is_rejected():
@pytest.mark.parametrize("api_base", ["ws://stt.internal.example", "http://198.51.100.7:8123"])
def test_cleartext_to_a_remote_host_is_rejected_because_the_key_is_a_connection_header(api_base: str):
with pytest.raises(ValueError, match="loopback"):
build_sarvam_connection(api_base, MODEL)
@pytest.mark.parametrize(
("api_base", "expected_netloc"),
[
("ws://localhost:8123", "localhost:8123"),
("ws://127.0.0.1:8123", "127.0.0.1:8123"),
("ws://[::1]:8123", "[::1]:8123"),
],
)
def test_a_loopback_test_double_stays_reachable_over_cleartext(api_base: str, expected_netloc: str):
url = build_sarvam_connection(api_base, MODEL).url
assert url.startswith(f"ws://{expected_netloc}/speech-to-text-realtime/ws?")
def test_a_model_the_registry_does_not_serve_on_the_realtime_endpoint_is_rejected():
with pytest.raises(SarvamProtocolError, match="model"):
build_sarvam_connection(None, "sarvam/whisper-large")
def test_a_model_registered_for_the_realtime_endpoint_is_accepted_without_a_code_change():
"""Support reads the model registry, so a new Sarvam realtime model ships as a cost map entry."""
import litellm
litellm.register_model(
{
"sarvam/saaras:v9-realtime": {
"litellm_provider": "sarvam",
"mode": "audio_transcription",
"supported_endpoints": ["/v1/realtime"],
}
}
)
assert build_sarvam_connection(None, "sarvam/saaras:v9-realtime").model == "saaras:v9-realtime"
def test_a_sarvam_chat_model_is_not_served_on_the_realtime_endpoint():
"""The registry entry has to say audio_transcription on /v1/realtime, not merely exist under sarvam/."""
with pytest.raises(SarvamProtocolError, match="model"):
build_sarvam_connection(None, "sarvam/sarvam-m")
@pytest.mark.parametrize(
("requested", "expected"),
[("hi", "hi-IN"), ("hi-IN", "hi-IN"), ("HI-in", "hi-IN"), ("en_IN", "en-IN"), ("auto", "auto")],
@ -272,6 +323,41 @@ def test_session_begin_becomes_an_openai_transcription_session():
assert session["audio"]["input"]["turn_detection"] == {"type": "server_vad"}
def test_a_later_update_that_only_changes_the_language_keeps_manual_endpointing():
"""Turn detection left out of a partial update means unchanged. Reverting to VAD here would silently turn
the client's audio buffer commits into no-ops."""
config = _configured(turn_detection=None)
language_only = _sent(config, _session_update(turn_detection="omit", language="ta"))
config.transform_realtime_request(_append(ONE_SECOND_OF_SILENCE), MODEL)
committed = _sent(config, _client_event("input_audio_buffer.commit"))
assert language_only == [{"event": "config.update", "language_code": "ta-IN"}]
assert [frame["event"] for frame in committed] == ["speech_end", "flush"]
def test_applied_settings_reach_the_client_as_a_session_updated_event():
"""Sarvam opens on its own defaults, so a client that configures the session after session.created only
learns the language and turn detection that took effect from the config.updated acknowledgement."""
config = _configured()
_backend(config, event="session.begin", request_id="req-42", config={})
config.transform_realtime_request(_session_update(language="hi", turn_detection=None), MODEL)
events = _backend(config, event="config.updated", applied=["language_code", "endpointing"])
assert [event["type"] for event in events] == ["session.updated"]
session = events[0]["session"]
assert session["id"] == "req-42"
assert session["audio"]["input"]["transcription"]["language"] == "hi-IN"
assert session["audio"]["input"]["turn_detection"] is None
def test_an_acknowledgement_before_the_session_exists_is_dropped():
config = _configured()
assert _backend(config, event="config.updated", applied=["language_code"]) == []
def test_session_begin_reports_manual_endpointing_as_disabled_turn_detection():
config = _configured(turn_detection=None)
@ -411,7 +497,7 @@ def test_provider_errors_reach_the_client_as_openai_error_events():
assert "language not supported" in events[0]["error"]["message"]
@pytest.mark.parametrize("event", ["config.updated", "pong", "something.new"])
@pytest.mark.parametrize("event", ["pong", "something.new"])
def test_backend_events_without_an_openai_equivalent_are_dropped(event: str):
config = _configured()