diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py
index aee0ae05738..757be04540d 100644
--- a/litellm/litellm_core_utils/realtime_streaming.py
+++ b/litellm/litellm_core_utils/realtime_streaming.py
@@ -1568,7 +1568,11 @@ class RealTimeStreaming:
return BackendClose(code=1011, reason="proxy failed while relaying the upstream websocket")
def _backend_refused_session(self, close: BackendClose) -> bool:
- return close.code != 1000 and not self._backend_sent_frames
+ return (
+ close.code != 1000
+ and not self._backend_sent_frames
+ and not self._holds_session_updates_until_first_content()
+ )
async def _log_session_end(self, transport_refusal: Exception | None) -> None:
provider_refusal: Final = None if self.provider_config is None else self.provider_config.session_refusal()
diff --git a/litellm/llms/gemini/realtime/transformation.py b/litellm/llms/gemini/realtime/transformation.py
index ec1ef4628b0..3134dfa1e4b 100644
--- a/litellm/llms/gemini/realtime/transformation.py
+++ b/litellm/llms/gemini/realtime/transformation.py
@@ -18,6 +18,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeSessionUpdateRejection
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
+ OPENAI_TO_GEMINI_TTS_VOICES,
VertexGeminiConfig,
)
from litellm.responses.litellm_completion_transformation.transformation import (
@@ -125,7 +126,7 @@ def _is_undocumented_live_voice(voice: object) -> bool:
return (
isinstance(voice, str)
and voice.lower() not in OPENAI_STOCK_REALTIME_VOICES
- and voice.lower() not in GEMINI_LIVE_DOCUMENTED_VOICES
+ and OPENAI_TO_GEMINI_TTS_VOICES.get(voice.lower(), voice).lower() not in GEMINI_LIVE_DOCUMENTED_VOICES
)
diff --git a/tests/e2e/llm_translation/realtime/realtime_client.py b/tests/e2e/llm_translation/realtime/realtime_client.py
index 4d8336775ef..6bd5d61d721 100644
--- a/tests/e2e/llm_translation/realtime/realtime_client.py
+++ b/tests/e2e/llm_translation/realtime/realtime_client.py
@@ -47,6 +47,7 @@ class RealtimeProvider:
id: str
alias: str
litellm_params: LiteLLMParamsBody
+ modalities: tuple[str, ...] = ("text",)
PROVIDERS = (
@@ -75,6 +76,7 @@ PROVIDERS = (
model="gemini/gemini-3.1-flash-live-preview",
api_key="os.environ/GEMINI_API_KEY",
),
+ modalities=("audio",),
),
RealtimeProvider(
"vertex_ai",
@@ -84,6 +86,7 @@ PROVIDERS = (
vertex_location="us-central1",
vertex_credentials="os.environ/VERTEXAI_CREDENTIALS",
),
+ modalities=("audio",),
),
RealtimeProvider(
"bedrock",
diff --git a/tests/e2e/llm_translation/realtime/test_realtime_e2e.py b/tests/e2e/llm_translation/realtime/test_realtime_e2e.py
index f5592abb34d..3b7ba3a2430 100644
--- a/tests/e2e/llm_translation/realtime/test_realtime_e2e.py
+++ b/tests/e2e/llm_translation/realtime/test_realtime_e2e.py
@@ -124,7 +124,8 @@ def test_text_conversation(
session.send(
SessionUpdate(
session=SessionConfig(
- instructions="You are a terse assistant. Reply in one short sentence."
+ modalities=list(provider.modalities),
+ instructions="You are a terse assistant. Reply in one short sentence.",
)
)
)
@@ -159,6 +160,7 @@ def test_tool_call_round_trip(
session.send(
SessionUpdate(
session=SessionConfig(
+ modalities=list(provider.modalities),
tools=[WEATHER_TOOL],
tool_choice="auto",
instructions=(
diff --git a/tests/integration/providers/test_realtime_transcription_guardrail_session_update.py b/tests/integration/providers/test_realtime_transcription_guardrail_session_update.py
index ad58a400cec..2712d6e30cd 100644
--- a/tests/integration/providers/test_realtime_transcription_guardrail_session_update.py
+++ b/tests/integration/providers/test_realtime_transcription_guardrail_session_update.py
@@ -355,7 +355,13 @@ def _voice_scenario_created_after_the_first_frame(transcript: str) -> RealtimeRe
def _live_scenario() -> RealtimeResponse:
- return RealtimeResponse(content_type="application/x-realtime", events=())
+ return RealtimeResponse(
+ content_type="application/x-realtime",
+ events=(
+ {"serverContent": {"modelTurn": {"parts": [{"text": "scripted $REQUEST_ID"}]}}},
+ {"serverContent": {"turnComplete": True}},
+ ),
+ )
def _realtime_update(turn_detection: JsonValue) -> dict[str, JsonValue]:
@@ -1348,10 +1354,14 @@ def test_gemini_ai_studio_voice_session_with_vad_tuning_connects_under_the_guard
_ws_base(_owned_url(guardrail_proxy)),
f"model={model}",
key,
- (_step(_update_frame(update), until="session.updated"),),
+ (
+ _step(_update_frame(update), until="session.updated"),
+ _step(_user_text_item(CLEAN_TRANSCRIPT), RESPONSE_CREATE, until=RESPONSE_DONE),
+ ),
)
observed: Final = _observed(upstream_url, handle.scenario_id)
- assert session.types == ("session.created", "session.updated"), session
+ assert session.types[:2] == ("session.created", "session.updated"), session
+ assert session.types[-1] == RESPONSE_DONE, session
assert _setup_detection(observed) == {"disabled": True}, _sent(observed)
@@ -1365,10 +1375,14 @@ def test_vertex_live_voice_session_keeps_vad_tuning_under_the_guardrail(guardrai
_ws_base(_owned_url(guardrail_proxy)),
f"model={model}",
key,
- (_step(_update_frame(_realtime_update(TUNED_SERVER_VAD)), until="session.updated"),),
+ (
+ _step(_update_frame(_realtime_update(TUNED_SERVER_VAD)), until="session.updated"),
+ _step(_user_text_item(CLEAN_TRANSCRIPT), RESPONSE_CREATE, until=RESPONSE_DONE),
+ ),
)
observed: Final = _observed(upstream_url, handle.scenario_id)
- assert session.types == ("session.created", "session.updated"), session
+ assert session.types[:2] == ("session.created", "session.updated"), session
+ assert session.types[-1] == RESPONSE_DONE, session
assert _setup_detection(observed) == GEMINI_TUNED_DETECTION, _sent(observed)
diff --git a/tests/unit/litellm_core_utils/test_realtime_streaming.py b/tests/unit/litellm_core_utils/test_realtime_streaming.py
index 9363ba0ddce..7c4b9d865e1 100644
--- a/tests/unit/litellm_core_utils/test_realtime_streaming.py
+++ b/tests/unit/litellm_core_utils/test_realtime_streaming.py
@@ -4174,6 +4174,24 @@ async def test_upstream_refusal_after_a_synthetic_session_created_still_logs_a_f
assert session.logging.logged_sessions == ()
+class _LazySetupRealtimeConfig(_PassthroughRealtimeConfig):
+ def builds_setup_from_session_update(self) -> bool:
+ return True
+
+
+@pytest.mark.asyncio
+async def test_upstream_outage_before_a_held_setup_was_sent_logs_the_session_not_a_refusal():
+ upstream_close: Final = ConnectionClosed(Close(1011, "upstream outage"), None)
+ session: Final = _relay_session(
+ _client_ws_that_never_sends(), _backend_ws_closing_with(upstream_close), _LazySetupRealtimeConfig()
+ )
+
+ await session.run()
+
+ assert session.logging.logged_failures == ()
+ assert len(session.logging.logged_sessions) == 1
+
+
@pytest.mark.asyncio
async def test_upstream_close_after_relayed_events_still_logs_the_session_as_success():
client_ws: Final = _client_ws_that_never_sends()
diff --git a/tests/unit/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py b/tests/unit/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py
index 9d7fba8b11e..7f0106616ce 100644
--- a/tests/unit/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py
+++ b/tests/unit/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py
@@ -18,6 +18,7 @@ import websockets.exceptions # registers websockets.exceptions on the websocket
import litellm
from litellm.llms.vertex_ai.common_utils import get_vertex_base_url
+from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import OPENAI_TO_GEMINI_TTS_VOICES
from litellm.llms.vertex_ai.realtime.transformation import VertexAIRealtimeConfig
# ---------------------------------------------------------------------------
@@ -699,3 +700,20 @@ def test_vertex_drops_an_undocumented_voice_with_drop_params():
json.dumps(session_update), "gemini-live-2.5-flash", session_configuration_request=None
)
assert "speechConfig" not in json.loads(messages[0])["setup"].get("generationConfig", {})
+
+
+@pytest.mark.parametrize("drop_params", [False, True])
+@pytest.mark.parametrize("voice", ["fable", "onyx"])
+def test_vertex_sends_the_documented_voice_an_openai_tts_voice_maps_to(voice, drop_params):
+ cfg: Final = VertexAIRealtimeConfig(
+ access_token="tok", project="my-proj", location="us-central1", drop_params=drop_params
+ )
+ session_update: Final = {"type": "session.update", "session": {"voice": voice}}
+
+ assert cfg.setup_session_update_rejection("gemini-live-2.5-flash", session_update["session"], setup_sent=False) is None
+ messages: Final = cfg.transform_realtime_request(
+ json.dumps(session_update), "gemini-live-2.5-flash", session_configuration_request=None
+ )
+ assert json.loads(messages[0])["setup"]["generationConfig"]["speechConfig"] == {
+ "voiceConfig": {"prebuiltVoiceConfig": {"voiceName": OPENAI_TO_GEMINI_TTS_VOICES[voice]}}
+ }
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/RealtimePlayground.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/RealtimePlayground.test.tsx
index 117796d658e..56135c779e9 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/RealtimePlayground.test.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/RealtimePlayground.test.tsx
@@ -173,6 +173,24 @@ describe("RealtimePlayground", () => {
expect(update.session.type).toBe("realtime");
});
+ it("starts recording on the configuration the session was created with", async () => {
+ const user = userEvent.setup();
+ render();
+
+ await connect(user);
+ await act(async () => {
+ latestSocket().emit({ type: "session.created" });
+ });
+ await user.click(screen.getByTitle("Start recording"));
+
+ const updates = latestSocket()
+ .sent.map((raw) => JSON.parse(raw))
+ .filter((payload) => payload.type === "session.update");
+ expect(updates).toHaveLength(2);
+ expect(updates[1]).toEqual(updates[0]);
+ expect(updates[0].session.turn_detection).toEqual({ type: "server_vad" });
+ });
+
it("sends what was typed and then asks for a response", async () => {
const user = userEvent.setup();
render();
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/RealtimePlayground.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/RealtimePlayground.tsx
index 01151203e65..408cbd58593 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/RealtimePlayground.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/RealtimePlayground.tsx
@@ -22,6 +22,20 @@ interface RealtimePlaygroundProps {
selectedGuardrails?: string[];
}
+const sessionUpdate = (voice: string) =>
+ JSON.stringify({
+ type: "session.update",
+ session: {
+ type: "realtime",
+ modalities: ["text", "audio"],
+ voice,
+ input_audio_format: "pcm16",
+ output_audio_format: "pcm16",
+ input_audio_transcription: { model: "gpt-4o-mini-transcribe" },
+ turn_detection: { type: "server_vad" },
+ },
+ });
+
const RealtimePlayground: React.FC = ({
accessToken,
selectedModel,
@@ -125,21 +139,7 @@ const RealtimePlayground: React.FC = ({
const type = data.type;
if (type === "session.created") {
- // GA: session.type is required ("realtime" | "transcription")
- ws.send(
- JSON.stringify({
- type: "session.update",
- session: {
- type: "realtime",
- modalities: ["text", "audio"],
- voice: selectedVoice,
- input_audio_format: "pcm16",
- output_audio_format: "pcm16",
- input_audio_transcription: { model: "gpt-4o-mini-transcribe" },
- turn_detection: null,
- },
- }),
- );
+ ws.send(sessionUpdate(selectedVoice));
} else if (type === "session.updated") {
// session configured
} else if (
@@ -231,22 +231,7 @@ const RealtimePlayground: React.FC = ({
const startRecording = useCallback(async () => {
if (!wsRef.current || wsRef.current.readyState !== WebSocket.OPEN) return;
- // Switch to server VAD mode for voice input
- // GA: session.type is required
- wsRef.current.send(
- JSON.stringify({
- type: "session.update",
- session: {
- type: "realtime",
- modalities: ["text", "audio"],
- voice: selectedVoice,
- input_audio_format: "pcm16",
- output_audio_format: "pcm16",
- input_audio_transcription: { model: "gpt-4o-mini-transcribe" },
- turn_detection: { type: "server_vad" },
- },
- }),
- );
+ wsRef.current.send(sessionUpdate(selectedVoice));
try {
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
@@ -301,7 +286,7 @@ const RealtimePlayground: React.FC = ({
} catch (err: any) {
addMessage("status", `Microphone error: ${err.message}`);
}
- }, [addMessage]);
+ }, [addMessage, selectedVoice]);
const stopRecording = useCallback(() => {
processorRef.current?.disconnect();
@@ -317,21 +302,7 @@ const RealtimePlayground: React.FC = ({
if (!wsRef.current || wsRef.current.readyState !== WebSocket.OPEN) return;
if (configureSessionRef.current) return;
configureSessionRef.current = true;
- // GA: session.type is required
- wsRef.current.send(
- JSON.stringify({
- type: "session.update",
- session: {
- type: "realtime",
- modalities: ["text", "audio"],
- voice: selectedVoice,
- input_audio_format: "pcm16",
- output_audio_format: "pcm16",
- input_audio_transcription: { model: "gpt-4o-mini-transcribe" },
- turn_detection: null,
- },
- }),
- );
+ wsRef.current.send(sessionUpdate(selectedVoice));
}, [selectedVoice]);
const sendTextMessage = useCallback(() => {