diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index aee0ae05738..757be04540d 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -1568,7 +1568,11 @@ class RealTimeStreaming: return BackendClose(code=1011, reason="proxy failed while relaying the upstream websocket") def _backend_refused_session(self, close: BackendClose) -> bool: - return close.code != 1000 and not self._backend_sent_frames + return ( + close.code != 1000 + and not self._backend_sent_frames + and not self._holds_session_updates_until_first_content() + ) async def _log_session_end(self, transport_refusal: Exception | None) -> None: provider_refusal: Final = None if self.provider_config is None else self.provider_config.session_refusal() diff --git a/litellm/llms/gemini/realtime/transformation.py b/litellm/llms/gemini/realtime/transformation.py index ec1ef4628b0..3134dfa1e4b 100644 --- a/litellm/llms/gemini/realtime/transformation.py +++ b/litellm/llms/gemini/realtime/transformation.py @@ -18,6 +18,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeSessionUpdateRejection from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + OPENAI_TO_GEMINI_TTS_VOICES, VertexGeminiConfig, ) from litellm.responses.litellm_completion_transformation.transformation import ( @@ -125,7 +126,7 @@ def _is_undocumented_live_voice(voice: object) -> bool: return ( isinstance(voice, str) and voice.lower() not in OPENAI_STOCK_REALTIME_VOICES - and voice.lower() not in GEMINI_LIVE_DOCUMENTED_VOICES + and OPENAI_TO_GEMINI_TTS_VOICES.get(voice.lower(), voice).lower() not in GEMINI_LIVE_DOCUMENTED_VOICES ) diff --git a/tests/e2e/llm_translation/realtime/realtime_client.py b/tests/e2e/llm_translation/realtime/realtime_client.py index 4d8336775ef..6bd5d61d721 100644 --- a/tests/e2e/llm_translation/realtime/realtime_client.py +++ b/tests/e2e/llm_translation/realtime/realtime_client.py @@ -47,6 +47,7 @@ class RealtimeProvider: id: str alias: str litellm_params: LiteLLMParamsBody + modalities: tuple[str, ...] = ("text",) PROVIDERS = ( @@ -75,6 +76,7 @@ PROVIDERS = ( model="gemini/gemini-3.1-flash-live-preview", api_key="os.environ/GEMINI_API_KEY", ), + modalities=("audio",), ), RealtimeProvider( "vertex_ai", @@ -84,6 +86,7 @@ PROVIDERS = ( vertex_location="us-central1", vertex_credentials="os.environ/VERTEXAI_CREDENTIALS", ), + modalities=("audio",), ), RealtimeProvider( "bedrock", diff --git a/tests/e2e/llm_translation/realtime/test_realtime_e2e.py b/tests/e2e/llm_translation/realtime/test_realtime_e2e.py index f5592abb34d..3b7ba3a2430 100644 --- a/tests/e2e/llm_translation/realtime/test_realtime_e2e.py +++ b/tests/e2e/llm_translation/realtime/test_realtime_e2e.py @@ -124,7 +124,8 @@ def test_text_conversation( session.send( SessionUpdate( session=SessionConfig( - instructions="You are a terse assistant. Reply in one short sentence." + modalities=list(provider.modalities), + instructions="You are a terse assistant. Reply in one short sentence.", ) ) ) @@ -159,6 +160,7 @@ def test_tool_call_round_trip( session.send( SessionUpdate( session=SessionConfig( + modalities=list(provider.modalities), tools=[WEATHER_TOOL], tool_choice="auto", instructions=( diff --git a/tests/integration/providers/test_realtime_transcription_guardrail_session_update.py b/tests/integration/providers/test_realtime_transcription_guardrail_session_update.py index ad58a400cec..2712d6e30cd 100644 --- a/tests/integration/providers/test_realtime_transcription_guardrail_session_update.py +++ b/tests/integration/providers/test_realtime_transcription_guardrail_session_update.py @@ -355,7 +355,13 @@ def _voice_scenario_created_after_the_first_frame(transcript: str) -> RealtimeRe def _live_scenario() -> RealtimeResponse: - return RealtimeResponse(content_type="application/x-realtime", events=()) + return RealtimeResponse( + content_type="application/x-realtime", + events=( + {"serverContent": {"modelTurn": {"parts": [{"text": "scripted $REQUEST_ID"}]}}}, + {"serverContent": {"turnComplete": True}}, + ), + ) def _realtime_update(turn_detection: JsonValue) -> dict[str, JsonValue]: @@ -1348,10 +1354,14 @@ def test_gemini_ai_studio_voice_session_with_vad_tuning_connects_under_the_guard _ws_base(_owned_url(guardrail_proxy)), f"model={model}", key, - (_step(_update_frame(update), until="session.updated"),), + ( + _step(_update_frame(update), until="session.updated"), + _step(_user_text_item(CLEAN_TRANSCRIPT), RESPONSE_CREATE, until=RESPONSE_DONE), + ), ) observed: Final = _observed(upstream_url, handle.scenario_id) - assert session.types == ("session.created", "session.updated"), session + assert session.types[:2] == ("session.created", "session.updated"), session + assert session.types[-1] == RESPONSE_DONE, session assert _setup_detection(observed) == {"disabled": True}, _sent(observed) @@ -1365,10 +1375,14 @@ def test_vertex_live_voice_session_keeps_vad_tuning_under_the_guardrail(guardrai _ws_base(_owned_url(guardrail_proxy)), f"model={model}", key, - (_step(_update_frame(_realtime_update(TUNED_SERVER_VAD)), until="session.updated"),), + ( + _step(_update_frame(_realtime_update(TUNED_SERVER_VAD)), until="session.updated"), + _step(_user_text_item(CLEAN_TRANSCRIPT), RESPONSE_CREATE, until=RESPONSE_DONE), + ), ) observed: Final = _observed(upstream_url, handle.scenario_id) - assert session.types == ("session.created", "session.updated"), session + assert session.types[:2] == ("session.created", "session.updated"), session + assert session.types[-1] == RESPONSE_DONE, session assert _setup_detection(observed) == GEMINI_TUNED_DETECTION, _sent(observed) diff --git a/tests/unit/litellm_core_utils/test_realtime_streaming.py b/tests/unit/litellm_core_utils/test_realtime_streaming.py index 9363ba0ddce..7c4b9d865e1 100644 --- a/tests/unit/litellm_core_utils/test_realtime_streaming.py +++ b/tests/unit/litellm_core_utils/test_realtime_streaming.py @@ -4174,6 +4174,24 @@ async def test_upstream_refusal_after_a_synthetic_session_created_still_logs_a_f assert session.logging.logged_sessions == () +class _LazySetupRealtimeConfig(_PassthroughRealtimeConfig): + def builds_setup_from_session_update(self) -> bool: + return True + + +@pytest.mark.asyncio +async def test_upstream_outage_before_a_held_setup_was_sent_logs_the_session_not_a_refusal(): + upstream_close: Final = ConnectionClosed(Close(1011, "upstream outage"), None) + session: Final = _relay_session( + _client_ws_that_never_sends(), _backend_ws_closing_with(upstream_close), _LazySetupRealtimeConfig() + ) + + await session.run() + + assert session.logging.logged_failures == () + assert len(session.logging.logged_sessions) == 1 + + @pytest.mark.asyncio async def test_upstream_close_after_relayed_events_still_logs_the_session_as_success(): client_ws: Final = _client_ws_that_never_sends() diff --git a/tests/unit/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py b/tests/unit/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py index 9d7fba8b11e..7f0106616ce 100644 --- a/tests/unit/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py +++ b/tests/unit/llms/vertex_ai/realtime/test_vertex_ai_realtime_transformation.py @@ -18,6 +18,7 @@ import websockets.exceptions # registers websockets.exceptions on the websocket import litellm from litellm.llms.vertex_ai.common_utils import get_vertex_base_url +from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import OPENAI_TO_GEMINI_TTS_VOICES from litellm.llms.vertex_ai.realtime.transformation import VertexAIRealtimeConfig # --------------------------------------------------------------------------- @@ -699,3 +700,20 @@ def test_vertex_drops_an_undocumented_voice_with_drop_params(): json.dumps(session_update), "gemini-live-2.5-flash", session_configuration_request=None ) assert "speechConfig" not in json.loads(messages[0])["setup"].get("generationConfig", {}) + + +@pytest.mark.parametrize("drop_params", [False, True]) +@pytest.mark.parametrize("voice", ["fable", "onyx"]) +def test_vertex_sends_the_documented_voice_an_openai_tts_voice_maps_to(voice, drop_params): + cfg: Final = VertexAIRealtimeConfig( + access_token="tok", project="my-proj", location="us-central1", drop_params=drop_params + ) + session_update: Final = {"type": "session.update", "session": {"voice": voice}} + + assert cfg.setup_session_update_rejection("gemini-live-2.5-flash", session_update["session"], setup_sent=False) is None + messages: Final = cfg.transform_realtime_request( + json.dumps(session_update), "gemini-live-2.5-flash", session_configuration_request=None + ) + assert json.loads(messages[0])["setup"]["generationConfig"]["speechConfig"] == { + "voiceConfig": {"prebuiltVoiceConfig": {"voiceName": OPENAI_TO_GEMINI_TTS_VOICES[voice]}} + } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/RealtimePlayground.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/RealtimePlayground.test.tsx index 117796d658e..56135c779e9 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/RealtimePlayground.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/RealtimePlayground.test.tsx @@ -173,6 +173,24 @@ describe("RealtimePlayground", () => { expect(update.session.type).toBe("realtime"); }); + it("starts recording on the configuration the session was created with", async () => { + const user = userEvent.setup(); + render(); + + await connect(user); + await act(async () => { + latestSocket().emit({ type: "session.created" }); + }); + await user.click(screen.getByTitle("Start recording")); + + const updates = latestSocket() + .sent.map((raw) => JSON.parse(raw)) + .filter((payload) => payload.type === "session.update"); + expect(updates).toHaveLength(2); + expect(updates[1]).toEqual(updates[0]); + expect(updates[0].session.turn_detection).toEqual({ type: "server_vad" }); + }); + it("sends what was typed and then asks for a response", async () => { const user = userEvent.setup(); render(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/RealtimePlayground.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/RealtimePlayground.tsx index 01151203e65..408cbd58593 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/RealtimePlayground.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/RealtimePlayground.tsx @@ -22,6 +22,20 @@ interface RealtimePlaygroundProps { selectedGuardrails?: string[]; } +const sessionUpdate = (voice: string) => + JSON.stringify({ + type: "session.update", + session: { + type: "realtime", + modalities: ["text", "audio"], + voice, + input_audio_format: "pcm16", + output_audio_format: "pcm16", + input_audio_transcription: { model: "gpt-4o-mini-transcribe" }, + turn_detection: { type: "server_vad" }, + }, + }); + const RealtimePlayground: React.FC = ({ accessToken, selectedModel, @@ -125,21 +139,7 @@ const RealtimePlayground: React.FC = ({ const type = data.type; if (type === "session.created") { - // GA: session.type is required ("realtime" | "transcription") - ws.send( - JSON.stringify({ - type: "session.update", - session: { - type: "realtime", - modalities: ["text", "audio"], - voice: selectedVoice, - input_audio_format: "pcm16", - output_audio_format: "pcm16", - input_audio_transcription: { model: "gpt-4o-mini-transcribe" }, - turn_detection: null, - }, - }), - ); + ws.send(sessionUpdate(selectedVoice)); } else if (type === "session.updated") { // session configured } else if ( @@ -231,22 +231,7 @@ const RealtimePlayground: React.FC = ({ const startRecording = useCallback(async () => { if (!wsRef.current || wsRef.current.readyState !== WebSocket.OPEN) return; - // Switch to server VAD mode for voice input - // GA: session.type is required - wsRef.current.send( - JSON.stringify({ - type: "session.update", - session: { - type: "realtime", - modalities: ["text", "audio"], - voice: selectedVoice, - input_audio_format: "pcm16", - output_audio_format: "pcm16", - input_audio_transcription: { model: "gpt-4o-mini-transcribe" }, - turn_detection: { type: "server_vad" }, - }, - }), - ); + wsRef.current.send(sessionUpdate(selectedVoice)); try { const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); @@ -301,7 +286,7 @@ const RealtimePlayground: React.FC = ({ } catch (err: any) { addMessage("status", `Microphone error: ${err.message}`); } - }, [addMessage]); + }, [addMessage, selectedVoice]); const stopRecording = useCallback(() => { processorRef.current?.disconnect(); @@ -317,21 +302,7 @@ const RealtimePlayground: React.FC = ({ if (!wsRef.current || wsRef.current.readyState !== WebSocket.OPEN) return; if (configureSessionRef.current) return; configureSessionRef.current = true; - // GA: session.type is required - wsRef.current.send( - JSON.stringify({ - type: "session.update", - session: { - type: "realtime", - modalities: ["text", "audio"], - voice: selectedVoice, - input_audio_format: "pcm16", - output_audio_format: "pcm16", - input_audio_transcription: { model: "gpt-4o-mini-transcribe" }, - turn_detection: null, - }, - }), - ); + wsRef.current.send(sessionUpdate(selectedVoice)); }, [selectedVoice]); const sendTextMessage = useCallback(() => {