diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index e6d165db221..ea4af565d7a 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -272,10 +272,14 @@ class RealTimeStreaming: ) sent = False for msg in transformed: - # Cache setup immediately once we send it so concurrent client - # session.update messages don't emit duplicate setup packets. - self._cache_session_configuration_request(msg) + # Send first; only cache the setup payload once the backend + # has actually accepted it. Caching before send would leave + # ``session_configuration_request`` populated after a failed + # send, causing subsequent client session.update messages to + # be treated as "subsequent" and dropped even though the + # backend never received the original setup. await self.backend_ws.send(msg) # type: ignore[union-attr, attr-defined] + self._cache_session_configuration_request(msg) sent = True return sent await self.backend_ws.send(message) # type: ignore[union-attr, attr-defined] @@ -924,8 +928,8 @@ class RealTimeStreaming: ) for msg in message: - self._cache_session_configuration_request(msg) await self.backend_ws.send(msg) # type: ignore[union-attr] + self._cache_session_configuration_request(msg) else: await self.backend_ws.send(message) # type: ignore[union-attr] diff --git a/litellm/llms/vertex_ai/realtime/transformation.py b/litellm/llms/vertex_ai/realtime/transformation.py index 4d5b9b7c779..94fdf82a6f7 100644 --- a/litellm/llms/vertex_ai/realtime/transformation.py +++ b/litellm/llms/vertex_ai/realtime/transformation.py @@ -12,10 +12,15 @@ Auth: OAuth2 Bearer token (not an API key). """ import json -from typing import List, Optional +from typing import List, Optional, cast from litellm import verbose_logger from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig +from litellm.types.llms.gemini import ( + BidiGenerateContentRealtimeInputConfig, + BidiGenerateContentSetup, +) +from litellm.types.llms.openai import OpenAIRealtimeTurnDetection class VertexAIRealtimeConfig(GeminiRealtimeConfig): @@ -140,6 +145,14 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig): # Request translation # ------------------------------------------------------------------ + def _vertex_model_path(self, model: str) -> str: + """Return the fully-qualified Vertex AI model resource path.""" + return ( + f"projects/{self._project}" + f"/locations/{self._location}" + f"/publishers/google/models/{model}" + ) + def _build_vertex_ai_setup_config(self, model: str, session_params: dict) -> dict: """Build Vertex AI setup configuration with proper model path and defaults.""" setup_config = self.map_openai_params( @@ -147,24 +160,26 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig): ) # Use full Vertex AI model path - setup_config["model"] = ( - f"projects/{self._project}" - f"/locations/{self._location}" - f"/publishers/google/models/{model}" - ) + setup_config["model"] = self._vertex_model_path(model) # Add Vertex AI specific defaults if not provided generation_config = setup_config.setdefault("generationConfig", {}) generation_config.setdefault("responseModalities", ["AUDIO"]) - setup_config.setdefault( - "realtimeInputConfig", - { - "automaticActivityDetection": { - "disabled": False, - "silenceDurationMs": 800, - } - }, + + # Ensure Vertex defaults for realtimeInputConfig apply even when + # the client provided a partial ``turn_detection`` (e.g. only + # ``silence_duration_ms``). ``map_openai_params`` defaults + # ``disabled=True`` when ``create_response`` is absent, which would + # otherwise silently disable VAD here. + realtime_input_config = setup_config.setdefault("realtimeInputConfig", {}) + automatic_detection = realtime_input_config.setdefault( + "automaticActivityDetection", {} ) + client_turn_detection = self._extract_turn_detection(session_params) or {} + if "create_response" not in client_turn_detection: + automatic_detection["disabled"] = False + automatic_detection.setdefault("silenceDurationMs", 800) + setup_config.setdefault("inputAudioTranscription", {}) setup_config.setdefault("outputAudioTranscription", {}) @@ -197,12 +212,33 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig): "Vertex AI Realtime: Sending initial setup with tools to backend" ) return [gemini_setup_msg] - else: - # Subsequent session.update - ignore - verbose_logger.debug( - "Vertex AI Realtime: Ignoring session.update (setup already sent)" + + # Subsequent session.update: forward turn_detection-only updates + # (e.g. guardrail-injected disable of VAD auto-response) as a + # follow-up setup. Other fields are dropped because Vertex AI + # doesn't support dynamic updates. + session_payload = json_message.get("session") or {} + turn_detection = self._extract_turn_detection(session_payload) + if turn_detection is not None: + transformed_audio_activity_config = self.map_automatic_turn_detection( + cast(OpenAIRealtimeTurnDetection, turn_detection) ) - return [] + if len(transformed_audio_activity_config) > 0: + follow_up_setup: BidiGenerateContentSetup = { + "model": self._vertex_model_path(model), + "realtimeInputConfig": BidiGenerateContentRealtimeInputConfig( + automaticActivityDetection=transformed_audio_activity_config + ), + } + verbose_logger.debug( + "Vertex AI Realtime: Forwarding turn_detection-only session.update as setup" + ) + return [json.dumps({"setup": follow_up_setup})] + + verbose_logger.debug( + "Vertex AI Realtime: Ignoring session.update (setup already sent)" + ) + return [] # For other message types, use parent's logic return super().transform_realtime_request(