fix greptile issues

litellm/litellm_core_utils/realtime_streaming.py

@@ -224,10 +224,24 @@ class RealTimeStreaming:
                 message, self.model, self.session_configuration_request
             )
             for msg in transformed:
+                # Cache setup immediately once we send it so concurrent client
+                # session.update messages don't emit duplicate setup packets.
+                self._cache_session_configuration_request(msg)
                 await self.backend_ws.send(msg)  # type: ignore[union-attr, attr-defined]
         else:
             await self.backend_ws.send(message)  # type: ignore[union-attr, attr-defined]
 
+    def _cache_session_configuration_request(self, transformed_message: str) -> None:
+        """Store setup payload once sent to backend."""
+        if self.session_configuration_request is not None:
+            return
+        try:
+            message_obj = json.loads(transformed_message)
+            if "setup" in message_obj:
+                self.session_configuration_request = transformed_message
+        except (json.JSONDecodeError, TypeError):
+            return
+
     def _has_realtime_guardrails(self) -> bool:
         """Return True if any callback is registered for realtime guardrail event types."""
         from litellm.integrations.custom_guardrail import CustomGuardrail
@@ -602,10 +616,11 @@ class RealTimeStreaming:
                 ## FORWARD TO BACKEND
                 if self.provider_config:
                     message = self.provider_config.transform_realtime_request(
-                        message, self.model
+                        message, self.model, self.session_configuration_request
                     )
 
                     for msg in message:
+                        self._cache_session_configuration_request(msg)
                         await self.backend_ws.send(msg)  # type: ignore[union-attr]
                 else:
                     await self.backend_ws.send(message)  # type: ignore[union-attr]

litellm/llms/gemini/realtime/transformation.py

@@ -348,17 +348,12 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
         if msg_type == "response.create":
             return []
 
-        ## HANDLE INPUT AUDIO BUFFER ##
-        if msg_type == "input_audio_buffer.append":
-            realtime_input_dict["audio"] = HttpxBlobType(
-                mimeType=self.get_audio_mime_type(), data=json_message["audio"]
-            )
         ## HANDLE conversation.item.create — extract user text or function call output ##
-        elif msg_type == "conversation.item.create":
+        if msg_type == "conversation.item.create":
             return self._handle_conversation_item(json_message)
         
         ## HANDLE INPUT AUDIO BUFFER - use realtimeInput for audio streaming ##
-        elif msg_type == "input_audio_buffer.append":
+        if msg_type == "input_audio_buffer.append":
             realtime_input_dict["audio"] = HttpxBlobType(
                 mimeType=self.get_audio_mime_type(), data=json_message["audio"]
             )
@@ -372,10 +367,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
             verbose_logger.debug("Gemini Realtime: Sending audio realtimeInput to backend")
             messages.append(gemini_msg)
             return messages
-        else:
-            # Unknown/unsupported OpenAI event type — drop silently rather than
-            # forwarding raw JSON as text input to the model.
-            return []
+        # Unknown/unsupported OpenAI event type — drop silently rather than
+        # forwarding raw JSON as text input to the model.
+        return []
 
     def transform_session_created_event(
         self,
@@ -446,14 +440,14 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
         delta_type: ALL_DELTA_TYPES,
         session_configuration_request: Optional[str] = None,
     ) -> List[OpenAIRealtimeEvents]:
-        if session_configuration_request is None:
-            raise ValueError(
-                "session_configuration_request is required for Gemini API calls"
-            )
-
-        session_configuration_request_dict: BidiGenerateContentSetup = json.loads(
-            session_configuration_request
-        ).get("setup", {})
+        session_configuration_request_dict: BidiGenerateContentSetup = {}
+        if session_configuration_request is not None:
+            try:
+                session_configuration_request_dict = json.loads(
+                    session_configuration_request
+                ).get("setup", {})
+            except json.JSONDecodeError:
+                session_configuration_request_dict = {}
         generation_config = session_configuration_request_dict.get(
             "generationConfig", {}
         )

litellm/llms/vertex_ai/realtime/transformation.py

@@ -153,6 +153,8 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig):
         )
         
         # Add Vertex AI specific defaults if not provided
+        generation_config = setup_config.setdefault("generationConfig", {})
+        generation_config.setdefault("responseModalities", ["AUDIO"])
         setup_config.setdefault(
             "realtimeInputConfig",
             {