scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-09 22:31:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 133

fix(realtime): consolidate send-and-cache, guard session.update lookup, preserve client turn_detection in GA remap

Browse source 
- Replace duplicated transform/send/cache logic in client_ack_messages with a call to _send_to_backend so future changes stay in one place.
- VertexAIRealtimeConfig.transform_realtime_request now uses .get('session') or {} for the first session.update so a malformed client payload no longer crashes the connection.
- Move the audio-transcription guardrail turn_detection injection to run BEFORE the beta->GA session remap. This lets the injected create_response ride along with any client-provided turn_detection fields (e.g. silence_duration_ms) into the nested audio.input.turn_detection path produced by the remap instead of being stranded as a separate root-level dict.
- Update the deferred-mode injection test to assert the GA-shaped location.

Co-authored-by: Yassin Kortam <yassin@berri.ai>
This commit is contained in:
Cursor Agent 2026-05-22 14:06:02 +00:00
parent cb49dd974e
commit abb7080496
No known key found for this signature in database
3 changed files with 37 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

58
litellm/litellm_core_utils/realtime_streaming.py
View file

@ -877,6 +877,31 @@ class RealTimeStreaming:
self._pending_guardrail_message = None
continue
## GUARDRAIL: Inject turn_detection into first session.update
# if needed. Done BEFORE the GA remap so the injected
# ``create_response`` rides along with any client-provided
# turn_detection fields (e.g. silence_duration_ms) into the
# nested ``audio.input.turn_detection`` path produced by the
# remap. Doing this after the remap would create a separate
# minimal root-level ``turn_detection`` and silently drop
# the client's nested settings.
if (
msg_type == "session.update"
and self.session_configuration_request is None
and not self._guardrail_turn_detection_update_sent
and self._has_audio_transcription_guardrails()
):
session = msg_obj.setdefault("session", {})
if isinstance(session, dict):
session.setdefault("turn_detection", {})[
"create_response"
] = False
message = json.dumps(msg_obj)
self._guardrail_turn_detection_update_sent = True
verbose_logger.debug(
"Injected turn_detection into first session.update for audio transcription guardrails"
)
# GA compatibility: remap beta-style session fields only when
# the upstream is in GA mode. Beta upstreams expect the flat
# session shape unchanged.
@ -893,28 +918,6 @@ class RealTimeStreaming:
except (json.JSONDecodeError, AttributeError):
pass
## GUARDRAIL: Inject turn_detection into first session.update if needed
try:
msg_obj = json.loads(message)
if (
msg_obj.get("type") == "session.update"
and self.session_configuration_request is None
and not self._guardrail_turn_detection_update_sent
and self._has_audio_transcription_guardrails()
):
# Inject turn_detection into the first session.update
session = msg_obj.setdefault("session", {})
session.setdefault("turn_detection", {})[
"create_response"
] = False
message = json.dumps(msg_obj)
self._guardrail_turn_detection_update_sent = True
verbose_logger.debug(
"Injected turn_detection into first session.update for audio transcription guardrails"
)
except (json.JSONDecodeError, AttributeError):
pass
## LOGGING
# Log after any in-place modifications (GA remap, guardrail
# turn_detection injection) so audit logs reflect what we
@ -922,16 +925,7 @@ class RealTimeStreaming:
self.store_input(message=message)
## FORWARD TO BACKEND
if self.provider_config:
message = self.provider_config.transform_realtime_request(
message, self.model, self.session_configuration_request
)
for msg in message:
await self.backend_ws.send(msg) # type: ignore[union-attr]
self._cache_session_configuration_request(msg)
else:
await self.backend_ws.send(message) # type: ignore[union-attr]
await self._send_to_backend(message)
except Exception as e:
verbose_logger.debug(f"Error in client ack messages: {e}")

2
litellm/llms/vertex_ai/realtime/transformation.py
View file

@ -204,7 +204,7 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig):
if session_configuration_request is None:
# First session.update - send the setup with Vertex AI configuration
setup_config = self._build_vertex_ai_setup_config(
model, json_message["session"]
model, json_message.get("session") or {}
)
gemini_setup_msg = json.dumps({"setup": setup_config})

13
tests/test_litellm/litellm_core_utils/test_realtime_streaming.py
View file

@ -1353,11 +1353,18 @@ async def test_guardrail_turn_detection_injected_into_first_session_update_defer
# Simulate first session.update in deferred mode
await streaming.client_ack_messages()
# Verify turn_detection was injected into the session.update
# Verify turn_detection was injected into the session.update. The
# injection runs before the GA remap, so the create_response flag ends
# up nested under audio.input.turn_detection in the GA-shaped payload.
assert len(transformed_messages) == 1
transformed_msg, session_config = transformed_messages[0]
msg_obj = json.loads(transformed_msg)
assert msg_obj["type"] == "session.update"
assert "turn_detection" in msg_obj["session"]
assert msg_obj["session"]["turn_detection"]["create_response"] is False
session_obj = msg_obj["session"]
injected_turn_detection = (
session_obj.get("turn_detection")
or session_obj.get("audio", {}).get("input", {}).get("turn_detection")
)
assert injected_turn_detection is not None
assert injected_turn_detection["create_response"] is False
assert streaming._guardrail_turn_detection_update_sent is True