From e2e96cd7c16c17a27ab00e7de6d476b70eb694c2 Mon Sep 17 00:00:00 2001 From: Marcin Juraszek <10106352+korasino@users.noreply.github.com> Date: Wed, 23 Sep 2026 14:36:37 +0000 Subject: [PATCH] fix(gemini realtime): send manual activity boundaries --- .../llms/gemini/realtime/transformation.py | 12 +++++- litellm/types/llms/gemini.py | 11 ++--- .../custom_httpx/test_llm_http_handler.py | 43 +++++++++++++++++++ .../test_gemini_realtime_transformation.py | 41 +++++++++++++++++- 4 files changed, 100 insertions(+), 7 deletions(-) diff --git a/litellm/llms/gemini/realtime/transformation.py b/litellm/llms/gemini/realtime/transformation.py index e64cbf88d95..bb18f1882d4 100644 --- a/litellm/llms/gemini/realtime/transformation.py +++ b/litellm/llms/gemini/realtime/transformation.py @@ -150,6 +150,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): # turns; buffer it here so the next response.done carries the token counts. self._pending_usage_metadata: dict | None = None self._unbilled_input_audio_bytes: int = 0 + self._manual_activity_started = False def is_setup_message(self, msg_obj: dict) -> bool: return "setup" in msg_obj @@ -265,8 +266,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): def _handle_input_audio_buffer_commit_or_end(self, session_configuration_request: str | None) -> list[str]: """Map OpenAI buffer commit/end to Gemini Live turn-boundary signals.""" if self._manual_turn_detection_enabled(session_configuration_request): + self._manual_activity_started = False realtime_input_dict: BidiGenerateContentRealtimeInput = { - "activityEnd": True, + "activityEnd": {}, } verbose_logger.debug("Gemini Realtime: Sending activityEnd realtimeInput to backend") else: @@ -616,6 +618,14 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): return self._handle_conversation_item(json_message) if msg_type == "input_audio_buffer.append": + if self._manual_turn_detection_enabled(session_configuration_request) and not self._manual_activity_started: + # Gemini Live requires activity boundaries as empty objects. + messages.append( + json.dumps( + {"realtimeInput": {"activityStart": {}}} # mutable-ok: [LIT002] required API shape + ) + ) + self._manual_activity_started = True audio_b64: Final = json_message["audio"] if isinstance(audio_b64, str): self._unbilled_input_audio_bytes += _base64_decoded_byte_count(audio_b64) diff --git a/litellm/types/llms/gemini.py b/litellm/types/llms/gemini.py index 57fb8b5b0cd..bc91beb448c 100644 --- a/litellm/types/llms/gemini.py +++ b/litellm/types/llms/gemini.py @@ -1,7 +1,8 @@ +from collections.abc import Mapping from enum import Enum from typing import Any, Literal -from typing_extensions import Required, TypedDict +from typing_extensions import Never, ReadOnly, Required, TypedDict from .vertex_ai import ( GenerationConfig, @@ -93,11 +94,11 @@ class BidiGenerateContentRealtimeInput(TypedDict, total=False): audioStreamEnd: bool """Output only. If true, indicates that the audio stream has ended.""" - activityStart: bool - """Output only. If true, indicates that the activity has started.""" + activityStart: ReadOnly[Mapping[str, Never]] + """Signals that manual activity has started with an empty object.""" - activityEnd: bool - """Output only. If true, indicates that the activity has ended.""" + activityEnd: ReadOnly[Mapping[str, Never]] + """Signals that manual activity has ended with an empty object.""" StartOfSpeechSensitivityEnum = Literal[ diff --git a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py index 67a8d045036..f3e7b190ceb 100644 --- a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py @@ -4009,6 +4009,49 @@ async def test_async_realtime_bridges_a_transcription_session_through_the_provid assert [bytes(request.audio) for request in speech_client.requests[1:]] == [b"\x00\x01" * 800, b"\x00\x01" * 800] +@pytest.mark.asyncio +async def test_gemini_realtime_forwards_manual_activity_boundaries(): + from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming + from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig + + backend_ws: Final = Mock() + backend_ws.send = AsyncMock() + streaming: Final = RealTimeStreaming( + Mock(scope={"headers": []}), + backend_ws, + Mock(), + provider_config=GeminiRealtimeConfig(), + model="gemini-live-2.5-flash-native-audio", + ) + session_configuration_request: Final = json.dumps( + { + "setup": { + "realtimeInputConfig": { + "automaticActivityDetection": {"disabled": True}, + } + } + } + ) + streaming.session_configuration_request = session_configuration_request + append: Final = json.dumps( + { + "type": "input_audio_buffer.append", + "audio": base64.b64encode(b"audio").decode(), + } + ) + + await streaming._send_to_backend(append) + await streaming._send_to_backend(append) + await streaming._send_to_backend(json.dumps({"type": "input_audio_buffer.commit"})) + + sent_messages: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list] + assert sent_messages[0] == {"realtimeInput": {"activityStart": {}}} + assert "audio" in sent_messages[1]["realtimeInput"] + assert "audio" in sent_messages[2]["realtimeInput"] + assert sent_messages[3] == {"realtimeInput": {"activityEnd": {}}} + + + @pytest.mark.asyncio async def test_responses_agentic_followup_does_not_repeat_request_params_from_plan_kwargs(monkeypatch): """A plan whose kwargs repeat a request param must not crash the Responses follow-up with a duplicate keyword""" diff --git a/tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py b/tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py index bcd5f3d8d19..91717e5a41c 100644 --- a/tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py +++ b/tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py @@ -1362,6 +1362,45 @@ def test_gemini_input_audio_buffer_clear_is_local_noop(): assert messages == [] +def test_gemini_manual_vad_sends_activity_start_once_per_turn(): + config = GeminiRealtimeConfig() + setup = { + "setup": { + "realtimeInputConfig": { + "automaticActivityDetection": {"disabled": True}, + } + } + } + session_configuration_request = json.dumps(setup) + + first = config.transform_realtime_request( + _input_audio_append_message(3200), + "gemini-live-2.5-flash-native-audio", + session_configuration_request=session_configuration_request, + ) + second = config.transform_realtime_request( + _input_audio_append_message(3200), + "gemini-live-2.5-flash-native-audio", + session_configuration_request=session_configuration_request, + ) + + assert json.loads(first[0]) == {"realtimeInput": {"activityStart": {}}} + assert len(first) == 2 + assert len(second) == 1 + + config.transform_realtime_request( + json.dumps({"type": "input_audio_buffer.commit"}), + "gemini-live-2.5-flash-native-audio", + session_configuration_request=session_configuration_request, + ) + next_turn = config.transform_realtime_request( + _input_audio_append_message(3200), + "gemini-live-2.5-flash-native-audio", + session_configuration_request=session_configuration_request, + ) + assert json.loads(next_turn[0]) == {"realtimeInput": {"activityStart": {}}} + + def test_gemini_input_audio_buffer_commit_maps_to_activity_end_when_manual_vad(): config = GeminiRealtimeConfig() setup = { @@ -1377,7 +1416,7 @@ def test_gemini_input_audio_buffer_commit_maps_to_activity_end_when_manual_vad() session_configuration_request=json.dumps(setup), ) assert len(messages) == 1 - assert json.loads(messages[0]) == {"realtimeInput": {"activityEnd": True}} + assert json.loads(messages[0]) == {"realtimeInput": {"activityEnd": {}}} def test_gemini_tool_call_id_to_name_evicts_oldest_when_capped():