fix(gemini realtime): send manual activity boundaries

This commit is contained in:
Marcin Juraszek 2026-09-23 14:36:37 +00:00
parent 2ea43214c9
commit e2e96cd7c1
4 changed files with 100 additions and 7 deletions

View file

@ -150,6 +150,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
# turns; buffer it here so the next response.done carries the token counts.
self._pending_usage_metadata: dict | None = None
self._unbilled_input_audio_bytes: int = 0
self._manual_activity_started = False
def is_setup_message(self, msg_obj: dict) -> bool:
return "setup" in msg_obj
@ -265,8 +266,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
def _handle_input_audio_buffer_commit_or_end(self, session_configuration_request: str | None) -> list[str]:
"""Map OpenAI buffer commit/end to Gemini Live turn-boundary signals."""
if self._manual_turn_detection_enabled(session_configuration_request):
self._manual_activity_started = False
realtime_input_dict: BidiGenerateContentRealtimeInput = {
"activityEnd": True,
"activityEnd": {},
}
verbose_logger.debug("Gemini Realtime: Sending activityEnd realtimeInput to backend")
else:
@ -616,6 +618,14 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
return self._handle_conversation_item(json_message)
if msg_type == "input_audio_buffer.append":
if self._manual_turn_detection_enabled(session_configuration_request) and not self._manual_activity_started:
# Gemini Live requires activity boundaries as empty objects.
messages.append(
json.dumps(
{"realtimeInput": {"activityStart": {}}} # mutable-ok: [LIT002] required API shape
)
)
self._manual_activity_started = True
audio_b64: Final = json_message["audio"]
if isinstance(audio_b64, str):
self._unbilled_input_audio_bytes += _base64_decoded_byte_count(audio_b64)

View file

@ -1,7 +1,8 @@
from collections.abc import Mapping
from enum import Enum
from typing import Any, Literal
from typing_extensions import Required, TypedDict
from typing_extensions import Never, ReadOnly, Required, TypedDict
from .vertex_ai import (
GenerationConfig,
@ -93,11 +94,11 @@ class BidiGenerateContentRealtimeInput(TypedDict, total=False):
audioStreamEnd: bool
"""Output only. If true, indicates that the audio stream has ended."""
activityStart: bool
"""Output only. If true, indicates that the activity has started."""
activityStart: ReadOnly[Mapping[str, Never]]
"""Signals that manual activity has started with an empty object."""
activityEnd: bool
"""Output only. If true, indicates that the activity has ended."""
activityEnd: ReadOnly[Mapping[str, Never]]
"""Signals that manual activity has ended with an empty object."""
StartOfSpeechSensitivityEnum = Literal[

View file

@ -4009,6 +4009,49 @@ async def test_async_realtime_bridges_a_transcription_session_through_the_provid
assert [bytes(request.audio) for request in speech_client.requests[1:]] == [b"\x00\x01" * 800, b"\x00\x01" * 800]
@pytest.mark.asyncio
async def test_gemini_realtime_forwards_manual_activity_boundaries():
from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming
from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig
backend_ws: Final = Mock()
backend_ws.send = AsyncMock()
streaming: Final = RealTimeStreaming(
Mock(scope={"headers": []}),
backend_ws,
Mock(),
provider_config=GeminiRealtimeConfig(),
model="gemini-live-2.5-flash-native-audio",
)
session_configuration_request: Final = json.dumps(
{
"setup": {
"realtimeInputConfig": {
"automaticActivityDetection": {"disabled": True},
}
}
}
)
streaming.session_configuration_request = session_configuration_request
append: Final = json.dumps(
{
"type": "input_audio_buffer.append",
"audio": base64.b64encode(b"audio").decode(),
}
)
await streaming._send_to_backend(append)
await streaming._send_to_backend(append)
await streaming._send_to_backend(json.dumps({"type": "input_audio_buffer.commit"}))
sent_messages: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list]
assert sent_messages[0] == {"realtimeInput": {"activityStart": {}}}
assert "audio" in sent_messages[1]["realtimeInput"]
assert "audio" in sent_messages[2]["realtimeInput"]
assert sent_messages[3] == {"realtimeInput": {"activityEnd": {}}}
@pytest.mark.asyncio
async def test_responses_agentic_followup_does_not_repeat_request_params_from_plan_kwargs(monkeypatch):
"""A plan whose kwargs repeat a request param must not crash the Responses follow-up with a duplicate keyword"""

View file

@ -1362,6 +1362,45 @@ def test_gemini_input_audio_buffer_clear_is_local_noop():
assert messages == []
def test_gemini_manual_vad_sends_activity_start_once_per_turn():
config = GeminiRealtimeConfig()
setup = {
"setup": {
"realtimeInputConfig": {
"automaticActivityDetection": {"disabled": True},
}
}
}
session_configuration_request = json.dumps(setup)
first = config.transform_realtime_request(
_input_audio_append_message(3200),
"gemini-live-2.5-flash-native-audio",
session_configuration_request=session_configuration_request,
)
second = config.transform_realtime_request(
_input_audio_append_message(3200),
"gemini-live-2.5-flash-native-audio",
session_configuration_request=session_configuration_request,
)
assert json.loads(first[0]) == {"realtimeInput": {"activityStart": {}}}
assert len(first) == 2
assert len(second) == 1
config.transform_realtime_request(
json.dumps({"type": "input_audio_buffer.commit"}),
"gemini-live-2.5-flash-native-audio",
session_configuration_request=session_configuration_request,
)
next_turn = config.transform_realtime_request(
_input_audio_append_message(3200),
"gemini-live-2.5-flash-native-audio",
session_configuration_request=session_configuration_request,
)
assert json.loads(next_turn[0]) == {"realtimeInput": {"activityStart": {}}}
def test_gemini_input_audio_buffer_commit_maps_to_activity_end_when_manual_vad():
config = GeminiRealtimeConfig()
setup = {
@ -1377,7 +1416,7 @@ def test_gemini_input_audio_buffer_commit_maps_to_activity_end_when_manual_vad()
session_configuration_request=json.dumps(setup),
)
assert len(messages) == 1
assert json.loads(messages[0]) == {"realtimeInput": {"activityEnd": True}}
assert json.loads(messages[0]) == {"realtimeInput": {"activityEnd": {}}}
def test_gemini_tool_call_id_to_name_evicts_oldest_when_capped():