mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(gemini realtime): send manual activity boundaries
This commit is contained in:
parent
2ea43214c9
commit
e2e96cd7c1
4 changed files with 100 additions and 7 deletions
|
|
@ -150,6 +150,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
# turns; buffer it here so the next response.done carries the token counts.
|
||||
self._pending_usage_metadata: dict | None = None
|
||||
self._unbilled_input_audio_bytes: int = 0
|
||||
self._manual_activity_started = False
|
||||
|
||||
def is_setup_message(self, msg_obj: dict) -> bool:
|
||||
return "setup" in msg_obj
|
||||
|
|
@ -265,8 +266,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
def _handle_input_audio_buffer_commit_or_end(self, session_configuration_request: str | None) -> list[str]:
|
||||
"""Map OpenAI buffer commit/end to Gemini Live turn-boundary signals."""
|
||||
if self._manual_turn_detection_enabled(session_configuration_request):
|
||||
self._manual_activity_started = False
|
||||
realtime_input_dict: BidiGenerateContentRealtimeInput = {
|
||||
"activityEnd": True,
|
||||
"activityEnd": {},
|
||||
}
|
||||
verbose_logger.debug("Gemini Realtime: Sending activityEnd realtimeInput to backend")
|
||||
else:
|
||||
|
|
@ -616,6 +618,14 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
return self._handle_conversation_item(json_message)
|
||||
|
||||
if msg_type == "input_audio_buffer.append":
|
||||
if self._manual_turn_detection_enabled(session_configuration_request) and not self._manual_activity_started:
|
||||
# Gemini Live requires activity boundaries as empty objects.
|
||||
messages.append(
|
||||
json.dumps(
|
||||
{"realtimeInput": {"activityStart": {}}} # mutable-ok: [LIT002] required API shape
|
||||
)
|
||||
)
|
||||
self._manual_activity_started = True
|
||||
audio_b64: Final = json_message["audio"]
|
||||
if isinstance(audio_b64, str):
|
||||
self._unbilled_input_audio_bytes += _base64_decoded_byte_count(audio_b64)
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
from collections.abc import Mapping
|
||||
from enum import Enum
|
||||
from typing import Any, Literal
|
||||
|
||||
from typing_extensions import Required, TypedDict
|
||||
from typing_extensions import Never, ReadOnly, Required, TypedDict
|
||||
|
||||
from .vertex_ai import (
|
||||
GenerationConfig,
|
||||
|
|
@ -93,11 +94,11 @@ class BidiGenerateContentRealtimeInput(TypedDict, total=False):
|
|||
audioStreamEnd: bool
|
||||
"""Output only. If true, indicates that the audio stream has ended."""
|
||||
|
||||
activityStart: bool
|
||||
"""Output only. If true, indicates that the activity has started."""
|
||||
activityStart: ReadOnly[Mapping[str, Never]]
|
||||
"""Signals that manual activity has started with an empty object."""
|
||||
|
||||
activityEnd: bool
|
||||
"""Output only. If true, indicates that the activity has ended."""
|
||||
activityEnd: ReadOnly[Mapping[str, Never]]
|
||||
"""Signals that manual activity has ended with an empty object."""
|
||||
|
||||
|
||||
StartOfSpeechSensitivityEnum = Literal[
|
||||
|
|
|
|||
|
|
@ -4009,6 +4009,49 @@ async def test_async_realtime_bridges_a_transcription_session_through_the_provid
|
|||
assert [bytes(request.audio) for request in speech_client.requests[1:]] == [b"\x00\x01" * 800, b"\x00\x01" * 800]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_gemini_realtime_forwards_manual_activity_boundaries():
|
||||
from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming
|
||||
from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig
|
||||
|
||||
backend_ws: Final = Mock()
|
||||
backend_ws.send = AsyncMock()
|
||||
streaming: Final = RealTimeStreaming(
|
||||
Mock(scope={"headers": []}),
|
||||
backend_ws,
|
||||
Mock(),
|
||||
provider_config=GeminiRealtimeConfig(),
|
||||
model="gemini-live-2.5-flash-native-audio",
|
||||
)
|
||||
session_configuration_request: Final = json.dumps(
|
||||
{
|
||||
"setup": {
|
||||
"realtimeInputConfig": {
|
||||
"automaticActivityDetection": {"disabled": True},
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
streaming.session_configuration_request = session_configuration_request
|
||||
append: Final = json.dumps(
|
||||
{
|
||||
"type": "input_audio_buffer.append",
|
||||
"audio": base64.b64encode(b"audio").decode(),
|
||||
}
|
||||
)
|
||||
|
||||
await streaming._send_to_backend(append)
|
||||
await streaming._send_to_backend(append)
|
||||
await streaming._send_to_backend(json.dumps({"type": "input_audio_buffer.commit"}))
|
||||
|
||||
sent_messages: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list]
|
||||
assert sent_messages[0] == {"realtimeInput": {"activityStart": {}}}
|
||||
assert "audio" in sent_messages[1]["realtimeInput"]
|
||||
assert "audio" in sent_messages[2]["realtimeInput"]
|
||||
assert sent_messages[3] == {"realtimeInput": {"activityEnd": {}}}
|
||||
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_agentic_followup_does_not_repeat_request_params_from_plan_kwargs(monkeypatch):
|
||||
"""A plan whose kwargs repeat a request param must not crash the Responses follow-up with a duplicate keyword"""
|
||||
|
|
|
|||
|
|
@ -1362,6 +1362,45 @@ def test_gemini_input_audio_buffer_clear_is_local_noop():
|
|||
assert messages == []
|
||||
|
||||
|
||||
def test_gemini_manual_vad_sends_activity_start_once_per_turn():
|
||||
config = GeminiRealtimeConfig()
|
||||
setup = {
|
||||
"setup": {
|
||||
"realtimeInputConfig": {
|
||||
"automaticActivityDetection": {"disabled": True},
|
||||
}
|
||||
}
|
||||
}
|
||||
session_configuration_request = json.dumps(setup)
|
||||
|
||||
first = config.transform_realtime_request(
|
||||
_input_audio_append_message(3200),
|
||||
"gemini-live-2.5-flash-native-audio",
|
||||
session_configuration_request=session_configuration_request,
|
||||
)
|
||||
second = config.transform_realtime_request(
|
||||
_input_audio_append_message(3200),
|
||||
"gemini-live-2.5-flash-native-audio",
|
||||
session_configuration_request=session_configuration_request,
|
||||
)
|
||||
|
||||
assert json.loads(first[0]) == {"realtimeInput": {"activityStart": {}}}
|
||||
assert len(first) == 2
|
||||
assert len(second) == 1
|
||||
|
||||
config.transform_realtime_request(
|
||||
json.dumps({"type": "input_audio_buffer.commit"}),
|
||||
"gemini-live-2.5-flash-native-audio",
|
||||
session_configuration_request=session_configuration_request,
|
||||
)
|
||||
next_turn = config.transform_realtime_request(
|
||||
_input_audio_append_message(3200),
|
||||
"gemini-live-2.5-flash-native-audio",
|
||||
session_configuration_request=session_configuration_request,
|
||||
)
|
||||
assert json.loads(next_turn[0]) == {"realtimeInput": {"activityStart": {}}}
|
||||
|
||||
|
||||
def test_gemini_input_audio_buffer_commit_maps_to_activity_end_when_manual_vad():
|
||||
config = GeminiRealtimeConfig()
|
||||
setup = {
|
||||
|
|
@ -1377,7 +1416,7 @@ def test_gemini_input_audio_buffer_commit_maps_to_activity_end_when_manual_vad()
|
|||
session_configuration_request=json.dumps(setup),
|
||||
)
|
||||
assert len(messages) == 1
|
||||
assert json.loads(messages[0]) == {"realtimeInput": {"activityEnd": True}}
|
||||
assert json.loads(messages[0]) == {"realtimeInput": {"activityEnd": {}}}
|
||||
|
||||
|
||||
def test_gemini_tool_call_id_to_name_evicts_oldest_when_capped():
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue