mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
Merge ef52e611b3 into a66dc50efb
This commit is contained in:
commit
a57f37a0e8
15 changed files with 1600 additions and 120 deletions
|
|
@ -416,6 +416,7 @@ REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS: Final = get_env_float(
|
|||
"REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS", 20.0
|
||||
)
|
||||
BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS: Final = get_env_float("BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS", 5.0)
|
||||
REALTIME_LAZY_SETUP_DEADLINE_SECONDS: Final = 240.0
|
||||
|
||||
# RFC 6455 caps the close frame payload at 125 bytes, 2 of which carry the status code
|
||||
WEBSOCKET_CLOSE_REASON_MAX_BYTES: Final = 123
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import asyncio
|
||||
import json
|
||||
import time
|
||||
import traceback
|
||||
from collections.abc import Callable, Coroutine, Mapping, MutableMapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
|
|
@ -15,12 +16,17 @@ from litellm._logging import redact_internal_details_from_client_message, verbos
|
|||
from litellm._uuid import uuid
|
||||
from litellm.constants import (
|
||||
BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY,
|
||||
REALTIME_LAZY_SETUP_DEADLINE_SECONDS,
|
||||
REALTIME_METERED_USAGE_KEY,
|
||||
REALTIME_SESSION_FAILURE_LOGGED_KEY,
|
||||
REALTIME_SESSION_SUCCESS_LOGGED_KEY,
|
||||
)
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeBackend
|
||||
from litellm.llms.base_llm.realtime.transformation import (
|
||||
BaseRealtimeConfig,
|
||||
RealtimeBackend,
|
||||
RealtimeSessionUpdateRejection,
|
||||
)
|
||||
from litellm.types.llms.openai import (
|
||||
OpenAIRealtimeEvents,
|
||||
OpenAIRealtimeOutputItemDone,
|
||||
|
|
@ -124,6 +130,65 @@ class _LoggingWorker(Protocol):
|
|||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
_SESSION_FIELDS: Final = TypeAdapter(dict[str, object])
|
||||
_NO_SESSION_FIELDS: Final[Mapping[str, object]] = MappingProxyType({})
|
||||
_SESSION_MERGE_MAX_DEPTH: Final = 8
|
||||
|
||||
|
||||
class _SessionUpdateFrame(TypedDict):
|
||||
type: ReadOnly[str]
|
||||
session: ReadOnly[Mapping[str, object]]
|
||||
|
||||
|
||||
def _merge_session_fields(
|
||||
base: Mapping[str, object], update: Mapping[str, object], depth: int = 0
|
||||
) -> Mapping[str, object]:
|
||||
def merged_value(key: str) -> object:
|
||||
if key not in update:
|
||||
return base[key]
|
||||
value: Final = update[key]
|
||||
base_value: Final = base.get(key)
|
||||
if depth < _SESSION_MERGE_MAX_DEPTH and isinstance(value, Mapping) and isinstance(base_value, Mapping):
|
||||
return _merge_session_fields(
|
||||
_SESSION_FIELDS.validate_python(base_value), _SESSION_FIELDS.validate_python(value), depth + 1
|
||||
)
|
||||
return update[key]
|
||||
|
||||
return MappingProxyType({key: merged_value(key) for key in dict.fromkeys((*base, *update))})
|
||||
|
||||
|
||||
def _session_update_frame(session: Mapping[str, object]) -> str:
|
||||
frame: Final[_SessionUpdateFrame] = {"type": "session.update", "session": session}
|
||||
return json.dumps(frame, default=_SESSION_FIELDS.validate_python)
|
||||
|
||||
|
||||
_HELD_SESSION_UPDATE_TOO_LARGE: Final = RealtimeSessionUpdateRejection(
|
||||
code="session_update_too_large",
|
||||
message=(
|
||||
"The session.update frames sent before the first conversation content add up to more than 10 MB, "
|
||||
"so this session.update was not applied"
|
||||
),
|
||||
param="session",
|
||||
)
|
||||
|
||||
|
||||
def _unscreened_audio_error_event(client_event_id: object) -> dict[str, object]:
|
||||
return {
|
||||
"type": "error",
|
||||
"event_id": f"event_{uuid.uuid4()}",
|
||||
"error": {
|
||||
"type": "invalid_request_error",
|
||||
"code": "audio_input_not_screenable",
|
||||
"message": (
|
||||
"A realtime_input_transcription guardrail is on, and Gemini Live only transcribes a turn after it "
|
||||
"starts answering it, so audio input was not sent. Send text with conversation.item.create instead"
|
||||
),
|
||||
"param": None,
|
||||
"event_id": client_event_id if isinstance(client_event_id, str) else None,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _decode_json_object(payload: str) -> Mapping[str, object]:
|
||||
"""Decode a realtime frame into its top-level field mapping."""
|
||||
return json.loads(payload)
|
||||
|
|
@ -242,8 +307,10 @@ class RealTimeStreaming:
|
|||
event_normalizer: RealtimeEventNormalizer | None = None,
|
||||
usage_meter: RealtimeUsageMeter | None = None,
|
||||
logging_worker: _LoggingWorker = GLOBAL_LOGGING_WORKER,
|
||||
lazy_setup_deadline_seconds: float = REALTIME_LAZY_SETUP_DEADLINE_SECONDS,
|
||||
):
|
||||
self.websocket: _ClientWebSocket = websocket
|
||||
self._lazy_setup_deadline: Final = time.monotonic() + lazy_setup_deadline_seconds
|
||||
self._replay_scope: Final = client_websocket_scope(websocket)
|
||||
self.backend_ws = backend_ws
|
||||
self.logging_obj = logging_obj
|
||||
|
|
@ -299,6 +366,10 @@ class RealTimeStreaming:
|
|||
# Gemini Live rejects a follow-up BidiGenerateContentSetup once any
|
||||
# content (realtimeInput / clientContent / toolResponse) has been sent.
|
||||
self._content_sent_after_setup: bool = False
|
||||
self._held_session_update: Mapping[str, object] | None = None
|
||||
self._applied_session_fields: Mapping[str, object] | None = None
|
||||
self._held_session_update_bytes: int = 0
|
||||
self._unscreened_audio_refused: bool = False
|
||||
# Whether this is a transcription-only session (session.type == "transcription",
|
||||
# e.g. gpt-realtime-whisper). Such sessions must not be sent response.create and
|
||||
# their input_audio_transcription.completed usage drives duration-based cost.
|
||||
|
|
@ -336,6 +407,15 @@ class RealTimeStreaming:
|
|||
]
|
||||
)
|
||||
_CLIENT_AUDIO_BUFFER_COMMIT_TYPES = frozenset(["input_audio_buffer.commit", "input_audio_buffer.end"])
|
||||
_CLIENT_CONTENT_TYPES = frozenset(
|
||||
[
|
||||
"conversation.item.create",
|
||||
"response.create",
|
||||
"input_audio_buffer.append",
|
||||
"input_audio_buffer.commit",
|
||||
"input_audio_buffer.end",
|
||||
]
|
||||
)
|
||||
_AUDIO_FORMAT_MAP: dict[str, dict[str, str | int]] = {
|
||||
"pcm16": {"type": "audio/pcm", "rate": 24000},
|
||||
"g711_ulaw": {"type": "audio/G711-ulaw", "rate": 8000},
|
||||
|
|
@ -376,7 +456,7 @@ class RealTimeStreaming:
|
|||
message_obj: dict[str, Any] = cast(dict[str, Any], message)
|
||||
else:
|
||||
message_obj = cast(dict[str, Any], json.loads(message))
|
||||
self._collect_tool_calls_from_response_done(cast(dict, message_obj))
|
||||
self._collect_tool_calls_from_response_done(message_obj)
|
||||
if not self._should_store_message(message_obj):
|
||||
return
|
||||
try:
|
||||
|
|
@ -501,7 +581,7 @@ class RealTimeStreaming:
|
|||
self.store_message(flush_event)
|
||||
self._capture_transcription_usage(flush_event)
|
||||
|
||||
def _collect_tool_calls_from_response_done(self, event_obj: dict | OpenAIRealtimeEvents) -> None:
|
||||
def _collect_tool_calls_from_response_done(self, event_obj: Mapping[str, object] | OpenAIRealtimeEvents) -> None:
|
||||
"""Extract function_call items from response.done events for spend logging."""
|
||||
try:
|
||||
if event_obj.get("type") != "response.done":
|
||||
|
|
@ -606,6 +686,130 @@ class RealTimeStreaming:
|
|||
self._usage_meter.observe_client_event(sent_event)
|
||||
return True
|
||||
|
||||
def _builds_setup_lazily(self) -> bool:
|
||||
if self.provider_config is None:
|
||||
return False
|
||||
return self.provider_config.builds_setup_from_session_update()
|
||||
|
||||
def _holds_session_updates_until_first_content(self) -> bool:
|
||||
return self.session_configuration_request is None and self._builds_setup_lazily()
|
||||
|
||||
async def _rejected_setup_session_update(self, message: str) -> bool:
|
||||
provider_config: Final = self.provider_config
|
||||
if provider_config is None or not provider_config.builds_setup_from_session_update():
|
||||
return False
|
||||
client_event: Final = _decode_json_object(message)
|
||||
session: Final = client_event.get("session")
|
||||
update: Final = _SESSION_FIELDS.validate_python(session) if isinstance(session, Mapping) else _NO_SESSION_FIELDS
|
||||
if self._leaves_applied_setup_unchanged(update):
|
||||
await self._acknowledge_session_setup(self._applied_session_fields or _NO_SESSION_FIELDS)
|
||||
return True
|
||||
rejection: Final = provider_config.setup_session_update_rejection(
|
||||
self.model, update, setup_sent=self.session_configuration_request is not None
|
||||
)
|
||||
if rejection is None:
|
||||
return False
|
||||
await self.websocket.send_text(json.dumps(rejection.error_event(client_event.get("event_id"))))
|
||||
return True
|
||||
|
||||
def _held_session_update_message(self) -> str:
|
||||
return _session_update_frame(self._held_session_update or _NO_SESSION_FIELDS)
|
||||
|
||||
async def _hold_session_update(self, message: str) -> None:
|
||||
client_event: Final = _decode_json_object(message)
|
||||
if self._held_session_update_bytes + len(message.encode("utf-8")) > RealTimeStreaming._MAX_BUFFERED_BYTES:
|
||||
await self.websocket.send_text(
|
||||
json.dumps(_HELD_SESSION_UPDATE_TOO_LARGE.error_event(client_event.get("event_id")))
|
||||
)
|
||||
return
|
||||
session: Final = client_event.get("session")
|
||||
update: Final = _SESSION_FIELDS.validate_python(session) if isinstance(session, Mapping) else _NO_SESSION_FIELDS
|
||||
held: Final = _merge_session_fields(self._held_session_update or _NO_SESSION_FIELDS, update)
|
||||
self._held_session_update = held
|
||||
self._held_session_update_bytes = len(_session_update_frame(held).encode("utf-8"))
|
||||
await self._acknowledge_session_setup(held)
|
||||
|
||||
def _setup_preview(self, session: Mapping[str, object]) -> str | None:
|
||||
if self.provider_config is None:
|
||||
return None
|
||||
preview: Final = self.provider_config.transform_realtime_request(
|
||||
_session_update_frame(session), self.model, None
|
||||
)
|
||||
return next(
|
||||
(frame for frame in preview if isinstance(frame, str) and "setup" in _decode_json_object(frame)), None
|
||||
)
|
||||
|
||||
def _leaves_applied_setup_unchanged(self, update: Mapping[str, object]) -> bool:
|
||||
applied: Final = self._applied_session_fields
|
||||
if self.session_configuration_request is None or applied is None:
|
||||
return False
|
||||
applied_setup: Final = self._setup_preview(applied)
|
||||
merged_setup: Final = self._setup_preview(_merge_session_fields(applied, update))
|
||||
return (
|
||||
applied_setup is not None
|
||||
and merged_setup is not None
|
||||
and _decode_json_object(applied_setup) == _decode_json_object(merged_setup)
|
||||
)
|
||||
|
||||
async def _acknowledge_session_setup(self, session: Mapping[str, object]) -> None:
|
||||
if self.provider_config is None:
|
||||
return
|
||||
setup: Final = self._setup_preview(session)
|
||||
if setup is None:
|
||||
return
|
||||
acknowledged: Final = self.provider_config.transform_session_created_event(
|
||||
self.model, str(self.logging_obj.litellm_trace_id), setup
|
||||
)
|
||||
if acknowledged is None:
|
||||
return
|
||||
event: Final[Mapping[str, object]] = {**acknowledged, "type": "session.updated"}
|
||||
event_str: Final = json.dumps(event)
|
||||
self.store_message(event_str)
|
||||
await self._send_event_to_client(event, event_str)
|
||||
|
||||
async def _send_held_session_update_as_setup(self) -> None:
|
||||
held: Final = self._held_session_update
|
||||
if held is None:
|
||||
return
|
||||
message: Final = self._held_session_update_message()
|
||||
self._held_session_update = None
|
||||
self._held_session_update_bytes = 0
|
||||
if await self._send_to_backend(message):
|
||||
self._applied_session_fields = held
|
||||
|
||||
async def _send_setup_before_content(self, msg_type: str | None) -> None:
|
||||
if self._holds_session_updates_until_first_content():
|
||||
await self._send_held_session_update_as_setup()
|
||||
await self._send_default_setup_before_first_client_message(msg_type)
|
||||
|
||||
async def _receive_client_message_before_setup_deadline(self) -> str | None:
|
||||
if not self._holds_session_updates_until_first_content():
|
||||
return await self._receive_client_message()
|
||||
receive: Final = asyncio.ensure_future(self._receive_client_message())
|
||||
try:
|
||||
done, _ = await asyncio.wait((receive,), timeout=self._lazy_setup_deadline - time.monotonic())
|
||||
if not done:
|
||||
await self._send_setup_before_content(None)
|
||||
return await receive
|
||||
finally:
|
||||
receive.cancel()
|
||||
|
||||
async def _send_default_setup_before_first_client_message(self, msg_type: str | None) -> None:
|
||||
if self.session_configuration_request is not None or self.provider_config is None:
|
||||
return
|
||||
if msg_type == "session.update" or (
|
||||
self._uses_deferred_backend_setup()
|
||||
and (msg_type in RealTimeStreaming._CLIENT_AUDIO_BUFFER_TYPES or self._pending_messages_until_setup)
|
||||
):
|
||||
return
|
||||
default_setup: Final = self.provider_config.session_configuration_request(self.model)
|
||||
if not default_setup:
|
||||
return
|
||||
setup: Final = self.maybe_inject_guardrail_auto_response_disable(default_setup)
|
||||
await self.backend_ws.send(setup)
|
||||
self._cache_session_configuration_request(setup)
|
||||
self._applied_session_fields = _NO_SESSION_FIELDS
|
||||
|
||||
def _enforce_transcription_session_model(self, message: str) -> str:
|
||||
"""Force client transcription session updates to the authorized model.
|
||||
|
||||
|
|
@ -957,6 +1161,21 @@ class RealTimeStreaming:
|
|||
"""Transcription-only sessions have no assistant turn to gate and reject realtime session updates."""
|
||||
return not self._is_transcription_session and self._has_audio_transcription_guardrails()
|
||||
|
||||
def _refuses_unscreened_audio(self, msg_type: str | None) -> bool:
|
||||
return (
|
||||
msg_type in RealTimeStreaming._CLIENT_AUDIO_BUFFER_TYPES
|
||||
and self._builds_setup_lazily()
|
||||
and self._should_disable_vad_auto_response()
|
||||
)
|
||||
|
||||
async def _refuse_unscreened_audio(self, msg_type: str | None, message: str) -> None:
|
||||
if self._unscreened_audio_refused and msg_type not in RealTimeStreaming._CLIENT_AUDIO_BUFFER_COMMIT_TYPES:
|
||||
return
|
||||
self._unscreened_audio_refused = True
|
||||
await self.websocket.send_text(
|
||||
json.dumps(_unscreened_audio_error_event(_decode_json_object(message).get("event_id")))
|
||||
)
|
||||
|
||||
async def run_realtime_guardrails(
|
||||
self,
|
||||
transcript: str,
|
||||
|
|
@ -1135,6 +1354,9 @@ class RealTimeStreaming:
|
|||
break
|
||||
finally:
|
||||
self._flushing_pending_messages_until_setup = False
|
||||
if self.session_created_sent_to_client and self._builds_setup_lazily():
|
||||
await self._maybe_send_guardrail_turn_detection_update()
|
||||
continue
|
||||
if self.session_created_sent_to_client:
|
||||
# A synthetic session.created (with placeholder defaults) was
|
||||
# already forwarded to the client when we connected. The
|
||||
|
|
@ -1482,7 +1704,7 @@ class RealTimeStreaming:
|
|||
client_event: _ClientEventFrame
|
||||
try:
|
||||
while True:
|
||||
message = await self._receive_client_message()
|
||||
message = await self._receive_client_message_before_setup_deadline()
|
||||
if message is None:
|
||||
return ClientLoopExit.CLIENT_DISCONNECTED
|
||||
|
||||
|
|
@ -1497,6 +1719,7 @@ class RealTimeStreaming:
|
|||
msg_type = client_event.get("type")
|
||||
|
||||
if msg_type == "conversation.item.create":
|
||||
await self._send_setup_before_content(msg_type)
|
||||
# Check user text messages for prompt injection
|
||||
item = client_event.get("item", {})
|
||||
# Check function_call_output first so a client cannot
|
||||
|
|
@ -1594,6 +1817,7 @@ class RealTimeStreaming:
|
|||
if (
|
||||
msg_type == "session.update"
|
||||
and self.session_configuration_request is None
|
||||
and self._held_session_update is None
|
||||
and not self._guardrail_turn_detection_update_sent
|
||||
and self._should_disable_vad_auto_response()
|
||||
and (
|
||||
|
|
@ -1657,12 +1881,19 @@ class RealTimeStreaming:
|
|||
except (json.JSONDecodeError, AttributeError):
|
||||
pass
|
||||
|
||||
if self._refuses_unscreened_audio(msg_type):
|
||||
await self._refuse_unscreened_audio(msg_type, message)
|
||||
continue
|
||||
|
||||
## LOGGING
|
||||
# Log after any in-place modifications (GA remap, guardrail
|
||||
# turn_detection injection) so audit logs reflect what we
|
||||
# actually forward to the backend.
|
||||
self.store_input(message=message)
|
||||
|
||||
if msg_type in RealTimeStreaming._CLIENT_CONTENT_TYPES and self._uses_deferred_backend_setup():
|
||||
await self._send_setup_before_content(msg_type)
|
||||
|
||||
if self._should_buffer_client_message_until_setup(message):
|
||||
self._buffer_pending_message_until_setup(message)
|
||||
continue
|
||||
|
|
@ -1691,6 +1922,15 @@ class RealTimeStreaming:
|
|||
# ``_maybe_send_guardrail_turn_detection_update`` would retry.
|
||||
if msg_type == "session.update" and self.provider_config is None:
|
||||
await self._send_session_gate_once()
|
||||
if msg_type == "session.update" and await self._rejected_setup_session_update(message):
|
||||
continue
|
||||
if msg_type == "session.update" and self._holds_session_updates_until_first_content():
|
||||
await self._hold_session_update(message)
|
||||
if self._pending_messages_until_setup:
|
||||
await self._send_held_session_update_as_setup()
|
||||
continue
|
||||
if msg_type in RealTimeStreaming._CLIENT_CONTENT_TYPES:
|
||||
await self._send_setup_before_content(msg_type)
|
||||
sent = await self._send_to_backend(message)
|
||||
if guardrail_turn_detection_injected and sent:
|
||||
self._guardrail_turn_detection_update_sent = True
|
||||
|
|
|
|||
|
|
@ -1,11 +1,13 @@
|
|||
import dataclasses
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Mapping, Sequence
|
||||
from types import TracebackType
|
||||
from typing import TYPE_CHECKING, Any, Protocol
|
||||
from typing import TYPE_CHECKING, Any, Literal, Protocol
|
||||
|
||||
import httpx
|
||||
from typing_extensions import Self
|
||||
|
||||
from litellm._uuid import uuid
|
||||
from litellm.types.llms.openai import OpenAIRealtimeStreamSessionEvents
|
||||
from litellm.types.realtime import (
|
||||
RealtimeInputAudioTranscriptionUsage,
|
||||
|
|
@ -23,6 +25,26 @@ else:
|
|||
LiteLLMLoggingObj = Any
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True, slots=True)
|
||||
class RealtimeSessionUpdateRejection:
|
||||
code: Literal["invalid_type", "invalid_value", "session_update_not_applied", "session_update_too_large"]
|
||||
message: str
|
||||
param: str
|
||||
|
||||
def error_event(self, client_event_id: object) -> dict[str, object]:
|
||||
return {
|
||||
"type": "error",
|
||||
"event_id": f"event_{uuid.uuid4()}",
|
||||
"error": {
|
||||
"type": "invalid_request_error",
|
||||
"code": self.code,
|
||||
"message": self.message,
|
||||
"param": self.param,
|
||||
"event_id": client_event_id if isinstance(client_event_id, str) else None,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class RealtimeBackend(Protocol):
|
||||
async def __aenter__(self) -> Self: ...
|
||||
|
||||
|
|
@ -94,6 +116,14 @@ class BaseRealtimeConfig(ABC):
|
|||
def session_configuration_request(self, model: str) -> str | None: # message sent to setup the realtime session
|
||||
return None
|
||||
|
||||
def builds_setup_from_session_update(self) -> bool:
|
||||
return False
|
||||
|
||||
def setup_session_update_rejection(
|
||||
self, model: str, session: Mapping[str, object], *, setup_sent: bool
|
||||
) -> RealtimeSessionUpdateRejection | None:
|
||||
return None
|
||||
|
||||
def unbilled_usage_on_session_close(self, model: str) -> RealtimeInputAudioTranscriptionUsage | None:
|
||||
return None
|
||||
|
||||
|
|
|
|||
|
|
@ -17,7 +17,6 @@ from pydantic import JsonValue, TypeAdapter
|
|||
|
||||
import litellm
|
||||
from litellm._logging import redact_string, verbose_proxy_logger
|
||||
from litellm._uuid import uuid
|
||||
from litellm.constants import (
|
||||
BEDROCK_REALTIME_CLOSE_TIMEOUT_SECONDS,
|
||||
BEDROCK_REALTIME_COMMITTED_FAILURE_SCOPE_KEY,
|
||||
|
|
@ -31,13 +30,14 @@ from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
|
|||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
||||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
from litellm.litellm_core_utils.realtime_streaming import DefaultLoggedRealTimeEventTypes
|
||||
from litellm.llms.base_llm.realtime.transformation import RealtimeSessionUpdateRejection
|
||||
from litellm.types.llms.bedrock import AwsAuthParams
|
||||
from litellm.types.llms.openai import OpenAIRealtimeEvents
|
||||
from litellm.types.realtime import RealtimeResponseTransformInput
|
||||
|
||||
from ..base_aws_llm import BaseAWSLLM, run_aws_signing
|
||||
from ..common_utils import BedrockError
|
||||
from .transformation import BedrockRealtimeConfig, BedrockSessionUpdateRejection, requested_modalities
|
||||
from .transformation import BedrockRealtimeConfig, requested_modalities
|
||||
|
||||
_CLIENT_MESSAGE_ADAPTER: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
|
||||
|
||||
|
|
@ -256,23 +256,11 @@ async def _ack_session_update(
|
|||
async def _reject_session_update(
|
||||
client_ws: RealtimeClientWebSocket,
|
||||
parsed_client_message: Mapping[str, JsonValue],
|
||||
rejection: BedrockSessionUpdateRejection,
|
||||
rejection: RealtimeSessionUpdateRejection,
|
||||
) -> bool:
|
||||
"""Send an error event for a session.update Bedrock cannot apply; False means the client is gone."""
|
||||
client_event_id: Final = parsed_client_message.get("event_id")
|
||||
error_event: Final = {
|
||||
"type": "error",
|
||||
"event_id": f"event_{uuid.uuid4()}",
|
||||
"error": {
|
||||
"type": "invalid_request_error",
|
||||
"code": rejection.code,
|
||||
"message": rejection.message,
|
||||
"param": rejection.param,
|
||||
"event_id": client_event_id if isinstance(client_event_id, str) else None,
|
||||
},
|
||||
}
|
||||
try:
|
||||
await client_ws.send_text(json.dumps(error_event))
|
||||
await client_ws.send_text(json.dumps(rejection.error_event(parsed_client_message.get("event_id"))))
|
||||
except Exception as e: # noqa: BLE001 # any send failure means the client is gone
|
||||
verbose_proxy_logger.debug("Client to Bedrock forwarding ended: %s", e, exc_info=True)
|
||||
return False
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ import math
|
|||
import uuid as uuid_lib
|
||||
from collections.abc import Mapping, Sequence
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Literal
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
from pydantic import ConfigDict, JsonValue, TypeAdapter
|
||||
|
|
@ -19,7 +19,7 @@ from pydantic import ConfigDict, JsonValue, TypeAdapter
|
|||
from litellm._logging import verbose_logger
|
||||
from litellm._uuid import uuid
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig
|
||||
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeSessionUpdateRejection
|
||||
from litellm.llms.bedrock.common_utils import BedrockError
|
||||
from litellm.llms.bedrock.realtime.trigger_audio import ready_trigger_pcm
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
|
|
@ -196,13 +196,6 @@ def _malformed_settings(session_config: Mapping[str, JsonValue]) -> tuple[tuple[
|
|||
return tuple((name, expected) for name, expected, is_malformed in malformed if is_malformed)
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True, slots=True)
|
||||
class BedrockSessionUpdateRejection:
|
||||
code: Literal["invalid_type", "invalid_value", "session_update_not_applied"]
|
||||
message: str
|
||||
param: str
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True, slots=True)
|
||||
class BedrockLockedSessionSettings:
|
||||
"""The session settings Bedrock Nova Sonic locks in once the prompt starts."""
|
||||
|
|
@ -533,7 +526,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
|
|||
max_tokens=max_tokens,
|
||||
)
|
||||
|
||||
def session_update_rejection(self, json_message: Mapping[str, JsonValue]) -> BedrockSessionUpdateRejection | None:
|
||||
def session_update_rejection(self, json_message: Mapping[str, JsonValue]) -> RealtimeSessionUpdateRejection | None:
|
||||
"""The error for a session.update this session cannot apply, or None when it applies."""
|
||||
session_config: Final = json_message.get("session")
|
||||
if not isinstance(session_config, Mapping):
|
||||
|
|
@ -541,7 +534,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
|
|||
malformed: Final = () if self.drop_params else _malformed_settings(session_config)
|
||||
if malformed:
|
||||
name, expected = malformed[0]
|
||||
return BedrockSessionUpdateRejection(
|
||||
return RealtimeSessionUpdateRejection(
|
||||
code="invalid_type",
|
||||
message=(
|
||||
f"Invalid type for 'session.{name}': Bedrock Nova Sonic needs {expected} there. "
|
||||
|
|
@ -551,7 +544,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
|
|||
)
|
||||
unplayable_voice: Final = None if self.drop_params else _unplayable_voice(session_config)
|
||||
if unplayable_voice is not None:
|
||||
return BedrockSessionUpdateRejection(
|
||||
return RealtimeSessionUpdateRejection(
|
||||
code="invalid_value",
|
||||
message=(
|
||||
f"Invalid value for 'session.voice': '{unplayable_voice}' is not a Bedrock Nova Sonic voice. "
|
||||
|
|
@ -562,7 +555,7 @@ class BedrockRealtimeConfig(BaseRealtimeConfig):
|
|||
if not self.prompt_started:
|
||||
return None
|
||||
if self._apply_locked_settings(session_config, self._started_settings) != self._started_settings:
|
||||
return BedrockSessionUpdateRejection(
|
||||
return RealtimeSessionUpdateRejection(
|
||||
code="session_update_not_applied",
|
||||
message="Bedrock Nova Sonic cannot change tools, instructions, voice, output audio format, temperature or max output tokens after the session has started, so this session.update was not applied",
|
||||
param="session",
|
||||
|
|
|
|||
|
|
@ -6849,14 +6849,19 @@ class BaseLLMHTTPHandler:
|
|||
),
|
||||
)
|
||||
|
||||
# Auto-send session setup if the provider requires it (e.g.
|
||||
# Gemini/Vertex AI Live needs a `setup` before any realtime_input).
|
||||
# Auto-send session setup at connect unless the provider builds it from the
|
||||
# client's first session.update (Gemini/Vertex AI Live accept exactly one
|
||||
# setup, so RealTimeStreaming sends theirs on the first client message).
|
||||
# Build the streaming handler first so a transcription guardrail's
|
||||
# auto-response disable can be folded into this one setup: Gemini
|
||||
# rejects a second setup, so a follow-up disable would be dropped
|
||||
# and the guardrail bypassed.
|
||||
_session_config: str | None = None
|
||||
if provider_config.requires_session_configuration():
|
||||
sends_setup_at_connect: Final = (
|
||||
provider_config.requires_session_configuration()
|
||||
and not provider_config.builds_setup_from_session_update()
|
||||
)
|
||||
if sends_setup_at_connect:
|
||||
_session_config = provider_config.session_configuration_request(model)
|
||||
if _session_config:
|
||||
_session_config = realtime_streaming.maybe_inject_guardrail_auto_response_disable(
|
||||
|
|
@ -6865,9 +6870,9 @@ class BaseLLMHTTPHandler:
|
|||
await backend_ws.send(_session_config)
|
||||
realtime_streaming.session_configuration_request = _session_config
|
||||
|
||||
# For providers that defer setup until client session.update, optionally
|
||||
# For providers that defer setup until the client's first message, optionally
|
||||
# send synthetic session.created to unblock clients waiting on connect.
|
||||
if not provider_config.requires_session_configuration():
|
||||
if not sends_setup_at_connect:
|
||||
synthetic_session: Final = provider_config.transform_session_created_event(
|
||||
model=model,
|
||||
logging_session_id=logging_obj.litellm_trace_id,
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ from collections import OrderedDict
|
|||
from collections.abc import Mapping, Sequence
|
||||
from typing import Final, cast
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
from typing_extensions import ReadOnly, Required, TypedDict
|
||||
|
||||
import litellm
|
||||
|
|
@ -15,7 +16,7 @@ from litellm._uuid import uuid
|
|||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig
|
||||
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeSessionUpdateRejection
|
||||
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
|
||||
VertexGeminiConfig,
|
||||
)
|
||||
|
|
@ -76,27 +77,82 @@ MAP_GEMINI_FIELD_TO_OPENAI_EVENT: Final[dict[str, OpenAIRealtimeEventTypes | Res
|
|||
|
||||
# Keys the main transform loop handles; siblings like ``usageMetadata`` are skipped.
|
||||
_KNOWN_GEMINI_TOP_LEVEL_KEYS: Final[set] = {map_key.split(".", 1)[0] for map_key in MAP_GEMINI_FIELD_TO_OPENAI_EVENT}
|
||||
_GEMINI_LIVE_RESPONSE_MODALITIES: Final = frozenset({"TEXT", "AUDIO"})
|
||||
|
||||
|
||||
OPENAI_STOCK_REALTIME_VOICES: Final[frozenset[str]] = frozenset(
|
||||
{"alloy", "ash", "ballad", "cedar", "coral", "echo", "marin", "sage", "shimmer", "verse"}
|
||||
)
|
||||
|
||||
# Per https://docs.cloud.google.com/vertex-ai/generative-ai/docs/live-api/configure-language-voice (2026-10-28)
|
||||
GEMINI_LIVE_DOCUMENTED_VOICES: Final[frozenset[str]] = frozenset(
|
||||
{
|
||||
"achernar",
|
||||
"achird",
|
||||
"algenib",
|
||||
"algieba",
|
||||
"alnilam",
|
||||
"aoede",
|
||||
"autonoe",
|
||||
"callirrhoe",
|
||||
"charon",
|
||||
"despina",
|
||||
"enceladus",
|
||||
"erinome",
|
||||
"fenrir",
|
||||
"gacrux",
|
||||
"iapetus",
|
||||
"kore",
|
||||
"laomedeia",
|
||||
"leda",
|
||||
"orus",
|
||||
"puck",
|
||||
"pulcherrima",
|
||||
"rasalgethi",
|
||||
"sadachbia",
|
||||
"sadaltager",
|
||||
"schedar",
|
||||
"sulafat",
|
||||
"umbriel",
|
||||
"vindemiatrix",
|
||||
"zephyr",
|
||||
"zubenelgenubi",
|
||||
}
|
||||
)
|
||||
|
||||
def _gemini_live_speech_config(voice: object) -> Mapping[str, object] | None:
|
||||
|
||||
def _is_undocumented_live_voice(voice: object) -> bool:
|
||||
return (
|
||||
isinstance(voice, str)
|
||||
and voice.lower() not in OPENAI_STOCK_REALTIME_VOICES
|
||||
and voice.lower() not in GEMINI_LIVE_DOCUMENTED_VOICES
|
||||
)
|
||||
|
||||
|
||||
def _gemini_live_speech_config(voice: object, drop_undocumented: bool) -> Mapping[str, object] | None:
|
||||
"""Build the Gemini Live speechConfig for a client-requested voice.
|
||||
|
||||
OpenAI stock voice names have no Gemini equivalent and Gemini Live closes
|
||||
the session on an unknown voice, so they are dropped with a warning and
|
||||
the model keeps its default voice. Every other name is forwarded verbatim.
|
||||
Gemini Live closes the session on a voice name it does not know. OpenAI
|
||||
stock voice names are always dropped, and so is any name outside Google's
|
||||
documented Live voices when ``drop_undocumented`` is set, keeping the
|
||||
model's default voice. Every other name is forwarded verbatim.
|
||||
"""
|
||||
if isinstance(voice, str) and voice.lower() in OPENAI_STOCK_REALTIME_VOICES:
|
||||
if not isinstance(voice, str):
|
||||
return VertexGeminiConfig().map_audio_params({"voice": voice})
|
||||
if voice.lower() in OPENAI_STOCK_REALTIME_VOICES:
|
||||
verbose_logger.warning(
|
||||
"Gemini Realtime: voice %s is an OpenAI voice with no Gemini equivalent; "
|
||||
"dropping it so the session keeps the model's default voice.",
|
||||
voice,
|
||||
)
|
||||
return None
|
||||
if drop_undocumented and _is_undocumented_live_voice(voice):
|
||||
verbose_logger.warning(
|
||||
"Gemini Realtime: voice %s is not a documented Gemini Live voice; "
|
||||
"drop_params is set, so it is dropped and the session keeps the model's default voice.",
|
||||
voice,
|
||||
)
|
||||
return None
|
||||
return VertexGeminiConfig().map_audio_params({"voice": voice})
|
||||
|
||||
|
||||
|
|
@ -142,16 +198,27 @@ def _base64_decoded_byte_count(data: str) -> int:
|
|||
return max(len(data) * 3 // 4 - padding, 0)
|
||||
|
||||
|
||||
_JSON_ARRAY: Final = TypeAdapter(tuple[object, ...])
|
||||
|
||||
|
||||
def _system_instruction_text(system_instruction: HttpxContentType | None) -> str | None:
|
||||
parts: Final = system_instruction.get("parts", []) if system_instruction else []
|
||||
texts: Final = tuple(text for part in parts if isinstance(text := part.get("text"), str))
|
||||
return "\n".join(texts) if texts else None
|
||||
|
||||
|
||||
class GeminiRealtimeConfig(BaseRealtimeConfig):
|
||||
_TOOL_CALL_ID_TO_NAME_MAX = 256 # LRU cap for call_id→name mapping
|
||||
|
||||
def __init__(self):
|
||||
def __init__(self, drop_params: bool = False):
|
||||
super().__init__()
|
||||
self.drop_params = drop_params
|
||||
self._tool_call_id_to_name: OrderedDict[str, str] = OrderedDict()
|
||||
# Gemini Live sometimes emits usageMetadata in a standalone frame between
|
||||
# turns; buffer it here so the next response.done carries the token counts.
|
||||
self._pending_usage_metadata: dict | None = None
|
||||
self._unbilled_input_audio_bytes: int = 0
|
||||
self._manual_activity_open: bool = False
|
||||
|
||||
def is_setup_message(self, msg_obj: dict) -> bool:
|
||||
return "setup" in msg_obj
|
||||
|
|
@ -167,6 +234,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
"""Google AI Studio closes the socket (1007) on padding or silence tuning next to ``disabled: true``."""
|
||||
return True
|
||||
|
||||
def _rejects_undocumented_voices(self) -> bool:
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _usage_detail_alias(details: object, defaults: dict[str, int]) -> dict[str, object]:
|
||||
detail_mapping: Final = as_str_mapping(details)
|
||||
|
|
@ -272,17 +342,26 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
except (json.JSONDecodeError, TypeError, AttributeError):
|
||||
return False
|
||||
|
||||
def _start_manual_activity(self, session_configuration_request: str | None) -> tuple[str, ...]:
|
||||
if self._manual_activity_open or not self._manual_turn_detection_enabled(session_configuration_request):
|
||||
return ()
|
||||
self._manual_activity_open = True
|
||||
activity_start: Final[BidiGenerateContentRealtimeInput] = {"activityStart": {}}
|
||||
return (json.dumps({"realtimeInput": activity_start}),)
|
||||
|
||||
def _end_manual_activity(self) -> tuple[str, ...]:
|
||||
if not self._manual_activity_open:
|
||||
return ()
|
||||
self._manual_activity_open = False
|
||||
activity_end: Final[BidiGenerateContentRealtimeInput] = {"activityEnd": {}}
|
||||
return (json.dumps({"realtimeInput": activity_end}),)
|
||||
|
||||
def _handle_input_audio_buffer_commit_or_end(self, session_configuration_request: str | None) -> list[str]:
|
||||
"""Map OpenAI buffer commit/end to Gemini Live turn-boundary signals."""
|
||||
if self._manual_turn_detection_enabled(session_configuration_request):
|
||||
realtime_input_dict: BidiGenerateContentRealtimeInput = {
|
||||
"activityEnd": True,
|
||||
}
|
||||
verbose_logger.debug("Gemini Realtime: Sending activityEnd realtimeInput to backend")
|
||||
else:
|
||||
realtime_input_dict = {"audioStreamEnd": True}
|
||||
verbose_logger.debug("Gemini Realtime: Sending audioStreamEnd realtimeInput to backend")
|
||||
return [json.dumps({"realtimeInput": realtime_input_dict})]
|
||||
return list(self._end_manual_activity())
|
||||
audio_stream_end: Final[BidiGenerateContentRealtimeInput] = {"audioStreamEnd": True}
|
||||
return [json.dumps({"realtimeInput": audio_stream_end})]
|
||||
|
||||
def map_automatic_turn_detection(self, value: OpenAIRealtimeTurnDetection) -> AutomaticActivityDetection:
|
||||
"""Map OpenAI ``server_vad`` to Gemini ``automaticActivityDetection``.
|
||||
|
|
@ -333,12 +412,12 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
optional_params["systemInstruction"] = HttpxContentType(role="user", parts=[{"text": value}])
|
||||
elif key == "temperature":
|
||||
optional_params["generationConfig"]["temperature"] = value
|
||||
elif key == "max_response_output_tokens":
|
||||
elif key == "max_response_output_tokens" and isinstance(value, int):
|
||||
optional_params["generationConfig"]["maxOutputTokens"] = value
|
||||
elif key == "modalities":
|
||||
optional_params["generationConfig"]["responseModalities"] = [
|
||||
modality.upper() for modality in cast(list[str], value)
|
||||
]
|
||||
optional_params["generationConfig"]["responseModalities"] = list(
|
||||
GeminiRealtimeConfig._requested_response_modalities(cast(object, value))
|
||||
)
|
||||
elif key == "tools":
|
||||
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
|
||||
VertexGeminiConfig,
|
||||
|
|
@ -351,30 +430,38 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
)
|
||||
elif key == "input_audio_transcription" and value is not None:
|
||||
optional_params["inputAudioTranscription"] = {}
|
||||
elif key == "turn_detection" and value is not None:
|
||||
value_typed = cast(OpenAIRealtimeTurnDetection, value)
|
||||
if (
|
||||
isinstance(value_typed, dict)
|
||||
and value_typed.get("type") == "semantic_vad"
|
||||
and "create_response" not in value_typed
|
||||
):
|
||||
# Pipecat/OpenAI GA semantic VAD — skip; Gemini uses its own VAD.
|
||||
# Only skip when there is no create_response override so that
|
||||
# a guardrail-injected create_response:false is not dropped.
|
||||
continue
|
||||
transformed_audio_activity_config = self.map_automatic_turn_detection(value_typed)
|
||||
if transformed_audio_activity_config:
|
||||
optional_params["realtimeInputConfig"] = BidiGenerateContentRealtimeInputConfig(
|
||||
automaticActivityDetection=transformed_audio_activity_config
|
||||
)
|
||||
elif key == "turn_detection":
|
||||
realtime_input_config = self._realtime_input_config(cast(object, value))
|
||||
if realtime_input_config is not None:
|
||||
optional_params["realtimeInputConfig"] = realtime_input_config
|
||||
elif key == "voice":
|
||||
speech_config = _gemini_live_speech_config(value)
|
||||
speech_config = _gemini_live_speech_config(
|
||||
value, self.drop_params and self._rejects_undocumented_voices()
|
||||
)
|
||||
if speech_config:
|
||||
optional_params["generationConfig"]["speechConfig"] = speech_config
|
||||
if len(optional_params["generationConfig"]) == 0:
|
||||
optional_params.pop("generationConfig")
|
||||
return optional_params
|
||||
|
||||
def _realtime_input_config(self, turn_detection: object) -> BidiGenerateContentRealtimeInputConfig | None:
|
||||
if turn_detection is None:
|
||||
return BidiGenerateContentRealtimeInputConfig(automaticActivityDetection={"disabled": True})
|
||||
value_typed: Final = cast(OpenAIRealtimeTurnDetection, turn_detection)
|
||||
if (
|
||||
isinstance(value_typed, dict)
|
||||
and value_typed.get("type") == "semantic_vad"
|
||||
and "create_response" not in value_typed
|
||||
):
|
||||
# Pipecat/OpenAI GA semantic VAD — skip; Gemini uses its own VAD.
|
||||
# Only skip when there is no create_response override so that
|
||||
# a guardrail-injected create_response:false is not dropped.
|
||||
return None
|
||||
transformed_audio_activity_config: Final = self.map_automatic_turn_detection(value_typed)
|
||||
if not transformed_audio_activity_config:
|
||||
return None
|
||||
return BidiGenerateContentRealtimeInputConfig(automaticActivityDetection=transformed_audio_activity_config)
|
||||
|
||||
@staticmethod
|
||||
def _extract_turn_detection(session: dict) -> dict | None:
|
||||
"""Extract turn_detection from a session.update payload.
|
||||
|
|
@ -423,6 +510,8 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
if isinstance(input_cfg, dict):
|
||||
if "input_audio_transcription" not in normalized and "transcription" in input_cfg:
|
||||
normalized["input_audio_transcription"] = input_cfg["transcription"]
|
||||
if "turn_detection" not in normalized and "turn_detection" in input_cfg:
|
||||
normalized["turn_detection"] = input_cfg["turn_detection"]
|
||||
output_cfg: Final = audio.get("output")
|
||||
if isinstance(output_cfg, dict) and output_cfg.get("voice"):
|
||||
normalized["voice"] = output_cfg["voice"]
|
||||
|
|
@ -455,18 +544,122 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
return "TEXT" if GeminiRealtimeConfig._is_text_only_live_model(model) else "AUDIO"
|
||||
|
||||
@staticmethod
|
||||
def _coerce_response_modalities(model: str, modalities: Sequence[object]) -> tuple[str, ...]:
|
||||
"""Swap responseModalities a Live model cannot produce: TEXT to AUDIO for
|
||||
audio-only models, AUDIO to TEXT for text-only ones (e.g. transcribe-live)."""
|
||||
normalized: Final = tuple(
|
||||
modality.upper() if isinstance(modality, str) else str(modality).upper() for modality in modalities
|
||||
def _requested_response_modalities(value: object) -> tuple[str, ...]:
|
||||
if isinstance(value, str):
|
||||
return (value.upper(),)
|
||||
if not isinstance(value, (list, tuple)):
|
||||
return ()
|
||||
return tuple(str(modality).upper() for modality in _JSON_ARRAY.validate_python(value))
|
||||
|
||||
@staticmethod
|
||||
def _unsupported_response_modalities(model: str, requested: tuple[str, ...]) -> tuple[str, ...]:
|
||||
unknown: Final = tuple(modality for modality in requested if modality not in _GEMINI_LIVE_RESPONSE_MODALITIES)
|
||||
if unknown:
|
||||
return unknown
|
||||
if GeminiRealtimeConfig._is_audio_only_live_model(model) and frozenset(requested) == frozenset(("TEXT",)):
|
||||
return ("TEXT",)
|
||||
if GeminiRealtimeConfig._is_text_only_live_model(model) and "AUDIO" in requested:
|
||||
return ("AUDIO",)
|
||||
return ()
|
||||
|
||||
def setup_session_update_rejection(
|
||||
self, model: str, session: Mapping[str, object], *, setup_sent: bool
|
||||
) -> RealtimeSessionUpdateRejection | None:
|
||||
if self.drop_params:
|
||||
return None
|
||||
if setup_sent:
|
||||
return RealtimeSessionUpdateRejection(
|
||||
code="session_update_not_applied",
|
||||
message=(
|
||||
"Gemini Live accepts one session configuration per connection, and this session already sent "
|
||||
"its configuration, so this session.update was not applied. Put every setting in session.update "
|
||||
"before the first conversation content, or set drop_params to drop it instead"
|
||||
),
|
||||
param="session",
|
||||
)
|
||||
return self._modality_rejection(model, session) or self._voice_rejection(model, session)
|
||||
|
||||
@staticmethod
|
||||
def _modality_rejection(model: str, session: Mapping[str, object]) -> RealtimeSessionUpdateRejection | None:
|
||||
param: Final = "output_modalities" if "output_modalities" in session else "modalities"
|
||||
unsupported: Final = GeminiRealtimeConfig._unsupported_response_modalities(
|
||||
model, GeminiRealtimeConfig._requested_response_modalities(session.get(param))
|
||||
)
|
||||
if not unsupported:
|
||||
return None
|
||||
return RealtimeSessionUpdateRejection(
|
||||
code="invalid_value",
|
||||
message=(
|
||||
f"Invalid value for 'session.{param}': {model} cannot produce "
|
||||
f"{', '.join(modality.lower() for modality in unsupported)} output. Set drop_params to drop it instead"
|
||||
),
|
||||
param=f"session.{param}",
|
||||
)
|
||||
|
||||
def _voice_rejection(self, model: str, session: Mapping[str, object]) -> RealtimeSessionUpdateRejection | None:
|
||||
audio_output: Final = as_str_mapping((as_str_mapping(session.get("audio")) or {}).get("output")) or {}
|
||||
output_voice: Final = audio_output.get("voice")
|
||||
voice: Final = output_voice or session.get("voice")
|
||||
if not self._rejects_undocumented_voices() or not _is_undocumented_live_voice(voice):
|
||||
return None
|
||||
param: Final = "session.audio.output.voice" if output_voice else "session.voice"
|
||||
return RealtimeSessionUpdateRejection(
|
||||
code="invalid_value",
|
||||
message=(
|
||||
f"Invalid value for '{param}': {voice} is not a Gemini Live voice {model} offers. "
|
||||
"Set drop_params to drop it instead"
|
||||
),
|
||||
param=param,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _coerce_response_modalities(model: str, modalities: Sequence[object]) -> tuple[str, ...]:
|
||||
"""Drop modalities Gemini Live cannot produce at all, swap the ones this
|
||||
model cannot (TEXT to AUDIO for audio-only models, AUDIO to TEXT for
|
||||
text-only ones such as transcribe-live), then keep a single modality
|
||||
because Gemini Live rejects a setup naming two."""
|
||||
return GeminiRealtimeConfig._single_response_modality(
|
||||
GeminiRealtimeConfig._swap_unsupported_response_modalities(
|
||||
model, GeminiRealtimeConfig._known_response_modalities(model, modalities)
|
||||
)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _known_response_modalities(model: str, modalities: Sequence[object]) -> tuple[str, ...]:
|
||||
normalized: Final = tuple(str(modality).upper() for modality in modalities)
|
||||
known: Final = tuple(modality for modality in normalized if modality in _GEMINI_LIVE_RESPONSE_MODALITIES)
|
||||
if len(known) < len(normalized):
|
||||
verbose_logger.warning(
|
||||
"Gemini Live: %s only produces TEXT or AUDIO; ignoring the requested %s",
|
||||
model,
|
||||
", ".join(modality for modality in normalized if modality not in _GEMINI_LIVE_RESPONSE_MODALITIES),
|
||||
)
|
||||
return known or (GeminiRealtimeConfig._default_response_modality(model),)
|
||||
|
||||
@staticmethod
|
||||
def _swap_unsupported_response_modalities(model: str, normalized: tuple[str, ...]) -> tuple[str, ...]:
|
||||
if GeminiRealtimeConfig._is_audio_only_live_model(model) and "TEXT" in normalized:
|
||||
verbose_logger.warning(
|
||||
"Gemini Live: %s only produces audio; downgrading the requested TEXT response modality to AUDIO",
|
||||
model,
|
||||
)
|
||||
return tuple(modality for modality in normalized if modality != "TEXT") or ("AUDIO",)
|
||||
if GeminiRealtimeConfig._is_text_only_live_model(model) and "AUDIO" in normalized:
|
||||
return tuple(modality for modality in normalized if modality != "AUDIO") or ("TEXT",)
|
||||
return normalized
|
||||
|
||||
@staticmethod
|
||||
def _single_response_modality(modalities: tuple[str, ...]) -> tuple[str, ...]:
|
||||
if len(modalities) <= 1:
|
||||
return modalities
|
||||
chosen: Final = "AUDIO" if "AUDIO" in modalities else modalities[0]
|
||||
verbose_logger.warning(
|
||||
"Gemini Live: at most one response modality is allowed per session; using %s for the requested %s",
|
||||
chosen,
|
||||
", ".join(modalities),
|
||||
)
|
||||
return (chosen,)
|
||||
|
||||
@staticmethod
|
||||
def _finalize_gemini_live_setup(model: str, setup: dict[str, object]) -> dict[str, object]:
|
||||
generation_config: Final = setup.get("generationConfig")
|
||||
|
|
@ -618,14 +811,13 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
message_str = str(message)
|
||||
raise ValueError(f"Invalid JSON message: {message_str}")
|
||||
|
||||
messages: Final[list[str]] = []
|
||||
msg_type: Final = json_message.get("type")
|
||||
|
||||
if msg_type == "session.update":
|
||||
return self._handle_session_update(json_message, model, session_configuration_request)
|
||||
|
||||
if msg_type == "response.create":
|
||||
return [] # Gemini responds automatically; nothing to forward
|
||||
return list(self._end_manual_activity())
|
||||
|
||||
if msg_type == "conversation.item.create":
|
||||
return self._handle_conversation_item(json_message)
|
||||
|
|
@ -643,8 +835,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
|
||||
gemini_msg: Final = json.dumps({"realtimeInput": realtime_input_dict})
|
||||
verbose_logger.debug("Gemini Realtime: Sending audio realtimeInput to backend")
|
||||
messages.append(gemini_msg)
|
||||
return messages
|
||||
return [*self._start_manual_activity(session_configuration_request), gemini_msg]
|
||||
|
||||
if msg_type in ("input_audio_buffer.commit", "input_audio_buffer.end"):
|
||||
return self._handle_input_audio_buffer_commit_or_end(session_configuration_request)
|
||||
|
|
@ -660,22 +851,21 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
logging_session_id: str,
|
||||
session_configuration_request: str | None = None,
|
||||
) -> OpenAIRealtimeStreamSessionEvents:
|
||||
if session_configuration_request:
|
||||
session_configuration_request_dict: BidiGenerateContentSetup = _parse_setup(session_configuration_request)
|
||||
else:
|
||||
session_configuration_request_dict = {}
|
||||
session_configuration_request_dict: Final = _parse_setup(
|
||||
session_configuration_request or self.session_configuration_request(model)
|
||||
)
|
||||
|
||||
_model: Final = session_configuration_request_dict.get("model") or model
|
||||
generation_config: Final = session_configuration_request_dict.get("generationConfig", {}) or {}
|
||||
gemini_modalities: Final = generation_config.get("responseModalities", ["AUDIO"])
|
||||
_modalities: Final = [modality.lower() for modality in cast(list[str], gemini_modalities)]
|
||||
_system_instruction: Final = session_configuration_request_dict.get("systemInstruction")
|
||||
instructions: Final = _system_instruction_text(session_configuration_request_dict.get("systemInstruction"))
|
||||
session: Final = OpenAIRealtimeStreamSession(
|
||||
id=logging_session_id,
|
||||
modalities=_modalities,
|
||||
)
|
||||
if _system_instruction is not None and isinstance(_system_instruction, str):
|
||||
session["instructions"] = _system_instruction
|
||||
if instructions is not None:
|
||||
session["instructions"] = instructions
|
||||
if _model is not None and isinstance(_model, str):
|
||||
# Strip Vertex/AI Studio path prefixes to expose the bare model name.
|
||||
if "/models/" in _model:
|
||||
|
|
@ -1662,6 +1852,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
# Deferred setup opt-in: litellm.gemini_live_defer_setup = True
|
||||
return not litellm.gemini_live_defer_setup
|
||||
|
||||
def builds_setup_from_session_update(self) -> bool:
|
||||
return True
|
||||
|
||||
def session_configuration_request(self, model: str) -> str:
|
||||
"""
|
||||
|
||||
|
|
|
|||
|
|
@ -39,8 +39,8 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig):
|
|||
(they require async I/O) and injected at construction time.
|
||||
"""
|
||||
|
||||
def __init__(self, access_token: str, project: str, location: str) -> None:
|
||||
super().__init__()
|
||||
def __init__(self, access_token: str, project: str, location: str, drop_params: bool = False) -> None:
|
||||
super().__init__(drop_params=drop_params)
|
||||
self._access_token = access_token
|
||||
self._project = project
|
||||
self._location = location
|
||||
|
|
@ -51,6 +51,9 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig):
|
|||
def _rejects_vad_tuning_with_detection_off(self) -> bool:
|
||||
return False
|
||||
|
||||
def _rejects_undocumented_voices(self) -> bool:
|
||||
return True
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# URL
|
||||
# ------------------------------------------------------------------
|
||||
|
|
@ -177,15 +180,16 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig):
|
|||
# ``silence_duration_ms``). ``map_automatic_turn_detection`` sets
|
||||
# ``disabled=True`` whenever ``create_response`` is absent or
|
||||
# ``False``. Force ``disabled=False`` only when the client did
|
||||
# not explicitly request ``create_response: False`` — that path
|
||||
# not explicitly request ``create_response: False`` or
|
||||
# ``turn_detection: null`` — the first of those paths
|
||||
# is how transcription guardrails suppress automatic responses,
|
||||
# and overriding it here would silently bypass the guardrail.
|
||||
# Vertex Live has no "VAD on, no auto-response" mode, so callers
|
||||
# that need that behaviour must accept that VAD is off.
|
||||
client_turn_detection: Final = session_params.get("turn_detection")
|
||||
client_disabled_auto_response: Final = (
|
||||
isinstance(client_turn_detection, dict) and client_turn_detection.get("create_response") is False
|
||||
)
|
||||
"turn_detection" in session_params and client_turn_detection is None
|
||||
) or (isinstance(client_turn_detection, dict) and client_turn_detection.get("create_response") is False)
|
||||
realtime_input_config: Final = setup_config.setdefault("realtimeInputConfig", {})
|
||||
automatic_detection: Final = realtime_input_config.setdefault("automaticActivityDetection", {})
|
||||
if not client_disabled_auto_response:
|
||||
|
|
@ -261,4 +265,5 @@ def vertex_realtime_config(
|
|||
access_token=access_token,
|
||||
project=project,
|
||||
location=VertexBase.get_vertex_region(vertex_region=location, model=model),
|
||||
drop_params=drop_params,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -81,6 +81,14 @@ class BidiGenerateContentServerMessage(TypedDict, total=False):
|
|||
"""Output only. The setup complete message."""
|
||||
|
||||
|
||||
class BidiGenerateContentActivityStart(TypedDict):
|
||||
pass
|
||||
|
||||
|
||||
class BidiGenerateContentActivityEnd(TypedDict):
|
||||
pass
|
||||
|
||||
|
||||
class BidiGenerateContentRealtimeInput(TypedDict, total=False):
|
||||
text: str
|
||||
"""The text to be sent to the model."""
|
||||
|
|
@ -94,11 +102,11 @@ class BidiGenerateContentRealtimeInput(TypedDict, total=False):
|
|||
audioStreamEnd: bool
|
||||
"""Output only. If true, indicates that the audio stream has ended."""
|
||||
|
||||
activityStart: bool
|
||||
"""Output only. If true, indicates that the activity has started."""
|
||||
activityStart: BidiGenerateContentActivityStart
|
||||
"""Marks the start of user activity when automatic activity detection is disabled."""
|
||||
|
||||
activityEnd: bool
|
||||
"""Output only. If true, indicates that the activity has ended."""
|
||||
activityEnd: BidiGenerateContentActivityEnd
|
||||
"""Marks the end of user activity when automatic activity detection is disabled."""
|
||||
|
||||
|
||||
StartOfSpeechSensitivityEnum = Literal[
|
||||
|
|
|
|||
|
|
@ -10561,7 +10561,7 @@ class ProviderConfigManager:
|
|||
if LlmProviders.GEMINI == provider:
|
||||
from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig
|
||||
|
||||
return GeminiRealtimeConfig()
|
||||
return GeminiRealtimeConfig(drop_params=drop_params)
|
||||
if LlmProviders.META == provider:
|
||||
from litellm.llms.meta.realtime.transformation import MetaRealtimeConfig
|
||||
|
||||
|
|
|
|||
|
|
@ -84,6 +84,7 @@ IGNORE_FUNCTIONS = [
|
|||
"_sort_processed_sets", # bounded by the nesting depth of the log-record extra it walks (a finite JSON tree, no cycles possible).
|
||||
"scrub_json_strings", # max depth set (MAX_SCRUB_DEPTH); fails closed by returning "[Filtered]" for anything nested past the cap.
|
||||
"_without_requester_ip_address", # walks only the fresh tree _sanitize_request_body_for_spend_logs_payload returns (mapping cycles already cut), so it never goes deeper than that sanitizer did.
|
||||
"_merge_session_fields", # max depth set (_SESSION_MERGE_MAX_DEPTH); deeper mappings are replaced wholesale by the later session.update.
|
||||
]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
from collections.abc import Coroutine, Mapping, Sequence
|
||||
import time
|
||||
from collections.abc import Callable, Coroutine, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Final, Literal
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
|
@ -21,6 +22,7 @@ from litellm.constants import (
|
|||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.litellm_core_utils.realtime_streaming import (
|
||||
RealTimeStreaming,
|
||||
_merge_session_fields,
|
||||
client_sent_openai_beta_realtime_header,
|
||||
)
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
|
|
@ -1256,11 +1258,14 @@ async def test_client_ack_caches_setup_to_prevent_duplicate_session_update_setup
|
|||
side_effect=[
|
||||
json.dumps({"type": "session.update", "session": {"tools": []}}),
|
||||
json.dumps({"type": "session.update", "session": {"tools": []}}),
|
||||
_TEXT_ITEM_CREATE,
|
||||
Exception("client done"),
|
||||
]
|
||||
)
|
||||
websocket.send_text = AsyncMock()
|
||||
|
||||
provider_config = MagicMock()
|
||||
provider_config.setup_session_update_rejection = MagicMock(return_value=None)
|
||||
|
||||
def _transform(message: str, model: str, session_configuration_request=None):
|
||||
if session_configuration_request is None:
|
||||
|
|
@ -2286,6 +2291,7 @@ async def test_guardrail_turn_detection_injected_into_first_session_update_defer
|
|||
logging_obj.success_handler = MagicMock()
|
||||
|
||||
provider_config = MagicMock()
|
||||
provider_config.builds_setup_from_session_update = MagicMock(return_value=False)
|
||||
transformed_messages = []
|
||||
|
||||
def mock_transform(msg, model, session_config):
|
||||
|
|
@ -2352,6 +2358,7 @@ async def test_guardrail_turn_detection_injection_tolerates_non_dict_value(
|
|||
logging_obj.success_handler = MagicMock()
|
||||
|
||||
provider_config = MagicMock()
|
||||
provider_config.builds_setup_from_session_update = MagicMock(return_value=False)
|
||||
transformed_messages = []
|
||||
|
||||
def mock_transform(msg, model, session_config):
|
||||
|
|
@ -2414,6 +2421,7 @@ async def test_subsequent_session_update_cannot_reenable_vad_when_guardrails_act
|
|||
logging_obj.success_handler = MagicMock()
|
||||
|
||||
provider_config = MagicMock()
|
||||
provider_config.setup_session_update_rejection = MagicMock(return_value=None)
|
||||
transformed_messages = []
|
||||
|
||||
def mock_transform(msg, model, session_config):
|
||||
|
|
@ -2466,6 +2474,7 @@ async def test_follow_up_setup_updates_cached_session_configuration_request():
|
|||
logging_obj.success_handler = MagicMock()
|
||||
|
||||
provider_config = MagicMock()
|
||||
provider_config.setup_session_update_rejection = MagicMock(return_value=None)
|
||||
follow_up_setup = json.dumps(
|
||||
{
|
||||
"setup": {
|
||||
|
|
@ -2499,6 +2508,559 @@ async def test_follow_up_setup_updates_cached_session_configuration_request():
|
|||
assert streaming.session_configuration_request == follow_up_setup
|
||||
|
||||
|
||||
def _gemini_default_mode_streaming(
|
||||
monkeypatch, client_messages, model="gemini-live-2.5-flash-native-audio", drop_params=False, defer_setup=False
|
||||
):
|
||||
monkeypatch.setattr(litellm, "gemini_live_defer_setup", defer_setup, raising=False)
|
||||
from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig
|
||||
|
||||
client_ws = MagicMock()
|
||||
client_ws.receive_text = AsyncMock(side_effect=[*client_messages, ConnectionClosed(None, None)])
|
||||
client_ws.send_text = AsyncMock()
|
||||
backend_ws = MagicMock()
|
||||
backend_ws.send = AsyncMock()
|
||||
streaming = RealTimeStreaming(
|
||||
client_ws,
|
||||
backend_ws,
|
||||
MagicMock(),
|
||||
provider_config=GeminiRealtimeConfig(drop_params=drop_params),
|
||||
model=model,
|
||||
)
|
||||
return streaming, backend_ws
|
||||
|
||||
|
||||
def _events_sent_to_client(streaming: RealTimeStreaming) -> list[dict]:
|
||||
return [json.loads(call.args[0]) for call in streaming.websocket.send_text.await_args_list]
|
||||
|
||||
|
||||
_TEXT_ITEM_CREATE = json.dumps(
|
||||
{
|
||||
"type": "conversation.item.create",
|
||||
"item": {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hi"}]},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_default_mode_sends_default_setup_before_first_non_session_client_message(monkeypatch):
|
||||
"""Gemini Live setup is no longer pre-sent at connect, so a client that never
|
||||
sends session.update still gets the provider default setup ahead of its
|
||||
first content frame."""
|
||||
streaming, backend_ws = _gemini_default_mode_streaming(monkeypatch, [_TEXT_ITEM_CREATE])
|
||||
|
||||
await streaming.client_ack_messages()
|
||||
|
||||
frames = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list]
|
||||
assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"]
|
||||
assert frames[0]["setup"]["generationConfig"]["responseModalities"] == ["AUDIO"]
|
||||
assert streaming.session_configuration_request == backend_ws.send.await_args_list[0].args[0]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_default_mode_session_update_first_builds_the_only_setup(monkeypatch):
|
||||
session_update = json.dumps({"type": "session.update", "session": {"modalities": ["text"]}})
|
||||
streaming, backend_ws = _gemini_default_mode_streaming(
|
||||
monkeypatch, [session_update, _TEXT_ITEM_CREATE], model="gemini-live-2.5-flash"
|
||||
)
|
||||
|
||||
await streaming.client_ack_messages()
|
||||
|
||||
frames = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list]
|
||||
assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"]
|
||||
assert frames[0]["setup"]["generationConfig"]["responseModalities"] == ["TEXT"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_default_mode_merges_every_session_update_sent_before_the_first_content_frame(monkeypatch):
|
||||
"""Clients configure a session over several session.update messages. Gemini
|
||||
Live takes one setup, so every update sent before the first content frame is
|
||||
merged into that setup instead of only the first one applying."""
|
||||
instructions_update = json.dumps({"type": "session.update", "session": {"instructions": "Be brief."}})
|
||||
modalities_update = json.dumps({"type": "session.update", "session": {"modalities": ["text"]}})
|
||||
streaming, backend_ws = _gemini_default_mode_streaming(
|
||||
monkeypatch, [instructions_update, modalities_update, _TEXT_ITEM_CREATE], model="gemini-live-2.5-flash"
|
||||
)
|
||||
|
||||
await streaming.client_ack_messages()
|
||||
|
||||
frames = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list]
|
||||
assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"]
|
||||
assert frames[0]["setup"]["generationConfig"]["responseModalities"] == ["TEXT"]
|
||||
assert frames[0]["setup"]["systemInstruction"]["parts"][0]["text"] == "Be brief."
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_default_mode_merges_nested_audio_settings_across_session_updates(monkeypatch):
|
||||
vad_update = json.dumps(
|
||||
{"type": "session.update", "session": {"audio": {"input": {"turn_detection": {"silence_duration_ms": 1500}}}}}
|
||||
)
|
||||
voice_update = json.dumps({"type": "session.update", "session": {"audio": {"output": {"voice": "Kore"}}}})
|
||||
streaming, backend_ws = _gemini_default_mode_streaming(
|
||||
monkeypatch, [vad_update, voice_update, _TEXT_ITEM_CREATE], model="gemini-live-2.5-flash"
|
||||
)
|
||||
|
||||
await streaming.client_ack_messages()
|
||||
|
||||
setup = json.loads(backend_ws.send.await_args_list[0].args[0])["setup"]
|
||||
assert setup["realtimeInputConfig"]["automaticActivityDetection"]["silenceDurationMs"] == 1500
|
||||
assert setup["generationConfig"]["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Kore"
|
||||
|
||||
|
||||
def test_merge_session_fields_replaces_nested_mappings_wholesale_past_the_depth_cap():
|
||||
def nested(levels: int, key: str) -> dict:
|
||||
return {key: levels, **({"child": nested(levels - 1, key)} if levels else {})}
|
||||
|
||||
def level(node, depth: int):
|
||||
return node if depth == 0 else level(node["child"], depth - 1)
|
||||
|
||||
merged = _merge_session_fields(nested(10, "keep"), nested(10, "new"))
|
||||
|
||||
assert all({"keep", "new"} <= set(level(merged, depth)) for depth in range(9))
|
||||
assert "keep" not in level(merged, 9)
|
||||
assert level(merged, 9)["new"] == 1
|
||||
|
||||
|
||||
|
||||
def _merge_cost(levels: int, leaves: int) -> float:
|
||||
def nested(depth: int) -> dict[str, object]:
|
||||
return {"child": nested(depth - 1)} if depth else {f"k{index}": index for index in range(leaves)}
|
||||
|
||||
session: Final = nested(levels)
|
||||
|
||||
def once() -> float:
|
||||
started: Final = time.perf_counter()
|
||||
_merge_session_fields(session, session)
|
||||
return time.perf_counter() - started
|
||||
|
||||
return min(once() for _ in range(3))
|
||||
|
||||
|
||||
def test_merge_session_fields_cost_stays_flat_as_matching_updates_nest_deeper():
|
||||
growth: Final = _merge_cost(7, 2000) / _merge_cost(1, 2000)
|
||||
|
||||
assert growth < 8.0, f"merging seven nested levels cost {growth:.1f}x one level"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_default_mode_acknowledges_each_held_session_update_before_any_backend_frame(monkeypatch):
|
||||
"""A client that waits for session.updated before sending content must not
|
||||
hang while its updates are held, so each one is acknowledged right away
|
||||
with the modalities the merged setup will carry."""
|
||||
instructions_update = json.dumps({"type": "session.update", "session": {"instructions": "Be brief."}})
|
||||
modalities_update = json.dumps({"type": "session.update", "session": {"modalities": ["text"]}})
|
||||
streaming, backend_ws = _gemini_default_mode_streaming(
|
||||
monkeypatch, [instructions_update, modalities_update], model="gemini-live-2.5-flash"
|
||||
)
|
||||
|
||||
await streaming.client_ack_messages()
|
||||
|
||||
acks = [json.loads(call.args[0]) for call in streaming.websocket.send_text.await_args_list]
|
||||
assert [ack["type"] for ack in acks] == ["session.updated", "session.updated"]
|
||||
assert [ack["session"].get("output_modalities") or ack["session"].get("modalities") for ack in acks] == [
|
||||
["audio"],
|
||||
["text"],
|
||||
]
|
||||
assert acks[1]["session"]["instructions"] == "Be brief."
|
||||
backend_ws.send.assert_not_awaited()
|
||||
|
||||
|
||||
_LATE_INSTRUCTIONS_UPDATE = json.dumps(
|
||||
{"type": "session.update", "event_id": "evt_late", "session": {"instructions": "Be brief."}}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_default_mode_answers_a_session_update_after_the_first_content_with_an_error(monkeypatch):
|
||||
streaming, backend_ws = _gemini_default_mode_streaming(monkeypatch, [_TEXT_ITEM_CREATE, _LATE_INSTRUCTIONS_UPDATE])
|
||||
|
||||
await streaming.client_ack_messages()
|
||||
|
||||
errors: Final = [event["error"] for event in _events_sent_to_client(streaming) if event["type"] == "error"]
|
||||
assert [(error["code"], error["event_id"]) for error in errors] == [("session_update_not_applied", "evt_late")]
|
||||
frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list]
|
||||
assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_default_mode_drops_a_late_session_update_quietly_under_drop_params(monkeypatch):
|
||||
streaming, backend_ws = _gemini_default_mode_streaming(
|
||||
monkeypatch, [_TEXT_ITEM_CREATE, _LATE_INSTRUCTIONS_UPDATE], drop_params=True
|
||||
)
|
||||
|
||||
await streaming.client_ack_messages()
|
||||
|
||||
assert _events_sent_to_client(streaming) == []
|
||||
assert backend_ws.send.await_count == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("model", "requested_output"),
|
||||
[
|
||||
("gemini-live-2.5-flash-native-audio", {"modalities": ["text"]}),
|
||||
("gemini-live-2.5-flash", {"output_modalities": ["image"]}),
|
||||
],
|
||||
)
|
||||
async def test_default_mode_rejects_an_output_modality_the_model_cannot_produce(monkeypatch, model, requested_output):
|
||||
update: Final = json.dumps(
|
||||
{"type": "session.update", "session": {**requested_output, "instructions": "Be brief."}}
|
||||
)
|
||||
streaming, backend_ws = _gemini_default_mode_streaming(monkeypatch, [update, _TEXT_ITEM_CREATE], model=model)
|
||||
|
||||
await streaming.client_ack_messages()
|
||||
|
||||
events: Final = _events_sent_to_client(streaming)
|
||||
assert [(event["type"], event["error"]["code"]) for event in events] == [("error", "invalid_value")]
|
||||
setup: Final = json.loads(backend_ws.send.await_args_list[0].args[0])["setup"]
|
||||
assert "systemInstruction" not in setup
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("modalities", "drop_params"),
|
||||
[(["text", "audio"], False), (["text"], True)],
|
||||
ids=["openai-default-pair", "text-dropped-under-drop-params"],
|
||||
)
|
||||
async def test_default_mode_applies_the_update_with_audio_output_on_an_audio_only_model(
|
||||
monkeypatch, modalities, drop_params
|
||||
):
|
||||
update: Final = json.dumps(
|
||||
{"type": "session.update", "session": {"modalities": modalities, "instructions": "Be brief."}}
|
||||
)
|
||||
streaming, backend_ws = _gemini_default_mode_streaming(
|
||||
monkeypatch, [update, _TEXT_ITEM_CREATE], drop_params=drop_params
|
||||
)
|
||||
|
||||
await streaming.client_ack_messages()
|
||||
|
||||
assert [event["type"] for event in _events_sent_to_client(streaming)] == ["session.updated"]
|
||||
setup: Final = json.loads(backend_ws.send.await_args_list[0].args[0])["setup"]
|
||||
assert setup["generationConfig"]["responseModalities"] == ["AUDIO"]
|
||||
assert setup["systemInstruction"]["parts"][0]["text"] == "Be brief."
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_default_mode_sends_no_extra_session_updated_when_gemini_confirms_the_setup(monkeypatch):
|
||||
"""Each held session.update was already acknowledged, so Gemini's setupComplete
|
||||
must not reach the client as one more session.updated nobody asked for."""
|
||||
monkeypatch.setattr(litellm, "gemini_live_defer_setup", False, raising=False)
|
||||
from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig
|
||||
|
||||
model: Final = "gemini-live-2.5-flash-native-audio"
|
||||
config: Final = GeminiRealtimeConfig()
|
||||
client_ws: Final = MagicMock()
|
||||
client_ws.send_text = AsyncMock()
|
||||
backend_ws: Final = MagicMock()
|
||||
backend_ws.send = AsyncMock()
|
||||
backend_ws.recv = AsyncMock(side_effect=[json.dumps({"setupComplete": {}}).encode(), ConnectionClosed(None, None)])
|
||||
logging_obj: Final = _mock_logging()
|
||||
logging_obj.litellm_trace_id = "trace_lazy"
|
||||
logging_obj.async_success_handler = AsyncMock()
|
||||
logging_obj.success_handler = MagicMock()
|
||||
streaming: Final = RealTimeStreaming(client_ws, backend_ws, logging_obj, provider_config=config, model=model)
|
||||
streaming.session_created_sent_to_client = True
|
||||
streaming.session_configuration_request = config.session_configuration_request(model)
|
||||
|
||||
await streaming.backend_to_client_send_messages()
|
||||
|
||||
assert [json.loads(call.args[0])["type"] for call in client_ws.send_text.await_args_list] == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_default_mode_folds_transcription_guardrail_vad_disable_into_the_merged_setup(monkeypatch):
|
||||
"""A transcription guardrail gates every turn, so its auto-response disable
|
||||
has to land in the one setup built from the held session.updates, injected
|
||||
once rather than into every update that gets merged."""
|
||||
monkeypatch.setattr(litellm, "callbacks", [_transcription_guardrail()])
|
||||
debug = MagicMock()
|
||||
monkeypatch.setattr("litellm.litellm_core_utils.realtime_streaming.verbose_logger.debug", debug)
|
||||
instructions_update = json.dumps({"type": "session.update", "session": {"instructions": "Be brief."}})
|
||||
vad_update = json.dumps(
|
||||
{"type": "session.update", "session": {"turn_detection": {"type": "server_vad", "silence_duration_ms": 900}}}
|
||||
)
|
||||
streaming, backend_ws = _gemini_default_mode_streaming(
|
||||
monkeypatch, [instructions_update, vad_update, _TEXT_ITEM_CREATE], model="gemini-live-2.5-flash"
|
||||
)
|
||||
|
||||
await streaming.client_ack_messages()
|
||||
|
||||
frames = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list]
|
||||
assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"]
|
||||
assert frames[0]["setup"]["realtimeInputConfig"]["automaticActivityDetection"] == {"disabled": True}
|
||||
assert frames[0]["setup"]["systemInstruction"]["parts"][0]["text"] == "Be brief."
|
||||
assert streaming._guardrail_turn_detection_update_sent is True
|
||||
injected = [call for call in debug.call_args_list if str(call.args[0]).startswith("Injected turn_detection")]
|
||||
assert len(injected) == 1
|
||||
|
||||
|
||||
def _blocking_text_guardrail() -> CustomGuardrail:
|
||||
class _BlockingTextGuardrail(CustomGuardrail):
|
||||
async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None):
|
||||
if any("zebrafruit" in text for text in inputs.get("texts", [])):
|
||||
raise ValueError("Content blocked")
|
||||
return inputs
|
||||
|
||||
return _BlockingTextGuardrail(
|
||||
guardrail_name="test_text_guard", event_hook=GuardrailEventHooks.pre_call, default_on=True
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_default_mode_sends_the_setup_before_a_guardrail_blocks_the_first_message(monkeypatch):
|
||||
"""Gemini Live closes the socket when content arrives before the setup, so a
|
||||
guardrail that blocks the first typed message must not reach the backend first."""
|
||||
monkeypatch.setattr(litellm, "callbacks", [_blocking_text_guardrail()])
|
||||
instructions_update = json.dumps({"type": "session.update", "session": {"instructions": "Be brief."}})
|
||||
blocked_item = json.dumps(
|
||||
{
|
||||
"type": "conversation.item.create",
|
||||
"item": {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "zebrafruit"}]},
|
||||
}
|
||||
)
|
||||
streaming, backend_ws = _gemini_default_mode_streaming(
|
||||
monkeypatch, [instructions_update, blocked_item], model="gemini-live-2.5-flash"
|
||||
)
|
||||
|
||||
await streaming.client_ack_messages()
|
||||
|
||||
frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list]
|
||||
assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"]
|
||||
assert frames[0]["setup"]["systemInstruction"]["parts"][0]["text"] == "Be brief."
|
||||
assert "zebrafruit" not in json.dumps(frames[1])
|
||||
errors: Final = [event["error"]["type"] for event in _events_sent_to_client(streaming) if event["type"] == "error"]
|
||||
assert errors == ["guardrail_violation"]
|
||||
|
||||
|
||||
def _client_ws_receiving(*steps: Callable[[], Coroutine[object, object, str]]) -> MagicMock:
|
||||
remaining: Final = iter(steps)
|
||||
|
||||
async def receive_text() -> str:
|
||||
return await next(remaining)()
|
||||
|
||||
client_ws: Final = MagicMock()
|
||||
client_ws.receive_text = receive_text
|
||||
client_ws.send_text = AsyncMock()
|
||||
return client_ws
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_default_mode_sends_the_held_setup_when_no_content_arrives_before_the_deadline(monkeypatch):
|
||||
"""Vertex AI refuses a setup sent minutes after connecting, so an idle session
|
||||
sends what it holds at the deadline and answers any later session.update as late."""
|
||||
monkeypatch.setattr(litellm, "gemini_live_defer_setup", False, raising=False)
|
||||
from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig
|
||||
|
||||
setup_sent: Final = asyncio.Event()
|
||||
|
||||
async def instructions_update() -> str:
|
||||
return json.dumps({"type": "session.update", "session": {"instructions": "Be brief."}})
|
||||
|
||||
async def late_update_once_the_setup_is_out() -> str:
|
||||
await setup_sent.wait()
|
||||
return json.dumps(
|
||||
{"type": "session.update", "event_id": "evt_late", "session": {"instructions": "Answer in French."}}
|
||||
)
|
||||
|
||||
async def disconnect() -> str:
|
||||
raise ConnectionClosed(None, None)
|
||||
|
||||
backend_ws: Final = MagicMock()
|
||||
backend_ws.send = AsyncMock(side_effect=lambda frame: setup_sent.set())
|
||||
streaming: Final = RealTimeStreaming(
|
||||
_client_ws_receiving(instructions_update, late_update_once_the_setup_is_out, disconnect),
|
||||
backend_ws,
|
||||
MagicMock(),
|
||||
provider_config=GeminiRealtimeConfig(),
|
||||
model="gemini-live-2.5-flash",
|
||||
lazy_setup_deadline_seconds=0.05,
|
||||
)
|
||||
|
||||
await asyncio.wait_for(streaming.client_ack_messages(), timeout=5)
|
||||
|
||||
frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list]
|
||||
assert [next(iter(frame)) for frame in frames] == ["setup"]
|
||||
assert frames[0]["setup"]["systemInstruction"]["parts"][0]["text"] == "Be brief."
|
||||
events: Final = _events_sent_to_client(streaming)
|
||||
assert [(event["type"], event.get("error", {}).get("code")) for event in events] == [
|
||||
("session.updated", None),
|
||||
("error", "session_update_not_applied"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_default_mode_refuses_held_session_updates_past_the_size_cap(monkeypatch):
|
||||
def instructions_update(text: str) -> str:
|
||||
return json.dumps({"type": "session.update", "event_id": f"evt_{text[0]}", "session": {"instructions": text}})
|
||||
|
||||
half_the_cap: Final = RealTimeStreaming._MAX_BUFFERED_BYTES // 2
|
||||
streaming, backend_ws = _gemini_default_mode_streaming(
|
||||
monkeypatch,
|
||||
[instructions_update("a" * half_the_cap), instructions_update("b" * half_the_cap), _TEXT_ITEM_CREATE],
|
||||
model="gemini-live-2.5-flash",
|
||||
)
|
||||
|
||||
await streaming.client_ack_messages()
|
||||
|
||||
events: Final = _events_sent_to_client(streaming)
|
||||
assert [(event["type"], event.get("error", {}).get("event_id")) for event in events] == [
|
||||
("session.updated", None),
|
||||
("error", "evt_b"),
|
||||
]
|
||||
assert events[1]["error"]["code"] == "session_update_too_large"
|
||||
setup: Final = json.loads(backend_ws.send.await_args_list[0].args[0])["setup"]
|
||||
assert setup["systemInstruction"]["parts"][0]["text"].startswith("a")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_default_mode_acknowledges_an_unchanged_session_update_after_content(monkeypatch):
|
||||
"""Pipecat resends its unchanged session.update right after its first
|
||||
conversation items and treats any error event as fatal."""
|
||||
session_update: Final = json.dumps(
|
||||
{"type": "session.update", "session": {"modalities": ["text"], "instructions": "Be brief."}}
|
||||
)
|
||||
streaming, backend_ws = _gemini_default_mode_streaming(
|
||||
monkeypatch, [session_update, _TEXT_ITEM_CREATE, session_update], model="gemini-live-2.5-flash"
|
||||
)
|
||||
|
||||
await streaming.client_ack_messages()
|
||||
|
||||
frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list]
|
||||
assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"]
|
||||
events: Final = _events_sent_to_client(streaming)
|
||||
assert [event["type"] for event in events] == ["session.updated", "session.updated"]
|
||||
assert events[0]["session"] == events[1]["session"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_deferred_setup_merges_session_updates_until_the_first_content(monkeypatch):
|
||||
"""Deferred setup merges every session.update sent before the first content,
|
||||
the same as the default mode, and answers a changed one after it as late."""
|
||||
monkeypatch.setattr(litellm, "gemini_live_defer_setup", True, raising=False)
|
||||
from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig
|
||||
|
||||
client_ws: Final = MagicMock()
|
||||
client_ws.receive_text = AsyncMock(
|
||||
side_effect=[
|
||||
json.dumps({"type": "session.update", "session": {"modalities": ["text"]}}),
|
||||
json.dumps({"type": "session.update", "session": {"instructions": "Be brief."}}),
|
||||
_TEXT_ITEM_CREATE,
|
||||
json.dumps(
|
||||
{"type": "session.update", "event_id": "evt_late", "session": {"instructions": "Answer in French."}}
|
||||
),
|
||||
ConnectionClosed(None, None),
|
||||
]
|
||||
)
|
||||
client_ws.send_text = AsyncMock()
|
||||
backend_ws: Final = MagicMock()
|
||||
backend_ws.send = AsyncMock()
|
||||
streaming: Final = RealTimeStreaming(
|
||||
client_ws, backend_ws, MagicMock(), provider_config=GeminiRealtimeConfig(), model="gemini-live-2.5-flash"
|
||||
)
|
||||
|
||||
await streaming.client_ack_messages()
|
||||
|
||||
frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list]
|
||||
assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"]
|
||||
assert frames[0]["setup"]["generationConfig"]["responseModalities"] == ["TEXT"]
|
||||
assert frames[0]["setup"]["systemInstruction"]["parts"][0]["text"] == "Be brief."
|
||||
errors: Final = [event["error"] for event in _events_sent_to_client(streaming) if event["type"] == "error"]
|
||||
assert [(error["code"], error["event_id"]) for error in errors] == [("session_update_not_applied", "evt_late")]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_deferred_setup_sends_the_default_setup_before_text_content_that_comes_first(monkeypatch):
|
||||
"""Gemini Live closes the socket on content sent before any setup, so text that arrives
|
||||
before a session.update in deferred mode gets the model's default setup first."""
|
||||
monkeypatch.setattr(litellm, "gemini_live_defer_setup", True, raising=False)
|
||||
from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig
|
||||
|
||||
client_ws: Final = MagicMock()
|
||||
client_ws.receive_text = AsyncMock(
|
||||
side_effect=[
|
||||
_TEXT_ITEM_CREATE,
|
||||
json.dumps({"type": "session.update", "event_id": "evt_late", "session": {"modalities": ["text"]}}),
|
||||
ConnectionClosed(None, None),
|
||||
]
|
||||
)
|
||||
client_ws.send_text = AsyncMock()
|
||||
backend_ws: Final = MagicMock()
|
||||
backend_ws.send = AsyncMock()
|
||||
streaming: Final = RealTimeStreaming(
|
||||
client_ws, backend_ws, MagicMock(), provider_config=GeminiRealtimeConfig(), model="gemini-live-2.5-flash"
|
||||
)
|
||||
|
||||
await streaming.client_ack_messages()
|
||||
|
||||
frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list]
|
||||
assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"]
|
||||
errors: Final = [event["error"] for event in _events_sent_to_client(streaming) if event["type"] == "error"]
|
||||
assert [(error["code"], error["event_id"]) for error in errors] == [("session_update_not_applied", "evt_late")]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_deferred_setup_sends_a_session_update_that_follows_buffered_audio_at_once(monkeypatch):
|
||||
append: Final = json.dumps({"type": "input_audio_buffer.append", "audio": "AAAA"})
|
||||
commit: Final = json.dumps({"type": "input_audio_buffer.commit"})
|
||||
session_update: Final = json.dumps({"type": "session.update", "session": {"instructions": "Be a pirate."}})
|
||||
streaming, backend_ws = _gemini_default_mode_streaming(
|
||||
monkeypatch, [append, commit, session_update], model="gemini-live-2.5-flash", defer_setup=True
|
||||
)
|
||||
|
||||
await streaming.client_ack_messages()
|
||||
|
||||
frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list]
|
||||
assert [next(iter(frame)) for frame in frames] == ["setup"]
|
||||
assert "Be a pirate." in json.dumps(frames[0]["setup"]["systemInstruction"])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("defer_setup", [False, True])
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_transcription_guardrail_refuses_gemini_audio_it_cannot_screen(monkeypatch, defer_setup):
|
||||
monkeypatch.setattr(litellm, "callbacks", [_transcription_guardrail()])
|
||||
session_update: Final = json.dumps(
|
||||
{"type": "session.update", "session": {"turn_detection": {"type": "server_vad", "create_response": False}}}
|
||||
)
|
||||
append: Final = json.dumps({"type": "input_audio_buffer.append", "audio": "AAAA"})
|
||||
commit: Final = json.dumps({"type": "input_audio_buffer.commit", "event_id": "commit_1"})
|
||||
streaming, backend_ws = _gemini_default_mode_streaming(
|
||||
monkeypatch,
|
||||
[session_update, append, append, commit, json.dumps({"type": "response.create"}), _TEXT_ITEM_CREATE],
|
||||
model="gemini-live-2.5-flash",
|
||||
defer_setup=defer_setup,
|
||||
)
|
||||
|
||||
await streaming.client_ack_messages()
|
||||
|
||||
frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list]
|
||||
assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"]
|
||||
refusals: Final = [
|
||||
event["error"]["event_id"]
|
||||
for event in _events_sent_to_client(streaming)
|
||||
if event["type"] == "error" and event["error"]["code"] == "audio_input_not_screenable"
|
||||
]
|
||||
assert refusals == [None, "commit_1"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("defer_setup", [False, True])
|
||||
@pytest.mark.parametrize("control_type", ["response.cancel", "conversation.item.delete", "input_audio_buffer.clear"])
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_control_event_before_session_update_leaves_the_setup_open(monkeypatch, defer_setup, control_type):
|
||||
control_event: Final = json.dumps({"type": control_type, "item_id": "item_x"})
|
||||
session_update: Final = json.dumps({"type": "session.update", "session": {"modalities": ["text"]}})
|
||||
streaming, backend_ws = _gemini_default_mode_streaming(
|
||||
monkeypatch,
|
||||
[control_event, session_update, _TEXT_ITEM_CREATE],
|
||||
model="gemini-live-2.5-flash",
|
||||
defer_setup=defer_setup,
|
||||
)
|
||||
|
||||
await streaming.client_ack_messages()
|
||||
|
||||
frames: Final = [json.loads(call.args[0]) for call in backend_ws.send.await_args_list]
|
||||
assert [next(iter(frame)) for frame in frames] == ["setup", "clientContent"]
|
||||
assert frames[0]["setup"]["generationConfig"]["responseModalities"] == ["TEXT"]
|
||||
assert [event["type"] for event in _events_sent_to_client(streaming) if event["type"] == "error"] == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_deferred_setup_buffers_audio_until_backend_setup_complete(monkeypatch):
|
||||
"""Pipecat may send audio before session.update when setup is deferred."""
|
||||
|
|
@ -2534,14 +3096,18 @@ async def test_deferred_setup_buffers_audio_until_backend_setup_complete(monkeyp
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_deferred_setup_sends_session_update_before_buffered_audio(monkeypatch):
|
||||
async def test_deferred_setup_sends_the_held_session_update_when_more_audio_arrives(monkeypatch):
|
||||
monkeypatch.setattr(litellm, "gemini_live_defer_setup", True, raising=False)
|
||||
from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig
|
||||
|
||||
client_ws = MagicMock()
|
||||
client_ws.send_text = AsyncMock()
|
||||
audio_msg = json.dumps({"type": "input_audio_buffer.append", "audio": "AA=="})
|
||||
session_update = json.dumps({"type": "session.update", "session": {"modalities": ["audio"]}})
|
||||
client_ws.receive_text = AsyncMock(side_effect=[audio_msg, session_update, ConnectionClosed(None, None)])
|
||||
later_audio_msg = json.dumps({"type": "input_audio_buffer.append", "audio": "AQ=="})
|
||||
client_ws.receive_text = AsyncMock(
|
||||
side_effect=[audio_msg, session_update, later_audio_msg, ConnectionClosed(None, None)]
|
||||
)
|
||||
backend_ws = MagicMock()
|
||||
backend_ws.send = AsyncMock()
|
||||
logging_obj: Final = _mock_logging()
|
||||
|
|
@ -2561,7 +3127,7 @@ async def test_deferred_setup_sends_session_update_before_buffered_audio(monkeyp
|
|||
sent_payload = json.loads(backend_ws.send.await_args_list[0].args[0])
|
||||
assert "setup" in sent_payload
|
||||
assert "realtimeInput" not in sent_payload
|
||||
assert streaming._pending_messages_until_setup == [audio_msg]
|
||||
assert streaming._pending_messages_until_setup == [audio_msg, later_audio_msg]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ from unittest.mock import AsyncMock, Mock, patch
|
|||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
import websockets.exceptions
|
||||
from botocore.credentials import RefreshableCredentials
|
||||
|
||||
import litellm
|
||||
|
|
@ -2785,6 +2786,193 @@ async def test_async_realtime_error_event_send_failure_still_closes():
|
|||
assert client_ws.events == [("close", (1011, "Internal server error: vertex token refresh exploded"))]
|
||||
|
||||
|
||||
_GEMINI_LIVE_SETUP_COMPLETE = json.dumps({"setupComplete": {}})
|
||||
_GEMINI_LIVE_TEXT_TURN = json.dumps({"serverContent": {"modelTurn": {"parts": [{"text": "Blue."}]}}})
|
||||
_GEMINI_LIVE_TURN_COMPLETE = json.dumps({"serverContent": {"turnComplete": True}})
|
||||
|
||||
|
||||
class _ScriptedRealtimeBackend:
|
||||
"""Backend socket that answers the first setup frame with the scripted replies
|
||||
and closes once the client is done, so both bridge loops run to completion."""
|
||||
|
||||
def __init__(self, replies_after_setup):
|
||||
self.sent = []
|
||||
self.closed = asyncio.Event()
|
||||
self._replies_after_setup = list(replies_after_setup)
|
||||
self._inbox = asyncio.Queue()
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc, tb):
|
||||
return None
|
||||
|
||||
async def send(self, payload):
|
||||
self.sent.append(payload)
|
||||
if "setup" in json.loads(payload):
|
||||
for reply in self._replies_after_setup:
|
||||
self._inbox.put_nowait(reply)
|
||||
|
||||
def close_after_replies(self):
|
||||
self._inbox.put_nowait(None)
|
||||
|
||||
async def recv(self, decode=True):
|
||||
reply = await self._inbox.get()
|
||||
if reply is None:
|
||||
self.closed.set()
|
||||
raise websockets.exceptions.ConnectionClosedOK(None, None)
|
||||
return reply
|
||||
|
||||
|
||||
class _ScriptedRealtimeClient:
|
||||
def __init__(self, messages, backend):
|
||||
self.scope = {"headers": []}
|
||||
self.exceptions = websockets.exceptions
|
||||
self.sent = []
|
||||
self._messages = list(messages)
|
||||
self._backend = backend
|
||||
|
||||
async def send_text(self, payload):
|
||||
self.sent.append(payload)
|
||||
|
||||
async def receive_text(self):
|
||||
if self._messages:
|
||||
return self._messages.pop(0)
|
||||
self._backend.close_after_replies()
|
||||
await self._backend.closed.wait()
|
||||
raise websockets.exceptions.ConnectionClosedOK(None, None)
|
||||
|
||||
async def close(self, code=None, reason=None):
|
||||
return None
|
||||
|
||||
|
||||
def _realtime_logging_obj():
|
||||
logging_obj = Mock()
|
||||
logging_obj.litellm_trace_id = "trace-lit-6974"
|
||||
logging_obj.async_success_handler = AsyncMock()
|
||||
logging_obj.dispatch_failure_handlers = AsyncMock()
|
||||
return logging_obj
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model, session, backend_modality, client_modality, backend_turn, drop_params",
|
||||
[
|
||||
(
|
||||
"gemini-live-2.5-flash",
|
||||
{"type": "realtime", "output_modalities": ["text"]},
|
||||
"TEXT",
|
||||
"text",
|
||||
[_GEMINI_LIVE_TEXT_TURN],
|
||||
False,
|
||||
),
|
||||
("gemini-live-2.5-flash", {"modalities": ["text"]}, "TEXT", "text", [_GEMINI_LIVE_TEXT_TURN], False),
|
||||
(
|
||||
"gemini-live-2.5-flash-native-audio",
|
||||
{"type": "realtime", "output_modalities": ["text"]},
|
||||
"AUDIO",
|
||||
"audio",
|
||||
[],
|
||||
True,
|
||||
),
|
||||
],
|
||||
)
|
||||
async def test_async_realtime_vertex_first_session_update_builds_the_only_setup(
|
||||
model, session, backend_modality, client_modality, backend_turn, drop_params: bool, monkeypatch
|
||||
):
|
||||
"""Regression for text-only Vertex Live sessions answered in audio: the
|
||||
connect-time default setup (AUDIO) used to go out before the client's
|
||||
session.update, and Gemini accepts only one setup per connection, so the
|
||||
requested modality was dropped and billed at the audio rate. The setup now
|
||||
goes out on the client's first message, built from session.update when that
|
||||
comes first. Under drop_params an audio-only model still gets AUDIO, and the
|
||||
client learns the downgrade from session.updated."""
|
||||
from litellm.llms.vertex_ai.realtime.transformation import VertexAIRealtimeConfig
|
||||
|
||||
monkeypatch.setattr(litellm, "gemini_live_defer_setup", False, raising=False)
|
||||
backend = _ScriptedRealtimeBackend([_GEMINI_LIVE_SETUP_COMPLETE, *backend_turn, _GEMINI_LIVE_TURN_COMPLETE])
|
||||
client_ws = _ScriptedRealtimeClient(
|
||||
[
|
||||
json.dumps({"type": "session.update", "session": session}),
|
||||
json.dumps(
|
||||
{
|
||||
"type": "conversation.item.create",
|
||||
"item": {
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [{"type": "input_text", "text": "What color is the sky? One word."}],
|
||||
},
|
||||
}
|
||||
),
|
||||
json.dumps({"type": "response.create"}),
|
||||
],
|
||||
backend,
|
||||
)
|
||||
handler = BaseLLMHTTPHandler()
|
||||
|
||||
with patch.object(handler, "_open_realtime_backend_ws", AsyncMock(return_value=backend)):
|
||||
await asyncio.wait_for(
|
||||
handler.async_realtime(
|
||||
model=model,
|
||||
websocket=client_ws,
|
||||
logging_obj=_realtime_logging_obj(),
|
||||
provider_config=VertexAIRealtimeConfig(
|
||||
access_token="fake-token", project="fake-project", location="us-central1", drop_params=drop_params
|
||||
),
|
||||
headers={},
|
||||
),
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
backend_frames = [json.loads(frame) for frame in backend.sent]
|
||||
setups = [frame["setup"] for frame in backend_frames if "setup" in frame]
|
||||
assert len(setups) == 1
|
||||
assert "setup" in backend_frames[0]
|
||||
assert setups[0]["generationConfig"]["responseModalities"] == [backend_modality]
|
||||
assert any("clientContent" in frame for frame in backend_frames[1:])
|
||||
|
||||
client_events = [json.loads(frame) for frame in client_ws.sent]
|
||||
event_types = [event["type"] for event in client_events]
|
||||
assert event_types[:2] == ["session.created", "session.updated"]
|
||||
assert client_events[1]["session"]["modalities"] == [client_modality]
|
||||
if backend_turn:
|
||||
assert "response.output_text.delta" in event_types
|
||||
assert "response.output_audio_transcript.delta" not in event_types
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_realtime_sends_setup_at_connect_when_provider_cannot_build_it_from_session_update():
|
||||
"""Bedrock Nova Sonic style providers keep the connect-time setup and get no
|
||||
synthetic session.created."""
|
||||
setup = json.dumps({"event": {"sessionStart": {}}})
|
||||
backend = _ScriptedRealtimeBackend([])
|
||||
client_ws = _ScriptedRealtimeClient([], backend)
|
||||
provider_config = Mock()
|
||||
provider_config.get_complete_url.return_value = "wss://backend.example/live"
|
||||
provider_config.validate_environment.return_value = {}
|
||||
provider_config.requires_session_configuration.return_value = True
|
||||
provider_config.builds_setup_from_session_update.return_value = False
|
||||
provider_config.session_configuration_request.return_value = setup
|
||||
provider_config.unbilled_usage_on_session_close.return_value = None
|
||||
provider_config.open_backend = AsyncMock(return_value=None)
|
||||
handler = BaseLLMHTTPHandler()
|
||||
|
||||
with patch.object(handler, "_open_realtime_backend_ws", AsyncMock(return_value=backend)):
|
||||
await asyncio.wait_for(
|
||||
handler.async_realtime(
|
||||
model="amazon.nova-sonic-v1:0",
|
||||
websocket=client_ws,
|
||||
logging_obj=_realtime_logging_obj(),
|
||||
provider_config=provider_config,
|
||||
headers={},
|
||||
),
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
assert backend.sent == [setup]
|
||||
assert client_ws.sent == []
|
||||
|
||||
|
||||
class _JSONBodyAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
|
||||
def get_supported_openai_params(self, model):
|
||||
return []
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import base64
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from typing import Final, cast
|
||||
|
|
@ -7,7 +8,7 @@ import pytest
|
|||
|
||||
|
||||
import litellm
|
||||
from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig
|
||||
from litellm.llms.gemini.realtime.transformation import GEMINI_LIVE_DOCUMENTED_VOICES, GeminiRealtimeConfig
|
||||
from litellm.types.llms.gemini import BidiGenerateContentServerMessage
|
||||
|
||||
|
||||
|
|
@ -613,6 +614,35 @@ def test_gemini_audio_only_live_models_drop_text_from_text_audio_combo(patch_gem
|
|||
assert setup["generationConfig"]["responseModalities"] == ["AUDIO"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("output_modalities", ["text", ["text"]])
|
||||
def test_gemini_session_update_accepts_a_single_string_output_modality(output_modalities):
|
||||
"""Regression: a bare string was iterated per character, so output_modalities
|
||||
"text" reached Vertex as responseModalities ["T"], which it rejects with 1007."""
|
||||
config = GeminiRealtimeConfig()
|
||||
session_update = {"type": "session.update", "session": {"type": "realtime", "output_modalities": output_modalities}}
|
||||
|
||||
messages = config.transform_realtime_request(
|
||||
json.dumps(session_update), "gemini-live-2.5-flash", session_configuration_request=None
|
||||
)
|
||||
|
||||
assert json.loads(messages[0])["setup"]["generationConfig"]["responseModalities"] == ["TEXT"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("output_modalities", "expected"),
|
||||
[(["video", "text"], ["TEXT"]), (["video"], ["AUDIO"]), (None, ["AUDIO"])],
|
||||
)
|
||||
def test_gemini_session_update_drops_modalities_live_cannot_produce(output_modalities, expected):
|
||||
config = GeminiRealtimeConfig()
|
||||
session_update = {"type": "session.update", "session": {"type": "realtime", "output_modalities": output_modalities}}
|
||||
|
||||
messages = config.transform_realtime_request(
|
||||
json.dumps(session_update), "gemini-live-2.5-flash", session_configuration_request=None
|
||||
)
|
||||
|
||||
assert json.loads(messages[0])["setup"]["generationConfig"]["responseModalities"] == expected
|
||||
|
||||
|
||||
def test_gemini_non_live_model_preserves_text_modality():
|
||||
config = GeminiRealtimeConfig()
|
||||
session_update = {
|
||||
|
|
@ -1396,22 +1426,71 @@ def test_gemini_input_audio_buffer_clear_is_local_noop():
|
|||
assert messages == []
|
||||
|
||||
|
||||
def test_gemini_input_audio_buffer_commit_maps_to_activity_end_when_manual_vad():
|
||||
_MANUAL_VAD_SETUP = json.dumps({"setup": {"realtimeInputConfig": {"automaticActivityDetection": {"disabled": True}}}})
|
||||
_APPEND = json.dumps({"type": "input_audio_buffer.append", "audio": base64.b64encode(b"\x00\x01" * 80).decode()})
|
||||
|
||||
|
||||
def _realtime_input_keys(config: GeminiRealtimeConfig, client_event: str, setup: str | None) -> list[str]:
|
||||
frames = config.transform_realtime_request(client_event, "gemini-live-2.5-flash-native-audio", setup)
|
||||
return [key for frame in frames for key in json.loads(frame)["realtimeInput"]]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("turn_end", ["input_audio_buffer.commit", "response.create"])
|
||||
def test_gemini_manual_vad_wraps_each_audio_turn_in_activity_start_and_end(turn_end):
|
||||
"""With activity detection off, Gemini Live answers only after activityEnd closes an
|
||||
activityStart, and it closes the socket on any other shape of those markers."""
|
||||
config = GeminiRealtimeConfig()
|
||||
setup = {
|
||||
"setup": {
|
||||
"realtimeInputConfig": {
|
||||
"automaticActivityDetection": {"disabled": True},
|
||||
}
|
||||
}
|
||||
}
|
||||
messages = config.transform_realtime_request(
|
||||
json.dumps({"type": "input_audio_buffer.commit"}),
|
||||
"gemini-live-2.5-flash-native-audio",
|
||||
session_configuration_request=json.dumps(setup),
|
||||
end = json.dumps({"type": turn_end})
|
||||
|
||||
turns = [
|
||||
[_realtime_input_keys(config, event, _MANUAL_VAD_SETUP) for event in (_APPEND, _APPEND, end, end)]
|
||||
for _ in range(2)
|
||||
]
|
||||
|
||||
assert turns == [[["activityStart", "audio"], ["audio"], ["activityEnd"], []]] * 2
|
||||
|
||||
|
||||
def test_gemini_manual_vad_activity_markers_are_empty_messages():
|
||||
config = GeminiRealtimeConfig()
|
||||
commit = json.dumps({"type": "input_audio_buffer.commit"})
|
||||
|
||||
first_append = config.transform_realtime_request(_APPEND, "gemini-live-2.5-flash", _MANUAL_VAD_SETUP)
|
||||
turn_end = config.transform_realtime_request(commit, "gemini-live-2.5-flash", _MANUAL_VAD_SETUP)
|
||||
|
||||
assert json.loads(first_append[0]) == {"realtimeInput": {"activityStart": {}}}
|
||||
assert [json.loads(frame) for frame in turn_end] == [{"realtimeInput": {"activityEnd": {}}}]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("session", "expected_keys"),
|
||||
[
|
||||
({"turn_detection": None}, [["activityStart", "audio"], ["activityEnd"]]),
|
||||
({"type": "realtime", "audio": {"input": {"turn_detection": None}}}, [["activityStart", "audio"], ["activityEnd"]]),
|
||||
({"type": "realtime", "audio": {"input": {}}}, [["audio"], ["audioStreamEnd"]]),
|
||||
],
|
||||
)
|
||||
def test_gemini_null_turn_detection_makes_the_client_mark_each_turn(session, expected_keys):
|
||||
"""OpenAI's turn_detection null turns server VAD off, so the client commits each turn itself."""
|
||||
config = GeminiRealtimeConfig()
|
||||
setup = config.transform_realtime_request(
|
||||
json.dumps({"type": "session.update", "session": session}), "gemini-live-2.5-flash-native-audio", None
|
||||
)
|
||||
assert len(messages) == 1
|
||||
assert json.loads(messages[0]) == {"realtimeInput": {"activityEnd": True}}
|
||||
commit = json.dumps({"type": "input_audio_buffer.commit"})
|
||||
|
||||
keys = [_realtime_input_keys(config, event, setup[0]) for event in (_APPEND, commit)]
|
||||
|
||||
assert keys == expected_keys
|
||||
|
||||
|
||||
def test_gemini_automatic_vad_sends_no_activity_markers():
|
||||
config = GeminiRealtimeConfig()
|
||||
commit = json.dumps({"type": "input_audio_buffer.commit"})
|
||||
response_create = json.dumps({"type": "response.create"})
|
||||
|
||||
keys = [_realtime_input_keys(config, event, None) for event in (_APPEND, commit)]
|
||||
|
||||
assert keys == [["audio"], ["audioStreamEnd"]]
|
||||
assert config.transform_realtime_request(response_create, "gemini-live-2.5-flash", None) == []
|
||||
|
||||
|
||||
def test_gemini_tool_call_id_to_name_evicts_oldest_when_capped():
|
||||
|
|
@ -1890,6 +1969,20 @@ def test_map_openai_params_drops_stock_voice_case_insensitively():
|
|||
assert passthrough["generationConfig"]["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Kore"
|
||||
|
||||
|
||||
_DOCUMENTED_VOICE: Final = sorted(GEMINI_LIVE_DOCUMENTED_VOICES)[0].capitalize()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("drop_params", [False, True])
|
||||
@pytest.mark.parametrize("voice", ["Nova", _DOCUMENTED_VOICE])
|
||||
def test_google_ai_studio_forwards_any_gemini_voice_name(voice, drop_params):
|
||||
"""AI Studio Live speaks voices outside Google's documented list, so the name is forwarded even with drop_params."""
|
||||
config = GeminiRealtimeConfig(drop_params=drop_params)
|
||||
mapped = config.map_openai_params(optional_params={}, non_default_params={"voice": voice})
|
||||
|
||||
assert mapped["generationConfig"]["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == voice
|
||||
assert config.setup_session_update_rejection("gemini-3.8-live", {"voice": voice}, setup_sent=False) is None
|
||||
|
||||
|
||||
@pytest.fixture(autouse=False)
|
||||
def patch_gemini_transcribe_live_cost_map_entry(monkeypatch):
|
||||
"""Inject the gemini-3.5-transcribe-live registry entry locally.
|
||||
|
|
@ -1954,6 +2047,42 @@ def test_gemini_transcribe_live_coerces_audio_modality_to_text(modalities, patch
|
|||
assert setup["generationConfig"]["responseModalities"] == ["TEXT"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["gemini-3.5-transcribe-live", "gemini/gemini-3.5-transcribe-live"])
|
||||
def test_gemini_transcribe_live_synthetic_session_created_reports_text_output(
|
||||
model, patch_gemini_transcribe_live_cost_map_entry
|
||||
):
|
||||
session = GeminiRealtimeConfig().transform_session_created_event(model, "sess_transcribe")["session"]
|
||||
|
||||
assert session["modalities"] == ["text"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("modalities", [["audio"], ["audio", "text"]])
|
||||
@pytest.mark.parametrize("drop_params", [False, True])
|
||||
def test_gemini_transcribe_live_audio_output_is_rejected_unless_drop_params(
|
||||
modalities, drop_params, patch_gemini_transcribe_live_cost_map_entry
|
||||
):
|
||||
rejection = GeminiRealtimeConfig(drop_params=drop_params).setup_session_update_rejection(
|
||||
"gemini-3.5-transcribe-live", {"output_modalities": modalities}, setup_sent=False
|
||||
)
|
||||
|
||||
assert (rejection is None) is drop_params
|
||||
assert rejection is None or rejection.param == "session.output_modalities"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("drop_params", [False, True])
|
||||
def test_gemini_provider_realtime_config_carries_drop_params(drop_params):
|
||||
from litellm.types.utils import LlmProviders
|
||||
from litellm.utils import ProviderConfigManager
|
||||
|
||||
model = "gemini-live-2.5-flash-native-audio"
|
||||
config = ProviderConfigManager.get_provider_realtime_config(
|
||||
model=model, provider=LlmProviders.GEMINI, drop_params=drop_params
|
||||
)
|
||||
|
||||
rejection = config.setup_session_update_rejection(model, {"modalities": ["text"]}, setup_sent=False)
|
||||
assert (rejection is None) is drop_params
|
||||
|
||||
|
||||
def test_gemini_chat_model_with_text_output_modalities_keeps_audio_eager_setup(
|
||||
patch_gemini_transcribe_live_cost_map_entry,
|
||||
):
|
||||
|
|
@ -2169,6 +2298,47 @@ def test_unbilled_usage_on_session_close_flushes_trailing_audio(patch_gemini_tra
|
|||
assert config.unbilled_usage_on_session_close("gemini-3.5-transcribe-live") is None
|
||||
|
||||
|
||||
def test_gemini_realtime_two_output_modalities_collapse_to_audio():
|
||||
"""Gemini Live closes the socket with 1007 when the setup names two response
|
||||
modalities, so a GA session.update asking for ["text", "audio"] must reach the
|
||||
backend as AUDIO only, the same choice the beta shim makes."""
|
||||
config = GeminiRealtimeConfig()
|
||||
session_update = {
|
||||
"type": "session.update",
|
||||
"session": {"output_modalities": ["text", "audio"], "instructions": "Be brief."},
|
||||
}
|
||||
|
||||
messages = config.transform_realtime_request(
|
||||
json.dumps(session_update),
|
||||
"gemini-live-2.5-flash",
|
||||
session_configuration_request=None,
|
||||
)
|
||||
|
||||
assert len(messages) == 1
|
||||
assert json.loads(messages[0])["setup"]["generationConfig"]["responseModalities"] == ["AUDIO"]
|
||||
|
||||
|
||||
def test_gemini_realtime_inf_max_response_output_tokens_is_not_forwarded():
|
||||
"""OpenAI's default max_response_output_tokens is the string "inf", which Gemini
|
||||
rejects with 1007 (TYPE_INT32), so only integer limits map to maxOutputTokens."""
|
||||
config = GeminiRealtimeConfig()
|
||||
|
||||
def generation_config_for(limit):
|
||||
session_update = {
|
||||
"type": "session.update",
|
||||
"session": {"modalities": ["text"], "max_response_output_tokens": limit},
|
||||
}
|
||||
messages = config.transform_realtime_request(
|
||||
json.dumps(session_update),
|
||||
"gemini-live-2.5-flash",
|
||||
session_configuration_request=None,
|
||||
)
|
||||
return json.loads(messages[0])["setup"]["generationConfig"]
|
||||
|
||||
assert "maxOutputTokens" not in generation_config_for("inf")
|
||||
assert generation_config_for(200)["maxOutputTokens"] == 200
|
||||
|
||||
|
||||
def _grounded_live_frame(grounding_metadata: Mapping[str, object] | None) -> Mapping[str, object]:
|
||||
"""One Live server frame. Grounding metadata and usageMetadata arrive together, as Vertex sends them."""
|
||||
from typing import Final
|
||||
|
|
|
|||
|
|
@ -179,6 +179,25 @@ def test_vertex_audio_only_live_model_coerces_text_modality_to_audio(
|
|||
assert setup["generationConfig"]["responseModalities"] == ["AUDIO"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("drop_params", [False, True])
|
||||
def test_vertex_realtime_config_carries_drop_params(drop_params, patch_native_audio_cost_map_entry):
|
||||
from litellm.llms.vertex_ai.realtime.transformation import vertex_realtime_config
|
||||
|
||||
cfg: Final = vertex_realtime_config(
|
||||
_NATIVE_AUDIO_MODEL,
|
||||
access_token="tok",
|
||||
resolve_access_token=AsyncMock(return_value="tok"),
|
||||
project="my-proj",
|
||||
location="us-central1",
|
||||
drop_params=drop_params,
|
||||
)
|
||||
|
||||
rejection: Final = cfg.setup_session_update_rejection(
|
||||
_NATIVE_AUDIO_MODEL, {"modalities": ["text"]}, setup_sent=False
|
||||
)
|
||||
assert (rejection is None) is drop_params
|
||||
|
||||
|
||||
def test_vertex_session_update_normalizes_ga_remapped_fields(
|
||||
patch_native_audio_cost_map_entry,
|
||||
):
|
||||
|
|
@ -253,6 +272,26 @@ def test_vertex_keeps_turn_detection_tuning_when_auto_response_is_off():
|
|||
}
|
||||
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("session", "disabled"),
|
||||
[
|
||||
({"turn_detection": None}, True),
|
||||
({"type": "realtime", "audio": {"input": {"turn_detection": None}}}, True),
|
||||
({"type": "realtime", "audio": {"input": {}}}, False),
|
||||
],
|
||||
)
|
||||
def test_vertex_turns_activity_detection_off_for_null_turn_detection(session, disabled):
|
||||
"""OpenAI's turn_detection null is push-to-talk, so Vertex must wait for the client's activity markers."""
|
||||
cfg: Final = VertexAIRealtimeConfig(access_token="tok", project="my-proj", location="us-central1")
|
||||
messages: Final = cfg.transform_realtime_request(
|
||||
json.dumps({"type": "session.update", "session": session}),
|
||||
_NATIVE_AUDIO_MODEL,
|
||||
session_configuration_request=None,
|
||||
)
|
||||
setup: Final = json.loads(messages[0])["setup"]
|
||||
assert setup["realtimeInputConfig"]["automaticActivityDetection"]["disabled"] is disabled
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Round-trip test: text-in / text-out via RealTimeStreaming
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -607,3 +646,56 @@ def test_vertex_native_audio_unmapped_voice_passes_through(patch_native_audio_co
|
|||
|
||||
generation_config = json.loads(messages[0])["setup"]["generationConfig"]
|
||||
assert generation_config["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Kore"
|
||||
|
||||
|
||||
def test_vertex_session_update_two_output_modalities_collapse_to_audio():
|
||||
"""Vertex Live rejects a setup naming two response modalities, so a GA
|
||||
["text", "audio"] request must be sent as AUDIO only."""
|
||||
cfg = VertexAIRealtimeConfig(access_token="tok", project="my-proj", location="global")
|
||||
session_update = {
|
||||
"type": "session.update",
|
||||
"session": {"output_modalities": ["text", "audio"], "instructions": "Be brief."},
|
||||
}
|
||||
|
||||
messages = cfg.transform_realtime_request(
|
||||
json.dumps(session_update),
|
||||
"gemini-live-2.5-flash",
|
||||
session_configuration_request=None,
|
||||
)
|
||||
|
||||
assert len(messages) == 1
|
||||
assert json.loads(messages[0])["setup"]["generationConfig"]["responseModalities"] == ["AUDIO"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("session", "param"),
|
||||
[
|
||||
({"voice": "Nova"}, "session.voice"),
|
||||
({"type": "realtime", "audio": {"output": {"voice": "Nova"}}}, "session.audio.output.voice"),
|
||||
],
|
||||
)
|
||||
def test_vertex_rejects_an_undocumented_voice_on_the_session_update(session, param):
|
||||
cfg: Final = VertexAIRealtimeConfig(access_token="tok", project="my-proj", location="us-central1")
|
||||
|
||||
rejection: Final = cfg.setup_session_update_rejection("gemini-live-2.5-flash", session, setup_sent=False)
|
||||
|
||||
assert rejection is not None
|
||||
assert (rejection.code, rejection.param) == ("invalid_value", param)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("voice", ["Kore", "alloy"])
|
||||
def test_vertex_accepts_documented_and_openai_stock_voices_on_the_session_update(voice):
|
||||
cfg: Final = VertexAIRealtimeConfig(access_token="tok", project="my-proj", location="us-central1")
|
||||
|
||||
assert cfg.setup_session_update_rejection("gemini-live-2.5-flash", {"voice": voice}, setup_sent=False) is None
|
||||
|
||||
|
||||
def test_vertex_drops_an_undocumented_voice_with_drop_params():
|
||||
cfg: Final = VertexAIRealtimeConfig(access_token="tok", project="my-proj", location="us-central1", drop_params=True)
|
||||
session_update: Final = {"type": "session.update", "session": {"voice": "Nova", "instructions": "Be brief."}}
|
||||
|
||||
assert cfg.setup_session_update_rejection("gemini-live-2.5-flash", session_update["session"], setup_sent=False) is None
|
||||
messages: Final = cfg.transform_realtime_request(
|
||||
json.dumps(session_update), "gemini-live-2.5-flash", session_configuration_request=None
|
||||
)
|
||||
assert "speechConfig" not in json.loads(messages[0])["setup"].get("generationConfig", {})
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue