diff --git a/litellm/llms/gemini/realtime/transformation.py b/litellm/llms/gemini/realtime/transformation.py index e64cbf88d95..01d8e594309 100644 --- a/litellm/llms/gemini/realtime/transformation.py +++ b/litellm/llms/gemini/realtime/transformation.py @@ -132,7 +132,15 @@ def _grounding_metadata_from_frame(frame: Mapping[str, object]) -> tuple[Mapping # 175 text tokens/min of output (ai.google.dev/gemini-api/docs/pricing). GEMINI_LIVE_TRANSCRIBE_AUDIO_TOKENS_PER_SECOND: Final = 25 GEMINI_LIVE_TRANSCRIBE_OUTPUT_TEXT_TOKENS_PER_MINUTE: Final = 175 -PCM16_INPUT_AUDIO_BYTES_PER_SECOND: Final = 48000 + +# The MIME rate controls server resampling and must describe the input bytes. +GEMINI_LIVE_INPUT_AUDIO_SAMPLE_RATE_HZ: Final = 16000 +PCM16_BYTES_PER_SAMPLE: Final = 2 +# Bound client-controlled rates because they affect the duration billed. +MIN_ACCEPTED_INPUT_AUDIO_SAMPLE_RATE_HZ: Final = 8000 +MAX_ACCEPTED_INPUT_AUDIO_SAMPLE_RATE_HZ: Final = 48000 +# The beta pcm16 codec implies 24kHz, matching the beta-to-GA conversion. +BETA_PCM16_INPUT_AUDIO_SAMPLE_RATE_HZ: Final = 24000 def _base64_decoded_byte_count(data: str) -> int: @@ -149,7 +157,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): # Gemini Live sometimes emits usageMetadata in a standalone frame between # turns; buffer it here so the next response.done carries the token counts. self._pending_usage_metadata: dict | None = None - self._unbilled_input_audio_bytes: int = 0 + # Store per-chunk duration so later rate changes cannot reprice earlier audio. + self._unbilled_input_audio_seconds: float = 0.0 + self._input_audio_sample_rate_hz: int = GEMINI_LIVE_INPUT_AUDIO_SAMPLE_RATE_HZ def is_setup_message(self, msg_obj: dict) -> bool: return "setup" in msg_obj @@ -243,7 +253,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): def get_audio_mime_type(self, input_audio_format: str = "pcm16"): mime_types: Final = { - "pcm16": "audio/pcm;rate=24000", + "pcm16": f"audio/pcm;rate={self._input_audio_sample_rate_hz}", "g711_ulaw": "audio/pcmu", "g711_alaw": "audio/pcma", } @@ -463,6 +473,45 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): ) return setup + @staticmethod + def _declared_rate_from_ga_shape(session_payload: Mapping[str, object]) -> object: + audio = session_payload.get("audio") + if not isinstance(audio, dict): + return None + audio_input = audio.get("input") + if not isinstance(audio_input, dict): + return None + audio_format = audio_input.get("format") + if not isinstance(audio_format, dict): + return None + return audio_format.get("rate") + + @staticmethod + def _declared_rate_from_beta_shape(session_payload: Mapping[str, object]) -> object: + """Only pcm16 implies a rate here because append frames are sent as pcm16.""" + if session_payload.get("input_audio_format") == "pcm16": + return BETA_PCM16_INPUT_AUDIO_SAMPLE_RATE_HZ + return None + + def _record_input_audio_sample_rate(self, session_payload: Mapping[str, object]) -> None: + """Read both shapes so the OpenAI-Beta header cannot change the audio's declared rate.""" + rate = self._declared_rate_from_ga_shape(session_payload) + if rate is None: + rate = self._declared_rate_from_beta_shape(session_payload) + if isinstance(rate, bool) or not isinstance(rate, int): + return + if not (MIN_ACCEPTED_INPUT_AUDIO_SAMPLE_RATE_HZ <= rate <= MAX_ACCEPTED_INPUT_AUDIO_SAMPLE_RATE_HZ): + verbose_logger.warning( + "Gemini Realtime: ignoring declared input audio rate %s, outside the accepted " + "%s-%s Hz range; keeping %s Hz", + rate, + MIN_ACCEPTED_INPUT_AUDIO_SAMPLE_RATE_HZ, + MAX_ACCEPTED_INPUT_AUDIO_SAMPLE_RATE_HZ, + self._input_audio_sample_rate_hz, + ) + return + self._input_audio_sample_rate_hz = rate + def _handle_session_update( self, json_message: _OpenAIRealtimeClientEvent, @@ -488,6 +537,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): # explicit modality / transcription / turn-detection settings # would be silently dropped because ``map_openai_params`` only # recognises the flat OpenAI-beta key names. + self._record_input_audio_sample_rate(session_payload) session_payload = self._normalize_session_payload_for_mapping(session_payload) new_overrides: Final = self.map_openai_params(optional_params={}, non_default_params=session_payload) @@ -618,7 +668,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): if msg_type == "input_audio_buffer.append": audio_b64: Final = json_message["audio"] if isinstance(audio_b64, str): - self._unbilled_input_audio_bytes += _base64_decoded_byte_count(audio_b64) + self._unbilled_input_audio_seconds += _base64_decoded_byte_count(audio_b64) / ( + self._input_audio_sample_rate_hz * PCM16_BYTES_PER_SAMPLE + ) realtime_input_dict["audio"] = HttpxBlobType(mimeType=self.get_audio_mime_type(), data=audio_b64) realtime_input_dict = cast( @@ -1213,10 +1265,10 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): def _consume_input_transcription_usage_estimate(self, model: str) -> RealtimeInputAudioTranscriptionUsage | None: """Gemini Live sends no usageMetadata for transcribe sessions; estimate billing from streamed audio duration.""" - if self._unbilled_input_audio_bytes <= 0 or not self._is_text_only_live_model(model): + if self._unbilled_input_audio_seconds <= 0 or not self._is_text_only_live_model(model): return None - audio_seconds: Final = self._unbilled_input_audio_bytes / PCM16_INPUT_AUDIO_BYTES_PER_SECOND - self._unbilled_input_audio_bytes = 0 + audio_seconds: Final = self._unbilled_input_audio_seconds + self._unbilled_input_audio_seconds = 0.0 audio_tokens: Final = round(audio_seconds * GEMINI_LIVE_TRANSCRIBE_AUDIO_TOKENS_PER_SECOND) output_tokens: Final = round(audio_seconds * GEMINI_LIVE_TRANSCRIBE_OUTPUT_TEXT_TOKENS_PER_MINUTE / 60) usage: Final[RealtimeInputAudioTranscriptionUsage] = { diff --git a/litellm/llms/vertex_ai/realtime/transformation.py b/litellm/llms/vertex_ai/realtime/transformation.py index 9fed6d52f0e..72aed733c1e 100644 --- a/litellm/llms/vertex_ai/realtime/transformation.py +++ b/litellm/llms/vertex_ai/realtime/transformation.py @@ -93,19 +93,6 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig): headers["x-goog-user-project"] = self._project return headers - # ------------------------------------------------------------------ - # Audio MIME type — Vertex AI needs the sample rate in the MIME string - # ------------------------------------------------------------------ - - def get_audio_mime_type(self, input_audio_format: str = "pcm16") -> str: - mime_types: Final = { - # Gemini Live native audio (OpenAI GA realtime default) is 24kHz PCM. - "pcm16": "audio/pcm;rate=24000", - "g711_ulaw": "audio/pcmu", - "g711_alaw": "audio/pcma", - } - return mime_types.get(input_audio_format, "application/octet-stream") - # ------------------------------------------------------------------ # Session setup message # ------------------------------------------------------------------ @@ -212,6 +199,10 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig): msg_type: Final = json_message.get("type") if msg_type == "session.update": + # Vertex handles session.update itself and never reaches the parent's handler, so the + # declared input audio rate has to be recorded here or the Vertex path silently keeps + # the default no matter what the client declares. + self._record_input_audio_sample_rate(json_message.get("session") or {}) if session_configuration_request is None: setup_config: Final = self._build_vertex_ai_setup_config(model, json_message.get("session") or {}) gemini_setup_msg: Final = json.dumps({"setup": setup_config}) diff --git a/tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py b/tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py index bcd5f3d8d19..7a6f6a23ade 100644 --- a/tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py +++ b/tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py @@ -1,9 +1,10 @@ import json from collections.abc import Mapping -from typing import cast +from typing import Final, cast from unittest.mock import MagicMock import pytest +from pydantic import TypeAdapter import litellm @@ -2007,10 +2008,219 @@ def _input_audio_append_message(raw_byte_count: int) -> str: ) +def _session_update_message(session: Mapping[str, object]) -> str: + return json.dumps({"type": "session.update", "session": dict(session)}) + + +def _sent_audio_mime_type(config: GeminiRealtimeConfig, raw_byte_count: int = 32000) -> str: + sent: Final = config.transform_realtime_request( + _input_audio_append_message(raw_byte_count), "gemini-3.5-transcribe-live" + ) + frame: Final = TypeAdapter(dict[str, dict[str, dict[str, str]]]).validate_json(sent[0]) + return frame["realtimeInput"]["audio"]["mimeType"] + + +def test_input_audio_mime_type_declares_the_native_16khz_input_rate(): + """The Live API resamples against the MIME rate, and its documented native input rate is + 16kHz; 24kHz is the output rate. Declaring the output rate on the input path mislabels + correctly-encoded audio (ai.google.dev/gemini-api/docs/live-api/capabilities).""" + from typing import Final + + config: Final = GeminiRealtimeConfig() + assert config.get_audio_mime_type() == "audio/pcm;rate=16000" + assert _sent_audio_mime_type(config) == "audio/pcm;rate=16000" + + +def test_vertex_realtime_sends_the_native_16khz_input_rate() -> None: + """Vertex's default input rate must match the serialized audio frame.""" + from typing import Final + + from litellm.llms.vertex_ai.realtime.transformation import VertexAIRealtimeConfig + + config: Final = VertexAIRealtimeConfig(access_token="t", project="p", location="us-central1") + assert config.get_audio_mime_type() == "audio/pcm;rate=16000" + assert _sent_audio_mime_type(config) == "audio/pcm;rate=16000" + + +def test_client_declared_input_audio_rate_is_honored_on_the_wire(): + """A GA client that declares a non-native input rate must have that rate forwarded, not the + default; the whole point of the MIME rate is to describe the bytes actually sent.""" + from typing import Final + + config: Final = GeminiRealtimeConfig() + config.transform_realtime_request( + _session_update_message({"audio": {"input": {"format": {"type": "audio/pcm", "rate": 24000}}}}), + "gemini-3.5-transcribe-live", + ) + assert _sent_audio_mime_type(config) == "audio/pcm;rate=24000" + + +@pytest.mark.parametrize( + "session", + [ + {}, + {"audio": {}}, + {"audio": {"input": {}}}, + {"audio": {"input": {"format": "audio/pcm"}}}, + {"audio": {"input": {"format": {"type": "audio/pcm"}}}}, + {"audio": {"input": {"format": {"type": "audio/pcm", "rate": 0}}}}, + {"audio": {"input": {"format": {"type": "audio/pcm", "rate": -1}}}}, + {"audio": {"input": {"format": {"type": "audio/pcm", "rate": 7999}}}}, + {"audio": {"input": {"format": {"type": "audio/pcm", "rate": 48001}}}}, + {"audio": {"input": {"format": {"type": "audio/pcm", "rate": 100_000_000}}}}, + {"audio": {"input": {"format": {"type": "audio/pcm", "rate": "24000"}}}}, + {"audio": {"input": {"format": {"type": "audio/pcm", "rate": True}}}}, + {"input_audio_format": "g711_ulaw"}, + ], +) +def test_malformed_or_absent_declared_rate_keeps_the_native_default(session): + """Anything that is not a plausible PCM rate, including a bool (which is an int subclass) and + rates outside 8000-48000, must leave the 16kHz default alone. The out-of-range cases matter + because the rate feeds the spend estimate: an unclamped 100MHz declaration would bill a long + session as a few milliseconds. A beta codec name other than pcm16 is also left alone, because + ``get_audio_mime_type`` labels every append pcm16 regardless.""" + from typing import Final + + config: Final = GeminiRealtimeConfig() + config.transform_realtime_request(_session_update_message(session), "gemini-3.5-transcribe-live") + assert _sent_audio_mime_type(config) == "audio/pcm;rate=16000" + + +def test_beta_input_audio_format_declares_its_specified_24khz_rate(): + """The beta shape has no rate field, but pcm16 is specified as 24kHz, so a client that sends + the flat codec name has declared 24kHz audio and the MIME label has to say so.""" + from typing import Final + + config: Final = GeminiRealtimeConfig() + config.transform_realtime_request( + _session_update_message({"input_audio_format": "pcm16"}), "gemini-3.5-transcribe-live" + ) + assert _sent_audio_mime_type(config) == "audio/pcm;rate=24000" + + +def test_beta_session_reaches_the_same_rate_through_the_ga_remap(): + """The proxy only forwards the flat beta shape untouched when the client sent the OpenAI-Beta + header. Without it, RealTimeStreaming rewrites the payload into the GA shape first. Driving the + real converter rather than hand-building the GA dict is what makes this able to fail: both + routes must land on the same rate, or an identical audio stream gets labelled 16kHz or 24kHz + depending on a header that says nothing about sample rates.""" + from typing import Final + + from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming + + remapped: Final = RealTimeStreaming._remap_beta_session_to_ga({"input_audio_format": "pcm16"}) + assert remapped["audio"]["input"]["format"] == {"type": "audio/pcm", "rate": 24000} + + via_remap: Final = GeminiRealtimeConfig() + via_remap.transform_realtime_request(_session_update_message(remapped), "gemini-3.5-transcribe-live") + + passthrough: Final = GeminiRealtimeConfig() + passthrough.transform_realtime_request( + _session_update_message({"input_audio_format": "pcm16"}), "gemini-3.5-transcribe-live" + ) + + assert _sent_audio_mime_type(via_remap) == _sent_audio_mime_type(passthrough) == "audio/pcm;rate=24000" + + +def test_ga_declared_rate_wins_over_the_beta_codec_name(): + """A session carrying both shapes has stated a rate outright; the name-implied one is a + fallback for when it has not.""" + from typing import Final + + config: Final = GeminiRealtimeConfig() + config.transform_realtime_request( + _session_update_message( + { + "input_audio_format": "pcm16", + "audio": {"input": {"format": {"type": "audio/pcm", "rate": 16000}}}, + } + ), + "gemini-3.5-transcribe-live", + ) + assert _sent_audio_mime_type(config) == "audio/pcm;rate=16000" + + +def test_declared_rate_also_drives_the_billed_audio_duration(patch_gemini_transcribe_live_cost_map_entry): + """The MIME label and the duration estimate read the same rate, so a client that declares + 24kHz is billed for 24kHz audio: 96000 pcm16 bytes = 2s -> 50 in / 6 out.""" + from typing import Final + + config: Final = GeminiRealtimeConfig() + config.transform_realtime_request( + _session_update_message({"audio": {"input": {"format": {"type": "audio/pcm", "rate": 24000}}}}), + "gemini-3.5-transcribe-live", + ) + config.transform_realtime_request(_input_audio_append_message(96000), "gemini-3.5-transcribe-live") + + usage: Final = config.unbilled_usage_on_session_close("gemini-3.5-transcribe-live") + assert usage is not None + assert usage["input_tokens"] == 50 + assert usage["output_tokens"] == 6 + + +def test_vertex_records_the_declared_rate_from_its_own_session_update(): + """VertexAIRealtimeConfig handles session.update itself and never calls the parent's handler, so + without recording the rate on that path a Vertex client's declaration is silently discarded.""" + from typing import Final + + from litellm.llms.vertex_ai.realtime.transformation import VertexAIRealtimeConfig + + config: Final = VertexAIRealtimeConfig(access_token="t", project="p", location="us-central1") + config.transform_realtime_request( + _session_update_message({"audio": {"input": {"format": {"type": "audio/pcm", "rate": 24000}}}}), + "gemini-3.5-transcribe-live", + ) + assert _sent_audio_mime_type(config) == "audio/pcm;rate=24000" + + +def test_a_later_rate_declaration_cannot_reprice_already_buffered_audio( + patch_gemini_transcribe_live_cost_map_entry, +): + """Audio is converted to seconds at the rate in force when it arrived. Otherwise a client could + stream at 16kHz and then declare 24kHz before the estimate is consumed, billing 2/3 of what it + actually sent while the backend still processed all of it.""" + from typing import Final + + config: Final = GeminiRealtimeConfig() + # 96000 bytes at the 16kHz default is 3s -> 75 in / 9 out. + config.transform_realtime_request(_input_audio_append_message(96000), "gemini-3.5-transcribe-live") + config.transform_realtime_request( + _session_update_message({"audio": {"input": {"format": {"type": "audio/pcm", "rate": 24000}}}}), + "gemini-3.5-transcribe-live", + ) + + usage: Final = config.unbilled_usage_on_session_close("gemini-3.5-transcribe-live") + assert usage is not None + assert usage["input_tokens"] == 75 + assert usage["output_tokens"] == 9 + + +def test_each_chunk_is_billed_at_the_rate_declared_when_it_arrived( + patch_gemini_transcribe_live_cost_map_entry, +): + """Mixed-rate sessions bill per chunk: 96000 bytes at 16kHz (3s) then 96000 at 24kHz (2s) is 5s + total, not 5s at either single rate.""" + from typing import Final + + config: Final = GeminiRealtimeConfig() + config.transform_realtime_request(_input_audio_append_message(96000), "gemini-3.5-transcribe-live") + config.transform_realtime_request( + _session_update_message({"audio": {"input": {"format": {"type": "audio/pcm", "rate": 24000}}}}), + "gemini-3.5-transcribe-live", + ) + config.transform_realtime_request(_input_audio_append_message(96000), "gemini-3.5-transcribe-live") + + usage: Final = config.unbilled_usage_on_session_close("gemini-3.5-transcribe-live") + assert usage is not None + assert usage["input_tokens"] == 125 # 5s * 25 audio tokens/sec + assert usage["output_tokens"] == 15 # round(5 * 175 / 60) + + def test_transcribe_live_completed_event_carries_estimated_usage(patch_gemini_transcribe_live_cost_map_entry): """Gemini Live sends no usageMetadata for transcribe sessions, so LiteLLM bills from streamed audio duration at Google's published estimate (25 audio tok/sec in, - 175 text tok/min out): 96000 pcm16 bytes = 2s at 24kHz -> 50 in / 6 out.""" + 175 text tok/min out): 96000 pcm16 bytes = 3s at the Live API's native 16kHz input + rate -> 75 in / 9 out.""" from typing import Final from litellm.types.llms.gemini import BidiGenerateContentServerMessage @@ -2048,10 +2258,10 @@ def test_transcribe_live_completed_event_carries_estimated_usage(patch_gemini_tr assert completed[0]["transcript"] == "ahoy there" expected_usage: Final[RealtimeInputAudioTranscriptionUsage] = { "type": "tokens", - "input_tokens": 50, - "output_tokens": 6, - "total_tokens": 56, - "input_token_details": {"text_tokens": 0, "audio_tokens": 50}, + "input_tokens": 75, + "output_tokens": 9, + "total_tokens": 84, + "input_token_details": {"text_tokens": 0, "audio_tokens": 75}, } assert completed[0]["usage"] == expected_usage @@ -2114,7 +2324,8 @@ def test_non_transcription_live_model_completed_event_has_no_usage(patch_gemini_ def test_unbilled_usage_on_session_close_flushes_trailing_audio(patch_gemini_transcribe_live_cost_map_entry): """Audio appended after the last transcript frame is still unbilled when the session closes; the session-close hook must hand back the estimate exactly once - so the streaming layer can bill it (144000 pcm16 bytes = 3s -> 75 in / 9 out).""" + so the streaming layer can bill it (144000 pcm16 bytes = 4.5s at the native 16kHz + input rate -> 112 in / 13 out; 112.5 rounds to even).""" from typing import Final from litellm.types.realtime import RealtimeInputAudioTranscriptionUsage @@ -2126,10 +2337,10 @@ def test_unbilled_usage_on_session_close_flushes_trailing_audio(patch_gemini_tra expected: Final[RealtimeInputAudioTranscriptionUsage] = { "type": "tokens", - "input_tokens": 75, - "output_tokens": 9, - "total_tokens": 84, - "input_token_details": {"text_tokens": 0, "audio_tokens": 75}, + "input_tokens": 112, + "output_tokens": 13, + "total_tokens": 125, + "input_token_details": {"text_tokens": 0, "audio_tokens": 112}, } assert usage == expected assert config.unbilled_usage_on_session_close("gemini-3.5-transcribe-live") is None