diff --git a/litellm/llms/gemini/realtime/transformation.py b/litellm/llms/gemini/realtime/transformation.py index a4e9f8e2627..337418bd0db 100644 --- a/litellm/llms/gemini/realtime/transformation.py +++ b/litellm/llms/gemini/realtime/transformation.py @@ -146,6 +146,11 @@ PCM16_BYTES_PER_SAMPLE: Final = 2 # native default stands, which bounds how far a bogus rate can move a bill. MIN_ACCEPTED_INPUT_AUDIO_SAMPLE_RATE_HZ: Final = 8000 MAX_ACCEPTED_INPUT_AUDIO_SAMPLE_RATE_HZ: Final = 48000 +# The beta session shape has no rate field, but its ``input_audio_format`` codec name carries one +# by definition. LiteLLM's own type stub for it says pcm16 input "must be 16-bit PCM at a 24kHz +# sample rate" (``OpenAIRealtimeSession.input_audio_format`` in litellm/types/llms/openai.py), and +# the beta-to-GA converter in realtime_streaming.py already expands the name to that rate. +BETA_PCM16_INPUT_AUDIO_SAMPLE_RATE_HZ: Final = 24000 def _base64_decoded_byte_count(data: str) -> int: @@ -481,30 +486,55 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): ) return setup + @staticmethod + def _declared_rate_from_ga_shape(session_payload: Mapping[str, object]) -> object: + """Read ``audio.input.format.rate`` out of the GA nested session shape.""" + audio = session_payload.get("audio") + if not isinstance(audio, dict): + return None + audio_input = audio.get("input") + if not isinstance(audio_input, dict): + return None + audio_format = audio_input.get("format") + if not isinstance(audio_format, dict): + return None + return audio_format.get("rate") + + @staticmethod + def _declared_rate_from_beta_shape(session_payload: Mapping[str, object]) -> object: + """Read the rate implied by the flat beta ``input_audio_format`` codec name. + + Only pcm16 is mapped. ``get_audio_mime_type`` labels every append as pcm16, so a rate + lifted from a g711 name would describe bytes with a codec they are not in. + """ + if session_payload.get("input_audio_format") == "pcm16": + return BETA_PCM16_INPUT_AUDIO_SAMPLE_RATE_HZ + return None + def _record_input_audio_sample_rate(self, session_payload: Mapping[str, object]) -> None: """ Remember the input sample rate the client declared on session.update. The rate reaches Gemini only through the per-blob MIME type, and the server resamples against whatever that MIME type claims, so it has to describe the bytes actually sent. - Only the GA session shape carries a rate (``audio.input.format.rate``); the beta shape's - ``input_audio_format`` is a bare codec name with no rate, and leaves the 16kHz default - in place. + + Both session shapes can declare a rate. The GA shape states it outright in + ``audio.input.format.rate``. The beta shape has no rate field, but its + ``input_audio_format`` codec name implies one, and pcm16 is specified as 24kHz. Both are + read here because which shape reaches this method is decided upstream by the + ``OpenAI-Beta`` header: without it, ``RealTimeStreaming._remap_beta_session_to_ga`` + rewrites the flat payload into the GA shape and supplies that same 24kHz for pcm16; with + it, the flat payload arrives untouched. Reading only the GA shape would label one + client's audio 24kHz and an identical client's 16kHz over a header that says nothing + about sample rates. A change here only affects audio that arrives after it: already-buffered audio was converted to seconds at the rate in force when it was appended, so a mid-stream redeclaration cannot retroactively reprice it. """ - audio = session_payload.get("audio") - if not isinstance(audio, dict): - return - audio_input = audio.get("input") - if not isinstance(audio_input, dict): - return - audio_format = audio_input.get("format") - if not isinstance(audio_format, dict): - return - rate = audio_format.get("rate") + rate = self._declared_rate_from_ga_shape(session_payload) + if rate is None: + rate = self._declared_rate_from_beta_shape(session_payload) # bool is an int subclass, so exclude it explicitly. if isinstance(rate, bool) or not isinstance(rate, int): return diff --git a/tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py b/tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py index fac04aca0de..07610b3d6de 100644 --- a/tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py +++ b/tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py @@ -2013,9 +2013,7 @@ def _session_update_message(session: dict) -> str: def _sent_audio_mime_type(config, raw_byte_count: int = 32000) -> str: """Round-trips one input_audio_buffer.append and returns the mimeType actually put on the wire.""" - sent = config.transform_realtime_request( - _input_audio_append_message(raw_byte_count), "gemini-3.5-transcribe-live" - ) + sent = config.transform_realtime_request(_input_audio_append_message(raw_byte_count), "gemini-3.5-transcribe-live") return json.loads(sent[0])["realtimeInput"]["audio"]["mimeType"] @@ -2070,14 +2068,15 @@ def test_client_declared_input_audio_rate_is_honored_on_the_wire(): {"audio": {"input": {"format": {"type": "audio/pcm", "rate": 100_000_000}}}}, {"audio": {"input": {"format": {"type": "audio/pcm", "rate": "24000"}}}}, {"audio": {"input": {"format": {"type": "audio/pcm", "rate": True}}}}, - {"input_audio_format": "pcm16"}, + {"input_audio_format": "g711_ulaw"}, ], ) def test_malformed_or_absent_declared_rate_keeps_the_native_default(session): - """Anything that is not a plausible PCM rate, including the rate-less beta shape, a bool (which - is an int subclass), and rates outside 8000-48000, must leave the 16kHz default alone. The - out-of-range cases matter because the rate feeds the spend estimate: an unclamped 100MHz - declaration would bill a long session as a few milliseconds.""" + """Anything that is not a plausible PCM rate, including a bool (which is an int subclass) and + rates outside 8000-48000, must leave the 16kHz default alone. The out-of-range cases matter + because the rate feeds the spend estimate: an unclamped 100MHz declaration would bill a long + session as a few milliseconds. A beta codec name other than pcm16 is also left alone, because + ``get_audio_mime_type`` labels every append pcm16 regardless.""" from typing import Final config: Final = GeminiRealtimeConfig() @@ -2085,6 +2084,60 @@ def test_malformed_or_absent_declared_rate_keeps_the_native_default(session): assert _sent_audio_mime_type(config) == "audio/pcm;rate=16000" +def test_beta_input_audio_format_declares_its_specified_24khz_rate(): + """The beta shape has no rate field, but pcm16 is specified as 24kHz, so a client that sends + the flat codec name has declared 24kHz audio and the MIME label has to say so.""" + from typing import Final + + config: Final = GeminiRealtimeConfig() + config.transform_realtime_request( + _session_update_message({"input_audio_format": "pcm16"}), "gemini-3.5-transcribe-live" + ) + assert _sent_audio_mime_type(config) == "audio/pcm;rate=24000" + + +def test_beta_session_reaches_the_same_rate_through_the_ga_remap(): + """The proxy only forwards the flat beta shape untouched when the client sent the OpenAI-Beta + header. Without it, RealTimeStreaming rewrites the payload into the GA shape first. Driving the + real converter rather than hand-building the GA dict is what makes this able to fail: both + routes must land on the same rate, or an identical audio stream gets labelled 16kHz or 24kHz + depending on a header that says nothing about sample rates.""" + from typing import Final + + from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming + + remapped: Final = RealTimeStreaming._remap_beta_session_to_ga({"input_audio_format": "pcm16"}) + assert remapped["audio"]["input"]["format"] == {"type": "audio/pcm", "rate": 24000} + + via_remap: Final = GeminiRealtimeConfig() + via_remap.transform_realtime_request(_session_update_message(remapped), "gemini-3.5-transcribe-live") + + passthrough: Final = GeminiRealtimeConfig() + passthrough.transform_realtime_request( + _session_update_message({"input_audio_format": "pcm16"}), "gemini-3.5-transcribe-live" + ) + + assert _sent_audio_mime_type(via_remap) == _sent_audio_mime_type(passthrough) == "audio/pcm;rate=24000" + + +def test_ga_declared_rate_wins_over_the_beta_codec_name(): + """A session carrying both shapes has stated a rate outright; the name-implied one is a + fallback for when it has not.""" + from typing import Final + + config: Final = GeminiRealtimeConfig() + config.transform_realtime_request( + _session_update_message( + { + "input_audio_format": "pcm16", + "audio": {"input": {"format": {"type": "audio/pcm", "rate": 16000}}}, + } + ), + "gemini-3.5-transcribe-live", + ) + assert _sent_audio_mime_type(config) == "audio/pcm;rate=16000" + + def test_declared_rate_also_drives_the_billed_audio_duration(patch_gemini_transcribe_live_cost_map_entry): """The MIME label and the duration estimate read the same rate, so a client that declares 24kHz is billed for 24kHz audio: 96000 pcm16 bytes = 2s -> 50 in / 6 out."""