From ac1057f06f47b2c41765886c53ca312a0e52327d Mon Sep 17 00:00:00 2001 From: Jeremy Schoemaker Date: Thu, 10 Sep 2026 15:51:49 -0500 Subject: [PATCH 1/4] fix(gemini/vertex realtime): declare the real input audio sample rate Fixes #40563. Both GeminiRealtimeConfig and VertexAIRealtimeConfig hardcoded the pcm16 input MIME type to audio/pcm;rate=24000. 24kHz is the Live API's *output* rate. Its documented native *input* rate is 16kHz, and the MIME rate is the only channel the caller has for telling the server what it is actually sending: "Audio output always uses a sample rate of 24kHz. Input audio is natively 16kHz ... To convey the sample rate of input audio, set the MIME type of each audio-containing Blob to a value like audio/pcm;rate=16000." https://ai.google.dev/gemini-api/docs/live-api/capabilities Because the server resamples against whatever the MIME type claims, a client streaming correct 16kHz PCM16 had it relabeled as 24kHz, which corrupts it server-side and degrades transcription with no error anywhere. Three changes: 1. The rate now comes from what the client declared. session.update carries it in the GA shape at audio.input.format.rate, so that value is recorded and used for every subsequent blob. The rate-less beta shape (input_audio_format is a bare codec name), a missing or malformed rate, and a bool (an int subclass, so excluded explicitly) all leave the default alone. 2. That default is now 16000, the documented native input rate, instead of the output rate. 3. VertexAIRealtimeConfig's byte-identical copy of get_audio_mime_type is deleted so it inherits the parent. The duplicate is why patching the parent alone had no effect on the Vertex path, which is the trap the report calls out; a test now asserts the override stays gone. The billed audio duration reads the same rate, so the label and the duration estimate cannot disagree. PCM16_INPUT_AUDIO_BYTES_PER_SECOND (48000, that is 24kHz x 2 bytes) is replaced by the declared rate x PCM16_BYTES_PER_SAMPLE. This does move the estimate for a transcribe-live caller who declares no rate: the same byte count is now billed as 1.5x the duration, because 16kHz audio takes 1.5x as long to send as the 24kHz the old constant assumed. The two existing estimate tests are updated for that, and a new test covers a caller that declares 24kHz and still bills at the old numbers. 14 tests added or updated, each verified to fail against unpatched sources. tests/test_litellm/llms/{gemini,vertex_ai}/realtime: 97 passed. --- .../llms/gemini/realtime/transformation.py | 44 ++++++- .../llms/vertex_ai/realtime/transformation.py | 13 -- .../test_gemini_realtime_transformation.py | 113 ++++++++++++++++-- 3 files changed, 144 insertions(+), 26 deletions(-) diff --git a/litellm/llms/gemini/realtime/transformation.py b/litellm/llms/gemini/realtime/transformation.py index c92af7de145..077da2517e5 100644 --- a/litellm/llms/gemini/realtime/transformation.py +++ b/litellm/llms/gemini/realtime/transformation.py @@ -119,7 +119,15 @@ def _parse_setup(session_configuration_request: str) -> BidiGenerateContentSetup # 175 text tokens/min of output (ai.google.dev/gemini-api/docs/pricing). GEMINI_LIVE_TRANSCRIBE_AUDIO_TOKENS_PER_SECOND: Final = 25 GEMINI_LIVE_TRANSCRIBE_OUTPUT_TEXT_TOKENS_PER_MINUTE: Final = 175 -PCM16_INPUT_AUDIO_BYTES_PER_SECOND: Final = 48000 + +# Live API input audio is natively 16kHz; 24kHz is the *output* rate. Per +# ai.google.dev/gemini-api/docs/live-api/capabilities: "Audio output always uses a sample +# rate of 24kHz. Input audio is natively 16kHz ... To convey the sample rate of input +# audio, set the MIME type of each audio-containing Blob to a value like +# audio/pcm;rate=16000." The MIME rate is what the server resamples against, so declaring +# the output rate on the input path mislabels correctly-encoded audio. +GEMINI_LIVE_INPUT_AUDIO_SAMPLE_RATE_HZ: Final = 16000 +PCM16_BYTES_PER_SAMPLE: Final = 2 def _base64_decoded_byte_count(data: str) -> int: @@ -137,6 +145,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): # turns; buffer it here so the next response.done carries the token counts. self._pending_usage_metadata: dict | None = None self._unbilled_input_audio_bytes: int = 0 + # Overwritten from session.update when the client declares a rate; see + # _record_input_audio_sample_rate. + self._input_audio_sample_rate_hz: int = GEMINI_LIVE_INPUT_AUDIO_SAMPLE_RATE_HZ def is_setup_message(self, msg_obj: dict) -> bool: return "setup" in msg_obj @@ -230,7 +241,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): def get_audio_mime_type(self, input_audio_format: str = "pcm16"): mime_types: Final = { - "pcm16": "audio/pcm;rate=24000", + "pcm16": f"audio/pcm;rate={self._input_audio_sample_rate_hz}", "g711_ulaw": "audio/pcmu", "g711_alaw": "audio/pcma", } @@ -450,6 +461,31 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): ) return setup + def _record_input_audio_sample_rate(self, session_payload: Mapping[str, object]) -> None: + """ + Remember the input sample rate the client declared on session.update. + + The rate reaches Gemini only through the per-blob MIME type, and the server resamples + against whatever that MIME type claims, so it has to describe the bytes actually sent. + Only the GA session shape carries a rate (``audio.input.format.rate``); the beta shape's + ``input_audio_format`` is a bare codec name with no rate, and leaves the 16kHz default + in place. + """ + audio = session_payload.get("audio") + if not isinstance(audio, dict): + return + audio_input = audio.get("input") + if not isinstance(audio_input, dict): + return + audio_format = audio_input.get("format") + if not isinstance(audio_format, dict): + return + rate = audio_format.get("rate") + # bool is an int subclass, so exclude it explicitly. + if isinstance(rate, bool) or not isinstance(rate, int) or rate <= 0: + return + self._input_audio_sample_rate_hz = rate + def _handle_session_update( self, json_message: _OpenAIRealtimeClientEvent, @@ -475,6 +511,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): # explicit modality / transcription / turn-detection settings # would be silently dropped because ``map_openai_params`` only # recognises the flat OpenAI-beta key names. + self._record_input_audio_sample_rate(session_payload) session_payload = self._normalize_session_payload_for_mapping(session_payload) new_overrides: Final = self.map_openai_params(optional_params={}, non_default_params=session_payload) @@ -1197,7 +1234,8 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): """Gemini Live sends no usageMetadata for transcribe sessions; estimate billing from streamed audio duration.""" if self._unbilled_input_audio_bytes <= 0 or not self._is_text_only_live_model(model): return None - audio_seconds: Final = self._unbilled_input_audio_bytes / PCM16_INPUT_AUDIO_BYTES_PER_SECOND + bytes_per_second: Final = self._input_audio_sample_rate_hz * PCM16_BYTES_PER_SAMPLE + audio_seconds: Final = self._unbilled_input_audio_bytes / bytes_per_second self._unbilled_input_audio_bytes = 0 audio_tokens: Final = round(audio_seconds * GEMINI_LIVE_TRANSCRIBE_AUDIO_TOKENS_PER_SECOND) output_tokens: Final = round(audio_seconds * GEMINI_LIVE_TRANSCRIBE_OUTPUT_TEXT_TOKENS_PER_MINUTE / 60) diff --git a/litellm/llms/vertex_ai/realtime/transformation.py b/litellm/llms/vertex_ai/realtime/transformation.py index fe59034c27b..0a8e9e52b31 100644 --- a/litellm/llms/vertex_ai/realtime/transformation.py +++ b/litellm/llms/vertex_ai/realtime/transformation.py @@ -87,19 +87,6 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig): headers["x-goog-user-project"] = self._project return headers - # ------------------------------------------------------------------ - # Audio MIME type — Vertex AI needs the sample rate in the MIME string - # ------------------------------------------------------------------ - - def get_audio_mime_type(self, input_audio_format: str = "pcm16") -> str: - mime_types: Final = { - # Gemini Live native audio (OpenAI GA realtime default) is 24kHz PCM. - "pcm16": "audio/pcm;rate=24000", - "g711_ulaw": "audio/pcmu", - "g711_alaw": "audio/pcma", - } - return mime_types.get(input_audio_format, "application/octet-stream") - # ------------------------------------------------------------------ # Session setup message # ------------------------------------------------------------------ diff --git a/tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py b/tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py index 2b3b6343fad..6da217db0fa 100644 --- a/tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py +++ b/tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py @@ -2052,10 +2052,102 @@ def _input_audio_append_message(raw_byte_count: int) -> str: ) +def _session_update_message(session: dict) -> str: + return json.dumps({"type": "session.update", "session": session}) + + +def _sent_audio_mime_type(config, raw_byte_count: int = 32000) -> str: + """Round-trips one input_audio_buffer.append and returns the mimeType actually put on the wire.""" + sent = config.transform_realtime_request( + _input_audio_append_message(raw_byte_count), "gemini-3.5-transcribe-live" + ) + return json.loads(sent[0])["realtimeInput"]["audio"]["mimeType"] + + +def test_input_audio_mime_type_declares_the_native_16khz_input_rate(): + """The Live API resamples against the MIME rate, and its documented native input rate is + 16kHz; 24kHz is the output rate. Declaring the output rate on the input path mislabels + correctly-encoded audio (ai.google.dev/gemini-api/docs/live-api/capabilities).""" + from typing import Final + + config: Final = GeminiRealtimeConfig() + assert config.get_audio_mime_type() == "audio/pcm;rate=16000" + assert _sent_audio_mime_type(config) == "audio/pcm;rate=16000" + + +def test_vertex_realtime_inherits_the_same_input_audio_rate(): + """VertexAIRealtimeConfig used to carry its own copy of get_audio_mime_type, so a fix to the + parent had no effect on the Vertex path. It must resolve through the parent now.""" + from typing import Final + + from litellm.llms.vertex_ai.realtime.transformation import VertexAIRealtimeConfig + + config: Final = VertexAIRealtimeConfig(access_token="t", project="p", location="us-central1") + assert "get_audio_mime_type" not in VertexAIRealtimeConfig.__dict__ + assert config.get_audio_mime_type() == "audio/pcm;rate=16000" + + +def test_client_declared_input_audio_rate_is_honored_on_the_wire(): + """A GA client that declares a non-native input rate must have that rate forwarded, not the + default; the whole point of the MIME rate is to describe the bytes actually sent.""" + from typing import Final + + config: Final = GeminiRealtimeConfig() + config.transform_realtime_request( + _session_update_message({"audio": {"input": {"format": {"type": "audio/pcm", "rate": 24000}}}}), + "gemini-3.5-transcribe-live", + ) + assert _sent_audio_mime_type(config) == "audio/pcm;rate=24000" + + +@pytest.mark.parametrize( + "session", + [ + {}, + {"audio": {}}, + {"audio": {"input": {}}}, + {"audio": {"input": {"format": "audio/pcm"}}}, + {"audio": {"input": {"format": {"type": "audio/pcm"}}}}, + {"audio": {"input": {"format": {"type": "audio/pcm", "rate": 0}}}}, + {"audio": {"input": {"format": {"type": "audio/pcm", "rate": -1}}}}, + {"audio": {"input": {"format": {"type": "audio/pcm", "rate": "24000"}}}}, + {"audio": {"input": {"format": {"type": "audio/pcm", "rate": True}}}}, + {"input_audio_format": "pcm16"}, + ], +) +def test_malformed_or_absent_declared_rate_keeps_the_native_default(session): + """Anything that is not a usable positive integer rate, including the rate-less beta shape and + a bool (which is an int subclass), must leave the 16kHz default alone rather than corrupt it.""" + from typing import Final + + config: Final = GeminiRealtimeConfig() + config.transform_realtime_request(_session_update_message(session), "gemini-3.5-transcribe-live") + assert _sent_audio_mime_type(config) == "audio/pcm;rate=16000" + + +def test_declared_rate_also_drives_the_billed_audio_duration(patch_gemini_transcribe_live_cost_map_entry): + """The MIME label and the duration estimate read the same rate, so a client that declares + 24kHz is billed for 24kHz audio: 96000 pcm16 bytes = 2s -> 50 in / 6 out.""" + from typing import Final + + config: Final = GeminiRealtimeConfig() + config.transform_realtime_request( + _session_update_message({"audio": {"input": {"format": {"type": "audio/pcm", "rate": 24000}}}}), + "gemini-3.5-transcribe-live", + ) + config.transform_realtime_request(_input_audio_append_message(96000), "gemini-3.5-transcribe-live") + + usage: Final = config.unbilled_usage_on_session_close("gemini-3.5-transcribe-live") + assert usage is not None + assert usage["input_tokens"] == 50 + assert usage["output_tokens"] == 6 + + def test_transcribe_live_completed_event_carries_estimated_usage(patch_gemini_transcribe_live_cost_map_entry): """Gemini Live sends no usageMetadata for transcribe sessions, so LiteLLM bills from streamed audio duration at Google's published estimate (25 audio tok/sec in, - 175 text tok/min out): 96000 pcm16 bytes = 2s at 24kHz -> 50 in / 6 out.""" + 175 text tok/min out): 96000 pcm16 bytes = 3s at the Live API's native 16kHz input + rate -> 75 in / 9 out.""" from typing import Final from litellm.types.llms.gemini import BidiGenerateContentServerMessage @@ -2093,10 +2185,10 @@ def test_transcribe_live_completed_event_carries_estimated_usage(patch_gemini_tr assert completed[0]["transcript"] == "ahoy there" expected_usage: Final[RealtimeInputAudioTranscriptionUsage] = { "type": "tokens", - "input_tokens": 50, - "output_tokens": 6, - "total_tokens": 56, - "input_token_details": {"text_tokens": 0, "audio_tokens": 50}, + "input_tokens": 75, + "output_tokens": 9, + "total_tokens": 84, + "input_token_details": {"text_tokens": 0, "audio_tokens": 75}, } assert completed[0]["usage"] == expected_usage @@ -2159,7 +2251,8 @@ def test_non_transcription_live_model_completed_event_has_no_usage(patch_gemini_ def test_unbilled_usage_on_session_close_flushes_trailing_audio(patch_gemini_transcribe_live_cost_map_entry): """Audio appended after the last transcript frame is still unbilled when the session closes; the session-close hook must hand back the estimate exactly once - so the streaming layer can bill it (144000 pcm16 bytes = 3s -> 75 in / 9 out).""" + so the streaming layer can bill it (144000 pcm16 bytes = 4.5s at the native 16kHz + input rate -> 112 in / 13 out; 112.5 rounds to even).""" from typing import Final from litellm.types.realtime import RealtimeInputAudioTranscriptionUsage @@ -2171,10 +2264,10 @@ def test_unbilled_usage_on_session_close_flushes_trailing_audio(patch_gemini_tra expected: Final[RealtimeInputAudioTranscriptionUsage] = { "type": "tokens", - "input_tokens": 75, - "output_tokens": 9, - "total_tokens": 84, - "input_token_details": {"text_tokens": 0, "audio_tokens": 75}, + "input_tokens": 112, + "output_tokens": 13, + "total_tokens": 125, + "input_token_details": {"text_tokens": 0, "audio_tokens": 112}, } assert usage == expected assert config.unbilled_usage_on_session_close("gemini-3.5-transcribe-live") is None From 231ba81961764349706670f269cfd0dd46915be6 Mon Sep 17 00:00:00 2001 From: Jeremy Schoemaker Date: Thu, 10 Sep 2026 16:59:04 -0500 Subject: [PATCH 2/4] fix(gemini realtime): close the two review findings on the declared rate Both from the automated review on #40617, both real. Vertex never recorded the declared rate. VertexAIRealtimeConfig handles session.update in its own transform_realtime_request and returns without reaching the parent's _handle_session_update, so the recording call added in the previous commit never ran on that path and a Vertex client's declaration was silently discarded. Recorded in the Vertex branch too, before the first-setup/subsequent-setup split, matching where the parent records it. A later declaration could reprice audio already sent. The estimate accumulated raw bytes and divided by the rate current at consume time, so a client could stream at 16kHz and then declare 24kHz before the estimate was read, billing two thirds of what it actually sent while the backend processed all of it. Audio is now converted to seconds at append time, at the rate in force when the chunk arrived, so a mid-stream redeclaration cannot reach backwards. Mixed-rate sessions bill each chunk at its own rate. Also bounded the accepted rate to 8000-48000 Hz. The value is client-controlled and now feeds the spend estimate directly; an unclamped declaration of 100 MHz would bill a long session as a few milliseconds. Out-of-range declarations are ignored with a warning and the native default stands. 6 tests added or extended, each verified to fail without these changes. tests/test_litellm/llms/{gemini,vertex_ai}/realtime: 103 passed. --- .../llms/gemini/realtime/transformation.py | 38 +++++++++-- .../llms/vertex_ai/realtime/transformation.py | 4 ++ .../test_gemini_realtime_transformation.py | 67 ++++++++++++++++++- 3 files changed, 100 insertions(+), 9 deletions(-) diff --git a/litellm/llms/gemini/realtime/transformation.py b/litellm/llms/gemini/realtime/transformation.py index 077da2517e5..f10d3ca6642 100644 --- a/litellm/llms/gemini/realtime/transformation.py +++ b/litellm/llms/gemini/realtime/transformation.py @@ -128,6 +128,11 @@ GEMINI_LIVE_TRANSCRIBE_OUTPUT_TEXT_TOKENS_PER_MINUTE: Final = 175 # the output rate on the input path mislabels correctly-encoded audio. GEMINI_LIVE_INPUT_AUDIO_SAMPLE_RATE_HZ: Final = 16000 PCM16_BYTES_PER_SAMPLE: Final = 2 +# The declared rate is client-controlled and feeds the transcription spend estimate, so only accept +# rates that real PCM audio actually uses. Outside this range the declaration is ignored and the +# native default stands, which bounds how far a bogus rate can move a bill. +MIN_ACCEPTED_INPUT_AUDIO_SAMPLE_RATE_HZ: Final = 8000 +MAX_ACCEPTED_INPUT_AUDIO_SAMPLE_RATE_HZ: Final = 48000 def _base64_decoded_byte_count(data: str) -> int: @@ -144,7 +149,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): # Gemini Live sometimes emits usageMetadata in a standalone frame between # turns; buffer it here so the next response.done carries the token counts. self._pending_usage_metadata: dict | None = None - self._unbilled_input_audio_bytes: int = 0 + # Seconds, not bytes: each chunk is converted at the rate declared when it arrived, so a + # later session.update cannot reprice audio the backend has already processed. + self._unbilled_input_audio_seconds: float = 0.0 # Overwritten from session.update when the client declares a rate; see # _record_input_audio_sample_rate. self._input_audio_sample_rate_hz: int = GEMINI_LIVE_INPUT_AUDIO_SAMPLE_RATE_HZ @@ -470,6 +477,10 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): Only the GA session shape carries a rate (``audio.input.format.rate``); the beta shape's ``input_audio_format`` is a bare codec name with no rate, and leaves the 16kHz default in place. + + A change here only affects audio that arrives after it: already-buffered audio was + converted to seconds at the rate in force when it was appended, so a mid-stream + redeclaration cannot retroactively reprice it. """ audio = session_payload.get("audio") if not isinstance(audio, dict): @@ -482,7 +493,19 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): return rate = audio_format.get("rate") # bool is an int subclass, so exclude it explicitly. - if isinstance(rate, bool) or not isinstance(rate, int) or rate <= 0: + if isinstance(rate, bool) or not isinstance(rate, int): + return + if not ( + MIN_ACCEPTED_INPUT_AUDIO_SAMPLE_RATE_HZ <= rate <= MAX_ACCEPTED_INPUT_AUDIO_SAMPLE_RATE_HZ + ): + verbose_logger.warning( + "Gemini Realtime: ignoring declared input audio rate %s, outside the accepted " + "%s-%s Hz range; keeping %s Hz", + rate, + MIN_ACCEPTED_INPUT_AUDIO_SAMPLE_RATE_HZ, + MAX_ACCEPTED_INPUT_AUDIO_SAMPLE_RATE_HZ, + self._input_audio_sample_rate_hz, + ) return self._input_audio_sample_rate_hz = rate @@ -642,7 +665,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): if msg_type == "input_audio_buffer.append": audio_b64: Final = json_message["audio"] if isinstance(audio_b64, str): - self._unbilled_input_audio_bytes += _base64_decoded_byte_count(audio_b64) + self._unbilled_input_audio_seconds += _base64_decoded_byte_count(audio_b64) / ( + self._input_audio_sample_rate_hz * PCM16_BYTES_PER_SAMPLE + ) realtime_input_dict["audio"] = HttpxBlobType(mimeType=self.get_audio_mime_type(), data=audio_b64) realtime_input_dict = cast( @@ -1232,11 +1257,10 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): def _consume_input_transcription_usage_estimate(self, model: str) -> RealtimeInputAudioTranscriptionUsage | None: """Gemini Live sends no usageMetadata for transcribe sessions; estimate billing from streamed audio duration.""" - if self._unbilled_input_audio_bytes <= 0 or not self._is_text_only_live_model(model): + if self._unbilled_input_audio_seconds <= 0 or not self._is_text_only_live_model(model): return None - bytes_per_second: Final = self._input_audio_sample_rate_hz * PCM16_BYTES_PER_SAMPLE - audio_seconds: Final = self._unbilled_input_audio_bytes / bytes_per_second - self._unbilled_input_audio_bytes = 0 + audio_seconds: Final = self._unbilled_input_audio_seconds + self._unbilled_input_audio_seconds = 0.0 audio_tokens: Final = round(audio_seconds * GEMINI_LIVE_TRANSCRIBE_AUDIO_TOKENS_PER_SECOND) output_tokens: Final = round(audio_seconds * GEMINI_LIVE_TRANSCRIBE_OUTPUT_TEXT_TOKENS_PER_MINUTE / 60) usage: Final[RealtimeInputAudioTranscriptionUsage] = { diff --git a/litellm/llms/vertex_ai/realtime/transformation.py b/litellm/llms/vertex_ai/realtime/transformation.py index 0a8e9e52b31..9e4b15ac375 100644 --- a/litellm/llms/vertex_ai/realtime/transformation.py +++ b/litellm/llms/vertex_ai/realtime/transformation.py @@ -193,6 +193,10 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig): msg_type: Final = json_message.get("type") if msg_type == "session.update": + # Vertex handles session.update itself and never reaches the parent's handler, so the + # declared input audio rate has to be recorded here or the Vertex path silently keeps + # the default no matter what the client declares. + self._record_input_audio_sample_rate(json_message.get("session") or {}) if session_configuration_request is None: setup_config: Final = self._build_vertex_ai_setup_config(model, json_message.get("session") or {}) gemini_setup_msg: Final = json.dumps({"setup": setup_config}) diff --git a/tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py b/tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py index 6da217db0fa..e92b3b23c07 100644 --- a/tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py +++ b/tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py @@ -2110,14 +2110,19 @@ def test_client_declared_input_audio_rate_is_honored_on_the_wire(): {"audio": {"input": {"format": {"type": "audio/pcm"}}}}, {"audio": {"input": {"format": {"type": "audio/pcm", "rate": 0}}}}, {"audio": {"input": {"format": {"type": "audio/pcm", "rate": -1}}}}, + {"audio": {"input": {"format": {"type": "audio/pcm", "rate": 7999}}}}, + {"audio": {"input": {"format": {"type": "audio/pcm", "rate": 48001}}}}, + {"audio": {"input": {"format": {"type": "audio/pcm", "rate": 100_000_000}}}}, {"audio": {"input": {"format": {"type": "audio/pcm", "rate": "24000"}}}}, {"audio": {"input": {"format": {"type": "audio/pcm", "rate": True}}}}, {"input_audio_format": "pcm16"}, ], ) def test_malformed_or_absent_declared_rate_keeps_the_native_default(session): - """Anything that is not a usable positive integer rate, including the rate-less beta shape and - a bool (which is an int subclass), must leave the 16kHz default alone rather than corrupt it.""" + """Anything that is not a plausible PCM rate, including the rate-less beta shape, a bool (which + is an int subclass), and rates outside 8000-48000, must leave the 16kHz default alone. The + out-of-range cases matter because the rate feeds the spend estimate: an unclamped 100MHz + declaration would bill a long session as a few milliseconds.""" from typing import Final config: Final = GeminiRealtimeConfig() @@ -2143,6 +2148,64 @@ def test_declared_rate_also_drives_the_billed_audio_duration(patch_gemini_transc assert usage["output_tokens"] == 6 +def test_vertex_records_the_declared_rate_from_its_own_session_update(): + """VertexAIRealtimeConfig handles session.update itself and never calls the parent's handler, so + without recording the rate on that path a Vertex client's declaration is silently discarded.""" + from typing import Final + + from litellm.llms.vertex_ai.realtime.transformation import VertexAIRealtimeConfig + + config: Final = VertexAIRealtimeConfig(access_token="t", project="p", location="us-central1") + config.transform_realtime_request( + _session_update_message({"audio": {"input": {"format": {"type": "audio/pcm", "rate": 24000}}}}), + "gemini-3.5-transcribe-live", + ) + assert _sent_audio_mime_type(config) == "audio/pcm;rate=24000" + + +def test_a_later_rate_declaration_cannot_reprice_already_buffered_audio( + patch_gemini_transcribe_live_cost_map_entry, +): + """Audio is converted to seconds at the rate in force when it arrived. Otherwise a client could + stream at 16kHz and then declare 24kHz before the estimate is consumed, billing 2/3 of what it + actually sent while the backend still processed all of it.""" + from typing import Final + + config: Final = GeminiRealtimeConfig() + # 96000 bytes at the 16kHz default is 3s -> 75 in / 9 out. + config.transform_realtime_request(_input_audio_append_message(96000), "gemini-3.5-transcribe-live") + config.transform_realtime_request( + _session_update_message({"audio": {"input": {"format": {"type": "audio/pcm", "rate": 24000}}}}), + "gemini-3.5-transcribe-live", + ) + + usage: Final = config.unbilled_usage_on_session_close("gemini-3.5-transcribe-live") + assert usage is not None + assert usage["input_tokens"] == 75 + assert usage["output_tokens"] == 9 + + +def test_each_chunk_is_billed_at_the_rate_declared_when_it_arrived( + patch_gemini_transcribe_live_cost_map_entry, +): + """Mixed-rate sessions bill per chunk: 96000 bytes at 16kHz (3s) then 96000 at 24kHz (2s) is 5s + total, not 5s at either single rate.""" + from typing import Final + + config: Final = GeminiRealtimeConfig() + config.transform_realtime_request(_input_audio_append_message(96000), "gemini-3.5-transcribe-live") + config.transform_realtime_request( + _session_update_message({"audio": {"input": {"format": {"type": "audio/pcm", "rate": 24000}}}}), + "gemini-3.5-transcribe-live", + ) + config.transform_realtime_request(_input_audio_append_message(96000), "gemini-3.5-transcribe-live") + + usage: Final = config.unbilled_usage_on_session_close("gemini-3.5-transcribe-live") + assert usage is not None + assert usage["input_tokens"] == 125 # 5s * 25 audio tokens/sec + assert usage["output_tokens"] == 15 # round(5 * 175 / 60) + + def test_transcribe_live_completed_event_carries_estimated_usage(patch_gemini_transcribe_live_cost_map_entry): """Gemini Live sends no usageMetadata for transcribe sessions, so LiteLLM bills from streamed audio duration at Google's published estimate (25 audio tok/sec in, From 274d268e28e915c384d69122da733ba9063d47d7 Mon Sep 17 00:00:00 2001 From: Jeremy Schoemaker Date: Thu, 10 Sep 2026 17:05:17 -0500 Subject: [PATCH 3/4] style(gemini realtime): satisfy ruff format on the rate-range guard CI runs ruff format --check over changed litellm/**/*.py; the multi-line condition on the accepted-rate check collapses to one line under the repo's 88-column config. --- litellm/llms/gemini/realtime/transformation.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/litellm/llms/gemini/realtime/transformation.py b/litellm/llms/gemini/realtime/transformation.py index f10d3ca6642..b7116cfc00a 100644 --- a/litellm/llms/gemini/realtime/transformation.py +++ b/litellm/llms/gemini/realtime/transformation.py @@ -495,9 +495,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): # bool is an int subclass, so exclude it explicitly. if isinstance(rate, bool) or not isinstance(rate, int): return - if not ( - MIN_ACCEPTED_INPUT_AUDIO_SAMPLE_RATE_HZ <= rate <= MAX_ACCEPTED_INPUT_AUDIO_SAMPLE_RATE_HZ - ): + if not (MIN_ACCEPTED_INPUT_AUDIO_SAMPLE_RATE_HZ <= rate <= MAX_ACCEPTED_INPUT_AUDIO_SAMPLE_RATE_HZ): verbose_logger.warning( "Gemini Realtime: ignoring declared input audio rate %s, outside the accepted " "%s-%s Hz range; keeping %s Hz", From 0d0e3b797cba6a42203db0f01b1ca8c27e2c2553 Mon Sep 17 00:00:00 2001 From: Jeremy Schoemaker Date: Fri, 11 Sep 2026 13:12:19 -0500 Subject: [PATCH 4/4] =?UTF-8?q?fix(gemini=20realtime):=20read=20the=20rate?= =?UTF-8?q?=20the=20beta=20codec=20name=20declares=20=F0=9F=8E=9A=EF=B8=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gap in this PR's own fix, found by auditing the other call paths that reach _record_input_audio_sample_rate. The method read the rate only from the GA shape (audio.input.format.rate), on the stated reasoning that "the beta shape's input_audio_format is a bare codec name with no rate". The bare codec name does carry a rate. LiteLLM's own type stub says so: "The format of input audio. Options are pcm16, g711_ulaw, or g711_alaw. For pcm16, input audio must be 16-bit PCM at a 24kHz sample rate, single channel (mono), and little-endian byte order." OpenAIRealtimeSession.input_audio_format, litellm/types/llms/openai.py:1898 And RealTimeStreaming._remap_beta_session_to_ga already expands it to exactly that, mapping "pcm16" to {"type": "audio/pcm", "rate": 24000} via _AUDIO_FORMAT_MAP (realtime_streaming.py:219). That remap runs on every client session.update when the backend is in GA mode (realtime_streaming.py:1501), before the payload reaches this config. So the declared rate for one client payload depended on the OpenAI-Beta header, which says nothing about sample rates: no header -> remap runs -> audio/pcm;rate=24000 header set -> remap skipped -> audio/pcm;rate=16000 Same bytes, two labels, and the server resamples against whichever it gets. The rate is now also read from the flat beta codec name, so both routes agree at 24000 for pcm16. A rate stated outright in the GA shape still wins; the name-implied one is only the fallback. Only pcm16 is mapped, because get_audio_mime_type labels every append as pcm16 regardless, so a rate lifted from a g711 name would describe bytes with a codec they are not in. The existing coverage could not catch this: it built the GA dict by hand and fed it straight to the config, sharing the same assumption as the code under test. The new test drives the real _remap_beta_session_to_ga instead and asserts both routes land on the same rate. Verified against the pre-fix source: assert 'audio/pcm;rate=24000' == 'audio/pcm;rate=16000' 3 tests added, the two new beta ones verified to fail before this change. The beta case is removed from the "keeps the native default" parametrize list, where it asserted the 16kHz the remapped path never produces, and replaced with g711_ulaw. tests/test_litellm/llms/{gemini,vertex_ai}/realtime plus tests/test_litellm/litellm_core_utils/test_realtime_streaming.py: 224 passed. --- .../llms/gemini/realtime/transformation.py | 56 +++++++++++---- .../test_gemini_realtime_transformation.py | 69 ++++++++++++++++--- 2 files changed, 104 insertions(+), 21 deletions(-) diff --git a/litellm/llms/gemini/realtime/transformation.py b/litellm/llms/gemini/realtime/transformation.py index b7116cfc00a..587d9c9b93c 100644 --- a/litellm/llms/gemini/realtime/transformation.py +++ b/litellm/llms/gemini/realtime/transformation.py @@ -133,6 +133,11 @@ PCM16_BYTES_PER_SAMPLE: Final = 2 # native default stands, which bounds how far a bogus rate can move a bill. MIN_ACCEPTED_INPUT_AUDIO_SAMPLE_RATE_HZ: Final = 8000 MAX_ACCEPTED_INPUT_AUDIO_SAMPLE_RATE_HZ: Final = 48000 +# The beta session shape has no rate field, but its ``input_audio_format`` codec name carries one +# by definition. LiteLLM's own type stub for it says pcm16 input "must be 16-bit PCM at a 24kHz +# sample rate" (``OpenAIRealtimeSession.input_audio_format`` in litellm/types/llms/openai.py), and +# the beta-to-GA converter in realtime_streaming.py already expands the name to that rate. +BETA_PCM16_INPUT_AUDIO_SAMPLE_RATE_HZ: Final = 24000 def _base64_decoded_byte_count(data: str) -> int: @@ -468,30 +473,55 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): ) return setup + @staticmethod + def _declared_rate_from_ga_shape(session_payload: Mapping[str, object]) -> object: + """Read ``audio.input.format.rate`` out of the GA nested session shape.""" + audio = session_payload.get("audio") + if not isinstance(audio, dict): + return None + audio_input = audio.get("input") + if not isinstance(audio_input, dict): + return None + audio_format = audio_input.get("format") + if not isinstance(audio_format, dict): + return None + return audio_format.get("rate") + + @staticmethod + def _declared_rate_from_beta_shape(session_payload: Mapping[str, object]) -> object: + """Read the rate implied by the flat beta ``input_audio_format`` codec name. + + Only pcm16 is mapped. ``get_audio_mime_type`` labels every append as pcm16, so a rate + lifted from a g711 name would describe bytes with a codec they are not in. + """ + if session_payload.get("input_audio_format") == "pcm16": + return BETA_PCM16_INPUT_AUDIO_SAMPLE_RATE_HZ + return None + def _record_input_audio_sample_rate(self, session_payload: Mapping[str, object]) -> None: """ Remember the input sample rate the client declared on session.update. The rate reaches Gemini only through the per-blob MIME type, and the server resamples against whatever that MIME type claims, so it has to describe the bytes actually sent. - Only the GA session shape carries a rate (``audio.input.format.rate``); the beta shape's - ``input_audio_format`` is a bare codec name with no rate, and leaves the 16kHz default - in place. + + Both session shapes can declare a rate. The GA shape states it outright in + ``audio.input.format.rate``. The beta shape has no rate field, but its + ``input_audio_format`` codec name implies one, and pcm16 is specified as 24kHz. Both are + read here because which shape reaches this method is decided upstream by the + ``OpenAI-Beta`` header: without it, ``RealTimeStreaming._remap_beta_session_to_ga`` + rewrites the flat payload into the GA shape and supplies that same 24kHz for pcm16; with + it, the flat payload arrives untouched. Reading only the GA shape would label one + client's audio 24kHz and an identical client's 16kHz over a header that says nothing + about sample rates. A change here only affects audio that arrives after it: already-buffered audio was converted to seconds at the rate in force when it was appended, so a mid-stream redeclaration cannot retroactively reprice it. """ - audio = session_payload.get("audio") - if not isinstance(audio, dict): - return - audio_input = audio.get("input") - if not isinstance(audio_input, dict): - return - audio_format = audio_input.get("format") - if not isinstance(audio_format, dict): - return - rate = audio_format.get("rate") + rate = self._declared_rate_from_ga_shape(session_payload) + if rate is None: + rate = self._declared_rate_from_beta_shape(session_payload) # bool is an int subclass, so exclude it explicitly. if isinstance(rate, bool) or not isinstance(rate, int): return diff --git a/tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py b/tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py index e92b3b23c07..ac3314abd95 100644 --- a/tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py +++ b/tests/test_litellm/llms/gemini/realtime/test_gemini_realtime_transformation.py @@ -2058,9 +2058,7 @@ def _session_update_message(session: dict) -> str: def _sent_audio_mime_type(config, raw_byte_count: int = 32000) -> str: """Round-trips one input_audio_buffer.append and returns the mimeType actually put on the wire.""" - sent = config.transform_realtime_request( - _input_audio_append_message(raw_byte_count), "gemini-3.5-transcribe-live" - ) + sent = config.transform_realtime_request(_input_audio_append_message(raw_byte_count), "gemini-3.5-transcribe-live") return json.loads(sent[0])["realtimeInput"]["audio"]["mimeType"] @@ -2115,14 +2113,15 @@ def test_client_declared_input_audio_rate_is_honored_on_the_wire(): {"audio": {"input": {"format": {"type": "audio/pcm", "rate": 100_000_000}}}}, {"audio": {"input": {"format": {"type": "audio/pcm", "rate": "24000"}}}}, {"audio": {"input": {"format": {"type": "audio/pcm", "rate": True}}}}, - {"input_audio_format": "pcm16"}, + {"input_audio_format": "g711_ulaw"}, ], ) def test_malformed_or_absent_declared_rate_keeps_the_native_default(session): - """Anything that is not a plausible PCM rate, including the rate-less beta shape, a bool (which - is an int subclass), and rates outside 8000-48000, must leave the 16kHz default alone. The - out-of-range cases matter because the rate feeds the spend estimate: an unclamped 100MHz - declaration would bill a long session as a few milliseconds.""" + """Anything that is not a plausible PCM rate, including a bool (which is an int subclass) and + rates outside 8000-48000, must leave the 16kHz default alone. The out-of-range cases matter + because the rate feeds the spend estimate: an unclamped 100MHz declaration would bill a long + session as a few milliseconds. A beta codec name other than pcm16 is also left alone, because + ``get_audio_mime_type`` labels every append pcm16 regardless.""" from typing import Final config: Final = GeminiRealtimeConfig() @@ -2130,6 +2129,60 @@ def test_malformed_or_absent_declared_rate_keeps_the_native_default(session): assert _sent_audio_mime_type(config) == "audio/pcm;rate=16000" +def test_beta_input_audio_format_declares_its_specified_24khz_rate(): + """The beta shape has no rate field, but pcm16 is specified as 24kHz, so a client that sends + the flat codec name has declared 24kHz audio and the MIME label has to say so.""" + from typing import Final + + config: Final = GeminiRealtimeConfig() + config.transform_realtime_request( + _session_update_message({"input_audio_format": "pcm16"}), "gemini-3.5-transcribe-live" + ) + assert _sent_audio_mime_type(config) == "audio/pcm;rate=24000" + + +def test_beta_session_reaches_the_same_rate_through_the_ga_remap(): + """The proxy only forwards the flat beta shape untouched when the client sent the OpenAI-Beta + header. Without it, RealTimeStreaming rewrites the payload into the GA shape first. Driving the + real converter rather than hand-building the GA dict is what makes this able to fail: both + routes must land on the same rate, or an identical audio stream gets labelled 16kHz or 24kHz + depending on a header that says nothing about sample rates.""" + from typing import Final + + from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming + + remapped: Final = RealTimeStreaming._remap_beta_session_to_ga({"input_audio_format": "pcm16"}) + assert remapped["audio"]["input"]["format"] == {"type": "audio/pcm", "rate": 24000} + + via_remap: Final = GeminiRealtimeConfig() + via_remap.transform_realtime_request(_session_update_message(remapped), "gemini-3.5-transcribe-live") + + passthrough: Final = GeminiRealtimeConfig() + passthrough.transform_realtime_request( + _session_update_message({"input_audio_format": "pcm16"}), "gemini-3.5-transcribe-live" + ) + + assert _sent_audio_mime_type(via_remap) == _sent_audio_mime_type(passthrough) == "audio/pcm;rate=24000" + + +def test_ga_declared_rate_wins_over_the_beta_codec_name(): + """A session carrying both shapes has stated a rate outright; the name-implied one is a + fallback for when it has not.""" + from typing import Final + + config: Final = GeminiRealtimeConfig() + config.transform_realtime_request( + _session_update_message( + { + "input_audio_format": "pcm16", + "audio": {"input": {"format": {"type": "audio/pcm", "rate": 16000}}}, + } + ), + "gemini-3.5-transcribe-live", + ) + assert _sent_audio_mime_type(config) == "audio/pcm;rate=16000" + + def test_declared_rate_also_drives_the_billed_audio_duration(patch_gemini_transcribe_live_cost_map_entry): """The MIME label and the duration estimate read the same rate, so a client that declares 24kHz is billed for 24kHz audio: 96000 pcm16 bytes = 2s -> 50 in / 6 out."""