diff --git a/litellm/llms/gemini/realtime/transformation.py b/litellm/llms/gemini/realtime/transformation.py index a305f5927f4..07d201a0933 100644 --- a/litellm/llms/gemini/realtime/transformation.py +++ b/litellm/llms/gemini/realtime/transformation.py @@ -141,6 +141,11 @@ GEMINI_LIVE_TRANSCRIBE_OUTPUT_TEXT_TOKENS_PER_MINUTE: Final = 175 # the output rate on the input path mislabels correctly-encoded audio. GEMINI_LIVE_INPUT_AUDIO_SAMPLE_RATE_HZ: Final = 16000 PCM16_BYTES_PER_SAMPLE: Final = 2 +# The declared rate is client-controlled and feeds the transcription spend estimate, so only accept +# rates that real PCM audio actually uses. Outside this range the declaration is ignored and the +# native default stands, which bounds how far a bogus rate can move a bill. +MIN_ACCEPTED_INPUT_AUDIO_SAMPLE_RATE_HZ: Final = 8000 +MAX_ACCEPTED_INPUT_AUDIO_SAMPLE_RATE_HZ: Final = 48000 def _base64_decoded_byte_count(data: str) -> int: @@ -157,7 +162,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): # Gemini Live sometimes emits usageMetadata in a standalone frame between # turns; buffer it here so the next response.done carries the token counts. self._pending_usage_metadata: dict | None = None - self._unbilled_input_audio_bytes: int = 0 + # Seconds, not bytes: each chunk is converted at the rate declared when it arrived, so a + # later session.update cannot reprice audio the backend has already processed. + self._unbilled_input_audio_seconds: float = 0.0 # Overwritten from session.update when the client declares a rate; see # _record_input_audio_sample_rate. self._input_audio_sample_rate_hz: int = GEMINI_LIVE_INPUT_AUDIO_SAMPLE_RATE_HZ @@ -483,6 +490,10 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): Only the GA session shape carries a rate (``audio.input.format.rate``); the beta shape's ``input_audio_format`` is a bare codec name with no rate, and leaves the 16kHz default in place. + + A change here only affects audio that arrives after it: already-buffered audio was + converted to seconds at the rate in force when it was appended, so a mid-stream + redeclaration cannot retroactively reprice it. """ audio = session_payload.get("audio") if not isinstance(audio, dict): @@ -495,7 +506,19 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): return rate = audio_format.get("rate") # bool is an int subclass, so exclude it explicitly. - if isinstance(rate, bool) or not isinstance(rate, int) or rate <= 0: + if isinstance(rate, bool) or not isinstance(rate, int): + return + if not ( + MIN_ACCEPTED_INPUT_AUDIO_SAMPLE_RATE_HZ <= rate <= MAX_ACCEPTED_INPUT_AUDIO_SAMPLE_RATE_HZ + ): + verbose_logger.warning( + "Gemini Realtime: ignoring declared input audio rate %s, outside the accepted " + "%s-%s Hz range; keeping %s Hz", + rate, + MIN_ACCEPTED_INPUT_AUDIO_SAMPLE_RATE_HZ, + MAX_ACCEPTED_INPUT_AUDIO_SAMPLE_RATE_HZ, + self._input_audio_sample_rate_hz, + ) return self._input_audio_sample_rate_hz = rate @@ -655,7 +678,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): if msg_type == "input_audio_buffer.append": audio_b64: Final = json_message["audio"] if isinstance(audio_b64, str): - self._unbilled_input_audio_bytes += _base64_decoded_byte_count(audio_b64) + self._unbilled_input_audio_seconds += _base64_decoded_byte_count(audio_b64) / ( + self._input_audio_sample_rate_hz * PCM16_BYTES_PER_SAMPLE + ) realtime_input_dict["audio"] = HttpxBlobType(mimeType=self.get_audio_mime_type(), data=audio_b64) realtime_input_dict = cast( @@ -1250,11 +1275,10 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): def _consume_input_transcription_usage_estimate(self, model: str) -> RealtimeInputAudioTranscriptionUsage | None: """Gemini Live sends no usageMetadata for transcribe sessions; estimate billing from streamed audio duration.""" - if self._unbilled_input_audio_bytes <= 0 or not self._is_text_only_live_model(model): + if self._unbilled_input_audio_seconds <= 0 or not self._is_text_only_live_model(model): return None - bytes_per_second: Final = self._input_audio_sample_rate_hz * PCM16_BYTES_PER_SAMPLE - audio_seconds: Final = self._unbilled_input_audio_bytes / bytes_per_second - self._unbilled_input_audio_bytes = 0 + audio_seconds: Final = self._unbilled_input_audio_seconds + self._unbilled_input_audio_seconds = 0.0 audio_tokens: Final = round(audio_seconds * GEMINI_LIVE_TRANSCRIBE_AUDIO_TOKENS_PER_SECOND) output_tokens: Final = round(audio_seconds * GEMINI_LIVE_TRANSCRIBE_OUTPUT_TEXT_TOKENS_PER_MINUTE / 60) usage: Final[RealtimeInputAudioTranscriptionUsage] = { diff --git a/litellm/llms/vertex_ai/realtime/transformation.py b/litellm/llms/vertex_ai/realtime/transformation.py index d52d7e7ad3a..72aed733c1e 100644 --- a/litellm/llms/vertex_ai/realtime/transformation.py +++ b/litellm/llms/vertex_ai/realtime/transformation.py @@ -199,6 +199,10 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig): msg_type: Final = json_message.get("type") if msg_type == "session.update": + # Vertex handles session.update itself and never reaches the parent's handler, so the + # declared input audio rate has to be recorded here or the Vertex path silently keeps + # the default no matter what the client declares. + self._record_input_audio_sample_rate(json_message.get("session") or {}) if session_configuration_request is None: setup_config: Final = self._build_vertex_ai_setup_config(model, json_message.get("session") or {}) gemini_setup_msg: Final = json.dumps({"setup": setup_config}) diff --git a/tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py b/tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py index 6ef4ae78bed..fac04aca0de 100644 --- a/tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py +++ b/tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py @@ -2065,14 +2065,19 @@ def test_client_declared_input_audio_rate_is_honored_on_the_wire(): {"audio": {"input": {"format": {"type": "audio/pcm"}}}}, {"audio": {"input": {"format": {"type": "audio/pcm", "rate": 0}}}}, {"audio": {"input": {"format": {"type": "audio/pcm", "rate": -1}}}}, + {"audio": {"input": {"format": {"type": "audio/pcm", "rate": 7999}}}}, + {"audio": {"input": {"format": {"type": "audio/pcm", "rate": 48001}}}}, + {"audio": {"input": {"format": {"type": "audio/pcm", "rate": 100_000_000}}}}, {"audio": {"input": {"format": {"type": "audio/pcm", "rate": "24000"}}}}, {"audio": {"input": {"format": {"type": "audio/pcm", "rate": True}}}}, {"input_audio_format": "pcm16"}, ], ) def test_malformed_or_absent_declared_rate_keeps_the_native_default(session): - """Anything that is not a usable positive integer rate, including the rate-less beta shape and - a bool (which is an int subclass), must leave the 16kHz default alone rather than corrupt it.""" + """Anything that is not a plausible PCM rate, including the rate-less beta shape, a bool (which + is an int subclass), and rates outside 8000-48000, must leave the 16kHz default alone. The + out-of-range cases matter because the rate feeds the spend estimate: an unclamped 100MHz + declaration would bill a long session as a few milliseconds.""" from typing import Final config: Final = GeminiRealtimeConfig() @@ -2098,6 +2103,64 @@ def test_declared_rate_also_drives_the_billed_audio_duration(patch_gemini_transc assert usage["output_tokens"] == 6 +def test_vertex_records_the_declared_rate_from_its_own_session_update(): + """VertexAIRealtimeConfig handles session.update itself and never calls the parent's handler, so + without recording the rate on that path a Vertex client's declaration is silently discarded.""" + from typing import Final + + from litellm.llms.vertex_ai.realtime.transformation import VertexAIRealtimeConfig + + config: Final = VertexAIRealtimeConfig(access_token="t", project="p", location="us-central1") + config.transform_realtime_request( + _session_update_message({"audio": {"input": {"format": {"type": "audio/pcm", "rate": 24000}}}}), + "gemini-3.5-transcribe-live", + ) + assert _sent_audio_mime_type(config) == "audio/pcm;rate=24000" + + +def test_a_later_rate_declaration_cannot_reprice_already_buffered_audio( + patch_gemini_transcribe_live_cost_map_entry, +): + """Audio is converted to seconds at the rate in force when it arrived. Otherwise a client could + stream at 16kHz and then declare 24kHz before the estimate is consumed, billing 2/3 of what it + actually sent while the backend still processed all of it.""" + from typing import Final + + config: Final = GeminiRealtimeConfig() + # 96000 bytes at the 16kHz default is 3s -> 75 in / 9 out. + config.transform_realtime_request(_input_audio_append_message(96000), "gemini-3.5-transcribe-live") + config.transform_realtime_request( + _session_update_message({"audio": {"input": {"format": {"type": "audio/pcm", "rate": 24000}}}}), + "gemini-3.5-transcribe-live", + ) + + usage: Final = config.unbilled_usage_on_session_close("gemini-3.5-transcribe-live") + assert usage is not None + assert usage["input_tokens"] == 75 + assert usage["output_tokens"] == 9 + + +def test_each_chunk_is_billed_at_the_rate_declared_when_it_arrived( + patch_gemini_transcribe_live_cost_map_entry, +): + """Mixed-rate sessions bill per chunk: 96000 bytes at 16kHz (3s) then 96000 at 24kHz (2s) is 5s + total, not 5s at either single rate.""" + from typing import Final + + config: Final = GeminiRealtimeConfig() + config.transform_realtime_request(_input_audio_append_message(96000), "gemini-3.5-transcribe-live") + config.transform_realtime_request( + _session_update_message({"audio": {"input": {"format": {"type": "audio/pcm", "rate": 24000}}}}), + "gemini-3.5-transcribe-live", + ) + config.transform_realtime_request(_input_audio_append_message(96000), "gemini-3.5-transcribe-live") + + usage: Final = config.unbilled_usage_on_session_close("gemini-3.5-transcribe-live") + assert usage is not None + assert usage["input_tokens"] == 125 # 5s * 25 audio tokens/sec + assert usage["output_tokens"] == 15 # round(5 * 175 / 60) + + def test_transcribe_live_completed_event_carries_estimated_usage(patch_gemini_transcribe_live_cost_map_entry): """Gemini Live sends no usageMetadata for transcribe sessions, so LiteLLM bills from streamed audio duration at Google's published estimate (25 audio tok/sec in,