scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-28 01:32:17 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 95

fix(gemini realtime): close the two review findings on the declared rate

Browse source 
Both from the automated review on #40617, both real.

Vertex never recorded the declared rate. VertexAIRealtimeConfig handles
session.update in its own transform_realtime_request and returns without
reaching the parent's _handle_session_update, so the recording call added in the
previous commit never ran on that path and a Vertex client's declaration was
silently discarded. Recorded in the Vertex branch too, before the
first-setup/subsequent-setup split, matching where the parent records it.

A later declaration could reprice audio already sent. The estimate accumulated
raw bytes and divided by the rate current at consume time, so a client could
stream at 16kHz and then declare 24kHz before the estimate was read, billing two
thirds of what it actually sent while the backend processed all of it. Audio is
now converted to seconds at append time, at the rate in force when the chunk
arrived, so a mid-stream redeclaration cannot reach backwards. Mixed-rate
sessions bill each chunk at its own rate.

Also bounded the accepted rate to 8000-48000 Hz. The value is client-controlled
and now feeds the spend estimate directly; an unclamped declaration of 100 MHz
would bill a long session as a few milliseconds. Out-of-range declarations are
ignored with a warning and the native default stands.

6 tests added or extended, each verified to fail without these changes.
tests/test_litellm/llms/{gemini,vertex_ai}/realtime: 103 passed.
This commit is contained in:
Jeremy Schoemaker 2026-09-10 16:59:04 -05:00
parent 39372b3922
commit 79577e1288
No known key found for this signature in database
3 changed files with 100 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

38
litellm/llms/gemini/realtime/transformation.py
View file

@ -141,6 +141,11 @@ GEMINI_LIVE_TRANSCRIBE_OUTPUT_TEXT_TOKENS_PER_MINUTE: Final = 175
# the output rate on the input path mislabels correctly-encoded audio.
GEMINI_LIVE_INPUT_AUDIO_SAMPLE_RATE_HZ: Final = 16000
PCM16_BYTES_PER_SAMPLE: Final = 2
# The declared rate is client-controlled and feeds the transcription spend estimate, so only accept
# rates that real PCM audio actually uses. Outside this range the declaration is ignored and the
# native default stands, which bounds how far a bogus rate can move a bill.
MIN_ACCEPTED_INPUT_AUDIO_SAMPLE_RATE_HZ: Final = 8000
MAX_ACCEPTED_INPUT_AUDIO_SAMPLE_RATE_HZ: Final = 48000
def _base64_decoded_byte_count(data: str) -> int:
@ -157,7 +162,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
# Gemini Live sometimes emits usageMetadata in a standalone frame between
# turns; buffer it here so the next response.done carries the token counts.
self._pending_usage_metadata: dict | None = None
self._unbilled_input_audio_bytes: int = 0
# Seconds, not bytes: each chunk is converted at the rate declared when it arrived, so a
# later session.update cannot reprice audio the backend has already processed.
self._unbilled_input_audio_seconds: float = 0.0
# Overwritten from session.update when the client declares a rate; see
# _record_input_audio_sample_rate.
self._input_audio_sample_rate_hz: int = GEMINI_LIVE_INPUT_AUDIO_SAMPLE_RATE_HZ
@ -483,6 +490,10 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
Only the GA session shape carries a rate (``audio.input.format.rate``); the beta shape's
``input_audio_format`` is a bare codec name with no rate, and leaves the 16kHz default
in place.
A change here only affects audio that arrives after it: already-buffered audio was
converted to seconds at the rate in force when it was appended, so a mid-stream
redeclaration cannot retroactively reprice it.
"""
audio = session_payload.get("audio")
if not isinstance(audio, dict):
@ -495,7 +506,19 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
return
rate = audio_format.get("rate")
# bool is an int subclass, so exclude it explicitly.
if isinstance(rate, bool) or not isinstance(rate, int) or rate <= 0:
if isinstance(rate, bool) or not isinstance(rate, int):
return
if not (
MIN_ACCEPTED_INPUT_AUDIO_SAMPLE_RATE_HZ <= rate <= MAX_ACCEPTED_INPUT_AUDIO_SAMPLE_RATE_HZ
):
verbose_logger.warning(
"Gemini Realtime: ignoring declared input audio rate %s, outside the accepted "
"%s-%s Hz range; keeping %s Hz",
rate,
MIN_ACCEPTED_INPUT_AUDIO_SAMPLE_RATE_HZ,
MAX_ACCEPTED_INPUT_AUDIO_SAMPLE_RATE_HZ,
self._input_audio_sample_rate_hz,
)
return
self._input_audio_sample_rate_hz = rate
@ -655,7 +678,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
if msg_type == "input_audio_buffer.append":
audio_b64: Final = json_message["audio"]
if isinstance(audio_b64, str):
self._unbilled_input_audio_bytes += _base64_decoded_byte_count(audio_b64)
self._unbilled_input_audio_seconds += _base64_decoded_byte_count(audio_b64) / (
self._input_audio_sample_rate_hz * PCM16_BYTES_PER_SAMPLE
)
realtime_input_dict["audio"] = HttpxBlobType(mimeType=self.get_audio_mime_type(), data=audio_b64)
realtime_input_dict = cast(
@ -1250,11 +1275,10 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
def _consume_input_transcription_usage_estimate(self, model: str) -> RealtimeInputAudioTranscriptionUsage | None:
"""Gemini Live sends no usageMetadata for transcribe sessions; estimate billing from streamed audio duration."""
if self._unbilled_input_audio_bytes <= 0 or not self._is_text_only_live_model(model):
if self._unbilled_input_audio_seconds <= 0 or not self._is_text_only_live_model(model):
return None
bytes_per_second: Final = self._input_audio_sample_rate_hz * PCM16_BYTES_PER_SAMPLE
audio_seconds: Final = self._unbilled_input_audio_bytes / bytes_per_second
self._unbilled_input_audio_bytes = 0
audio_seconds: Final = self._unbilled_input_audio_seconds
self._unbilled_input_audio_seconds = 0.0
audio_tokens: Final = round(audio_seconds * GEMINI_LIVE_TRANSCRIBE_AUDIO_TOKENS_PER_SECOND)
output_tokens: Final = round(audio_seconds * GEMINI_LIVE_TRANSCRIBE_OUTPUT_TEXT_TOKENS_PER_MINUTE / 60)
usage: Final[RealtimeInputAudioTranscriptionUsage] = {

4
litellm/llms/vertex_ai/realtime/transformation.py
View file

@ -199,6 +199,10 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig):
msg_type: Final = json_message.get("type")
if msg_type == "session.update":
# Vertex handles session.update itself and never reaches the parent's handler, so the
# declared input audio rate has to be recorded here or the Vertex path silently keeps
# the default no matter what the client declares.
self._record_input_audio_sample_rate(json_message.get("session") or {})
if session_configuration_request is None:
setup_config: Final = self._build_vertex_ai_setup_config(model, json_message.get("session") or {})
gemini_setup_msg: Final = json.dumps({"setup": setup_config})

67
tests/unit/llms/gemini/realtime/test_gemini_realtime_transformation.py
View file

@ -2065,14 +2065,19 @@ def test_client_declared_input_audio_rate_is_honored_on_the_wire():
{"audio": {"input": {"format": {"type": "audio/pcm"}}}},
{"audio": {"input": {"format": {"type": "audio/pcm", "rate": 0}}}},
{"audio": {"input": {"format": {"type": "audio/pcm", "rate": -1}}}},
{"audio": {"input": {"format": {"type": "audio/pcm", "rate": 7999}}}},
{"audio": {"input": {"format": {"type": "audio/pcm", "rate": 48001}}}},
{"audio": {"input": {"format": {"type": "audio/pcm", "rate": 100_000_000}}}},
{"audio": {"input": {"format": {"type": "audio/pcm", "rate": "24000"}}}},
{"audio": {"input": {"format": {"type": "audio/pcm", "rate": True}}}},
{"input_audio_format": "pcm16"},
],
)
def test_malformed_or_absent_declared_rate_keeps_the_native_default(session):
"""Anything that is not a usable positive integer rate, including the rate-less beta shape and
a bool (which is an int subclass), must leave the 16kHz default alone rather than corrupt it."""
"""Anything that is not a plausible PCM rate, including the rate-less beta shape, a bool (which
is an int subclass), and rates outside 8000-48000, must leave the 16kHz default alone. The
out-of-range cases matter because the rate feeds the spend estimate: an unclamped 100MHz
declaration would bill a long session as a few milliseconds."""
from typing import Final
config: Final = GeminiRealtimeConfig()
@ -2098,6 +2103,64 @@ def test_declared_rate_also_drives_the_billed_audio_duration(patch_gemini_transc
assert usage["output_tokens"] == 6
def test_vertex_records_the_declared_rate_from_its_own_session_update():
"""VertexAIRealtimeConfig handles session.update itself and never calls the parent's handler, so
without recording the rate on that path a Vertex client's declaration is silently discarded."""
from typing import Final
from litellm.llms.vertex_ai.realtime.transformation import VertexAIRealtimeConfig
config: Final = VertexAIRealtimeConfig(access_token="t", project="p", location="us-central1")
config.transform_realtime_request(
_session_update_message({"audio": {"input": {"format": {"type": "audio/pcm", "rate": 24000}}}}),
"gemini-3.5-transcribe-live",
)
assert _sent_audio_mime_type(config) == "audio/pcm;rate=24000"
def test_a_later_rate_declaration_cannot_reprice_already_buffered_audio(
patch_gemini_transcribe_live_cost_map_entry,
):
"""Audio is converted to seconds at the rate in force when it arrived. Otherwise a client could
stream at 16kHz and then declare 24kHz before the estimate is consumed, billing 2/3 of what it
actually sent while the backend still processed all of it."""
from typing import Final
config: Final = GeminiRealtimeConfig()
# 96000 bytes at the 16kHz default is 3s -> 75 in / 9 out.
config.transform_realtime_request(_input_audio_append_message(96000), "gemini-3.5-transcribe-live")
config.transform_realtime_request(
_session_update_message({"audio": {"input": {"format": {"type": "audio/pcm", "rate": 24000}}}}),
"gemini-3.5-transcribe-live",
)
usage: Final = config.unbilled_usage_on_session_close("gemini-3.5-transcribe-live")
assert usage is not None
assert usage["input_tokens"] == 75
assert usage["output_tokens"] == 9
def test_each_chunk_is_billed_at_the_rate_declared_when_it_arrived(
patch_gemini_transcribe_live_cost_map_entry,
):
"""Mixed-rate sessions bill per chunk: 96000 bytes at 16kHz (3s) then 96000 at 24kHz (2s) is 5s
total, not 5s at either single rate."""
from typing import Final
config: Final = GeminiRealtimeConfig()
config.transform_realtime_request(_input_audio_append_message(96000), "gemini-3.5-transcribe-live")
config.transform_realtime_request(
_session_update_message({"audio": {"input": {"format": {"type": "audio/pcm", "rate": 24000}}}}),
"gemini-3.5-transcribe-live",
)
config.transform_realtime_request(_input_audio_append_message(96000), "gemini-3.5-transcribe-live")
usage: Final = config.unbilled_usage_on_session_close("gemini-3.5-transcribe-live")
assert usage is not None
assert usage["input_tokens"] == 125 # 5s * 25 audio tokens/sec
assert usage["output_tokens"] == 15 # round(5 * 175 / 60)
def test_transcribe_live_completed_event_carries_estimated_usage(patch_gemini_transcribe_live_cost_map_entry):
"""Gemini Live sends no usageMetadata for transcribe sessions, so LiteLLM bills
from streamed audio duration at Google's published estimate (25 audio tok/sec in,