mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-28 01:32:17 +00:00
fix(gemini/vertex realtime): declare the real input audio sample rate
Fixes #40563. Both GeminiRealtimeConfig and VertexAIRealtimeConfig hardcoded the pcm16 input MIME type to audio/pcm;rate=24000. 24kHz is the Live API's *output* rate. Its documented native *input* rate is 16kHz, and the MIME rate is the only channel the caller has for telling the server what it is actually sending: "Audio output always uses a sample rate of 24kHz. Input audio is natively 16kHz ... To convey the sample rate of input audio, set the MIME type of each audio-containing Blob to a value like audio/pcm;rate=16000." https://ai.google.dev/gemini-api/docs/live-api/capabilities Because the server resamples against whatever the MIME type claims, a client streaming correct 16kHz PCM16 had it relabeled as 24kHz, which corrupts it server-side and degrades transcription with no error anywhere. Three changes: 1. The rate now comes from what the client declared. session.update carries it in the GA shape at audio.input.format.rate, so that value is recorded and used for every subsequent blob. The rate-less beta shape (input_audio_format is a bare codec name), a missing or malformed rate, and a bool (an int subclass, so excluded explicitly) all leave the default alone. 2. That default is now 16000, the documented native input rate, instead of the output rate. 3. VertexAIRealtimeConfig's byte-identical copy of get_audio_mime_type is deleted so it inherits the parent. The duplicate is why patching the parent alone had no effect on the Vertex path, which is the trap the report calls out; a test now asserts the override stays gone. The billed audio duration reads the same rate, so the label and the duration estimate cannot disagree. PCM16_INPUT_AUDIO_BYTES_PER_SECOND (48000, that is 24kHz x 2 bytes) is replaced by the declared rate x PCM16_BYTES_PER_SAMPLE. This does move the estimate for a transcribe-live caller who declares no rate: the same byte count is now billed as 1.5x the duration, because 16kHz audio takes 1.5x as long to send as the 24kHz the old constant assumed. The two existing estimate tests are updated for that, and a new test covers a caller that declares 24kHz and still bills at the old numbers. 14 tests added or updated, each verified to fail against unpatched sources. tests/test_litellm/llms/{gemini,vertex_ai}/realtime: 97 passed.
This commit is contained in:
Jeremy Schoemaker
parent
5db8543817
commit
39372b3922
3 changed files with 144 additions and 26 deletions
|
|
@ -132,7 +132,15 @@ def _grounding_metadata_from_frame(frame: Mapping[str, object]) -> tuple[Mapping
|
|||
# 175 text tokens/min of output (ai.google.dev/gemini-api/docs/pricing).
|
||||
GEMINI_LIVE_TRANSCRIBE_AUDIO_TOKENS_PER_SECOND: Final = 25
|
||||
GEMINI_LIVE_TRANSCRIBE_OUTPUT_TEXT_TOKENS_PER_MINUTE: Final = 175
|
||||
PCM16_INPUT_AUDIO_BYTES_PER_SECOND: Final = 48000
|
||||
|
||||
# Live API input audio is natively 16kHz; 24kHz is the *output* rate. Per
|
||||
# ai.google.dev/gemini-api/docs/live-api/capabilities: "Audio output always uses a sample
|
||||
# rate of 24kHz. Input audio is natively 16kHz ... To convey the sample rate of input
|
||||
# audio, set the MIME type of each audio-containing Blob to a value like
|
||||
# audio/pcm;rate=16000." The MIME rate is what the server resamples against, so declaring
|
||||
# the output rate on the input path mislabels correctly-encoded audio.
|
||||
GEMINI_LIVE_INPUT_AUDIO_SAMPLE_RATE_HZ: Final = 16000
|
||||
PCM16_BYTES_PER_SAMPLE: Final = 2
|
||||
|
||||
|
||||
def _base64_decoded_byte_count(data: str) -> int:
|
||||
|
|
@ -150,6 +158,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
# turns; buffer it here so the next response.done carries the token counts.
|
||||
self._pending_usage_metadata: dict | None = None
|
||||
self._unbilled_input_audio_bytes: int = 0
|
||||
# Overwritten from session.update when the client declares a rate; see
|
||||
# _record_input_audio_sample_rate.
|
||||
self._input_audio_sample_rate_hz: int = GEMINI_LIVE_INPUT_AUDIO_SAMPLE_RATE_HZ
|
||||
|
||||
def is_setup_message(self, msg_obj: dict) -> bool:
|
||||
return "setup" in msg_obj
|
||||
|
|
@ -243,7 +254,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
|
||||
def get_audio_mime_type(self, input_audio_format: str = "pcm16"):
|
||||
mime_types: Final = {
|
||||
"pcm16": "audio/pcm;rate=24000",
|
||||
"pcm16": f"audio/pcm;rate={self._input_audio_sample_rate_hz}",
|
||||
"g711_ulaw": "audio/pcmu",
|
||||
"g711_alaw": "audio/pcma",
|
||||
}
|
||||
|
|
@ -463,6 +474,31 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
)
|
||||
return setup
|
||||
|
||||
def _record_input_audio_sample_rate(self, session_payload: Mapping[str, object]) -> None:
|
||||
"""
|
||||
Remember the input sample rate the client declared on session.update.
|
||||
|
||||
The rate reaches Gemini only through the per-blob MIME type, and the server resamples
|
||||
against whatever that MIME type claims, so it has to describe the bytes actually sent.
|
||||
Only the GA session shape carries a rate (``audio.input.format.rate``); the beta shape's
|
||||
``input_audio_format`` is a bare codec name with no rate, and leaves the 16kHz default
|
||||
in place.
|
||||
"""
|
||||
audio = session_payload.get("audio")
|
||||
if not isinstance(audio, dict):
|
||||
return
|
||||
audio_input = audio.get("input")
|
||||
if not isinstance(audio_input, dict):
|
||||
return
|
||||
audio_format = audio_input.get("format")
|
||||
if not isinstance(audio_format, dict):
|
||||
return
|
||||
rate = audio_format.get("rate")
|
||||
# bool is an int subclass, so exclude it explicitly.
|
||||
if isinstance(rate, bool) or not isinstance(rate, int) or rate <= 0:
|
||||
return
|
||||
self._input_audio_sample_rate_hz = rate
|
||||
|
||||
def _handle_session_update(
|
||||
self,
|
||||
json_message: _OpenAIRealtimeClientEvent,
|
||||
|
|
@ -488,6 +524,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
# explicit modality / transcription / turn-detection settings
|
||||
# would be silently dropped because ``map_openai_params`` only
|
||||
# recognises the flat OpenAI-beta key names.
|
||||
self._record_input_audio_sample_rate(session_payload)
|
||||
session_payload = self._normalize_session_payload_for_mapping(session_payload)
|
||||
new_overrides: Final = self.map_openai_params(optional_params={}, non_default_params=session_payload)
|
||||
|
||||
|
|
@ -1215,7 +1252,8 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
|
|||
"""Gemini Live sends no usageMetadata for transcribe sessions; estimate billing from streamed audio duration."""
|
||||
if self._unbilled_input_audio_bytes <= 0 or not self._is_text_only_live_model(model):
|
||||
return None
|
||||
audio_seconds: Final = self._unbilled_input_audio_bytes / PCM16_INPUT_AUDIO_BYTES_PER_SECOND
|
||||
bytes_per_second: Final = self._input_audio_sample_rate_hz * PCM16_BYTES_PER_SAMPLE
|
||||
audio_seconds: Final = self._unbilled_input_audio_bytes / bytes_per_second
|
||||
self._unbilled_input_audio_bytes = 0
|
||||
audio_tokens: Final = round(audio_seconds * GEMINI_LIVE_TRANSCRIBE_AUDIO_TOKENS_PER_SECOND)
|
||||
output_tokens: Final = round(audio_seconds * GEMINI_LIVE_TRANSCRIBE_OUTPUT_TEXT_TOKENS_PER_MINUTE / 60)
|
||||
|
|
|
|||
|
|
@ -93,19 +93,6 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig):
|
|||
headers["x-goog-user-project"] = self._project
|
||||
return headers
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Audio MIME type — Vertex AI needs the sample rate in the MIME string
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def get_audio_mime_type(self, input_audio_format: str = "pcm16") -> str:
|
||||
mime_types: Final = {
|
||||
# Gemini Live native audio (OpenAI GA realtime default) is 24kHz PCM.
|
||||
"pcm16": "audio/pcm;rate=24000",
|
||||
"g711_ulaw": "audio/pcmu",
|
||||
"g711_alaw": "audio/pcma",
|
||||
}
|
||||
return mime_types.get(input_audio_format, "application/octet-stream")
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Session setup message
|
||||
# ------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -2007,10 +2007,102 @@ def _input_audio_append_message(raw_byte_count: int) -> str:
|
|||
)
|
||||
|
||||
|
||||
def _session_update_message(session: dict) -> str:
|
||||
return json.dumps({"type": "session.update", "session": session})
|
||||
|
||||
|
||||
def _sent_audio_mime_type(config, raw_byte_count: int = 32000) -> str:
|
||||
"""Round-trips one input_audio_buffer.append and returns the mimeType actually put on the wire."""
|
||||
sent = config.transform_realtime_request(
|
||||
_input_audio_append_message(raw_byte_count), "gemini-3.5-transcribe-live"
|
||||
)
|
||||
return json.loads(sent[0])["realtimeInput"]["audio"]["mimeType"]
|
||||
|
||||
|
||||
def test_input_audio_mime_type_declares_the_native_16khz_input_rate():
|
||||
"""The Live API resamples against the MIME rate, and its documented native input rate is
|
||||
16kHz; 24kHz is the output rate. Declaring the output rate on the input path mislabels
|
||||
correctly-encoded audio (ai.google.dev/gemini-api/docs/live-api/capabilities)."""
|
||||
from typing import Final
|
||||
|
||||
config: Final = GeminiRealtimeConfig()
|
||||
assert config.get_audio_mime_type() == "audio/pcm;rate=16000"
|
||||
assert _sent_audio_mime_type(config) == "audio/pcm;rate=16000"
|
||||
|
||||
|
||||
def test_vertex_realtime_inherits_the_same_input_audio_rate():
|
||||
"""VertexAIRealtimeConfig used to carry its own copy of get_audio_mime_type, so a fix to the
|
||||
parent had no effect on the Vertex path. It must resolve through the parent now."""
|
||||
from typing import Final
|
||||
|
||||
from litellm.llms.vertex_ai.realtime.transformation import VertexAIRealtimeConfig
|
||||
|
||||
config: Final = VertexAIRealtimeConfig(access_token="t", project="p", location="us-central1")
|
||||
assert "get_audio_mime_type" not in VertexAIRealtimeConfig.__dict__
|
||||
assert config.get_audio_mime_type() == "audio/pcm;rate=16000"
|
||||
|
||||
|
||||
def test_client_declared_input_audio_rate_is_honored_on_the_wire():
|
||||
"""A GA client that declares a non-native input rate must have that rate forwarded, not the
|
||||
default; the whole point of the MIME rate is to describe the bytes actually sent."""
|
||||
from typing import Final
|
||||
|
||||
config: Final = GeminiRealtimeConfig()
|
||||
config.transform_realtime_request(
|
||||
_session_update_message({"audio": {"input": {"format": {"type": "audio/pcm", "rate": 24000}}}}),
|
||||
"gemini-3.5-transcribe-live",
|
||||
)
|
||||
assert _sent_audio_mime_type(config) == "audio/pcm;rate=24000"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"session",
|
||||
[
|
||||
{},
|
||||
{"audio": {}},
|
||||
{"audio": {"input": {}}},
|
||||
{"audio": {"input": {"format": "audio/pcm"}}},
|
||||
{"audio": {"input": {"format": {"type": "audio/pcm"}}}},
|
||||
{"audio": {"input": {"format": {"type": "audio/pcm", "rate": 0}}}},
|
||||
{"audio": {"input": {"format": {"type": "audio/pcm", "rate": -1}}}},
|
||||
{"audio": {"input": {"format": {"type": "audio/pcm", "rate": "24000"}}}},
|
||||
{"audio": {"input": {"format": {"type": "audio/pcm", "rate": True}}}},
|
||||
{"input_audio_format": "pcm16"},
|
||||
],
|
||||
)
|
||||
def test_malformed_or_absent_declared_rate_keeps_the_native_default(session):
|
||||
"""Anything that is not a usable positive integer rate, including the rate-less beta shape and
|
||||
a bool (which is an int subclass), must leave the 16kHz default alone rather than corrupt it."""
|
||||
from typing import Final
|
||||
|
||||
config: Final = GeminiRealtimeConfig()
|
||||
config.transform_realtime_request(_session_update_message(session), "gemini-3.5-transcribe-live")
|
||||
assert _sent_audio_mime_type(config) == "audio/pcm;rate=16000"
|
||||
|
||||
|
||||
def test_declared_rate_also_drives_the_billed_audio_duration(patch_gemini_transcribe_live_cost_map_entry):
|
||||
"""The MIME label and the duration estimate read the same rate, so a client that declares
|
||||
24kHz is billed for 24kHz audio: 96000 pcm16 bytes = 2s -> 50 in / 6 out."""
|
||||
from typing import Final
|
||||
|
||||
config: Final = GeminiRealtimeConfig()
|
||||
config.transform_realtime_request(
|
||||
_session_update_message({"audio": {"input": {"format": {"type": "audio/pcm", "rate": 24000}}}}),
|
||||
"gemini-3.5-transcribe-live",
|
||||
)
|
||||
config.transform_realtime_request(_input_audio_append_message(96000), "gemini-3.5-transcribe-live")
|
||||
|
||||
usage: Final = config.unbilled_usage_on_session_close("gemini-3.5-transcribe-live")
|
||||
assert usage is not None
|
||||
assert usage["input_tokens"] == 50
|
||||
assert usage["output_tokens"] == 6
|
||||
|
||||
|
||||
def test_transcribe_live_completed_event_carries_estimated_usage(patch_gemini_transcribe_live_cost_map_entry):
|
||||
"""Gemini Live sends no usageMetadata for transcribe sessions, so LiteLLM bills
|
||||
from streamed audio duration at Google's published estimate (25 audio tok/sec in,
|
||||
175 text tok/min out): 96000 pcm16 bytes = 2s at 24kHz -> 50 in / 6 out."""
|
||||
175 text tok/min out): 96000 pcm16 bytes = 3s at the Live API's native 16kHz input
|
||||
rate -> 75 in / 9 out."""
|
||||
from typing import Final
|
||||
|
||||
from litellm.types.llms.gemini import BidiGenerateContentServerMessage
|
||||
|
|
@ -2048,10 +2140,10 @@ def test_transcribe_live_completed_event_carries_estimated_usage(patch_gemini_tr
|
|||
assert completed[0]["transcript"] == "ahoy there"
|
||||
expected_usage: Final[RealtimeInputAudioTranscriptionUsage] = {
|
||||
"type": "tokens",
|
||||
"input_tokens": 50,
|
||||
"output_tokens": 6,
|
||||
"total_tokens": 56,
|
||||
"input_token_details": {"text_tokens": 0, "audio_tokens": 50},
|
||||
"input_tokens": 75,
|
||||
"output_tokens": 9,
|
||||
"total_tokens": 84,
|
||||
"input_token_details": {"text_tokens": 0, "audio_tokens": 75},
|
||||
}
|
||||
assert completed[0]["usage"] == expected_usage
|
||||
|
||||
|
|
@ -2114,7 +2206,8 @@ def test_non_transcription_live_model_completed_event_has_no_usage(patch_gemini_
|
|||
def test_unbilled_usage_on_session_close_flushes_trailing_audio(patch_gemini_transcribe_live_cost_map_entry):
|
||||
"""Audio appended after the last transcript frame is still unbilled when the
|
||||
session closes; the session-close hook must hand back the estimate exactly once
|
||||
so the streaming layer can bill it (144000 pcm16 bytes = 3s -> 75 in / 9 out)."""
|
||||
so the streaming layer can bill it (144000 pcm16 bytes = 4.5s at the native 16kHz
|
||||
input rate -> 112 in / 13 out; 112.5 rounds to even)."""
|
||||
from typing import Final
|
||||
|
||||
from litellm.types.realtime import RealtimeInputAudioTranscriptionUsage
|
||||
|
|
@ -2126,10 +2219,10 @@ def test_unbilled_usage_on_session_close_flushes_trailing_audio(patch_gemini_tra
|
|||
|
||||
expected: Final[RealtimeInputAudioTranscriptionUsage] = {
|
||||
"type": "tokens",
|
||||
"input_tokens": 75,
|
||||
"output_tokens": 9,
|
||||
"total_tokens": 84,
|
||||
"input_token_details": {"text_tokens": 0, "audio_tokens": 75},
|
||||
"input_tokens": 112,
|
||||
"output_tokens": 13,
|
||||
"total_tokens": 125,
|
||||
"input_token_details": {"text_tokens": 0, "audio_tokens": 112},
|
||||
}
|
||||
assert usage == expected
|
||||
assert config.unbilled_usage_on_session_close("gemini-3.5-transcribe-live") is None
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue