scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-08-28 05:25:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 36

fix(cost): price gemini-live-2.5-flash-native-audio realtime sessions

Browse source 
The GA vertex model had no cost map entry, and the realtime cost handler
accepted the router's price-less auto-registered deployment entry for the
session.created model at zero-defaulted rates, so sessions billed 0.0 even
when base_model pointed at the priced preview key. Adds the GA entry at its
published rates and makes the handler fall through zero-defaulted candidates
unless their cost map entry explicitly declares pricing.
This commit is contained in:
mateo-berri 2026-08-26 15:53:37 -07:00
parent f7220556e1
commit ab1b7bf3b6
4 changed files with 239 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

80
litellm/cost_calculator.py
View file

@ -2,6 +2,7 @@
## File for 'response_cost' calculation in Logging
import logging
import time
from collections.abc import Sequence
from functools import lru_cache
from typing import TYPE_CHECKING, Any, Final, Literal, cast
@ -2370,6 +2371,61 @@ class RealtimeAPITokenUsageProcessor(BaseTokenUsageProcessor):
_TRANSCRIPTION_COMPLETED_EVENT_TYPE: Final = "conversation.item.input_audio_transcription.completed"
def _candidate_realtime_token_costs(
model_name: str,
combined_usage_object: Usage,
custom_llm_provider: str,
data_residency: str | None,
) -> tuple[float, float] | None:
try:
return generic_cost_per_token(
model=model_name,
usage=combined_usage_object,
custom_llm_provider=custom_llm_provider,
data_residency=data_residency,
)
except Exception:
return None
def _cost_map_entry_declares_pricing(model_name: str, custom_llm_provider: str) -> bool:
entries: Final = (
litellm.model_cost.get(model_name),
litellm.model_cost.get(f"{custom_llm_provider}/{model_name}"),
)
return any(entry is not None and any("cost_per" in field for field in entry) for entry in entries)
def _first_priced_realtime_token_costs(
potential_model_names: Sequence[str | None],
combined_usage_object: Usage,
custom_llm_provider: str,
data_residency: str | None,
) -> tuple[float, float]:
candidate_costs: Final = (
(model_name, costs)
for model_name in potential_model_names
if model_name is not None
and (
costs := _candidate_realtime_token_costs(
model_name=model_name,
combined_usage_object=combined_usage_object,
custom_llm_provider=custom_llm_provider,
data_residency=data_residency,
)
)
is not None
)
return next(
(
costs
for model_name, costs in candidate_costs
if sum(costs) > 0 or _cost_map_entry_declares_pricing(model_name, custom_llm_provider)
),
(0.0, 0.0),
)
def handle_realtime_stream_cost_calculation(
results: OpenAIRealtimeStreamList,
combined_usage_object: Usage,
@ -2394,24 +2450,12 @@ def handle_realtime_stream_cost_calculation(
potential_model_names.append(received_model)
potential_model_names.append(litellm_model_name)
input_cost_per_token = 0.0
output_cost_per_token = 0.0
for model_name in potential_model_names:
try:
if model_name is None:
continue
_input_cost_per_token, _output_cost_per_token = generic_cost_per_token(
model=model_name,
usage=combined_usage_object,
custom_llm_provider=custom_llm_provider,
data_residency=data_residency,
)
except Exception:
continue
input_cost_per_token += _input_cost_per_token
output_cost_per_token += _output_cost_per_token
break # exit if we find a valid model
input_cost_per_token, output_cost_per_token = _first_priced_realtime_token_costs(
potential_model_names=potential_model_names,
combined_usage_object=combined_usage_object,
custom_llm_provider=custom_llm_provider,
data_residency=data_residency,
)
transcription_cost: Final = (
handle_realtime_transcription_cost_calculation(
results=results,

43
litellm/model_prices_and_context_window_backup.json
View file

@ -20370,6 +20370,49 @@
},
"supports_image_size": false
},
"gemini-live-2.5-flash-native-audio": {
"input_cost_per_audio_token": 3e-06,
"input_cost_per_token": 5e-07,
"litellm_provider": "vertex_ai-language-models",
"max_input_tokens": 1048576,
"max_output_tokens": 65535,
"max_tokens": 65535,
"mode": "realtime",
"output_cost_per_audio_token": 1.2e-05,
"output_cost_per_token": 2e-06,
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
"supported_endpoints": [
"/vertex_ai/live"
],
"supported_modalities": [
"text",
"image",
"audio",
"video"
],
"supported_output_modalities": [
"text",
"audio"
],
"supports_audio_input": true,
"supports_audio_output": true,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_url_context": true,
"supports_vision": true,
"supports_web_search": true,
"search_context_cost_per_query": {
"search_context_size_low": 0.035,
"search_context_size_medium": 0.035,
"search_context_size_high": 0.035
},
"gemini_native_audio": true
},
"gemini-live-2.5-flash-preview-native-audio-09-2025": {
"cache_read_input_token_cost": 7.5e-08,
"input_cost_per_audio_token": 3e-06,

43
model_prices_and_context_window.json
View file

@ -20370,6 +20370,49 @@
},
"supports_image_size": false
},
"gemini-live-2.5-flash-native-audio": {
"input_cost_per_audio_token": 3e-06,
"input_cost_per_token": 5e-07,
"litellm_provider": "vertex_ai-language-models",
"max_input_tokens": 1048576,
"max_output_tokens": 65535,
"max_tokens": 65535,
"mode": "realtime",
"output_cost_per_audio_token": 1.2e-05,
"output_cost_per_token": 2e-06,
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
"supported_endpoints": [
"/vertex_ai/live"
],
"supported_modalities": [
"text",
"image",
"audio",
"video"
],
"supported_output_modalities": [
"text",
"audio"
],
"supports_audio_input": true,
"supports_audio_output": true,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_url_context": true,
"supports_vision": true,
"supports_web_search": true,
"search_context_cost_per_query": {
"search_context_size_low": 0.035,
"search_context_size_medium": 0.035,
"search_context_size_high": 0.035
},
"gemini_native_audio": true
},
"gemini-live-2.5-flash-preview-native-audio-09-2025": {
"cache_read_input_token_cost": 7.5e-08,
"input_cost_per_audio_token": 3e-06,

91
tests/test_litellm/test_cost_calculator.py
View file

@ -4116,3 +4116,94 @@ def test_every_one_hour_cache_write_rate_is_double_its_input_rate():
}
assert deviations == {}
def test_gemini_live_native_audio_ga_realtime_cost(_local_model_cost_map: None) -> None:
"""Regression for https://github.com/BerriAI/litellm/issues/31087: realtime sessions on the
GA vertex model gemini-live-2.5-flash-native-audio must bill at its published rates instead
of logging zero spend because only the preview-09-2025 key existed in the cost map."""
from litellm.types.utils import CompletionTokensDetailsWrapper
results: OpenAIRealtimeStreamList = [
{"type": "session.created", "session": {"model": "gemini-live-2.5-flash-native-audio"}},
]
combined_usage_object = Usage(
prompt_tokens=8,
completion_tokens=25,
total_tokens=33,
prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=8, audio_tokens=0),
completion_tokens_details=CompletionTokensDetailsWrapper(text_tokens=2, audio_tokens=23),
)
cost = handle_realtime_stream_cost_calculation(
results=results,
combined_usage_object=combined_usage_object,
custom_llm_provider="vertex_ai",
litellm_model_name="vertex_ai/gemini-live-2.5-flash-native-audio",
)
expected_cost = 8 * 5e-07 + 2 * 2e-06 + 23 * 1.2e-05
assert cost == pytest.approx(expected_cost, rel=1e-9)
def test_realtime_priceless_deployment_entry_falls_through_to_priced_model(
_local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch
) -> None:
"""Regression for https://github.com/BerriAI/litellm/issues/31087: the router registers every
deployment's backend key into litellm.model_cost without price fields, and the realtime cost
handler used to accept that zero-defaulted entry for the session.created model and stop, so a
configured base_model never priced the session."""
monkeypatch.setitem(
litellm.model_cost,
"vertex_ai/some-unmapped-live-model",
{"litellm_provider": "vertex_ai", "mode": "realtime"},
)
priced_model = "vertex_ai/gemini-live-2.5-flash-preview-native-audio-09-2025"
priced_entry = litellm.model_cost["gemini-live-2.5-flash-preview-native-audio-09-2025"]
results: OpenAIRealtimeStreamList = [
{"type": "session.created", "session": {"model": "some-unmapped-live-model"}},
]
combined_usage_object = Usage(prompt_tokens=8, completion_tokens=25, total_tokens=33)
cost = handle_realtime_stream_cost_calculation(
results=results,
combined_usage_object=combined_usage_object,
custom_llm_provider="vertex_ai",
litellm_model_name=priced_model,
)
expected_cost = 8 * priced_entry["input_cost_per_token"] + 25 * priced_entry["output_cost_per_token"]
assert cost == pytest.approx(expected_cost, rel=1e-9)
assert cost > 0
def test_realtime_explicitly_free_session_model_still_bills_zero(
_local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch
) -> None:
"""A session model whose cost map entry explicitly declares zero rates is genuinely free, so
the handler must keep billing it at zero instead of falling through to a priced fallback."""
monkeypatch.setitem(
litellm.model_cost,
"vertex_ai/free-live-model",
{
"litellm_provider": "vertex_ai",
"mode": "realtime",
"input_cost_per_token": 0.0,
"output_cost_per_token": 0.0,
},
)
results: OpenAIRealtimeStreamList = [
{"type": "session.created", "session": {"model": "free-live-model"}},
]
combined_usage_object = Usage(prompt_tokens=8, completion_tokens=25, total_tokens=33)
cost = handle_realtime_stream_cost_calculation(
results=results,
combined_usage_object=combined_usage_object,
custom_llm_provider="vertex_ai",
litellm_model_name="vertex_ai/gemini-live-2.5-flash-preview-native-audio-09-2025",
)
assert cost == 0.0