From ab1b7bf3b68850de00ff3686bb1edbbbd6482c1a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 26 Aug 2026 15:53:37 -0700 Subject: [PATCH 1/3] fix(cost): price gemini-live-2.5-flash-native-audio realtime sessions The GA vertex model had no cost map entry, and the realtime cost handler accepted the router's price-less auto-registered deployment entry for the session.created model at zero-defaulted rates, so sessions billed 0.0 even when base_model pointed at the priced preview key. Adds the GA entry at its published rates and makes the handler fall through zero-defaulted candidates unless their cost map entry explicitly declares pricing. --- litellm/cost_calculator.py | 80 ++++++++++++---- ...odel_prices_and_context_window_backup.json | 43 +++++++++ model_prices_and_context_window.json | 43 +++++++++ tests/test_litellm/test_cost_calculator.py | 91 +++++++++++++++++++ 4 files changed, 239 insertions(+), 18 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 4cd292b2416..128060815f6 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -2,6 +2,7 @@ ## File for 'response_cost' calculation in Logging import logging import time +from collections.abc import Sequence from functools import lru_cache from typing import TYPE_CHECKING, Any, Final, Literal, cast @@ -2370,6 +2371,61 @@ class RealtimeAPITokenUsageProcessor(BaseTokenUsageProcessor): _TRANSCRIPTION_COMPLETED_EVENT_TYPE: Final = "conversation.item.input_audio_transcription.completed" +def _candidate_realtime_token_costs( + model_name: str, + combined_usage_object: Usage, + custom_llm_provider: str, + data_residency: str | None, +) -> tuple[float, float] | None: + try: + return generic_cost_per_token( + model=model_name, + usage=combined_usage_object, + custom_llm_provider=custom_llm_provider, + data_residency=data_residency, + ) + except Exception: + return None + + +def _cost_map_entry_declares_pricing(model_name: str, custom_llm_provider: str) -> bool: + entries: Final = ( + litellm.model_cost.get(model_name), + litellm.model_cost.get(f"{custom_llm_provider}/{model_name}"), + ) + return any(entry is not None and any("cost_per" in field for field in entry) for entry in entries) + + +def _first_priced_realtime_token_costs( + potential_model_names: Sequence[str | None], + combined_usage_object: Usage, + custom_llm_provider: str, + data_residency: str | None, +) -> tuple[float, float]: + candidate_costs: Final = ( + (model_name, costs) + for model_name in potential_model_names + if model_name is not None + and ( + costs := _candidate_realtime_token_costs( + model_name=model_name, + combined_usage_object=combined_usage_object, + custom_llm_provider=custom_llm_provider, + data_residency=data_residency, + ) + ) + is not None + ) + return next( + ( + costs + for model_name, costs in candidate_costs + if sum(costs) > 0 or _cost_map_entry_declares_pricing(model_name, custom_llm_provider) + ), + (0.0, 0.0), + ) + + def handle_realtime_stream_cost_calculation( results: OpenAIRealtimeStreamList, combined_usage_object: Usage, @@ -2394,24 +2450,12 @@ def handle_realtime_stream_cost_calculation( potential_model_names.append(received_model) potential_model_names.append(litellm_model_name) - input_cost_per_token = 0.0 - output_cost_per_token = 0.0 - - for model_name in potential_model_names: - try: - if model_name is None: - continue - _input_cost_per_token, _output_cost_per_token = generic_cost_per_token( - model=model_name, - usage=combined_usage_object, - custom_llm_provider=custom_llm_provider, - data_residency=data_residency, - ) - except Exception: - continue - input_cost_per_token += _input_cost_per_token - output_cost_per_token += _output_cost_per_token - break # exit if we find a valid model + input_cost_per_token, output_cost_per_token = _first_priced_realtime_token_costs( + potential_model_names=potential_model_names, + combined_usage_object=combined_usage_object, + custom_llm_provider=custom_llm_provider, + data_residency=data_residency, + ) transcription_cost: Final = ( handle_realtime_transcription_cost_calculation( results=results, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index a6da2c1fb09..38098fa7082 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -20370,6 +20370,49 @@ }, "supports_image_size": false }, + "gemini-live-2.5-flash-native-audio": { + "input_cost_per_audio_token": 3e-06, + "input_cost_per_token": 5e-07, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 1048576, + "max_output_tokens": 65535, + "max_tokens": 65535, + "mode": "realtime", + "output_cost_per_audio_token": 1.2e-05, + "output_cost_per_token": 2e-06, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "supported_endpoints": [ + "/vertex_ai/live" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_vision": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + }, + "gemini_native_audio": true + }, "gemini-live-2.5-flash-preview-native-audio-09-2025": { "cache_read_input_token_cost": 7.5e-08, "input_cost_per_audio_token": 3e-06, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index a6da2c1fb09..38098fa7082 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -20370,6 +20370,49 @@ }, "supports_image_size": false }, + "gemini-live-2.5-flash-native-audio": { + "input_cost_per_audio_token": 3e-06, + "input_cost_per_token": 5e-07, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 1048576, + "max_output_tokens": 65535, + "max_tokens": 65535, + "mode": "realtime", + "output_cost_per_audio_token": 1.2e-05, + "output_cost_per_token": 2e-06, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "supported_endpoints": [ + "/vertex_ai/live" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_vision": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + }, + "gemini_native_audio": true + }, "gemini-live-2.5-flash-preview-native-audio-09-2025": { "cache_read_input_token_cost": 7.5e-08, "input_cost_per_audio_token": 3e-06, diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index dc2fbe3ed73..cfd07171556 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -4116,3 +4116,94 @@ def test_every_one_hour_cache_write_rate_is_double_its_input_rate(): } assert deviations == {} + + +def test_gemini_live_native_audio_ga_realtime_cost(_local_model_cost_map: None) -> None: + """Regression for https://github.com/BerriAI/litellm/issues/31087: realtime sessions on the + GA vertex model gemini-live-2.5-flash-native-audio must bill at its published rates instead + of logging zero spend because only the preview-09-2025 key existed in the cost map.""" + from litellm.types.utils import CompletionTokensDetailsWrapper + + results: OpenAIRealtimeStreamList = [ + {"type": "session.created", "session": {"model": "gemini-live-2.5-flash-native-audio"}}, + ] + combined_usage_object = Usage( + prompt_tokens=8, + completion_tokens=25, + total_tokens=33, + prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=8, audio_tokens=0), + completion_tokens_details=CompletionTokensDetailsWrapper(text_tokens=2, audio_tokens=23), + ) + + cost = handle_realtime_stream_cost_calculation( + results=results, + combined_usage_object=combined_usage_object, + custom_llm_provider="vertex_ai", + litellm_model_name="vertex_ai/gemini-live-2.5-flash-native-audio", + ) + + expected_cost = 8 * 5e-07 + 2 * 2e-06 + 23 * 1.2e-05 + assert cost == pytest.approx(expected_cost, rel=1e-9) + + +def test_realtime_priceless_deployment_entry_falls_through_to_priced_model( + _local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + """Regression for https://github.com/BerriAI/litellm/issues/31087: the router registers every + deployment's backend key into litellm.model_cost without price fields, and the realtime cost + handler used to accept that zero-defaulted entry for the session.created model and stop, so a + configured base_model never priced the session.""" + monkeypatch.setitem( + litellm.model_cost, + "vertex_ai/some-unmapped-live-model", + {"litellm_provider": "vertex_ai", "mode": "realtime"}, + ) + priced_model = "vertex_ai/gemini-live-2.5-flash-preview-native-audio-09-2025" + priced_entry = litellm.model_cost["gemini-live-2.5-flash-preview-native-audio-09-2025"] + + results: OpenAIRealtimeStreamList = [ + {"type": "session.created", "session": {"model": "some-unmapped-live-model"}}, + ] + combined_usage_object = Usage(prompt_tokens=8, completion_tokens=25, total_tokens=33) + + cost = handle_realtime_stream_cost_calculation( + results=results, + combined_usage_object=combined_usage_object, + custom_llm_provider="vertex_ai", + litellm_model_name=priced_model, + ) + + expected_cost = 8 * priced_entry["input_cost_per_token"] + 25 * priced_entry["output_cost_per_token"] + assert cost == pytest.approx(expected_cost, rel=1e-9) + assert cost > 0 + + +def test_realtime_explicitly_free_session_model_still_bills_zero( + _local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + """A session model whose cost map entry explicitly declares zero rates is genuinely free, so + the handler must keep billing it at zero instead of falling through to a priced fallback.""" + monkeypatch.setitem( + litellm.model_cost, + "vertex_ai/free-live-model", + { + "litellm_provider": "vertex_ai", + "mode": "realtime", + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + }, + ) + + results: OpenAIRealtimeStreamList = [ + {"type": "session.created", "session": {"model": "free-live-model"}}, + ] + combined_usage_object = Usage(prompt_tokens=8, completion_tokens=25, total_tokens=33) + + cost = handle_realtime_stream_cost_calculation( + results=results, + combined_usage_object=combined_usage_object, + custom_llm_provider="vertex_ai", + litellm_model_name="vertex_ai/gemini-live-2.5-flash-preview-native-audio-09-2025", + ) + + assert cost == 0.0 From b48bff7b5444711678504ec069cd13ad4ac769be Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 26 Aug 2026 16:27:28 -0700 Subject: [PATCH 2/3] fix(cost_calculator): require real values when detecting declared realtime pricing --- litellm/cost_calculator.py | 5 ++++- tests/test_litellm/test_cost_calculator.py | 25 +++++++++++++++++----- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 128060815f6..9ac2b425961 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -2393,7 +2393,10 @@ def _cost_map_entry_declares_pricing(model_name: str, custom_llm_provider: str) litellm.model_cost.get(model_name), litellm.model_cost.get(f"{custom_llm_provider}/{model_name}"), ) - return any(entry is not None and any("cost_per" in field for field in entry) for entry in entries) + return any( + entry is not None and any("cost_per" in field and value is not None for field, value in entry.items()) + for entry in entries + ) def _first_priced_realtime_token_costs( diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index cfd07171556..6d292cb5f94 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -4146,17 +4146,32 @@ def test_gemini_live_native_audio_ga_realtime_cost(_local_model_cost_map: None) assert cost == pytest.approx(expected_cost, rel=1e-9) +@pytest.mark.parametrize( + "priceless_entry", + [ + {"litellm_provider": "vertex_ai", "mode": "realtime"}, + { + "litellm_provider": "vertex_ai", + "mode": "realtime", + "input_cost_per_token": None, + "output_cost_per_token": None, + "input_cost_per_audio_token": None, + }, + ], + ids=["registered_without_price_fields", "registered_with_none_valued_price_fields"], +) def test_realtime_priceless_deployment_entry_falls_through_to_priced_model( - _local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch + _local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch, priceless_entry: dict ) -> None: """Regression for https://github.com/BerriAI/litellm/issues/31087: the router registers every - deployment's backend key into litellm.model_cost without price fields, and the realtime cost - handler used to accept that zero-defaulted entry for the session.created model and stop, so a - configured base_model never priced the session.""" + deployment's backend key into litellm.model_cost without price fields (and merges a None-valued + ModelInfo skeleton into mapped entries), and the realtime cost handler used to accept that + zero-defaulted entry for the session.created model and stop, so a configured base_model never + priced the session.""" monkeypatch.setitem( litellm.model_cost, "vertex_ai/some-unmapped-live-model", - {"litellm_provider": "vertex_ai", "mode": "realtime"}, + priceless_entry, ) priced_model = "vertex_ai/gemini-live-2.5-flash-preview-native-audio-09-2025" priced_entry = litellm.model_cost["gemini-live-2.5-flash-preview-native-audio-09-2025"] From e7b843d69b1ed8d23228ef5012a5aba9a1164a0f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 26 Aug 2026 16:33:02 -0700 Subject: [PATCH 3/3] test: trim realtime cost test docstrings to one line --- tests/test_litellm/test_cost_calculator.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 6d292cb5f94..3e127a23aa6 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -4119,9 +4119,7 @@ def test_every_one_hour_cache_write_rate_is_double_its_input_rate(): def test_gemini_live_native_audio_ga_realtime_cost(_local_model_cost_map: None) -> None: - """Regression for https://github.com/BerriAI/litellm/issues/31087: realtime sessions on the - GA vertex model gemini-live-2.5-flash-native-audio must bill at its published rates instead - of logging zero spend because only the preview-09-2025 key existed in the cost map.""" + """Regression for https://github.com/BerriAI/litellm/issues/31087.""" from litellm.types.utils import CompletionTokensDetailsWrapper results: OpenAIRealtimeStreamList = [ @@ -4163,11 +4161,7 @@ def test_gemini_live_native_audio_ga_realtime_cost(_local_model_cost_map: None) def test_realtime_priceless_deployment_entry_falls_through_to_priced_model( _local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch, priceless_entry: dict ) -> None: - """Regression for https://github.com/BerriAI/litellm/issues/31087: the router registers every - deployment's backend key into litellm.model_cost without price fields (and merges a None-valued - ModelInfo skeleton into mapped entries), and the realtime cost handler used to accept that - zero-defaulted entry for the session.created model and stop, so a configured base_model never - priced the session.""" + """Regression for https://github.com/BerriAI/litellm/issues/31087 (router-registered priceless entries).""" monkeypatch.setitem( litellm.model_cost, "vertex_ai/some-unmapped-live-model", @@ -4196,8 +4190,6 @@ def test_realtime_priceless_deployment_entry_falls_through_to_priced_model( def test_realtime_explicitly_free_session_model_still_bills_zero( _local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch ) -> None: - """A session model whose cost map entry explicitly declares zero rates is genuinely free, so - the handler must keep billing it at zero instead of falling through to a priced fallback.""" monkeypatch.setitem( litellm.model_cost, "vertex_ai/free-live-model",