diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 572d76ad555..37a79e2f6d4 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -2,6 +2,7 @@ ## File for 'response_cost' calculation in Logging import logging import time +from collections.abc import Sequence from functools import lru_cache from typing import TYPE_CHECKING, Any, Final, Literal, cast @@ -2373,6 +2374,64 @@ class RealtimeAPITokenUsageProcessor(BaseTokenUsageProcessor): _TRANSCRIPTION_COMPLETED_EVENT_TYPE: Final = "conversation.item.input_audio_transcription.completed" +def _candidate_realtime_token_costs( + model_name: str, + combined_usage_object: Usage, + custom_llm_provider: str, + data_residency: str | None, +) -> tuple[float, float] | None: + try: + return generic_cost_per_token( + model=model_name, + usage=combined_usage_object, + custom_llm_provider=custom_llm_provider, + data_residency=data_residency, + ) + except Exception: + return None + + +def _cost_map_entry_declares_pricing(model_name: str, custom_llm_provider: str) -> bool: + entries: Final = ( + litellm.model_cost.get(model_name), + litellm.model_cost.get(f"{custom_llm_provider}/{model_name}"), + ) + return any( + entry is not None and any("cost_per" in field and value is not None for field, value in entry.items()) + for entry in entries + ) + + +def _first_priced_realtime_token_costs( + potential_model_names: Sequence[str | None], + combined_usage_object: Usage, + custom_llm_provider: str, + data_residency: str | None, +) -> tuple[float, float]: + candidate_costs: Final = ( + (model_name, costs) + for model_name in potential_model_names + if model_name is not None + and ( + costs := _candidate_realtime_token_costs( + model_name=model_name, + combined_usage_object=combined_usage_object, + custom_llm_provider=custom_llm_provider, + data_residency=data_residency, + ) + ) + is not None + ) + return next( + ( + costs + for model_name, costs in candidate_costs + if sum(costs) > 0 or _cost_map_entry_declares_pricing(model_name, custom_llm_provider) + ), + (0.0, 0.0), + ) + + def handle_realtime_stream_cost_calculation( results: OpenAIRealtimeStreamList, combined_usage_object: Usage, @@ -2397,24 +2456,12 @@ def handle_realtime_stream_cost_calculation( potential_model_names.append(received_model) potential_model_names.append(litellm_model_name) - input_cost_per_token = 0.0 - output_cost_per_token = 0.0 - - for model_name in potential_model_names: - try: - if model_name is None: - continue - _input_cost_per_token, _output_cost_per_token = generic_cost_per_token( - model=model_name, - usage=combined_usage_object, - custom_llm_provider=custom_llm_provider, - data_residency=data_residency, - ) - except Exception: - continue - input_cost_per_token += _input_cost_per_token - output_cost_per_token += _output_cost_per_token - break # exit if we find a valid model + input_cost_per_token, output_cost_per_token = _first_priced_realtime_token_costs( + potential_model_names=potential_model_names, + combined_usage_object=combined_usage_object, + custom_llm_provider=custom_llm_provider, + data_residency=data_residency, + ) transcription_cost: Final = ( handle_realtime_transcription_cost_calculation( results=results, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index d30cc9f17ab..abb137a960c 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -20377,6 +20377,49 @@ "google_maps_grounding_cost_per_query": 0.025, "supports_image_size": false }, + "gemini-live-2.5-flash-native-audio": { + "input_cost_per_audio_token": 3e-06, + "input_cost_per_token": 5e-07, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 1048576, + "max_output_tokens": 65535, + "max_tokens": 65535, + "mode": "realtime", + "output_cost_per_audio_token": 1.2e-05, + "output_cost_per_token": 2e-06, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "supported_endpoints": [ + "/vertex_ai/live" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_vision": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + }, + "gemini_native_audio": true + }, "gemini-live-2.5-flash-preview-native-audio-09-2025": { "cache_read_input_token_cost": 7.5e-08, "input_cost_per_audio_token": 3e-06, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index d30cc9f17ab..abb137a960c 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -20377,6 +20377,49 @@ "google_maps_grounding_cost_per_query": 0.025, "supports_image_size": false }, + "gemini-live-2.5-flash-native-audio": { + "input_cost_per_audio_token": 3e-06, + "input_cost_per_token": 5e-07, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 1048576, + "max_output_tokens": 65535, + "max_tokens": 65535, + "mode": "realtime", + "output_cost_per_audio_token": 1.2e-05, + "output_cost_per_token": 2e-06, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", + "supported_endpoints": [ + "/vertex_ai/live" + ], + "supported_modalities": [ + "text", + "image", + "audio", + "video" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_url_context": true, + "supports_vision": true, + "supports_web_search": true, + "search_context_cost_per_query": { + "search_context_size_low": 0.035, + "search_context_size_medium": 0.035, + "search_context_size_high": 0.035 + }, + "gemini_native_audio": true + }, "gemini-live-2.5-flash-preview-native-audio-09-2025": { "cache_read_input_token_cost": 7.5e-08, "input_cost_per_audio_token": 3e-06, diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 4117b2fce6d..8fce9ba080c 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -4159,3 +4159,101 @@ def test_every_one_hour_cache_write_rate_is_double_its_input_rate(): } assert deviations == {} + + +def test_gemini_live_native_audio_ga_realtime_cost(_local_model_cost_map: None) -> None: + """Regression for https://github.com/BerriAI/litellm/issues/31087.""" + from litellm.types.utils import CompletionTokensDetailsWrapper + + results: OpenAIRealtimeStreamList = [ + {"type": "session.created", "session": {"model": "gemini-live-2.5-flash-native-audio"}}, + ] + combined_usage_object = Usage( + prompt_tokens=8, + completion_tokens=25, + total_tokens=33, + prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=8, audio_tokens=0), + completion_tokens_details=CompletionTokensDetailsWrapper(text_tokens=2, audio_tokens=23), + ) + + cost = handle_realtime_stream_cost_calculation( + results=results, + combined_usage_object=combined_usage_object, + custom_llm_provider="vertex_ai", + litellm_model_name="vertex_ai/gemini-live-2.5-flash-native-audio", + ) + + expected_cost = 8 * 5e-07 + 2 * 2e-06 + 23 * 1.2e-05 + assert cost == pytest.approx(expected_cost, rel=1e-9) + + +@pytest.mark.parametrize( + "priceless_entry", + [ + {"litellm_provider": "vertex_ai", "mode": "realtime"}, + { + "litellm_provider": "vertex_ai", + "mode": "realtime", + "input_cost_per_token": None, + "output_cost_per_token": None, + "input_cost_per_audio_token": None, + }, + ], + ids=["registered_without_price_fields", "registered_with_none_valued_price_fields"], +) +def test_realtime_priceless_deployment_entry_falls_through_to_priced_model( + _local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch, priceless_entry: dict +) -> None: + """Regression for https://github.com/BerriAI/litellm/issues/31087 (router-registered priceless entries).""" + monkeypatch.setitem( + litellm.model_cost, + "vertex_ai/some-unmapped-live-model", + priceless_entry, + ) + priced_model = "vertex_ai/gemini-live-2.5-flash-preview-native-audio-09-2025" + priced_entry = litellm.model_cost["gemini-live-2.5-flash-preview-native-audio-09-2025"] + + results: OpenAIRealtimeStreamList = [ + {"type": "session.created", "session": {"model": "some-unmapped-live-model"}}, + ] + combined_usage_object = Usage(prompt_tokens=8, completion_tokens=25, total_tokens=33) + + cost = handle_realtime_stream_cost_calculation( + results=results, + combined_usage_object=combined_usage_object, + custom_llm_provider="vertex_ai", + litellm_model_name=priced_model, + ) + + expected_cost = 8 * priced_entry["input_cost_per_token"] + 25 * priced_entry["output_cost_per_token"] + assert cost == pytest.approx(expected_cost, rel=1e-9) + assert cost > 0 + + +def test_realtime_explicitly_free_session_model_still_bills_zero( + _local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setitem( + litellm.model_cost, + "vertex_ai/free-live-model", + { + "litellm_provider": "vertex_ai", + "mode": "realtime", + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + }, + ) + + results: OpenAIRealtimeStreamList = [ + {"type": "session.created", "session": {"model": "free-live-model"}}, + ] + combined_usage_object = Usage(prompt_tokens=8, completion_tokens=25, total_tokens=33) + + cost = handle_realtime_stream_cost_calculation( + results=results, + combined_usage_object=combined_usage_object, + custom_llm_provider="vertex_ai", + litellm_model_name="vertex_ai/gemini-live-2.5-flash-preview-native-audio-09-2025", + ) + + assert cost == 0.0