From 0c9957988089d52561a343bee74f51048cac78f2 Mon Sep 17 00:00:00 2001 From: shivam Date: Wed, 22 Jul 2026 03:20:14 +0000 Subject: [PATCH 1/3] fix(cost): bill Gemini audio/video input tokens at the tiered input rate Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../litellm_core_utils/llm_cost_calc/utils.py | 47 +++++++++++++------ .../llm_cost_calc/test_llm_cost_calc_utils.py | 47 +++++++++++++++++++ 2 files changed, 79 insertions(+), 15 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 33bf546c239..d0787156be3 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -375,6 +375,28 @@ def calculate_cost_component(model_info: ModelInfo, cost_key: str, usage_value: return 0.0 +def _calculate_modality_token_cost( + model_info: ModelInfo, + cost_key: str, + tokens: int, + fallback_cost_per_token: float, +) -> float: + """ + Cost for a modality (audio/image/video) input token count. + + Uses the modality-specific per-token rate when the model defines one, otherwise falls back to + the resolved input per-token rate (``fallback_cost_per_token``), which is already tier-aware + (e.g. long-context ``_above_200k_tokens``) and service-tier-aware. Providers like Gemini bill + audio/video input at the standard input rate and expose no modality-specific key, so dropping + these tokens (or billing them at the untiered base rate) severely undercounts multimodal spend. + """ + if tokens <= 0: + return 0.0 + if model_info.get(cost_key) is None: + return float(tokens) * fallback_cost_per_token + return calculate_cost_component(model_info, cost_key, tokens) + + def _get_cost_per_unit(model_info: ModelInfo, cost_key: str, default_value: Optional[float] = 0.0) -> Optional[float]: # Sometimes the cost per unit is a string (e.g.: If a value like "3e-7" was read from the config.yaml) cost_per_unit = model_info.get(cost_key) @@ -579,25 +601,20 @@ def _calculate_input_cost( prompt_cost += float(prompt_tokens_details["cache_hit_tokens"]) * cache_read_cost ### AUDIO COST - if prompt_tokens_details["audio_tokens"]: - audio_cost_key = _get_service_tier_cost_key("input_cost_per_audio_token", service_tier) - prompt_cost += calculate_cost_component(model_info, audio_cost_key, prompt_tokens_details["audio_tokens"]) + audio_cost_key = _get_service_tier_cost_key("input_cost_per_audio_token", service_tier) + prompt_cost += _calculate_modality_token_cost( + model_info, audio_cost_key, prompt_tokens_details["audio_tokens"], prompt_base_cost + ) ### IMAGE TOKEN COST - if prompt_tokens_details["image_tokens"]: - # For image token costs: - # First check if input_cost_per_image_token is available. If not, default to generic input_cost_per_token. - image_token_cost_key = "input_cost_per_image_token" - if model_info.get(image_token_cost_key) is None: - image_token_cost_key = "input_cost_per_token" - prompt_cost += calculate_cost_component(model_info, image_token_cost_key, prompt_tokens_details["image_tokens"]) + prompt_cost += _calculate_modality_token_cost( + model_info, "input_cost_per_image_token", prompt_tokens_details["image_tokens"], prompt_base_cost + ) ### VIDEO TOKEN COST - if prompt_tokens_details["video_tokens"]: - video_token_cost_key = "input_cost_per_video_token" - if model_info.get(video_token_cost_key) is None: - video_token_cost_key = "input_cost_per_token" - prompt_cost += calculate_cost_component(model_info, video_token_cost_key, prompt_tokens_details["video_tokens"]) + prompt_cost += _calculate_modality_token_cost( + model_info, "input_cost_per_video_token", prompt_tokens_details["video_tokens"], prompt_base_cost + ) ### CACHE WRITING COST - Now uses tiered pricing if ( diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 9ff67a82f40..d40d569f943 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -400,6 +400,53 @@ def test_generic_cost_per_token_above_200k_tokens(): ) +def test_multimodal_input_tokens_use_tiered_rate_gemini_3_1_pro(): + """Regression test for LIT-4681. + + Gemini bills audio/video input at the standard input rate and exposes no + input_cost_per_audio_token / input_cost_per_video_token. For a >200k-token multimodal + request, audio tokens must not be dropped and video tokens must be billed at the + long-context (_above_200k_tokens) input rate, not the untiered base rate. + """ + model = "gemini-3.1-pro-preview" + custom_llm_provider = "vertex_ai" + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + model_cost_map = litellm.model_cost[f"{custom_llm_provider}/{model}"] + assert model_cost_map.get("input_cost_per_audio_token") is None + assert model_cost_map.get("input_cost_per_video_token") is None + above_200k_rate = model_cost_map["input_cost_per_token_above_200k_tokens"] + + text_tokens = 9033 + audio_tokens = 14999 + video_tokens = 792000 + prompt_tokens = text_tokens + audio_tokens + video_tokens + completion_tokens = 4559 + usage = Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=text_tokens, + audio_tokens=audio_tokens, + video_tokens=video_tokens, + ), + ) + + prompt_cost, completion_cost = generic_cost_per_token( + model=model, + usage=usage, + custom_llm_provider=custom_llm_provider, + ) + + assert round(prompt_cost, 10) == round(prompt_tokens * above_200k_rate, 10) + assert round(completion_cost, 10) == round( + model_cost_map["output_cost_per_token_above_200k_tokens"] * completion_tokens, + 10, + ) + + def test_get_token_base_cost_picks_highest_crossed_tier(): """Regression test for #30345. From ed7d5ed3f79d51d18db06580f3e45c29b90f8003 Mon Sep 17 00:00:00 2001 From: shivam Date: Wed, 22 Jul 2026 03:30:11 +0000 Subject: [PATCH 2/3] fix(cost): resolve service-tier audio rate and avoid double-billing count/duration modalities Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../litellm_core_utils/llm_cost_calc/utils.py | 48 ++++++++--- .../llm_cost_calc/test_llm_cost_calc_utils.py | 84 +++++++++++++++++++ 2 files changed, 121 insertions(+), 11 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index d0787156be3..0ebaefc915c 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -380,21 +380,35 @@ def _calculate_modality_token_cost( cost_key: str, tokens: int, fallback_cost_per_token: float, + alternative_pricing: Tuple[Tuple[float, str], ...] = (), ) -> float: """ Cost for a modality (audio/image/video) input token count. - Uses the modality-specific per-token rate when the model defines one, otherwise falls back to - the resolved input per-token rate (``fallback_cost_per_token``), which is already tier-aware - (e.g. long-context ``_above_200k_tokens``) and service-tier-aware. Providers like Gemini bill - audio/video input at the standard input rate and expose no modality-specific key, so dropping - these tokens (or billing them at the untiered base rate) severely undercounts multimodal spend. + Resolution order: + 1. Modality-specific per-token rate (``cost_key``), including the service-tier suffix + fallback to the base key handled by ``_get_cost_per_unit``. A present-but-malformed rate + resolves to 0 here, preserving prior behavior. + 2. If this request is already billed for the modality by count/duration + (``alternative_pricing`` pairs of ``(usage_amount, cost_key)`` such as the image count + with ``input_cost_per_image``), charge nothing here so the dedicated count/duration + component is not double-billed. Only a measurement actually present in this usage suppresses + the fallback; a model that merely lists a count price still bills reported tokens. + 3. Otherwise fall back to ``fallback_cost_per_token``, the resolved input per-token rate that + is already tier-aware (e.g. long-context ``_above_200k_tokens``) and service-tier-aware. + Providers like Gemini bill audio/video input at the standard input rate and expose no + modality-specific key, so dropping these tokens severely undercounts multimodal spend. """ if tokens <= 0: return 0.0 - if model_info.get(cost_key) is None: - return float(tokens) * fallback_cost_per_token - return calculate_cost_component(model_info, cost_key, tokens) + if model_info.get(cost_key) is not None: + return calculate_cost_component(model_info, cost_key, tokens) + resolved_cost_per_unit = _get_cost_per_unit(model_info, cost_key, None) + if resolved_cost_per_unit is not None: + return float(tokens) * resolved_cost_per_unit + if any(amount and model_info.get(alt_key) is not None for amount, alt_key in alternative_pricing): + return 0.0 + return float(tokens) * fallback_cost_per_token def _get_cost_per_unit(model_info: ModelInfo, cost_key: str, default_value: Optional[float] = 0.0) -> Optional[float]: @@ -603,17 +617,29 @@ def _calculate_input_cost( ### AUDIO COST audio_cost_key = _get_service_tier_cost_key("input_cost_per_audio_token", service_tier) prompt_cost += _calculate_modality_token_cost( - model_info, audio_cost_key, prompt_tokens_details["audio_tokens"], prompt_base_cost + model_info, + audio_cost_key, + prompt_tokens_details["audio_tokens"], + prompt_base_cost, + alternative_pricing=((prompt_tokens_details["audio_length_seconds"], "input_cost_per_audio_per_second"),), ) ### IMAGE TOKEN COST prompt_cost += _calculate_modality_token_cost( - model_info, "input_cost_per_image_token", prompt_tokens_details["image_tokens"], prompt_base_cost + model_info, + "input_cost_per_image_token", + prompt_tokens_details["image_tokens"], + prompt_base_cost, + alternative_pricing=((prompt_tokens_details["image_count"], "input_cost_per_image"),), ) ### VIDEO TOKEN COST prompt_cost += _calculate_modality_token_cost( - model_info, "input_cost_per_video_token", prompt_tokens_details["video_tokens"], prompt_base_cost + model_info, + "input_cost_per_video_token", + prompt_tokens_details["video_tokens"], + prompt_base_cost, + alternative_pricing=((prompt_tokens_details["video_length_seconds"], "input_cost_per_video_per_second"),), ) ### CACHE WRITING COST - Now uses tiered pricing diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index d40d569f943..18654d5dec1 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -447,6 +447,90 @@ def test_multimodal_input_tokens_use_tiered_rate_gemini_3_1_pro(): ) +def test_service_tier_audio_tokens_use_base_audio_rate(): + """A service-tier request must still bill audio tokens at the base input_cost_per_audio_token. + + When only the base audio rate exists (no _priority variant), the modality helper must resolve + it via the service-tier suffix fallback instead of dropping to the generic prompt rate. + """ + from unittest.mock import patch + + mock_model_info = { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "input_cost_per_audio_token": 5e-6, + } + + audio_tokens = 100 + text_tokens = 10 + usage = Usage( + prompt_tokens=text_tokens + audio_tokens, + completion_tokens=20, + total_tokens=text_tokens + audio_tokens + 20, + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=text_tokens, audio_tokens=audio_tokens + ), + ) + + with patch( + "litellm.litellm_core_utils.llm_cost_calc.utils.get_model_info", + return_value=mock_model_info, + ): + prompt_cost, _ = generic_cost_per_token( + model="test-model", + usage=usage, + custom_llm_provider="test-provider", + service_tier="priority", + ) + + # audio billed at the base audio rate, not the generic prompt rate + assert round(prompt_cost, 12) == round(text_tokens * 1e-6 + audio_tokens * 5e-6, 12) + + +def test_count_and_duration_priced_modalities_not_double_billed(): + """Count/duration-priced modalities must not also incur a generic per-token charge. + + A model priced by input_cost_per_image (count) and input_cost_per_video_per_second (duration) + exposes no per-token modality rate. The token fallback must stay silent so only the dedicated + count/duration components bill, avoiding double-charging the same modality. + """ + from unittest.mock import patch + + mock_model_info = { + "input_cost_per_token": 1e-6, + "output_cost_per_token": 2e-6, + "input_cost_per_image": 0.005, + "input_cost_per_video_per_second": 0.001, + } + + text_tokens = 10 + image_tokens = 1000 + video_tokens = 500 + usage = Usage( + prompt_tokens=text_tokens + image_tokens + video_tokens, + completion_tokens=20, + total_tokens=text_tokens + image_tokens + video_tokens + 20, + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=text_tokens, + image_tokens=image_tokens, + video_tokens=video_tokens, + image_count=2, + video_length_seconds=10.0, + ), + ) + + with patch( + "litellm.litellm_core_utils.llm_cost_calc.utils.get_model_info", + return_value=mock_model_info, + ): + prompt_cost, _ = generic_cost_per_token( + model="test-model", usage=usage, custom_llm_provider="test-provider" + ) + + expected = (text_tokens * 1e-6) + (2 * 0.005) + (10.0 * 0.001) + assert round(prompt_cost, 12) == round(expected, 12) + + def test_get_token_base_cost_picks_highest_crossed_tier(): """Regression test for #30345. From 186e04b2d9a0cadc252ccd0e7036fc12431f70ef Mon Sep 17 00:00:00 2001 From: yassin Date: Wed, 2 Sep 2026 15:57:56 +0000 Subject: [PATCH 3/3] test: justify pricing helper patches Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../llm_cost_calc/test_llm_cost_calc_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index ea74734dde9..00d35452cb6 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -461,7 +461,7 @@ def test_service_tier_audio_tokens_use_base_audio_rate(): ), ) - with patch( + with patch( # test-quality-ok: custom model info isolates the pricing behavior under test "litellm.litellm_core_utils.llm_cost_calc.utils.get_model_info", return_value=mock_model_info, ): @@ -508,7 +508,7 @@ def test_count_and_duration_priced_modalities_not_double_billed(): ), ) - with patch( + with patch( # test-quality-ok: custom model info isolates the pricing behavior under test "litellm.litellm_core_utils.llm_cost_calc.utils.get_model_info", return_value=mock_model_info, ):