Merge 186e04b2d9 into 252c71c0b2

litellm/litellm_core_utils/llm_cost_calc/utils.py

@@ -705,6 +705,42 @@ def calculate_cost_component(model_info: ModelInfo, cost_key: str, usage_value:
     return 0.0
 
 
+def _calculate_modality_token_cost(
+    model_info: ModelInfo,
+    cost_key: str,
+    tokens: int,
+    fallback_cost_per_token: float,
+    alternative_pricing: tuple[tuple[float, str], ...] = (),
+) -> float:
+    """
+    Cost for a modality (audio/image/video) input token count.
+
+    Resolution order:
+      1. Modality-specific per-token rate (``cost_key``), including the service-tier suffix
+         fallback to the base key handled by ``_get_cost_per_unit``. A present-but-malformed rate
+         resolves to 0 here, preserving prior behavior.
+      2. If this request is already billed for the modality by count/duration
+         (``alternative_pricing`` pairs of ``(usage_amount, cost_key)`` such as the image count
+         with ``input_cost_per_image``), charge nothing here so the dedicated count/duration
+         component is not double-billed. Only a measurement actually present in this usage suppresses
+         the fallback; a model that merely lists a count price still bills reported tokens.
+      3. Otherwise fall back to ``fallback_cost_per_token``, the resolved input per-token rate that
+         is already tier-aware (e.g. long-context ``_above_200k_tokens``) and service-tier-aware.
+         Providers like Gemini bill audio/video input at the standard input rate and expose no
+         modality-specific key, so dropping these tokens severely undercounts multimodal spend.
+    """
+    if tokens <= 0:
+        return 0.0
+    if model_info.get(cost_key) is not None:
+        return calculate_cost_component(model_info, cost_key, tokens)
+    resolved_cost_per_unit: Final = _get_cost_per_unit(model_info, cost_key, None)
+    if resolved_cost_per_unit is not None:
+        return float(tokens) * resolved_cost_per_unit
+    if any(amount and model_info.get(alt_key) is not None for amount, alt_key in alternative_pricing):
+        return 0.0
+    return float(tokens) * fallback_cost_per_token
+
+
 def _get_cost_per_unit(model_info: ModelInfo, cost_key: str, default_value: float | None = 0.0) -> float | None:
     # Sometimes the cost per unit is a string (e.g.: If a value like "3e-7" was read from the config.yaml)
     cost_per_unit: Final = model_info.get(cost_key)
@@ -921,25 +957,32 @@ def _calculate_input_cost(
     prompt_cost += float(prompt_tokens_details["cache_hit_tokens"]) * cache_read_cost
 
     ### AUDIO COST
-    if prompt_tokens_details["audio_tokens"]:
-        audio_cost_key: Final = _get_service_tier_cost_key("input_cost_per_audio_token", service_tier)
-        prompt_cost += calculate_cost_component(model_info, audio_cost_key, prompt_tokens_details["audio_tokens"])
+    audio_cost_key: Final = _get_service_tier_cost_key("input_cost_per_audio_token", service_tier)
+    prompt_cost += _calculate_modality_token_cost(
+        model_info,
+        audio_cost_key,
+        prompt_tokens_details["audio_tokens"],
+        prompt_base_cost,
+        alternative_pricing=((prompt_tokens_details["audio_length_seconds"], "input_cost_per_audio_per_second"),),
+    )
 
     ### IMAGE TOKEN COST
-    if prompt_tokens_details["image_tokens"]:
-        # For image token costs:
-        # First check if input_cost_per_image_token is available. If not, default to generic input_cost_per_token.
-        image_token_cost_key = "input_cost_per_image_token"
-        if model_info.get(image_token_cost_key) is None:
-            image_token_cost_key = "input_cost_per_token"
-        prompt_cost += calculate_cost_component(model_info, image_token_cost_key, prompt_tokens_details["image_tokens"])
+    prompt_cost += _calculate_modality_token_cost(
+        model_info,
+        "input_cost_per_image_token",
+        prompt_tokens_details["image_tokens"],
+        prompt_base_cost,
+        alternative_pricing=((prompt_tokens_details["image_count"], "input_cost_per_image"),),
+    )
 
     ### VIDEO TOKEN COST
-    if prompt_tokens_details["video_tokens"]:
-        video_token_cost_key = "input_cost_per_video_token"
-        if model_info.get(video_token_cost_key) is None:
-            video_token_cost_key = "input_cost_per_token"
-        prompt_cost += calculate_cost_component(model_info, video_token_cost_key, prompt_tokens_details["video_tokens"])
+    prompt_cost += _calculate_modality_token_cost(
+        model_info,
+        "input_cost_per_video_token",
+        prompt_tokens_details["video_tokens"],
+        prompt_base_cost,
+        alternative_pricing=((prompt_tokens_details["video_length_seconds"], "input_cost_per_video_per_second"),),
+    )
 
     ### CACHE WRITING COST - Now uses tiered pricing
     if (

tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py

@@ -434,6 +434,135 @@ def test_generic_cost_per_token_above_200k_tokens(_local_model_cost_map):
     )
 
 
+def test_multimodal_input_tokens_use_tiered_rate_gemini_3_1_pro(_local_model_cost_map):
+    """Regression test for LIT-4681.
+
+    Gemini bills audio/video input at the standard input rate and exposes no
+    input_cost_per_audio_token / input_cost_per_video_token. For a >200k-token multimodal
+    request, audio tokens must not be dropped and video tokens must be billed at the
+    long-context (_above_200k_tokens) input rate, not the untiered base rate.
+    """
+    model = "gemini-3.1-pro-preview"
+    custom_llm_provider = "vertex_ai"
+
+    model_cost_map = litellm.model_cost[f"{custom_llm_provider}/{model}"]
+    assert model_cost_map.get("input_cost_per_audio_token") is None
+    assert model_cost_map.get("input_cost_per_video_token") is None
+    above_200k_rate = model_cost_map["input_cost_per_token_above_200k_tokens"]
+
+    text_tokens = 9033
+    audio_tokens = 14999
+    video_tokens = 792000
+    prompt_tokens = text_tokens + audio_tokens + video_tokens
+    completion_tokens = 4559
+    usage = Usage(
+        prompt_tokens=prompt_tokens,
+        completion_tokens=completion_tokens,
+        total_tokens=prompt_tokens + completion_tokens,
+        prompt_tokens_details=PromptTokensDetailsWrapper(
+            text_tokens=text_tokens,
+            audio_tokens=audio_tokens,
+            video_tokens=video_tokens,
+        ),
+    )
+
+    prompt_cost, completion_cost = generic_cost_per_token(
+        model=model,
+        usage=usage,
+        custom_llm_provider=custom_llm_provider,
+    )
+
+    assert round(prompt_cost, 10) == round(prompt_tokens * above_200k_rate, 10)
+    assert round(completion_cost, 10) == round(
+        model_cost_map["output_cost_per_token_above_200k_tokens"] * completion_tokens,
+        10,
+    )
+
+
+def test_service_tier_audio_tokens_use_base_audio_rate():
+    """A service-tier request must still bill audio tokens at the base input_cost_per_audio_token.
+
+    When only the base audio rate exists (no _priority variant), the modality helper must resolve
+    it via the service-tier suffix fallback instead of dropping to the generic prompt rate.
+    """
+    from unittest.mock import patch
+
+    mock_model_info = {
+        "input_cost_per_token": 1e-6,
+        "output_cost_per_token": 2e-6,
+        "input_cost_per_audio_token": 5e-6,
+    }
+
+    audio_tokens = 100
+    text_tokens = 10
+    usage = Usage(
+        prompt_tokens=text_tokens + audio_tokens,
+        completion_tokens=20,
+        total_tokens=text_tokens + audio_tokens + 20,
+        prompt_tokens_details=PromptTokensDetailsWrapper(
+            text_tokens=text_tokens, audio_tokens=audio_tokens
+        ),
+    )
+
+    with patch(  # test-quality-ok: custom model info isolates the pricing behavior under test
+        "litellm.litellm_core_utils.llm_cost_calc.utils.get_model_info",
+        return_value=mock_model_info,
+    ):
+        prompt_cost, _ = generic_cost_per_token(
+            model="test-model",
+            usage=usage,
+            custom_llm_provider="test-provider",
+            service_tier="priority",
+        )
+
+    # audio billed at the base audio rate, not the generic prompt rate
+    assert round(prompt_cost, 12) == round(text_tokens * 1e-6 + audio_tokens * 5e-6, 12)
+
+
+def test_count_and_duration_priced_modalities_not_double_billed():
+    """Count/duration-priced modalities must not also incur a generic per-token charge.
+
+    A model priced by input_cost_per_image (count) and input_cost_per_video_per_second (duration)
+    exposes no per-token modality rate. The token fallback must stay silent so only the dedicated
+    count/duration components bill, avoiding double-charging the same modality.
+    """
+    from unittest.mock import patch
+
+    mock_model_info = {
+        "input_cost_per_token": 1e-6,
+        "output_cost_per_token": 2e-6,
+        "input_cost_per_image": 0.005,
+        "input_cost_per_video_per_second": 0.001,
+    }
+
+    text_tokens = 10
+    image_tokens = 1000
+    video_tokens = 500
+    usage = Usage(
+        prompt_tokens=text_tokens + image_tokens + video_tokens,
+        completion_tokens=20,
+        total_tokens=text_tokens + image_tokens + video_tokens + 20,
+        prompt_tokens_details=PromptTokensDetailsWrapper(
+            text_tokens=text_tokens,
+            image_tokens=image_tokens,
+            video_tokens=video_tokens,
+            image_count=2,
+            video_length_seconds=10.0,
+        ),
+    )
+
+    with patch(  # test-quality-ok: custom model info isolates the pricing behavior under test
+        "litellm.litellm_core_utils.llm_cost_calc.utils.get_model_info",
+        return_value=mock_model_info,
+    ):
+        prompt_cost, _ = generic_cost_per_token(
+            model="test-model", usage=usage, custom_llm_provider="test-provider"
+        )
+
+    expected = (text_tokens * 1e-6) + (2 * 0.005) + (10.0 * 0.001)
+    assert round(prompt_cost, 12) == round(expected, 12)
+
+
 def test_get_token_base_cost_picks_highest_crossed_tier():
     """Regression test for #30345.