diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 3f533de4f44..a082af4b05e 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -889,11 +889,22 @@ def generic_cost_per_token( total_details: Final = text_tokens + cache_hit + audio_tokens + cache_creation + image_tokens + video_tokens has_double_counting: Final = (cache_hit > 0 or cache_creation > 0) and total_details > usage.prompt_tokens - if (text_tokens == 0 and prompt_tokens_details["image_count"] == 0) or has_double_counting: - text_tokens = usage.prompt_tokens - cache_hit - audio_tokens - cache_creation - image_tokens - video_tokens + if has_double_counting: + # cached and per-modality counts are both subsets of prompt_tokens and may overlap, so a + # modality can only bill what the cache did not already cover or the overlap is billed twice + uncached_budget: Final = max(usage.prompt_tokens - cache_hit - cache_creation, 0) + billable_audio: Final = min(audio_tokens, uncached_budget) + billable_image: Final = min(image_tokens, uncached_budget - billable_audio) + billable_video: Final = min(video_tokens, uncached_budget - billable_audio - billable_image) + prompt_tokens_details["audio_tokens"] = billable_audio + prompt_tokens_details["image_tokens"] = billable_image + prompt_tokens_details["video_tokens"] = billable_video + prompt_tokens_details["text_tokens"] = uncached_budget - billable_audio - billable_image - billable_video + elif text_tokens == 0 and prompt_tokens_details["image_count"] == 0: # Clamp to zero: inconsistent streaming usage - text_tokens = max(text_tokens, 0) - prompt_tokens_details["text_tokens"] = text_tokens + prompt_tokens_details["text_tokens"] = max( + usage.prompt_tokens - cache_hit - audio_tokens - cache_creation - image_tokens - video_tokens, 0 + ) ( prompt_base_cost, diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 7c23cd038a4..ce90719789a 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -1552,6 +1552,76 @@ def test_string_cost_values(): assert round(completion_cost, 12) == round(expected_completion_cost, 12) +def test_generic_cost_per_token_overlapping_cached_and_image_tokens(): + """Some providers report cached_tokens and image_tokens as overlapping subsets of + prompt_tokens. Billing each in full charged the overlap twice, once at the cache rate + and again at the input rate.""" + model = "litellm-test-overlapping-cached-image" + litellm.register_model( + { + model: { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 1e-6, + "cache_read_input_token_cost": 1e-7, + "output_cost_per_token": 2e-6, + } + } + ) + usage = Usage( + prompt_tokens=100, + completion_tokens=10, + total_tokens=110, + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=None, cached_tokens=90, image_tokens=80 + ), + ) + + prompt_cost, completion_cost = generic_cost_per_token( + model=model, usage=usage, custom_llm_provider="openai" + ) + + # 90 cached at 1e-7, the remaining 10 uncached tokens once at 1e-6 + assert prompt_cost == pytest.approx(90 * 1e-7 + 10 * 1e-6) + assert completion_cost == pytest.approx(10 * 2e-6) + + +def test_generic_cost_per_token_warm_prefix_cache_spanning_text_and_image_tokens(): + """xAI reports text_tokens + image_tokens = prompt_tokens with cached_tokens overlapping + both, so a warm prefix cache covering the whole image exceeds the text-only count. + Observed live on grok-4.6 (issue #37281): the image tokens were billed a second time at + the full input rate on top of the cache-read bucket, 0.003500 in vs the provider's own + 0.001274 bill.""" + model = "litellm-test-warm-prefix-cache-overlap" + litellm.register_model( + { + model: { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 2e-6, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_token": 6e-6, + } + } + ) + usage = Usage( + prompt_tokens=2461, + completion_tokens=440, + total_tokens=2901, + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=1319, cached_tokens=2432, image_tokens=1142 + ), + ) + + prompt_cost, completion_cost = generic_cost_per_token( + model=model, usage=usage, custom_llm_provider="openai" + ) + + # 2432 cached at the cache-read rate, the 29 uncached tokens once at the input rate + assert prompt_cost == pytest.approx(2432 * 5e-7 + 29 * 2e-6) + assert completion_cost == pytest.approx(440 * 6e-6) + + def test_calculate_cost_component_with_string_values(): """Test the calculate_cost_component function directly with string cost values.""" from litellm.litellm_core_utils.llm_cost_calc.utils import calculate_cost_component