From cd7cdb3e3a8e61b207cb187a6e1a204273acccfd Mon Sep 17 00:00:00 2001 From: Srivatsa03 Date: Tue, 18 Aug 2026 20:44:45 -0500 Subject: [PATCH 1/2] fix(cost): stop double-billing cached tokens that overlap a modality Providers report cached_tokens and image_tokens as overlapping subsets of prompt_tokens rather than a disjoint partition, so a request whose images were served from cache paid for them twice, once at the cache-read rate and again at the image or input rate. The synthetic case in the issue came out at 109e-6 against a correct 39e-6. Clamp each modality to the part of the request the cache did not already cover, so the billed components still sum to prompt_tokens Fixes #37281 --- .../litellm_core_utils/llm_cost_calc/utils.py | 19 ++++++++--- .../llm_cost_calc/test_llm_cost_calc_utils.py | 34 +++++++++++++++++++ 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 9d6ad8b6e39..37774822565 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -854,11 +854,22 @@ def generic_cost_per_token( total_details: Final = text_tokens + cache_hit + audio_tokens + cache_creation + image_tokens + video_tokens has_double_counting: Final = (cache_hit > 0 or cache_creation > 0) and total_details > usage.prompt_tokens - if (text_tokens == 0 and prompt_tokens_details["image_count"] == 0) or has_double_counting: - text_tokens = usage.prompt_tokens - cache_hit - audio_tokens - cache_creation - image_tokens - video_tokens + if has_double_counting: + # cached and per-modality counts are both subsets of prompt_tokens and may overlap, so a + # modality can only bill what the cache did not already cover or the overlap is billed twice + uncached_budget: Final = max(usage.prompt_tokens - cache_hit - cache_creation, 0) + billable_audio: Final = min(audio_tokens, uncached_budget) + billable_image: Final = min(image_tokens, uncached_budget - billable_audio) + billable_video: Final = min(video_tokens, uncached_budget - billable_audio - billable_image) + prompt_tokens_details["audio_tokens"] = billable_audio + prompt_tokens_details["image_tokens"] = billable_image + prompt_tokens_details["video_tokens"] = billable_video + prompt_tokens_details["text_tokens"] = uncached_budget - billable_audio - billable_image - billable_video + elif text_tokens == 0 and prompt_tokens_details["image_count"] == 0: # Clamp to zero: inconsistent streaming usage - text_tokens = max(text_tokens, 0) - prompt_tokens_details["text_tokens"] = text_tokens + prompt_tokens_details["text_tokens"] = max( + usage.prompt_tokens - cache_hit - audio_tokens - cache_creation - image_tokens - video_tokens, 0 + ) ( prompt_base_cost, diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 4d157e74482..486f3a81331 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -1358,6 +1358,40 @@ def test_string_cost_values(): assert round(completion_cost, 12) == round(expected_completion_cost, 12) +def test_generic_cost_per_token_overlapping_cached_and_image_tokens(): + """Some providers report cached_tokens and image_tokens as overlapping subsets of + prompt_tokens. Billing each in full charged the overlap twice, once at the cache rate + and again at the input rate.""" + model = "litellm-test-overlapping-cached-image" + litellm.register_model( + { + model: { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 1e-6, + "cache_read_input_token_cost": 1e-7, + "output_cost_per_token": 2e-6, + } + } + ) + usage = Usage( + prompt_tokens=100, + completion_tokens=10, + total_tokens=110, + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=None, cached_tokens=90, image_tokens=80 + ), + ) + + prompt_cost, completion_cost = generic_cost_per_token( + model=model, usage=usage, custom_llm_provider="openai" + ) + + # 90 cached at 1e-7, the remaining 10 uncached tokens once at 1e-6 + assert prompt_cost == pytest.approx(90 * 1e-7 + 10 * 1e-6) + assert completion_cost == pytest.approx(10 * 2e-6) + + def test_calculate_cost_component_with_string_values(): """Test the calculate_cost_component function directly with string cost values.""" from litellm.litellm_core_utils.llm_cost_calc.utils import calculate_cost_component From e8a683e7a853fcfd9a1cce43573ebc260b7c6c13 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 26 Aug 2026 17:48:43 -0700 Subject: [PATCH 2/2] test(cost): cover warm prefix cache spanning text and image tokens --- .../llm_cost_calc/test_llm_cost_calc_utils.py | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index e32743a7e47..ee8527c487c 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -1586,6 +1586,42 @@ def test_generic_cost_per_token_overlapping_cached_and_image_tokens(): assert completion_cost == pytest.approx(10 * 2e-6) +def test_generic_cost_per_token_warm_prefix_cache_spanning_text_and_image_tokens(): + """xAI reports text_tokens + image_tokens = prompt_tokens with cached_tokens overlapping + both, so a warm prefix cache covering the whole image exceeds the text-only count. + Observed live on grok-4.6 (issue #37281): the image tokens were billed a second time at + the full input rate on top of the cache-read bucket, 0.003500 in vs the provider's own + 0.001274 bill.""" + model = "litellm-test-warm-prefix-cache-overlap" + litellm.register_model( + { + model: { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 2e-6, + "cache_read_input_token_cost": 5e-7, + "output_cost_per_token": 6e-6, + } + } + ) + usage = Usage( + prompt_tokens=2461, + completion_tokens=440, + total_tokens=2901, + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=1319, cached_tokens=2432, image_tokens=1142 + ), + ) + + prompt_cost, completion_cost = generic_cost_per_token( + model=model, usage=usage, custom_llm_provider="openai" + ) + + # 2432 cached at the cache-read rate, the 29 uncached tokens once at the input rate + assert prompt_cost == pytest.approx(2432 * 5e-7 + 29 * 2e-6) + assert completion_cost == pytest.approx(440 * 6e-6) + + def test_calculate_cost_component_with_string_values(): """Test the calculate_cost_component function directly with string cost values.""" from litellm.litellm_core_utils.llm_cost_calc.utils import calculate_cost_component