From 39b84433e728492f4e2bb5d3ffa3dbe6256dcac3 Mon Sep 17 00:00:00 2001 From: Anuj7411 Date: Wed, 26 Aug 2026 19:29:01 +0530 Subject: [PATCH 1/2] fix(cost): discount cached tokens in batch_cost_calculator when input_cost_per_token_batches is set batch_cost_calculator billed every prompt token, including cache-read and cache-creation tokens, at the flat input_cost_per_token_batches rate whenever a model declared that field. The sibling code path (models priced only via input_cost_per_token) already discounted cached tokens correctly, but models with an explicit batches rate skipped that logic entirely. Roughly 100 models in the pricing map set input_cost_per_token_batches (gpt-4.1, gpt-4o, gpt-5.x, gemini-3.x, and Claude on Bedrock/Vertex among them), always at exactly half of input_cost_per_token, so any Batches API call against them with prompt caching enabled was overbilled for the cached portion of the prompt. --- litellm/cost_calculator.py | 22 +++++++---- .../test_litellm/batches/test_batch_utils.py | 6 ++- tests/test_litellm/test_cost_calculator.py | 39 +++++++++++++++++++ 3 files changed, 58 insertions(+), 9 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 6536941a094..231cb231664 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -2201,9 +2201,7 @@ def batch_cost_calculator( output_cost_per_token: Final = model_info.get("output_cost_per_token") total_prompt_cost = 0.0 total_completion_cost = 0.0 - if input_cost_per_token_batches is not None: - total_prompt_cost = usage.prompt_tokens * input_cost_per_token_batches - elif input_cost_per_token: + if input_cost_per_token_batches is not None or input_cost_per_token: details: Final = parse_prompt_tokens_details(usage) cache_read_tokens: Final = details["cache_hit_tokens"] cache_creation_tokens: Final = details["cache_creation_tokens"] @@ -2211,15 +2209,23 @@ def batch_cost_calculator( # Subtract cached tokens from prompt_tokens before calculating cost # Fixes issue where cached tokens are being charged again base_input_tokens: Final = get_billable_input_tokens(usage) - cache_creation_tokens - total_prompt_cost = ( - base_input_tokens * (input_cost_per_token) / 2 - ) # batch cost is usually half of the regular token cost + if input_cost_per_token_batches is not None: + # An explicit batches rate is already the discounted per-token price + total_prompt_cost = base_input_tokens * input_cost_per_token_batches + else: + total_prompt_cost = ( + base_input_tokens * (input_cost_per_token) / 2 + ) # batch cost is usually half of the regular token cost - # Add cache read cost if applicable + # Add cache read cost if applicable. There is no dedicated batches cache-read + # rate, so the standard cache_read_input_token_cost is halved like the base rate. cache_read_cost_key: Final = _get_service_tier_cost_key("cache_read_input_token_cost", None) total_prompt_cost += calculate_cost_component(model_info, cache_read_cost_key, cache_read_tokens) / 2 - cache_creation_cost: Final = model_info.get("cache_creation_input_token_cost") or input_cost_per_token + non_batch_input_cost: Final = ( + input_cost_per_token if input_cost_per_token else (input_cost_per_token_batches or 0.0) * 2 + ) + cache_creation_cost: Final = model_info.get("cache_creation_input_token_cost") or non_batch_input_cost total_prompt_cost += cache_creation_tokens * cache_creation_cost / 2 if output_cost_per_token_batches is not None: total_completion_cost = usage.completion_tokens * output_cost_per_token_batches diff --git a/tests/test_litellm/batches/test_batch_utils.py b/tests/test_litellm/batches/test_batch_utils.py index 41b4bb8cf76..8fdec9bcc2d 100644 --- a/tests/test_litellm/batches/test_batch_utils.py +++ b/tests/test_litellm/batches/test_batch_utils.py @@ -431,7 +431,11 @@ def test_total_usage_and_cost_normalize_mixed_responses_and_chat(): assert usage.completion_tokens == 12 assert usage.total_tokens == 42 assert usage.cache_read_input_tokens == 3 - assert cost == pytest.approx((30 * 0.00125) + (12 * 0.005)) + # model_info declares no cache_read_input_token_cost, so the 3 cached tokens + # are excluded from the base input_cost_per_token_batches charge (consistent + # with how generic_cost_per_token prices an undeclared cache rate at $0) + # rather than billed at the full non-cached batches rate. + assert cost == pytest.approx((27 * 0.00125) + (12 * 0.005)) def test_total_usage_empty_is_zero(): diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 1f1c9be973f..1dd90ce4a8c 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -3584,6 +3584,45 @@ def test_batch_cost_calculator_cache_creation_falls_back_to_input_rate(): assert prompt_cost == pytest.approx((1000 * 3e-6 + 8000 * 3e-7 + 2000 * 3e-6) / 2) +def test_batch_cost_calculator_discounts_cache_tokens_with_explicit_batches_rate(): + """ + Regression test: when a model declares an explicit input_cost_per_token_batches + (e.g. gpt-4.1, gpt-4o, gemini-3-pro-preview, Claude on Bedrock/Vertex all do), + cached and cache-creation tokens must still get their cache discount instead of + being billed at the full non-cached batches rate. + + Before the fix, batch_cost_calculator billed every prompt token (including the + 8000 cached and 2000 cache-creation tokens below) at the flat batches rate, + overcharging by more than 2x on cache-heavy batch requests. + """ + from litellm.cost_calculator import batch_cost_calculator + + model_info: ModelInfo = { + "supported_openai_params": [], + "input_cost_per_token": 3e-6, + "input_cost_per_token_batches": 1.5e-6, # exactly half of input_cost_per_token + "output_cost_per_token": 15e-6, + "output_cost_per_token_batches": 7.5e-6, + "cache_read_input_token_cost": 3e-7, + "cache_creation_input_token_cost": 3.75e-6, + } + prompt_cost, completion_cost_value = batch_cost_calculator( + usage=_batch_cache_usage(), + model="claude-sonnet-4-5-20250929", + custom_llm_provider="anthropic", + model_info=model_info, + ) + + expected_prompt_cost = (1000 * 3e-6 + 8000 * 3e-7 + 2000 * 3.75e-6) / 2 + assert prompt_cost == pytest.approx(expected_prompt_cost) + assert completion_cost_value == pytest.approx(200 * 7.5e-6) + + # The bug billed every prompt token (cached ones included) at the flat batches + # rate, so guard against regressing back to that overcharge. + buggy_prompt_cost = 11000 * 1.5e-6 + assert prompt_cost < buggy_prompt_cost + + def test_completion_cost_bills_interactions_api_response(): from litellm.types.interactions import InteractionsAPIResponse From 7d6f28c5caeb1c0705e5d0a7602ebf9af48d3632 Mon Sep 17 00:00:00 2001 From: Anuj7411 Date: Fri, 28 Aug 2026 08:13:06 +0530 Subject: [PATCH 2/2] refactor(cost): compute typed batch input rate to satisfy reportOperatorIssue budget --- litellm/cost_calculator.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 231cb231664..690e7b1913b 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -2209,16 +2209,13 @@ def batch_cost_calculator( # Subtract cached tokens from prompt_tokens before calculating cost # Fixes issue where cached tokens are being charged again base_input_tokens: Final = get_billable_input_tokens(usage) - cache_creation_tokens - if input_cost_per_token_batches is not None: - # An explicit batches rate is already the discounted per-token price - total_prompt_cost = base_input_tokens * input_cost_per_token_batches - else: - total_prompt_cost = ( - base_input_tokens * (input_cost_per_token) / 2 - ) # batch cost is usually half of the regular token cost + batch_input_cost_per_token: Final = ( + input_cost_per_token_batches + if input_cost_per_token_batches is not None + else (input_cost_per_token or 0.0) / 2 + ) + total_prompt_cost = base_input_tokens * batch_input_cost_per_token - # Add cache read cost if applicable. There is no dedicated batches cache-read - # rate, so the standard cache_read_input_token_cost is halved like the base rate. cache_read_cost_key: Final = _get_service_tier_cost_key("cache_read_input_token_cost", None) total_prompt_cost += calculate_cost_component(model_info, cache_read_cost_key, cache_read_tokens) / 2