From 263281ac89c9b81b17451f0c5c84126ccb2f6213 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 10 Nov 2026 19:49:15 +0000 Subject: [PATCH] feat(cost): bill batch cached image input tokens for OpenAI image models (#50997) * feat(cost): bill batch cached image input tokens for OpenAI image models * fix(cost): bill batch cached audio and image tokens without a text cache price * fix(cost): honor modality cache prices in the half-price batch fallback --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: kerry --- .../crates/model-catalog/src/model_info.rs | 1 + litellm/cost_calculator.py | 132 ++++++++++----- .../litellm_core_utils/llm_cost_calc/utils.py | 3 + ...odel_prices_and_context_window_backup.json | 6 + litellm/types/utils.py | 2 + litellm/utils.py | 3 + model_prices_and_context_window.json | 6 + .../llm_cost_calc/test_llm_cost_calc_utils.py | 17 ++ tests/unit/test_cost_calculator.py | 155 +++++++++++++++++- tests/unit/test_utils.py | 1 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 4 + 11 files changed, 290 insertions(+), 40 deletions(-) diff --git a/litellm-rust/crates/model-catalog/src/model_info.rs b/litellm-rust/crates/model-catalog/src/model_info.rs index afbbfec59e6..fcd0c027323 100644 --- a/litellm-rust/crates/model-catalog/src/model_info.rs +++ b/litellm-rust/crates/model-catalog/src/model_info.rs @@ -28,6 +28,7 @@ pub struct ModelInfoFields { pub cache_read_input_audio_token_cost_flex: Option, pub cache_read_input_audio_token_cost_priority: Option, pub cache_read_input_audio_token_cost_batches: Option, + pub cache_read_input_image_token_cost_batches: Option, pub explicit_cache_read_input_token_cost: Option, pub prompt_cache_min_tokens: Option, pub cache_read_input_token_cost_above_200k_tokens: Option, diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 58947f9d0c2..53137b73f8b 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -49,7 +49,6 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import ( BilledTokenRates, CostCalculatorUtils, calculate_cache_writing_cost, - calculate_cost_component, generic_cost_per_character, generic_cost_per_token, get_batch_cost_rates, @@ -3177,6 +3176,21 @@ def default_video_cost_calculator( return 0.0 +def _optional_batch_rate( + model_info: ModelInfo, + key: Literal[ + "input_cost_per_audio_token_batches", + "cache_read_input_audio_token_cost_batches", + "input_cost_per_image_token_batches", + "input_cost_per_video_token_batches", + ], + prompt_tokens: int, +) -> float | None: + context_key: Final = get_modality_token_cost_key(model_info, key, prompt_tokens) + context_rate: Final = get_cost_per_unit(model_info, context_key, None) + return model_info.get(key) if context_rate is None else context_rate + + def _batch_rate( model_info: ModelInfo, key: Literal[ @@ -3189,9 +3203,7 @@ def _batch_rate( *, prompt_tokens: int = 0, ) -> float: - context_key: Final = get_modality_token_cost_key(model_info, key, prompt_tokens) - context_rate: Final = get_cost_per_unit(model_info, context_key, None) - rate: Final = model_info.get(key) if context_rate is None else context_rate + rate: Final = _optional_batch_rate(model_info, key, prompt_tokens) return fallback if rate is None else rate @@ -3275,30 +3287,14 @@ def batch_cost_calculator( model_info, batch_rates.input, batch_rates.cache_read, + batch_rates.cache_read_image, batch_rates.cache_creation, batch_rates.cache_creation_rates_by_ttl, ) elif input_cost_per_token is not None: - details: Final = parse_prompt_tokens_details(usage) - cache_read_tokens: Final = details["cache_hit_tokens"] - cache_creation_tokens: Final = details["cache_creation_tokens"] - - # Subtract cached tokens from prompt_tokens before calculating cost - # Fixes issue where cached tokens are being charged again - base_input_tokens: Final = get_billable_input_tokens(usage) - cache_creation_tokens - total_prompt_cost = ( - base_input_tokens * (input_cost_per_token) / 2 - ) # batch cost is usually half of the regular token cost - - # Add cache read cost if applicable - cache_read_cost_key: Final = get_service_tier_cost_key("cache_read_input_token_cost", None) - total_prompt_cost += calculate_cost_component(model_info, cache_read_cost_key, cache_read_tokens) / 2 - - configured_cache_creation_cost: Final = model_info.get("cache_creation_input_token_cost") - cache_creation_cost: Final = ( - input_cost_per_token if configured_cache_creation_cost is None else configured_cache_creation_cost + total_prompt_cost = _half_price_prompt_cost( + usage, model_info, input_cost_per_token, batch_rates.cache_read_image ) - total_prompt_cost += cache_creation_tokens * cache_creation_cost / 2 text_rate: Final = _batch_or_half(batch_rates.output, output_cost_per_token) image_rate: Final = _batch_or_half( model_info.get("output_cost_per_image_token_batches"), @@ -3324,15 +3320,54 @@ def _standard_rate(model_info: ModelInfo, key: str, prompt_tokens: int) -> float return get_cost_per_unit(model_info, get_modality_token_cost_key(model_info, key, prompt_tokens), None) -def _cached_audio_batch_fallback( - model_info: ModelInfo, batch_cache_read_rate: float | None, cache_read_rate: float | None, prompt_tokens: int -) -> float: +def _cached_modality_rate( + model_info: ModelInfo, + standard_key: Literal["cache_read_input_audio_token_cost", "cache_read_input_image_token_cost"], + batch_modality_rate: float | None, + batch_cache_read_rate: float | None, + cache_read_rate: float | None, + prompt_tokens: int, +) -> float | None: + if batch_modality_rate is not None: + return batch_modality_rate if batch_cache_read_rate is not None: return batch_cache_read_rate - standard_audio_rate: Final = _standard_rate(model_info, "cache_read_input_audio_token_cost", prompt_tokens) - if standard_audio_rate is not None: - return standard_audio_rate - return 0.0 if cache_read_rate is None else cache_read_rate + standard_modality_rate: Final = _standard_rate(model_info, standard_key, prompt_tokens) + return cache_read_rate if standard_modality_rate is None else standard_modality_rate + + +def _half_price_prompt_cost( + usage: Usage, model_info: ModelInfo, input_cost_per_token: float, batch_cache_read_image_rate: float | None +) -> float: + details: Final = parse_prompt_tokens_details(usage) + cache_creation_tokens: Final = details["cache_creation_tokens"] + base_input_tokens: Final = get_billable_input_tokens(usage) - cache_creation_tokens + cache_read_cost_key: Final = get_service_tier_cost_key("cache_read_input_token_cost", None) + half_cache_read_rate: Final = (get_cost_per_unit(model_info, cache_read_cost_key, 0.0) or 0.0) / 2 + cached_audio_rate: Final = _batch_or_half( + _optional_batch_rate(model_info, "cache_read_input_audio_token_cost_batches", usage.prompt_tokens), + _standard_rate(model_info, "cache_read_input_audio_token_cost", usage.prompt_tokens), + default=half_cache_read_rate, + ) + cached_image_rate: Final = _batch_or_half( + batch_cache_read_image_rate, + _standard_rate(model_info, "cache_read_input_image_token_cost", usage.prompt_tokens), + default=half_cache_read_rate, + ) + cached_audio_tokens: Final = details["cache_hit_audio_tokens"] + cached_image_tokens: Final = details["cached_image_tokens"] + cached_text_tokens: Final = max(details["cache_hit_tokens"] - cached_audio_tokens - cached_image_tokens, 0) + configured_cache_creation_cost: Final = model_info.get("cache_creation_input_token_cost") + cache_creation_cost: Final = ( + input_cost_per_token if configured_cache_creation_cost is None else configured_cache_creation_cost + ) + return ( + base_input_tokens * input_cost_per_token / 2 + + cached_text_tokens * half_cache_read_rate + + cached_audio_tokens * cached_audio_rate + + cached_image_tokens * cached_image_rate + + cache_creation_tokens * cache_creation_cost / 2 + ) def _batch_prompt_cost( @@ -3340,6 +3375,7 @@ def _batch_prompt_cost( model_info: ModelInfo, input_rate: float, batch_cache_read_rate: float | None, + batch_cache_read_image_rate: float | None, cache_creation_rate: float | None, cache_creation_rates_by_ttl: Mapping[str, float], ) -> float: @@ -3349,8 +3385,30 @@ def _batch_prompt_cost( if batch_cache_read_rate is None else batch_cache_read_rate ) - cached_tokens: Final = details["cache_hit_tokens"] if cache_read_rate is not None else 0 - cached_audio_tokens: Final = details["cache_hit_audio_tokens"] if cache_read_rate is not None else 0 + cached_audio_rate: Final = _cached_modality_rate( + model_info, + "cache_read_input_audio_token_cost", + _optional_batch_rate(model_info, "cache_read_input_audio_token_cost_batches", usage.prompt_tokens), + batch_cache_read_rate, + cache_read_rate, + usage.prompt_tokens, + ) + cached_image_rate: Final = _cached_modality_rate( + model_info, + "cache_read_input_image_token_cost", + batch_cache_read_image_rate, + batch_cache_read_rate, + cache_read_rate, + usage.prompt_tokens, + ) + cached_audio_tokens: Final = details["cache_hit_audio_tokens"] if cached_audio_rate is not None else 0 + cached_image_tokens: Final = details["cached_image_tokens"] if cached_image_rate is not None else 0 + cached_text_tokens: Final = ( + max(details["cache_hit_tokens"] - details["cache_hit_audio_tokens"] - details["cached_image_tokens"], 0) + if cache_read_rate is not None + else 0 + ) + cached_tokens: Final = cached_text_tokens + cached_audio_tokens + cached_image_tokens bills_cache_writes: Final = cache_creation_rate is not None or bool(cache_creation_rates_by_ttl) written_tokens: Final = details["cache_creation_tokens"] if bills_cache_writes else 0 audio_tokens, image_tokens, video_tokens = ( @@ -3367,13 +3425,9 @@ def _batch_prompt_cost( * _batch_rate(model_info, "input_cost_per_audio_token_batches", input_rate, prompt_tokens=usage.prompt_tokens) + image_tokens * _batch_rate(model_info, "input_cost_per_image_token_batches", input_rate) + video_tokens * _batch_rate(model_info, "input_cost_per_video_token_batches", input_rate) - + (cached_tokens - cached_audio_tokens) * (cache_read_rate or 0.0) - + cached_audio_tokens - * _batch_rate( - model_info, - "cache_read_input_audio_token_cost_batches", - _cached_audio_batch_fallback(model_info, batch_cache_read_rate, cache_read_rate, usage.prompt_tokens), - ) + + cached_text_tokens * (cache_read_rate or 0.0) + + cached_audio_tokens * (cached_audio_rate or 0.0) + + cached_image_tokens * (cached_image_rate or 0.0) + ( calculate_cache_writing_cost( written_tokens, diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index ae7c4b47ece..2eb204d151b 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -112,6 +112,7 @@ _BATCH_RATE_PREFIXES: Final = ( "input_cost_per_token", "output_cost_per_token", "cache_read_input_token_cost", + "cache_read_input_image_token_cost", "cache_creation_input_token_cost", ) _BATCH_TIER_PREFIXES: Final = (*_BATCH_RATE_PREFIXES, *_TTL_PRICED_CACHE_WRITE_RATE_KEYS.values()) @@ -326,6 +327,7 @@ class BatchCostRates: cache_read: float | None cache_creation: float | None cache_creation_rates_by_ttl: Mapping[str, float] = field(default_factory=lambda: MappingProxyType({})) + cache_read_image: float | None = None def _batch_rate(model_info: ModelInfo, key: str) -> float | None: @@ -388,6 +390,7 @@ def get_batch_cost_rates(model_info: ModelInfo, usage: Usage, custom_llm_provide if (rate := _batch_rate_for_prefix(model_info, key, usage, inclusive)) is not None } ), + cache_read_image=_batch_rate_for_prefix(model_info, "cache_read_input_image_token_cost", usage, inclusive), ) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index a4c16fbeaa3..b37a5197483 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -36449,6 +36449,7 @@ }, "gpt-image-1.5": { "cache_read_input_image_token_cost": 2e-06, + "cache_read_input_image_token_cost_batches": 1e-06, "cache_read_input_token_cost": 1.25e-06, "cache_read_input_token_cost_batches": 6.3e-07, "deprecation_date": "2026-12-01", @@ -36497,6 +36498,7 @@ }, "gpt-image-2": { "cache_read_input_image_token_cost": 2e-06, + "cache_read_input_image_token_cost_batches": 1e-06, "cache_read_input_token_cost": 1.25e-06, "cache_read_input_token_cost_batches": 6.25e-07, "deprecation_date": "2027-02-10", @@ -36539,6 +36541,7 @@ }, "gpt-image-2.5-flare": { "cache_read_input_image_token_cost": 2e-06, + "cache_read_input_image_token_cost_batches": 1e-06, "cache_read_input_token_cost": 1.25e-06, "cache_read_input_token_cost_batches": 6.25e-07, "input_cost_per_token": 5e-06, @@ -36575,6 +36578,7 @@ }, "gpt-image-2.5-sunburst": { "cache_read_input_image_token_cost": 2e-06, + "cache_read_input_image_token_cost_batches": 1e-06, "cache_read_input_token_cost": 1.25e-06, "cache_read_input_token_cost_batches": 6.25e-07, "input_cost_per_token": 5e-06, @@ -39936,6 +39940,7 @@ }, "gpt-image-1-mini": { "cache_read_input_image_token_cost": 2.5e-07, + "cache_read_input_image_token_cost_batches": 1.3e-07, "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_batches": 1e-07, "deprecation_date": "2026-12-01", @@ -60402,6 +60407,7 @@ }, "chatgpt-image-latest": { "cache_read_input_image_token_cost": 2e-06, + "cache_read_input_image_token_cost_batches": 1e-06, "cache_read_input_token_cost": 1.25e-06, "cache_read_input_token_cost_batches": 6.3e-07, "deprecation_date": "2026-12-01", diff --git a/litellm/types/utils.py b/litellm/types/utils.py index ffc9e915ea4..6325de96839 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -372,6 +372,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): cache_read_input_token_cost_above_512k_tokens: float | None cache_read_input_token_cost_batches: ReadOnly[float | None] cache_read_input_audio_token_cost_batches: ReadOnly[float | None] + cache_read_input_image_token_cost_batches: ReadOnly[float | None] cache_read_input_token_cost_above_200k_tokens_batches: ReadOnly[float | None] cache_read_input_token_cost_above_100k_tokens_batches: ReadOnly[float | None] cache_read_input_token_cost_above_272k_tokens_batches: ReadOnly[float | None] @@ -4190,6 +4191,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams): cache_read_input_token_cost_above_200k_tokens_batches: float | None = None cache_read_input_token_cost_above_272k_tokens_batches: float | None = None cache_read_input_audio_token_cost_batches: float | None = None + cache_read_input_image_token_cost_batches: float | None = None cache_creation_input_token_cost_batches: float | None = None cache_creation_input_token_cost_above_100k_tokens_batches: float | None = None cache_creation_input_token_cost_above_200k_tokens_batches: float | None = None diff --git a/litellm/utils.py b/litellm/utils.py index ca916e049d1..717b1b9009a 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -7266,6 +7266,9 @@ def get_model_info_helper( cache_read_input_audio_token_cost_batches=_model_info.get( "cache_read_input_audio_token_cost_batches", None ), + cache_read_input_image_token_cost_batches=_model_info.get( + "cache_read_input_image_token_cost_batches", None + ), explicit_cache_read_input_token_cost=_model_info.get("explicit_cache_read_input_token_cost", None), cache_read_input_image_token_cost=_model_info.get("cache_read_input_image_token_cost", None), prompt_cache_min_tokens=_model_info.get("prompt_cache_min_tokens", None), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index a4c16fbeaa3..b37a5197483 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -36449,6 +36449,7 @@ }, "gpt-image-1.5": { "cache_read_input_image_token_cost": 2e-06, + "cache_read_input_image_token_cost_batches": 1e-06, "cache_read_input_token_cost": 1.25e-06, "cache_read_input_token_cost_batches": 6.3e-07, "deprecation_date": "2026-12-01", @@ -36497,6 +36498,7 @@ }, "gpt-image-2": { "cache_read_input_image_token_cost": 2e-06, + "cache_read_input_image_token_cost_batches": 1e-06, "cache_read_input_token_cost": 1.25e-06, "cache_read_input_token_cost_batches": 6.25e-07, "deprecation_date": "2027-02-10", @@ -36539,6 +36541,7 @@ }, "gpt-image-2.5-flare": { "cache_read_input_image_token_cost": 2e-06, + "cache_read_input_image_token_cost_batches": 1e-06, "cache_read_input_token_cost": 1.25e-06, "cache_read_input_token_cost_batches": 6.25e-07, "input_cost_per_token": 5e-06, @@ -36575,6 +36578,7 @@ }, "gpt-image-2.5-sunburst": { "cache_read_input_image_token_cost": 2e-06, + "cache_read_input_image_token_cost_batches": 1e-06, "cache_read_input_token_cost": 1.25e-06, "cache_read_input_token_cost_batches": 6.25e-07, "input_cost_per_token": 5e-06, @@ -39936,6 +39940,7 @@ }, "gpt-image-1-mini": { "cache_read_input_image_token_cost": 2.5e-07, + "cache_read_input_image_token_cost_batches": 1.3e-07, "cache_read_input_token_cost": 2e-07, "cache_read_input_token_cost_batches": 1e-07, "deprecation_date": "2026-12-01", @@ -60402,6 +60407,7 @@ }, "chatgpt-image-latest": { "cache_read_input_image_token_cost": 2e-06, + "cache_read_input_image_token_cost_batches": 1e-06, "cache_read_input_token_cost": 1.25e-06, "cache_read_input_token_cost_batches": 6.3e-07, "deprecation_date": "2026-12-01", diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index fd7ec0d6bca..0aa76deb9d4 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -4163,6 +4163,23 @@ def test_get_batch_cost_rates_parses_string_rates(): assert (rates.input, rates.output, rates.cache_read) == (2e-6, 6e-6, 2e-7) +def test_get_batch_cost_rates_reads_the_batch_cached_image_rate_for_the_crossed_tier(): + from litellm.litellm_core_utils.llm_cost_calc.utils import get_batch_cost_rates + + rates = get_batch_cost_rates( + _batch_rates_model_info( + input_cost_per_token_batches=2.5e-6, + cache_read_input_image_token_cost=2e-6, + cache_read_input_image_token_cost_batches=1e-6, + cache_read_input_image_token_cost_above_272k_tokens_batches=1.5e-6, + ), + Usage(prompt_tokens=300_000, completion_tokens=1, total_tokens=300_001), + "openai", + ) + + assert rates.cache_read_image == 1.5e-6 + + def test_get_batch_cost_rates_falls_back_to_the_flat_rates_when_tier_rates_are_unparsable(): from litellm.litellm_core_utils.llm_cost_calc.utils import get_batch_cost_rates diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index 9f3bfc2de86..8af9ae29c67 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -3,6 +3,7 @@ import asyncio import datetime import time from collections.abc import Iterator, Mapping +from pathlib import Path from types import MappingProxyType, SimpleNamespace from typing import Final, Literal, cast @@ -10,7 +11,7 @@ import httpx import pytest import respx from openai.types.completion_usage import CompletionUsage -from pydantic import BaseModel +from pydantic import BaseModel, TypeAdapter import litellm from litellm import TranscriptionResponse, model_cost @@ -4580,6 +4581,158 @@ def test_batch_cost_calculator_bills_cached_tokens_at_the_standard_cache_price_w assert prompt_cost == pytest.approx(1000 * expected_rate) +def _cached_image_batch_usage() -> Usage: + return Usage( + prompt_tokens=1000, + completion_tokens=0, + total_tokens=1000, + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=1000, image_tokens=1000, cached_tokens_details=CachedTokensDetails(image_tokens=1000) + ), + ) + + +@pytest.mark.parametrize( + ("image_batch_cache_price", "text_batch_cache_price", "expected_rate"), + [ + pytest.param(1e-6, 6.25e-7, 1e-6, id="the-batch-cached-image-price-wins"), + pytest.param(None, 6.25e-7, 6.25e-7, id="without-it-cached-images-bill-the-batch-cache-price"), + pytest.param(None, None, 2e-6, id="without-any-batch-cache-price-cached-images-bill-the-standard-image-cache-price"), + ], +) +def test_batch_cost_calculator_bills_cached_image_tokens_at_the_batch_cached_image_price( + image_batch_cache_price: float | None, text_batch_cache_price: float | None, expected_rate: float +) -> None: + model_info: Final = cast( + ModelInfo, + { + "input_cost_per_token": 5e-6, + "input_cost_per_image_token": 8e-6, + "cache_read_input_token_cost": 1.25e-6, + "cache_read_input_image_token_cost": 2e-6, + "input_cost_per_token_batches": 2.5e-6, + "input_cost_per_image_token_batches": 4e-6, + "cache_read_input_token_cost_batches": text_batch_cache_price, + "cache_read_input_image_token_cost_batches": image_batch_cache_price, + }, + ) + + prompt_cost, _ = batch_cost_calculator( + usage=_cached_image_batch_usage(), model="gpt-image-2", custom_llm_provider="openai", model_info=model_info + ) + + assert prompt_cost == pytest.approx(1000 * expected_rate) + + +_COST_MAP_ROWS: Final = TypeAdapter(dict[str, dict[str, object]]) + + +def _rows_with_a_batch_cached_image_price() -> tuple[tuple[str, str, float, ModelInfo], ...]: + rows: Final = _COST_MAP_ROWS.validate_json( + Path(litellm.__file__).with_name("model_prices_and_context_window_backup.json").read_bytes() + ) + return tuple( + (name, str(entry["litellm_provider"]), rate, cast(ModelInfo, entry)) + for name, entry in rows.items() + if isinstance(rate := entry.get("cache_read_input_image_token_cost_batches"), float) + ) + + +def _batch_cached_image_cost(name: str, provider: str, entry: ModelInfo) -> float: + return batch_cost_calculator(_cached_image_batch_usage(), name, provider, model_info=entry)[0] + + +@pytest.mark.parametrize( + ("usage", "modality_batch_cache_key"), + [ + pytest.param(_cached_audio_batch_usage(), "cache_read_input_audio_token_cost_batches", id="cached-audio"), + pytest.param(_cached_image_batch_usage(), "cache_read_input_image_token_cost_batches", id="cached-image"), + ], +) +def test_batch_cost_calculator_bills_cached_modality_tokens_at_their_batch_cache_price_without_a_text_cache_price( + usage: Usage, modality_batch_cache_key: str +) -> None: + model_info: Final = cast( + ModelInfo, + { + "input_cost_per_token": 5e-6, + "input_cost_per_audio_token": 8e-6, + "input_cost_per_image_token": 8e-6, + "input_cost_per_token_batches": 2.5e-6, + "input_cost_per_audio_token_batches": 4e-6, + "input_cost_per_image_token_batches": 4e-6, + modality_batch_cache_key: 1e-6, + }, + ) + + prompt_cost, _ = batch_cost_calculator( + usage=usage, model="gpt-image-2", custom_llm_provider="openai", model_info=model_info + ) + + assert prompt_cost == pytest.approx(1000 * 1e-6) + + +@pytest.mark.parametrize( + ("usage", "modality_cache_key", "expected_prompt_cost"), + [ + pytest.param( + _cached_audio_batch_usage(), + "cache_read_input_audio_token_cost_batches", + 1000 * 1e-6, + id="cached-audio-batch-price", + ), + pytest.param( + _cached_image_batch_usage(), + "cache_read_input_image_token_cost_batches", + 1000 * 1e-6, + id="cached-image-batch-price", + ), + pytest.param( + _cached_audio_batch_usage(), + "cache_read_input_audio_token_cost", + 1000 * 1e-6 / 2, + id="cached-audio-standard-price-halved", + ), + pytest.param( + _cached_image_batch_usage(), + "cache_read_input_image_token_cost", + 1000 * 1e-6 / 2, + id="cached-image-standard-price-halved", + ), + ], +) +def test_batch_cost_calculator_half_price_fallback_bills_cached_modality_tokens_at_their_own_cache_price( + usage: Usage, modality_cache_key: str, expected_prompt_cost: float +) -> None: + model_info: Final = cast( + ModelInfo, + { + "input_cost_per_token": 5e-6, + "cache_read_input_token_cost": 2.5e-6, + modality_cache_key: 1e-6, + }, + ) + + prompt_cost, _ = batch_cost_calculator( + usage=usage, model="gpt-image-2", custom_llm_provider="openai", model_info=model_info + ) + + assert prompt_cost == pytest.approx(expected_prompt_cost) + + +def test_batch_cached_image_tokens_bill_the_rows_batch_cached_image_price_on_every_row_that_has_one() -> None: + rows: Final = _rows_with_a_batch_cached_image_price() + assert rows, "no cost map row prices Batch cached image input" + + mispriced: Final = tuple( + (name, _batch_cached_image_cost(name, provider, entry), 1000 * rate) + for name, provider, rate, entry in rows + if _batch_cached_image_cost(name, provider, entry) != pytest.approx(1000 * rate) + ) + + assert mispriced == () + + def test_batch_cost_calculator_bills_long_context_cached_tokens_at_the_standard_long_context_cache_price() -> None: model_info: Final[ModelInfo] = { "input_cost_per_token": 1.25e-6, diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index eaa0b3970bd..045d78fc1e7 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -1023,6 +1023,7 @@ _COST_MAP_ROW_SCHEMA: Final = { "cache_creation_input_token_cost_above_1hr_above_200k_tokens": {"type": "number"}, "cache_read_input_audio_token_cost": {"type": "number"}, "cache_read_input_audio_token_cost_batches": {"type": "number"}, + "cache_read_input_image_token_cost_batches": {"type": "number"}, "cache_read_input_audio_token_cost_flex": {"type": "number"}, "cache_read_input_audio_token_cost_priority": {"type": "number"}, "cache_read_input_image_token_cost": {"type": "number"}, diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index e1b0104e089..ddf6eb21c63 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -36614,6 +36614,8 @@ export interface components { cache_read_input_audio_token_cost_priority?: number | null; /** Cache Read Input Image Token Cost */ cache_read_input_image_token_cost?: number | null; + /** Cache Read Input Image Token Cost Batches */ + cache_read_input_image_token_cost_batches?: number | null; /** Cache Read Input Token Cost */ cache_read_input_token_cost?: number | null; /** Cache Read Input Token Cost Above 100K Tokens */ @@ -52804,6 +52806,8 @@ export interface components { cache_read_input_audio_token_cost_priority?: number | null; /** Cache Read Input Image Token Cost */ cache_read_input_image_token_cost?: number | null; + /** Cache Read Input Image Token Cost Batches */ + cache_read_input_image_token_cost_batches?: number | null; /** Cache Read Input Token Cost */ cache_read_input_token_cost?: number | null; /** Cache Read Input Token Cost Above 100K Tokens */