diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index e4b91dcf453..ebe9df3a44e 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -39,6 +39,13 @@ _VALID_DATA_RESIDENCIES = frozenset(r.value for r in DataResidency) # of being rebuilt for every model_info key on every call. _SERVICE_TIER_SUFFIXES: tuple[str, ...] = tuple(f"_{st.value}" for st in ServiceTier) +_ABOVE_THRESHOLD_KEY_PREFIXES: tuple[str, ...] = ( + "input_cost_per_token_above_", + "output_cost_per_token_above_", + "cache_creation_input_token_cost_above_", + "cache_read_input_token_cost_above_", +) + def _get_token_detail_value(details: object, key: str) -> Optional[int]: if isinstance(details, dict): @@ -239,7 +246,12 @@ def _get_token_base_cost( # so that the threshold detection loop only processes standard keys. The # service_tier-specific above-threshold key is resolved later via _get_service_tier_cost_key. threshold_keys = [ - k for k in model_info if k.startswith("input_cost_per_token_above_") and not k.endswith(_SERVICE_TIER_SUFFIXES) + k + for k in model_info + if k.startswith(_ABOVE_THRESHOLD_KEY_PREFIXES) + and k.endswith("_tokens") + and "_above_1hr" not in k + and not k.endswith(_SERVICE_TIER_SUFFIXES) ] if not threshold_keys: return ( @@ -270,7 +282,7 @@ def _get_token_base_cost( service_tier, ) if service_tier - else key + else f"input_cost_per_token_above_{threshold_str}_tokens" ) prompt_base_cost = cast( float, diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index a2eba8f27a2..890f625f90f 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -1016,6 +1016,50 @@ def test_generic_cost_per_token_prices_cache_write_at_above_272k_tier(): assert prompt_cost == pytest.approx(expected_prompt_cost, rel=1e-9) +def test_generic_cost_per_token_activates_cache_creation_tier_without_input_tier(): + """Regression for LIT-4725 / #33772: a long-context cache-creation tier must + activate on its own. When only cache_creation_input_token_cost_above_272k_tokens + is configured (no matching input_cost_per_token_above_272k_tokens), threshold + discovery must still find the tier so cache-write tokens past 272k bill at the + tiered rate while plain input stays at the base rate.""" + model = "litellm-test-cache-write-272k-cache-only" + custom_llm_provider = "openai" + litellm.register_model( + { + model: { + "litellm_provider": custom_llm_provider, + "mode": "chat", + "input_cost_per_token": 1e-6, + "output_cost_per_token": 4e-6, + "cache_creation_input_token_cost": 1.25e-6, + "cache_creation_input_token_cost_above_272k_tokens": 2.5e-6, + } + } + ) + + plain_tokens, cache_write = 200_000, 100_000 + prompt_tokens_details = PromptTokensDetailsWrapper() + prompt_tokens_details.cache_write_tokens = cache_write + usage = Usage( + prompt_tokens=plain_tokens + cache_write, + completion_tokens=100, + total_tokens=plain_tokens + cache_write + 100, + prompt_tokens_details=prompt_tokens_details, + ) + + try: + prompt_cost, _ = generic_cost_per_token( + model=model, + usage=usage, + custom_llm_provider=custom_llm_provider, + ) + finally: + litellm.model_cost.pop(model, None) + + expected_prompt_cost = plain_tokens * 1e-6 + cache_write * 2.5e-6 + assert prompt_cost == pytest.approx(expected_prompt_cost, rel=1e-9) + + def test_string_cost_values(): """Test that cost values defined as strings are properly converted to floats.""" from unittest.mock import patch diff --git a/tests/test_litellm/responses/test_responses_utils.py b/tests/test_litellm/responses/test_responses_utils.py index bc2a4a26104..e4db8022f01 100644 --- a/tests/test_litellm/responses/test_responses_utils.py +++ b/tests/test_litellm/responses/test_responses_utils.py @@ -464,13 +464,16 @@ class TestResponseAPILoggingUtils: def test_transform_response_api_usage_maps_cache_write_tokens_dict(self): """Regression for LIT-4725 / #33772: the Responses API reports cache writes under input_tokens_details.cache_write_tokens. The chat-shaped usage must carry them on - cache_creation_tokens so cost is computed identically to /chat/completions.""" - usage = { - "input_tokens": 10_000, - "output_tokens": 20, - "total_tokens": 10_020, - "input_tokens_details": {"cached_tokens": 2_000, "cache_write_tokens": 8_000}, - } + cache_creation_tokens so cost is computed identically to /chat/completions. + model_construct keeps input_tokens_details a raw dict so the dict branch runs.""" + from litellm.types.llms.openai import ResponseAPIUsage + + usage = ResponseAPIUsage.model_construct( + input_tokens=10_000, + output_tokens=20, + total_tokens=10_020, + input_tokens_details={"cached_tokens": 2_000, "cache_write_tokens": 8_000}, + ) result = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(usage) @@ -482,12 +485,14 @@ class TestResponseAPILoggingUtils: def test_transform_response_api_usage_cache_creation_tokens_precedence_dict(self): """When both cache_creation_tokens and cache_write_tokens are present, the explicit cache_creation_tokens wins (they describe the same tokens under different names).""" - usage = { - "input_tokens": 10_000, - "output_tokens": 20, - "total_tokens": 10_020, - "input_tokens_details": {"cache_creation_tokens": 5_000, "cache_write_tokens": 8_000}, - } + from litellm.types.llms.openai import ResponseAPIUsage + + usage = ResponseAPIUsage.model_construct( + input_tokens=10_000, + output_tokens=20, + total_tokens=10_020, + input_tokens_details={"cache_creation_tokens": 5_000, "cache_write_tokens": 8_000}, + ) result = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(usage)