From b96fdda76f3aad1f2182403fda3eaa52251aa7d1 Mon Sep 17 00:00:00 2001 From: Ryan Crabbe Date: Thu, 5 Feb 2026 15:37:33 -0800 Subject: [PATCH 1/4] perf: skip threshold pricing loop for models without threshold keys Collect threshold keys first and return early if none exist, avoiding expensive sorted iteration over all 66+ model_info keys for most models. --- .../litellm_core_utils/llm_cost_calc/utils.py | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 2308dc7beca..dae1cf1cace 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -189,9 +189,25 @@ def _get_token_base_cost( cache_read_cost = cast(float, _get_cost_per_unit(model_info, cache_read_cost_key)) ## CHECK IF ABOVE THRESHOLD + # Optimization: collect threshold keys first to avoid sorting all model_info keys. + # Most models don't have threshold pricing, so we can return early. + threshold_keys = [ + k for k in model_info if k.startswith("input_cost_per_token_above_") + ] + if not threshold_keys: + return ( + prompt_base_cost, + completion_base_cost, + cache_creation_cost, + cache_creation_cost_above_1hr, + cache_read_cost, + ) + + # Only sort the threshold keys (typically 1-2 keys instead of 66+) threshold: Optional[float] = None - for key, value in sorted(model_info.items(), reverse=True): - if key.startswith("input_cost_per_token_above_") and value is not None: + for key in sorted(threshold_keys, reverse=True): + value = model_info.get(key) + if value is not None: try: # Handle both formats: _above_128k_tokens and _above_128_tokens threshold_str = key.split("_above_")[1].split("_tokens")[0] From 00a1ba7bdcb4945d4c0d41962026fec463f42eec Mon Sep 17 00:00:00 2001 From: Ryan Crabbe Date: Thu, 5 Feb 2026 15:55:27 -0800 Subject: [PATCH 2/4] perf: skip cost component calculations when usage values are zero Add early-out checks in _calculate_input_cost() to skip calculate_cost_component() calls when audio_tokens, image_tokens, character_count, image_count, video_length_seconds, or cache_creation_tokens are 0. --- .../litellm_core_utils/llm_cost_calc/utils.py | 67 ++++++++++--------- 1 file changed, 36 insertions(+), 31 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index dae1cf1cace..d1a47efde7e 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -518,47 +518,52 @@ def _calculate_input_cost( prompt_cost += float(prompt_tokens_details["cache_hit_tokens"]) * cache_read_cost ### AUDIO COST - prompt_cost += calculate_cost_component( - model_info, "input_cost_per_audio_token", prompt_tokens_details["audio_tokens"] - ) + if prompt_tokens_details["audio_tokens"]: + prompt_cost += calculate_cost_component( + model_info, "input_cost_per_audio_token", prompt_tokens_details["audio_tokens"] + ) ### IMAGE TOKEN COST - # For image token costs: - # First check if input_cost_per_image_token is available. If not, default to generic input_cost_per_token. - image_token_cost_key = "input_cost_per_image_token" - if model_info.get(image_token_cost_key) is None: - image_token_cost_key = "input_cost_per_token" - prompt_cost += calculate_cost_component( - model_info, image_token_cost_key, prompt_tokens_details["image_tokens"] - ) + if prompt_tokens_details["image_tokens"]: + # For image token costs: + # First check if input_cost_per_image_token is available. If not, default to generic input_cost_per_token. + image_token_cost_key = "input_cost_per_image_token" + if model_info.get(image_token_cost_key) is None: + image_token_cost_key = "input_cost_per_token" + prompt_cost += calculate_cost_component( + model_info, image_token_cost_key, prompt_tokens_details["image_tokens"] + ) ### CACHE WRITING COST - Now uses tiered pricing - prompt_cost += calculate_cache_writing_cost( - cache_creation_tokens=prompt_tokens_details["cache_creation_tokens"], - cache_creation_token_details=prompt_tokens_details[ - "cache_creation_token_details" - ], - cache_creation_cost_above_1hr=cache_creation_cost_above_1hr, - cache_creation_cost=cache_creation_cost, - ) + if prompt_tokens_details["cache_creation_tokens"]: + prompt_cost += calculate_cache_writing_cost( + cache_creation_tokens=prompt_tokens_details["cache_creation_tokens"], + cache_creation_token_details=prompt_tokens_details[ + "cache_creation_token_details" + ], + cache_creation_cost_above_1hr=cache_creation_cost_above_1hr, + cache_creation_cost=cache_creation_cost, + ) ### CHARACTER COST - - prompt_cost += calculate_cost_component( - model_info, "input_cost_per_character", prompt_tokens_details["character_count"] - ) + if prompt_tokens_details["character_count"]: + prompt_cost += calculate_cost_component( + model_info, "input_cost_per_character", prompt_tokens_details["character_count"] + ) ### IMAGE COUNT COST - prompt_cost += calculate_cost_component( - model_info, "input_cost_per_image", prompt_tokens_details["image_count"] - ) + if prompt_tokens_details["image_count"]: + prompt_cost += calculate_cost_component( + model_info, "input_cost_per_image", prompt_tokens_details["image_count"] + ) ### VIDEO LENGTH COST - prompt_cost += calculate_cost_component( - model_info, - "input_cost_per_video_per_second", - prompt_tokens_details["video_length_seconds"], - ) + if prompt_tokens_details["video_length_seconds"]: + prompt_cost += calculate_cost_component( + model_info, + "input_cost_per_video_per_second", + prompt_tokens_details["video_length_seconds"], + ) return prompt_cost From eab8bb4e061091c093c43bf52586edcba2052b39 Mon Sep 17 00:00:00 2001 From: Ryan Crabbe Date: Thu, 5 Feb 2026 16:12:00 -0800 Subject: [PATCH 3/4] perf: skip output cost lookups when no audio/reasoning/image tokens MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move _get_cost_per_unit() calls for output audio/reasoning/image costs inside their conditionals — only lookup when we actually have those token types. --- .../litellm_core_utils/llm_cost_calc/utils.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index d1a47efde7e..38cd69110d5 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -688,18 +688,11 @@ def generic_cost_per_token( # noqa: PLR0915 ## TEXT COST completion_cost = float(text_tokens) * completion_base_cost - _output_cost_per_audio_token = _get_cost_per_unit( - model_info, "output_cost_per_audio_token", None - ) - _output_cost_per_reasoning_token = _get_cost_per_unit( - model_info, "output_cost_per_reasoning_token", None - ) - _output_cost_per_image_token = _get_cost_per_unit( - model_info, "output_cost_per_image_token", None - ) - ## AUDIO COST if not is_text_tokens_total and audio_tokens is not None and audio_tokens > 0: + _output_cost_per_audio_token = _get_cost_per_unit( + model_info, "output_cost_per_audio_token", None + ) _output_cost_per_audio_token = ( _output_cost_per_audio_token if _output_cost_per_audio_token is not None @@ -709,6 +702,9 @@ def generic_cost_per_token( # noqa: PLR0915 ## REASONING COST if not is_text_tokens_total and reasoning_tokens and reasoning_tokens > 0: + _output_cost_per_reasoning_token = _get_cost_per_unit( + model_info, "output_cost_per_reasoning_token", None + ) _output_cost_per_reasoning_token = ( _output_cost_per_reasoning_token if _output_cost_per_reasoning_token is not None @@ -718,6 +714,9 @@ def generic_cost_per_token( # noqa: PLR0915 ## IMAGE COST if not is_text_tokens_total and image_tokens and image_tokens > 0: + _output_cost_per_image_token = _get_cost_per_unit( + model_info, "output_cost_per_image_token", None + ) _output_cost_per_image_token = ( _output_cost_per_image_token if _output_cost_per_image_token is not None From 1d8e3a6f2814f28cb20d351a816657c128251c6a Mon Sep 17 00:00:00 2001 From: Ryan Crabbe Date: Sat, 7 Feb 2026 12:07:28 -0800 Subject: [PATCH 4/4] fix: include cache_creation_token_details in cost guard to prevent undercharging ephemeral tokens --- .../litellm_core_utils/llm_cost_calc/utils.py | 2 +- .../llm_cost_calc/test_llm_cost_calc_utils.py | 44 +++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 38cd69110d5..f35096717cc 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -535,7 +535,7 @@ def _calculate_input_cost( ) ### CACHE WRITING COST - Now uses tiered pricing - if prompt_tokens_details["cache_creation_tokens"]: + if prompt_tokens_details["cache_creation_tokens"] or prompt_tokens_details["cache_creation_token_details"] is not None: prompt_cost += calculate_cache_writing_cost( cache_creation_tokens=prompt_tokens_details["cache_creation_tokens"], cache_creation_token_details=prompt_tokens_details[ diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index 9e70f3e08d5..8c6dfca63b4 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -23,6 +23,8 @@ sys.path.insert( ) # Adds the parent directory to the system path from litellm.litellm_core_utils.llm_cost_calc.utils import ( + _calculate_input_cost, + PromptTokensDetailsResult, calculate_cache_writing_cost, generic_cost_per_token, ) @@ -559,6 +561,48 @@ def test_calculate_cache_writing_cost(): assert result_zero == 0.0 +def test_cache_writing_cost_with_zero_creation_tokens_and_ephemeral_details(): + """ + Regression test: when cache_creation_tokens is 0 but cache_creation_token_details + has non-zero ephemeral tokens, the cost must still be calculated. + This ensures the guard in _calculate_input_cost doesn't skip + calculate_cache_writing_cost when only ephemeral token details are present. + """ + cache_creation_cost = 3.75e-06 + cache_creation_cost_above_1hr = 6e-06 + + prompt_tokens_details: PromptTokensDetailsResult = { + "cache_hit_tokens": 0, + "cache_creation_tokens": 0, + "cache_creation_token_details": CacheCreationTokenDetails( + ephemeral_5m_input_tokens=100, + ephemeral_1h_input_tokens=200, + ), + "text_tokens": 0, + "audio_tokens": 0, + "image_tokens": 0, + "character_count": 0, + "image_count": 0, + "video_length_seconds": 0.0, + } + + model_info: ModelInfo = {} + + result = _calculate_input_cost( + prompt_tokens_details=prompt_tokens_details, + model_info=model_info, + prompt_base_cost=0.0, + cache_read_cost=0.0, + cache_creation_cost=cache_creation_cost, + cache_creation_cost_above_1hr=cache_creation_cost_above_1hr, + ) + + # Expected: (100 * 3.75e-06) + (200 * 6e-06) = 0.000375 + 0.0012 = 0.001575 + expected = (100 * cache_creation_cost) + (200 * cache_creation_cost_above_1hr) + assert result > 0, "Cost should not be zero when ephemeral token details are present" + assert round(result, 6) == round(expected, 6) + + def test_service_tier_flex_pricing(): """Test that flex service tier uses correct pricing (approximately 50% of standard).""" # Set up environment for local model cost map