From 4f38a6315251fa1843ee8782c6018d5e2e2d54a4 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 19 Aug 2025 16:40:12 +0530 Subject: [PATCH] Add long context support for claude-4-sonnet --- .../litellm_core_utils/llm_cost_calc/utils.py | 44 ++++++++++++------- ...odel_prices_and_context_window_backup.json | 10 +++-- model_prices_and_context_window.json | 10 +++-- 3 files changed, 43 insertions(+), 21 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 4b6cffd06c9..fe747788243 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -113,15 +113,20 @@ def _generic_cost_per_character( return prompt_cost, completion_cost -def _get_token_base_cost(model_info: ModelInfo, usage: Usage) -> Tuple[float, float]: +def _get_token_base_cost(model_info: ModelInfo, usage: Usage) -> Tuple[float, float, float, float]: """ - Return prompt cost for a given model and usage. + Return prompt cost, completion cost, and cache costs for a given model and usage. If input_tokens > threshold and `input_cost_per_token_above_[x]k_tokens` or `input_cost_per_token_above_[x]_tokens` is set, - then we use the corresponding threshold cost. + then we use the corresponding threshold cost for all token types. + + Returns: + Tuple[float, float, float, float] - (prompt_cost, completion_cost, cache_creation_cost, cache_read_cost) """ prompt_base_cost = cast(float, _get_cost_per_unit(model_info, "input_cost_per_token")) completion_base_cost = cast(float, _get_cost_per_unit(model_info, "output_cost_per_token")) + cache_creation_cost = cast(float, _get_cost_per_unit(model_info, "cache_creation_input_token_cost")) + cache_read_cost = cast(float, _get_cost_per_unit(model_info, "cache_read_input_token_cost")) ## CHECK IF ABOVE THRESHOLD threshold: Optional[float] = None @@ -141,13 +146,28 @@ def _get_token_base_cost(model_info: ModelInfo, usage: Usage) -> Tuple[float, fl f"output_cost_per_token_above_{threshold_str}_tokens", completion_base_cost, )) + + # Apply tiered pricing to cache costs + cache_creation_tiered_key = f"cache_creation_input_token_cost_above_{threshold_str}_tokens" + cache_read_tiered_key = f"cache_read_input_token_cost_above_{threshold_str}_tokens" + + if cache_creation_tiered_key in model_info: + cache_creation_cost = cast(float, _get_cost_per_unit( + model_info, cache_creation_tiered_key, cache_creation_cost + )) + + if cache_read_tiered_key in model_info: + cache_read_cost = cast(float, _get_cost_per_unit( + model_info, cache_read_tiered_key, cache_read_cost + )) + break except (IndexError, ValueError): continue except Exception: continue - return prompt_base_cost, completion_base_cost + return prompt_base_cost, completion_base_cost, cache_creation_cost, cache_read_cost def calculate_cost_component( @@ -262,28 +282,22 @@ def generic_cost_per_token( if text_tokens == 0: text_tokens = usage.prompt_tokens - cache_hit_tokens - audio_tokens - prompt_base_cost, completion_base_cost = _get_token_base_cost( + prompt_base_cost, completion_base_cost, cache_creation_cost, cache_read_cost = _get_token_base_cost( model_info=model_info, usage=usage ) prompt_cost = float(text_tokens) * prompt_base_cost - ### CACHE READ COST - prompt_cost += calculate_cost_component( - model_info, "cache_read_input_token_cost", cache_hit_tokens - ) + ### CACHE READ COST - Now uses tiered pricing + prompt_cost += float(cache_hit_tokens) * cache_read_cost ### AUDIO COST prompt_cost += calculate_cost_component( model_info, "input_cost_per_audio_token", audio_tokens ) - ### CACHE WRITING COST - prompt_cost += calculate_cost_component( - model_info, - "cache_creation_input_token_cost", - usage._cache_creation_input_tokens, - ) + ### CACHE WRITING COST - Now uses tiered pricing + prompt_cost += float(usage._cache_creation_input_tokens or 0) * cache_creation_cost ### CHARACTER COST diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 96680538888..4c6bff53c4c 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -6437,11 +6437,13 @@ "supports_computer_use": true }, "claude-4-sonnet-20250514": { - "max_tokens": 64000, - "max_input_tokens": 200000, - "max_output_tokens": 64000, + "max_tokens": 1000000, + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, "input_cost_per_token": 3e-06, "output_cost_per_token": 1.5e-05, + "input_cost_per_token_above_200k_tokens": 6e-06, + "output_cost_per_token_above_200k_tokens": 2.25e-05, "search_context_cost_per_query": { "search_context_size_low": 0.01, "search_context_size_medium": 0.01, @@ -6449,6 +6451,8 @@ }, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, + "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, + "cache_read_input_token_cost_above_200k_tokens": 6e-07, "litellm_provider": "anthropic", "mode": "chat", "supports_function_calling": true, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 96680538888..4c6bff53c4c 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -6437,11 +6437,13 @@ "supports_computer_use": true }, "claude-4-sonnet-20250514": { - "max_tokens": 64000, - "max_input_tokens": 200000, - "max_output_tokens": 64000, + "max_tokens": 1000000, + "max_input_tokens": 1000000, + "max_output_tokens": 1000000, "input_cost_per_token": 3e-06, "output_cost_per_token": 1.5e-05, + "input_cost_per_token_above_200k_tokens": 6e-06, + "output_cost_per_token_above_200k_tokens": 2.25e-05, "search_context_cost_per_query": { "search_context_size_low": 0.01, "search_context_size_medium": 0.01, @@ -6449,6 +6451,8 @@ }, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, + "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, + "cache_read_input_token_cost_above_200k_tokens": 6e-07, "litellm_provider": "anthropic", "mode": "chat", "supports_function_calling": true,