diff --git a/litellm/constants.py b/litellm/constants.py index 8e0a5cfe50f..e104c937a9b 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -2,7 +2,7 @@ import os import sys from typing import List, Literal, Optional -from litellm.litellm_core_utils.env_utils import get_env_int +from litellm.litellm_core_utils.env_utils import get_env_int, get_env_int_or_none DEFAULT_HEALTH_CHECK_PROMPT = str(os.getenv("DEFAULT_HEALTH_CHECK_PROMPT", "test from litellm")) AZURE_DEFAULT_RESPONSES_API_VERSION = str(os.getenv("AZURE_DEFAULT_RESPONSES_API_VERSION", "preview")) @@ -269,9 +269,18 @@ TOOL_POLICY_CACHE_TTL_SECONDS = int(os.getenv("TOOL_POLICY_CACHE_TTL_SECONDS", 6 MAX_SIZE_IN_MEMORY_QUEUE = int(os.getenv("MAX_SIZE_IN_MEMORY_QUEUE", int(LITELLM_ASYNCIO_QUEUE_MAXSIZE * 0.8))) MAX_IN_MEMORY_QUEUE_FLUSH_COUNT = int(os.getenv("MAX_IN_MEMORY_QUEUE_FLUSH_COUNT", 1000)) ############################################################################################### -MINIMUM_PROMPT_CACHE_TOKEN_COUNT = int( - os.getenv("MINIMUM_PROMPT_CACHE_TOKEN_COUNT", 1024) -) # minimum number of tokens to cache a prompt by Anthropic +# Providers will not cache a prefix below a minimum size. That minimum is per-model, not global: +# Anthropic's ranges from 512 to 4096 depending on the model, and can differ per platform for the +# same model. The real minimum is resolved from `prompt_cache_min_tokens` in the model cost map; +# this value is only the fallback for models the cost map has no entry for, and doubles as a global +# escape hatch when `MINIMUM_PROMPT_CACHE_TOKEN_COUNT` is explicitly set. +MINIMUM_PROMPT_CACHE_TOKEN_COUNT_OVERRIDE: int | None = get_env_int_or_none("MINIMUM_PROMPT_CACHE_TOKEN_COUNT") +DEFAULT_MINIMUM_PROMPT_CACHE_TOKEN_COUNT = 1024 +MINIMUM_PROMPT_CACHE_TOKEN_COUNT = ( + MINIMUM_PROMPT_CACHE_TOKEN_COUNT_OVERRIDE + if MINIMUM_PROMPT_CACHE_TOKEN_COUNT_OVERRIDE is not None + else DEFAULT_MINIMUM_PROMPT_CACHE_TOKEN_COUNT +) DEFAULT_TRIM_RATIO = float( os.getenv("DEFAULT_TRIM_RATIO", 0.75) ) # default ratio of tokens to trim from the end of a prompt diff --git a/litellm/litellm_core_utils/env_utils.py b/litellm/litellm_core_utils/env_utils.py index 34c65275331..3a64f44fb25 100644 --- a/litellm/litellm_core_utils/env_utils.py +++ b/litellm/litellm_core_utils/env_utils.py @@ -19,3 +19,19 @@ def get_env_int(env_var: str, default: int) -> int: return int(raw) except (ValueError, TypeError): return default + + +def get_env_int_or_none(env_var: str) -> int | None: + """Parse an environment variable as an integer, returning None when it is unset or unusable. + + Use this instead of `get_env_int` when callers must distinguish "explicitly configured" + from "left at the default", for example when an override should take precedence over a + value resolved from somewhere else. + """ + raw = os.getenv(env_var) + if raw is None: + return None + try: + return int(raw.strip()) + except (ValueError, TypeError): + return None diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index dedb9bbf40a..1a24088396f 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -721,7 +721,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "prompt_cache_min_tokens": 2048 }, "anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.25e-06, @@ -745,7 +746,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "anthropic.claude-haiku-4-5@20251001": { "cache_creation_input_token_cost": 1.25e-06, @@ -770,7 +772,8 @@ "supports_vision": true, "supports_native_streaming": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "anthropic.claude-3-5-sonnet-20240620-v1:0": { "input_cost_per_token": 3e-06, @@ -935,7 +938,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "anthropic.claude-opus-4-20250514-v1:0": { "cache_creation_input_token_cost": 1.875e-05, @@ -960,7 +964,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "anthropic.claude-opus-4-5-20251101-v1:0": { "cache_creation_input_token_cost": 6.25e-06, @@ -990,7 +995,8 @@ "supports_native_structured_output": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "high", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, @@ -1022,7 +1028,8 @@ "supports_output_config": true, "supports_max_reasoning_effort": true, "bedrock_output_config_effort_ceiling": "max", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "global.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, @@ -1054,7 +1061,8 @@ "supports_output_config": true, "supports_max_reasoning_effort": true, "bedrock_output_config_effort_ceiling": "max", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "us.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, @@ -1086,7 +1094,8 @@ "supports_output_config": true, "supports_max_reasoning_effort": true, "bedrock_output_config_effort_ceiling": "max", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "eu.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, @@ -1118,7 +1127,8 @@ "supports_output_config": true, "supports_max_reasoning_effort": true, "bedrock_output_config_effort_ceiling": "max", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "au.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, @@ -1150,7 +1160,8 @@ "supports_output_config": true, "supports_max_reasoning_effort": true, "bedrock_output_config_effort_ceiling": "max", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, @@ -1185,7 +1196,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 2048 }, "anthropic.claude-mythos-preview": { "input_cost_per_token": 0, @@ -1235,7 +1247,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 2048 }, "us.anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, @@ -1270,7 +1283,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 2048 }, "eu.anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, @@ -1305,7 +1319,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 2048 }, "au.anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, @@ -1340,7 +1355,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 2048 }, "anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.25e-05, @@ -1375,7 +1391,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "global.anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.25e-05, @@ -1410,7 +1427,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "us.anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.375e-05, @@ -1445,7 +1463,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "eu.anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.375e-05, @@ -1480,7 +1499,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -1516,7 +1536,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "global.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -1552,7 +1573,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "us.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -1588,7 +1610,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "eu.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -1624,7 +1647,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "au.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -1660,7 +1684,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "jp.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -1696,7 +1721,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "jp.anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, @@ -1729,7 +1755,8 @@ "supports_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_minimal_reasoning_effort": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 2048 }, "anthropic.claude-sonnet-5": { "cache_creation_input_token_cost": 2.5e-06, @@ -1764,7 +1791,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "global.anthropic.claude-sonnet-5": { "cache_creation_input_token_cost": 2.5e-06, @@ -1799,7 +1827,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "us.anthropic.claude-sonnet-5": { "cache_creation_input_token_cost": 2.75e-06, @@ -1834,7 +1863,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "eu.anthropic.claude-sonnet-5": { "cache_creation_input_token_cost": 2.75e-06, @@ -1869,7 +1899,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "au.anthropic.claude-sonnet-5": { "cache_creation_input_token_cost": 2.75e-06, @@ -1904,7 +1935,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "jp.anthropic.claude-sonnet-5": { "cache_creation_input_token_cost": 2.75e-06, @@ -1939,7 +1971,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -1970,7 +2003,8 @@ "supports_vision": true, "supports_native_structured_output": true, "supports_output_config": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "global.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -2001,7 +2035,8 @@ "supports_vision": true, "supports_native_structured_output": true, "supports_output_config": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "us.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -2032,7 +2067,8 @@ "supports_vision": true, "supports_native_structured_output": true, "supports_output_config": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "eu.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -2063,7 +2099,8 @@ "supports_vision": true, "supports_native_structured_output": true, "supports_output_config": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "au.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -2094,7 +2131,8 @@ "supports_vision": true, "supports_native_structured_output": true, "supports_output_config": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "jp.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -2125,7 +2163,8 @@ "supports_vision": true, "supports_native_structured_output": true, "supports_output_config": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -2155,7 +2194,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "bedrock_converse_supports_strict_tools": false + "bedrock_converse_supports_strict_tools": false, + "prompt_cache_min_tokens": 1024 }, "anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -2188,7 +2228,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "anthropic.claude-v1": { "input_cost_per_token": 8e-06, @@ -2439,7 +2480,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "apac.anthropic.claude-3-sonnet-20240229-v1:0": { "input_cost_per_token": 3e-06, @@ -2485,7 +2527,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "bedrock_converse_supports_strict_tools": false + "bedrock_converse_supports_strict_tools": false, + "prompt_cache_min_tokens": 1024 }, "assemblyai/best": { "input_cost_per_second": 3.333e-05, @@ -2530,7 +2573,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "azure/ada": { "input_cost_per_token": 1e-07, @@ -10490,7 +10534,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "bedrock/us-gov-east-1/claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.5e-06, @@ -10513,7 +10558,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "bedrock/us-gov-east-1/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 2.65e-06, @@ -10667,7 +10713,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "bedrock/us-gov-west-1/claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.5e-06, @@ -10690,7 +10737,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "bedrock/us-gov-west-1/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 2.65e-06, @@ -10940,7 +10988,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "prompt_cache_min_tokens": 2048 }, "black_forest_labs/flux-kontext-pro": { "litellm_provider": "black_forest_labs", @@ -11160,7 +11209,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 4096 }, "claude-haiku-4-5": { "cache_creation_input_token_cost": 1.25e-06, @@ -11181,7 +11231,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 4096 }, "claude-3-7-sonnet-20250219": { "cache_creation_input_token_cost": 3.75e-06, @@ -11271,7 +11322,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "claude-4-sonnet-20250514": { "cache_creation_input_token_cost": 3.75e-06, @@ -11301,7 +11353,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "prompt_cache_min_tokens": 1024 }, "claude-sonnet-4-5": { "cache_creation_input_token_cost": 3.75e-06, @@ -11333,7 +11386,8 @@ "supports_response_schema": true, "supports_native_structured_output": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "claude-sonnet-4-5-20250929": { "cache_creation_input_token_cost": 3.75e-06, @@ -11366,7 +11420,8 @@ "supports_native_structured_output": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "prompt_cache_min_tokens": 1024 }, "claude-sonnet-5": { "cache_creation_input_token_cost": 2.5e-06, @@ -11400,7 +11455,8 @@ "provider_specific_entry": { "us": 1.1 }, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 1024 }, "claude-sonnet-4-6": { "cache_creation_input_token_cost": 3.75e-06, @@ -11430,7 +11486,8 @@ "supports_max_reasoning_effort": true, "supports_tool_choice": true, "supports_vision": true, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 1024 }, "claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -11457,7 +11514,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "claude-opus-4-1": { "cache_creation_input_token_cost": 1.875e-05, @@ -11484,7 +11542,8 @@ "supports_response_schema": true, "supports_native_structured_output": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "claude-opus-4-1-20250805": { "cache_creation_input_token_cost": 1.875e-05, @@ -11512,7 +11571,8 @@ "supports_response_schema": true, "supports_native_structured_output": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "claude-opus-4-20250514": { "cache_creation_input_token_cost": 1.875e-05, @@ -11539,7 +11599,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "claude-opus-4-5-20251101": { "cache_creation_input_token_cost": 6.25e-06, @@ -11567,7 +11628,8 @@ "supports_native_structured_output": true, "supports_tool_choice": true, "supports_vision": true, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 4096 }, "claude-opus-4-5": { "cache_creation_input_token_cost": 6.25e-06, @@ -11595,7 +11657,8 @@ "supports_native_structured_output": true, "supports_tool_choice": true, "supports_vision": true, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 4096 }, "claude-opus-4-6": { "cache_creation_input_token_cost": 6.25e-06, @@ -11630,7 +11693,8 @@ }, "supports_output_config": true, "supports_max_reasoning_effort": true, - "supports_speed": true + "supports_speed": true, + "prompt_cache_min_tokens": 4096 }, "claude-opus-4-6-20260205": { "cache_creation_input_token_cost": 6.25e-06, @@ -11665,7 +11729,8 @@ }, "supports_max_reasoning_effort": true, "supports_output_config": true, - "supports_speed": true + "supports_speed": true, + "prompt_cache_min_tokens": 4096 }, "claude-opus-4-7": { "cache_creation_input_token_cost": 6.25e-06, @@ -11702,7 +11767,8 @@ "fast": 6.0 }, "supports_output_config": true, - "supports_speed": true + "supports_speed": true, + "prompt_cache_min_tokens": 2048 }, "claude-opus-4-7-20260416": { "cache_creation_input_token_cost": 6.25e-06, @@ -11739,7 +11805,8 @@ "fast": 6.0 }, "supports_output_config": true, - "supports_speed": true + "supports_speed": true, + "prompt_cache_min_tokens": 2048 }, "claude-fable-5": { "cache_creation_input_token_cost": 1.25e-05, @@ -11773,7 +11840,8 @@ "provider_specific_entry": { "us": 1.1 }, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 512 }, "claude-opus-4-8": { "cache_creation_input_token_cost": 6.25e-06, @@ -11810,7 +11878,8 @@ "fast": 2.0 }, "supports_output_config": true, - "supports_speed": true + "supports_speed": true, + "prompt_cache_min_tokens": 1024 }, "claude-sonnet-4-20250514": { "deprecation_date": "2026-05-14", @@ -11841,7 +11910,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "cloudflare/@cf/meta/llama-2-7b-chat-fp16": { "input_cost_per_token": 1.923e-06, @@ -15514,7 +15584,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "cache_read_input_token_cost": 2.5e-08, - "cache_creation_input_token_cost": 3.125e-07 + "cache_creation_input_token_cost": 3.125e-07, + "prompt_cache_min_tokens": 2048 }, "eu.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, @@ -15539,7 +15610,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "eu.anthropic.claude-3-5-sonnet-20240620-v1:0": { "input_cost_per_token": 3e-06, @@ -15666,7 +15738,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "eu.anthropic.claude-opus-4-20250514-v1:0": { "cache_creation_input_token_cost": 1.875e-05, @@ -15691,7 +15764,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "eu.anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -15721,7 +15795,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "bedrock_converse_supports_strict_tools": false + "bedrock_converse_supports_strict_tools": false, + "prompt_cache_min_tokens": 1024 }, "eu.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.125e-06, @@ -15754,7 +15829,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "eu.meta.llama3-2-1b-instruct-v1:0": { "input_cost_per_token": 1.3e-07, @@ -21105,7 +21181,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "global.anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -21135,7 +21212,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "bedrock_converse_supports_strict_tools": false + "bedrock_converse_supports_strict_tools": false, + "prompt_cache_min_tokens": 1024 }, "global.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.25e-06, @@ -21159,7 +21237,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "global.amazon.nova-2-lite-v1:0": { "cache_read_input_token_cost": 7.5e-08, @@ -25511,7 +25590,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "jp.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, @@ -25535,7 +25615,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "crusoe/deepseek-ai/DeepSeek-R1-0528": { "input_cost_per_token": 3e-06, @@ -34163,7 +34244,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "prompt_cache_min_tokens": 2048 }, "us.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, @@ -34187,7 +34269,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "us.anthropic.claude-3-5-sonnet-20240620-v1:0": { "input_cost_per_token": 3e-06, @@ -34314,7 +34397,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "us.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.125e-06, @@ -34347,7 +34431,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "us-gov.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.5e-06, @@ -34375,7 +34460,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "au.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, @@ -34398,7 +34484,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "us.anthropic.claude-opus-4-20250514-v1:0": { "cache_creation_input_token_cost": 1.875e-05, @@ -34423,7 +34510,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "us.anthropic.claude-opus-4-5-20251101-v1:0": { "cache_creation_input_token_cost": 6.875e-06, @@ -34453,7 +34541,8 @@ "supports_native_structured_output": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "high", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "global.anthropic.claude-opus-4-5-20251101-v1:0": { "cache_creation_input_token_cost": 6.25e-06, @@ -34483,7 +34572,8 @@ "supports_native_structured_output": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "high", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "eu.anthropic.claude-opus-4-5-20251101-v1:0": { "cache_creation_input_token_cost": 6.25e-06, @@ -34512,7 +34602,8 @@ "supports_native_structured_output": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "high", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "us.anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -34542,7 +34633,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "bedrock_converse_supports_strict_tools": false + "bedrock_converse_supports_strict_tools": false, + "prompt_cache_min_tokens": 1024 }, "us.deepseek.r1-v1:0": { "input_cost_per_token": 1.35e-06, @@ -36064,7 +36156,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_native_streaming": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-haiku-4-5@20251001": { "cache_creation_input_token_cost": 1.25e-06, @@ -36086,7 +36179,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_native_streaming": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-3-5-sonnet": { "input_cost_per_token": 3e-06, @@ -36241,7 +36335,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-opus-4-1": { "cache_creation_input_token_cost": 1.875e-05, @@ -36304,7 +36399,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-opus-4-5@20251101": { "cache_creation_input_token_cost": 6.25e-06, @@ -36332,7 +36428,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_streaming": true, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-opus-4-6": { "supports_adaptive_thinking": true, @@ -36361,7 +36458,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_output_config": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-opus-4-6@default": { "supports_adaptive_thinking": true, @@ -36390,7 +36488,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_output_config": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-opus-4-7": { "supports_adaptive_thinking": true, @@ -36420,7 +36519,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 2048 }, "vertex_ai/claude-opus-4-7@default": { "supports_adaptive_thinking": true, @@ -36450,7 +36550,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 2048 }, "vertex_ai/claude-fable-5": { "cache_creation_input_token_cost": 1.25e-05, @@ -36540,7 +36641,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-opus-4-8@default": { "supports_adaptive_thinking": true, @@ -36570,7 +36672,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-4-5": { "cache_creation_input_token_cost": 3.75e-06, @@ -36597,7 +36700,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-5": { "cache_creation_input_token_cost": 2.5e-06, @@ -36627,7 +36731,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -36656,7 +36761,8 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-4-5@20250929": { "cache_creation_input_token_cost": 3.75e-06, @@ -36684,7 +36790,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_native_streaming": true + "supports_native_streaming": true, + "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-opus-4@20250514": { "cache_creation_input_token_cost": 1.875e-05, @@ -36710,7 +36817,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-4": { "cache_creation_input_token_cost": 3.75e-06, @@ -36740,7 +36848,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-4@20250514": { "cache_creation_input_token_cost": 3.75e-06, @@ -36770,7 +36879,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "vertex_ai/mistralai/codestral-2@001": { "input_cost_per_token": 3e-07, @@ -44154,7 +44264,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-4-6@default": { "supports_adaptive_thinking": true, @@ -44183,7 +44294,8 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 1024 }, "duckduckgo/search": { "litellm_provider": "duckduckgo", @@ -44679,7 +44791,8 @@ "supports_vision": true, "supports_native_structured_output": true, "supports_pdf_input": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "bedrock/us-gov-west-1/anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.5e-06, @@ -44703,7 +44816,8 @@ "supports_vision": true, "supports_native_structured_output": true, "supports_pdf_input": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "snowflake/claude-sonnet-4-5": { "max_tokens": 16384, diff --git a/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py b/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py index 581783980fd..d6412c95da0 100644 --- a/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py +++ b/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py @@ -8,14 +8,40 @@ from typing import List, Optional, cast from litellm import verbose_logger from litellm.caching.dual_cache import DualCache +from litellm.constants import DEFAULT_MINIMUM_PROMPT_CACHE_TOKEN_COUNT from litellm.integrations.custom_logger import CustomLogger, Span from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import CallTypes, StandardLoggingPayload -from litellm.utils import is_prompt_caching_valid_prompt +from litellm.utils import get_prompt_cache_min_tokens, is_prompt_caching_valid_prompt from ..prompt_caching_cache import PromptCachingCache +def _get_min_token_count_for_deployments(healthy_deployments: list[dict]) -> int: + """ + Returns the lowest minimum cacheable prefix across a model group. + + This gate only decides whether the cache lookup is worth doing. It cannot cause a wrong pin, + because a deployment is only pinned when the cache already holds an entry for the prefix, and + entries are written by `async_log_success_event` against the deployment's real model. A model + that will not cache a prefix never records one, so there is nothing to pin it to. + + That makes the lowest minimum in the group the correct threshold rather than the highest. + `model` here is the model-group alias the operator chose, not a model name, so the threshold + has to come from the deployments themselves, and a group may mix models whose minimums differ. + Taking the highest would skip the lookup for a prefix a lower-minimum member genuinely cached, + losing a cache hit it had earned. The lowest can only cost a lookup that finds nothing. + """ + return min( + ( + get_prompt_cache_min_tokens(model=deployment["litellm_params"]["model"]) + for deployment in healthy_deployments + if deployment.get("litellm_params", {}).get("model") + ), + default=DEFAULT_MINIMUM_PROMPT_CACHE_TOKEN_COUNT, + ) + + class PromptCachingDeploymentCheck(CustomLogger): def __init__(self, cache: DualCache): self.cache = cache @@ -31,7 +57,8 @@ class PromptCachingDeploymentCheck(CustomLogger): if messages is not None and is_prompt_caching_valid_prompt( messages=messages, model=model, - ): # prompt > 1024 tokens + min_token_count=_get_min_token_count_for_deployments(healthy_deployments), + ): prompt_cache = PromptCachingCache( cache=self.cache, ) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index e2f1bdfc486..7e372ca3c68 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -197,6 +197,9 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): cache_read_input_token_cost_above_272k_tokens: Optional[float] cache_read_input_token_cost_above_272k_tokens_priority: Optional[float] cache_read_input_token_cost_above_512k_tokens: Optional[float] + # Smallest prefix this model will actually cache, whatever caching mechanism its provider uses. + # Absent means the provider-agnostic default applies; see MINIMUM_PROMPT_CACHE_TOKEN_COUNT. + prompt_cache_min_tokens: Optional[int] input_cost_per_character: Optional[float] # only for vertex ai models input_cost_per_audio_token: Optional[float] input_cost_per_token_above_128k_tokens: Optional[float] # only for vertex ai models diff --git a/litellm/utils.py b/litellm/utils.py index 0636d3683b7..e19d2b36a52 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -73,7 +73,8 @@ from litellm.constants import ( JITTER, MAX_RETRY_DELAY, MAX_TOKEN_TRIMMING_ATTEMPTS, - MINIMUM_PROMPT_CACHE_TOKEN_COUNT, + DEFAULT_MINIMUM_PROMPT_CACHE_TOKEN_COUNT, + MINIMUM_PROMPT_CACHE_TOKEN_COUNT_OVERRIDE, OPENAI_EMBEDDING_PARAMS, TOOL_CHOICE_OBJECT_TOKEN_COUNT, ) @@ -5402,6 +5403,7 @@ def _get_model_info_helper( "cache_creation_input_token_cost_above_200k_tokens", None ), cache_read_input_token_cost=_model_info.get("cache_read_input_token_cost", None), + prompt_cache_min_tokens=_model_info.get("prompt_cache_min_tokens", None), cache_read_input_token_cost_above_200k_tokens=_model_info.get( "cache_read_input_token_cost_above_200k_tokens", None ), @@ -9039,16 +9041,46 @@ def should_use_cohere_v1_client(api_base: Optional[str], present_version_params: return api_base.endswith("/v1/rerank") or (uses_v1_params and not api_base.endswith("/v2/rerank")) +def get_prompt_cache_min_tokens(model: str) -> int: + """ + Returns the smallest prefix `model` will actually cache. + + Resolution order is an explicitly configured `MINIMUM_PROMPT_CACHE_TOKEN_COUNT`, then the + model's `prompt_cache_min_tokens` in the cost map, then the provider-agnostic default. The + cost map is the source of truth because the real minimum is per-model and per-platform: + Anthropic's ranges from 512 to 4096 and moves in both directions across releases, and the + same model can differ by platform. + + Never raises. An unresolvable model falls back to the default rather than propagating, so a + caller cannot mistake "no entry for this model" for "this prompt is not cacheable". + """ + if MINIMUM_PROMPT_CACHE_TOKEN_COUNT_OVERRIDE is not None: + return MINIMUM_PROMPT_CACHE_TOKEN_COUNT_OVERRIDE + try: + min_tokens = get_model_info(model=model).get("prompt_cache_min_tokens") + except Exception: + return DEFAULT_MINIMUM_PROMPT_CACHE_TOKEN_COUNT + if min_tokens is None: + return DEFAULT_MINIMUM_PROMPT_CACHE_TOKEN_COUNT + return min_tokens + + def is_prompt_caching_valid_prompt( model: str, messages: Optional[List[AllMessageValues]], tools: Optional[List[ChatCompletionToolParam]] = None, custom_llm_provider: Optional[str] = None, + min_token_count: int | None = None, ) -> bool: """ Returns true if the prompt is valid for prompt caching. - OpenAI + Anthropic providers have a minimum token count of 1024 for prompt caching. + The minimum cacheable prefix is per-model, so it is resolved from `model` unless the caller + passes `min_token_count`. Callers that only hold a model-group alias (the router's deployment + checks) must resolve the threshold themselves and pass it, because an alias resolves to + nothing here and would silently fall back to the default. + + OpenAI's minimum is a flat 1024 across models, which the default already covers. """ try: if messages is None and tools is None: @@ -9061,7 +9093,9 @@ def is_prompt_caching_valid_prompt( model=model, use_default_image_token_count=True, ) - return token_count >= MINIMUM_PROMPT_CACHE_TOKEN_COUNT + if min_token_count is None: + min_token_count = get_prompt_cache_min_tokens(model=model) + return token_count >= min_token_count except Exception as e: verbose_logger.error(f"Error in is_prompt_caching_valid_prompt: {e}") return False diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index e10dde793d1..ffbc0dcd098 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -721,7 +721,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "prompt_cache_min_tokens": 2048 }, "anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.25e-06, @@ -745,7 +746,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "anthropic.claude-haiku-4-5@20251001": { "cache_creation_input_token_cost": 1.25e-06, @@ -770,7 +772,8 @@ "supports_vision": true, "supports_native_streaming": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "anthropic.claude-3-5-sonnet-20240620-v1:0": { "input_cost_per_token": 3e-06, @@ -935,7 +938,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "anthropic.claude-opus-4-20250514-v1:0": { "cache_creation_input_token_cost": 1.875e-05, @@ -960,7 +964,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "anthropic.claude-opus-4-5-20251101-v1:0": { "cache_creation_input_token_cost": 6.25e-06, @@ -990,7 +995,8 @@ "supports_native_structured_output": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "high", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, @@ -1022,7 +1028,8 @@ "supports_output_config": true, "supports_max_reasoning_effort": true, "bedrock_output_config_effort_ceiling": "max", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "global.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, @@ -1054,7 +1061,8 @@ "supports_output_config": true, "supports_max_reasoning_effort": true, "bedrock_output_config_effort_ceiling": "max", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "us.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, @@ -1086,7 +1094,8 @@ "supports_output_config": true, "supports_max_reasoning_effort": true, "bedrock_output_config_effort_ceiling": "max", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "eu.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, @@ -1118,7 +1127,8 @@ "supports_output_config": true, "supports_max_reasoning_effort": true, "bedrock_output_config_effort_ceiling": "max", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "au.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, @@ -1150,7 +1160,8 @@ "supports_output_config": true, "supports_max_reasoning_effort": true, "bedrock_output_config_effort_ceiling": "max", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, @@ -1185,7 +1196,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 2048 }, "anthropic.claude-mythos-preview": { "input_cost_per_token": 0, @@ -1235,7 +1247,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 2048 }, "us.anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, @@ -1270,7 +1283,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 2048 }, "eu.anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, @@ -1305,7 +1319,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 2048 }, "au.anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, @@ -1340,7 +1355,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 2048 }, "anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.25e-05, @@ -1375,7 +1391,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "global.anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.25e-05, @@ -1410,7 +1427,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "us.anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.375e-05, @@ -1445,7 +1463,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "eu.anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.375e-05, @@ -1480,7 +1499,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -1516,7 +1536,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "global.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -1552,7 +1573,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "us.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -1588,7 +1610,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "eu.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -1624,7 +1647,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "au.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -1660,7 +1684,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "jp.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -1696,7 +1721,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "jp.anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, @@ -1729,7 +1755,8 @@ "supports_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_minimal_reasoning_effort": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 2048 }, "anthropic.claude-sonnet-5": { "cache_creation_input_token_cost": 2.5e-06, @@ -1764,7 +1791,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "global.anthropic.claude-sonnet-5": { "cache_creation_input_token_cost": 2.5e-06, @@ -1799,7 +1827,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "us.anthropic.claude-sonnet-5": { "cache_creation_input_token_cost": 2.75e-06, @@ -1834,7 +1863,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "eu.anthropic.claude-sonnet-5": { "cache_creation_input_token_cost": 2.75e-06, @@ -1869,7 +1899,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "au.anthropic.claude-sonnet-5": { "cache_creation_input_token_cost": 2.75e-06, @@ -1904,7 +1935,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "jp.anthropic.claude-sonnet-5": { "cache_creation_input_token_cost": 2.75e-06, @@ -1939,7 +1971,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -1970,7 +2003,8 @@ "supports_vision": true, "supports_native_structured_output": true, "supports_output_config": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "global.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -2001,7 +2035,8 @@ "supports_vision": true, "supports_native_structured_output": true, "supports_output_config": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "us.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -2032,7 +2067,8 @@ "supports_vision": true, "supports_native_structured_output": true, "supports_output_config": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "eu.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -2063,7 +2099,8 @@ "supports_vision": true, "supports_native_structured_output": true, "supports_output_config": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "au.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -2094,7 +2131,8 @@ "supports_vision": true, "supports_native_structured_output": true, "supports_output_config": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "jp.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -2125,7 +2163,8 @@ "supports_vision": true, "supports_native_structured_output": true, "supports_output_config": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -2155,7 +2194,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "bedrock_converse_supports_strict_tools": false + "bedrock_converse_supports_strict_tools": false, + "prompt_cache_min_tokens": 1024 }, "anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -2188,7 +2228,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "anthropic.claude-v1": { "input_cost_per_token": 8e-06, @@ -2439,7 +2480,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "apac.anthropic.claude-3-sonnet-20240229-v1:0": { "input_cost_per_token": 3e-06, @@ -2485,7 +2527,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "bedrock_converse_supports_strict_tools": false + "bedrock_converse_supports_strict_tools": false, + "prompt_cache_min_tokens": 1024 }, "assemblyai/best": { "input_cost_per_second": 3.333e-05, @@ -2530,7 +2573,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "azure/ada": { "input_cost_per_token": 1e-07, @@ -10490,7 +10534,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "bedrock/us-gov-east-1/claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.5e-06, @@ -10513,7 +10558,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "bedrock/us-gov-east-1/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 2.65e-06, @@ -10667,7 +10713,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "bedrock/us-gov-west-1/claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.5e-06, @@ -10690,7 +10737,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "bedrock/us-gov-west-1/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 2.65e-06, @@ -10940,7 +10988,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "prompt_cache_min_tokens": 2048 }, "black_forest_labs/flux-kontext-pro": { "litellm_provider": "black_forest_labs", @@ -11160,7 +11209,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 4096 }, "claude-haiku-4-5": { "cache_creation_input_token_cost": 1.25e-06, @@ -11181,7 +11231,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 4096 }, "claude-3-7-sonnet-20250219": { "cache_creation_input_token_cost": 3.75e-06, @@ -11271,7 +11322,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "claude-4-sonnet-20250514": { "cache_creation_input_token_cost": 3.75e-06, @@ -11301,7 +11353,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "prompt_cache_min_tokens": 1024 }, "claude-sonnet-4-5": { "cache_creation_input_token_cost": 3.75e-06, @@ -11333,7 +11386,8 @@ "supports_response_schema": true, "supports_native_structured_output": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "claude-sonnet-4-5-20250929": { "cache_creation_input_token_cost": 3.75e-06, @@ -11366,7 +11420,8 @@ "supports_native_structured_output": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "prompt_cache_min_tokens": 1024 }, "claude-sonnet-5": { "cache_creation_input_token_cost": 2.5e-06, @@ -11400,7 +11455,8 @@ "provider_specific_entry": { "us": 1.1 }, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 1024 }, "claude-sonnet-4-6": { "cache_creation_input_token_cost": 3.75e-06, @@ -11430,7 +11486,8 @@ "supports_max_reasoning_effort": true, "supports_tool_choice": true, "supports_vision": true, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 1024 }, "claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -11457,7 +11514,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "claude-opus-4-1": { "cache_creation_input_token_cost": 1.875e-05, @@ -11484,7 +11542,8 @@ "supports_response_schema": true, "supports_native_structured_output": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "claude-opus-4-1-20250805": { "cache_creation_input_token_cost": 1.875e-05, @@ -11512,7 +11571,8 @@ "supports_response_schema": true, "supports_native_structured_output": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "claude-opus-4-20250514": { "cache_creation_input_token_cost": 1.875e-05, @@ -11539,7 +11599,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "claude-opus-4-5-20251101": { "cache_creation_input_token_cost": 6.25e-06, @@ -11567,7 +11628,8 @@ "supports_native_structured_output": true, "supports_tool_choice": true, "supports_vision": true, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 4096 }, "claude-opus-4-5": { "cache_creation_input_token_cost": 6.25e-06, @@ -11595,7 +11657,8 @@ "supports_native_structured_output": true, "supports_tool_choice": true, "supports_vision": true, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 4096 }, "claude-opus-4-6": { "cache_creation_input_token_cost": 6.25e-06, @@ -11630,7 +11693,8 @@ }, "supports_output_config": true, "supports_max_reasoning_effort": true, - "supports_speed": true + "supports_speed": true, + "prompt_cache_min_tokens": 4096 }, "claude-opus-4-6-20260205": { "cache_creation_input_token_cost": 6.25e-06, @@ -11665,7 +11729,8 @@ }, "supports_max_reasoning_effort": true, "supports_output_config": true, - "supports_speed": true + "supports_speed": true, + "prompt_cache_min_tokens": 4096 }, "claude-opus-4-7": { "cache_creation_input_token_cost": 6.25e-06, @@ -11702,7 +11767,8 @@ "fast": 6.0 }, "supports_output_config": true, - "supports_speed": true + "supports_speed": true, + "prompt_cache_min_tokens": 2048 }, "claude-opus-4-7-20260416": { "cache_creation_input_token_cost": 6.25e-06, @@ -11739,7 +11805,8 @@ "fast": 6.0 }, "supports_output_config": true, - "supports_speed": true + "supports_speed": true, + "prompt_cache_min_tokens": 2048 }, "claude-fable-5": { "cache_creation_input_token_cost": 1.25e-05, @@ -11773,7 +11840,8 @@ "provider_specific_entry": { "us": 1.1 }, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 512 }, "claude-opus-4-8": { "cache_creation_input_token_cost": 6.25e-06, @@ -11810,7 +11878,8 @@ "fast": 2.0 }, "supports_output_config": true, - "supports_speed": true + "supports_speed": true, + "prompt_cache_min_tokens": 1024 }, "claude-sonnet-4-20250514": { "deprecation_date": "2026-05-14", @@ -11841,7 +11910,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "cloudflare/@cf/meta/llama-2-7b-chat-fp16": { "input_cost_per_token": 1.923e-06, @@ -15514,7 +15584,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "cache_read_input_token_cost": 2.5e-08, - "cache_creation_input_token_cost": 3.125e-07 + "cache_creation_input_token_cost": 3.125e-07, + "prompt_cache_min_tokens": 2048 }, "eu.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, @@ -15539,7 +15610,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "eu.anthropic.claude-3-5-sonnet-20240620-v1:0": { "input_cost_per_token": 3e-06, @@ -15666,7 +15738,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "eu.anthropic.claude-opus-4-20250514-v1:0": { "cache_creation_input_token_cost": 1.875e-05, @@ -15691,7 +15764,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "eu.anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -15721,7 +15795,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "bedrock_converse_supports_strict_tools": false + "bedrock_converse_supports_strict_tools": false, + "prompt_cache_min_tokens": 1024 }, "eu.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.125e-06, @@ -15754,7 +15829,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "eu.meta.llama3-2-1b-instruct-v1:0": { "input_cost_per_token": 1.3e-07, @@ -21180,7 +21256,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "global.anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -21210,7 +21287,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "bedrock_converse_supports_strict_tools": false + "bedrock_converse_supports_strict_tools": false, + "prompt_cache_min_tokens": 1024 }, "global.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.25e-06, @@ -21234,7 +21312,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "global.amazon.nova-2-lite-v1:0": { "cache_read_input_token_cost": 7.5e-08, @@ -25586,7 +25665,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "jp.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, @@ -25610,7 +25690,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "crusoe/deepseek-ai/DeepSeek-R1-0528": { "input_cost_per_token": 3e-06, @@ -34254,7 +34335,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "prompt_cache_min_tokens": 2048 }, "us.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, @@ -34278,7 +34360,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "us.anthropic.claude-3-5-sonnet-20240620-v1:0": { "input_cost_per_token": 3e-06, @@ -34405,7 +34488,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "us.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.125e-06, @@ -34438,7 +34522,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "us-gov.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.5e-06, @@ -34466,7 +34551,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "au.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, @@ -34489,7 +34575,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "us.anthropic.claude-opus-4-20250514-v1:0": { "cache_creation_input_token_cost": 1.875e-05, @@ -34514,7 +34601,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "us.anthropic.claude-opus-4-5-20251101-v1:0": { "cache_creation_input_token_cost": 6.875e-06, @@ -34544,7 +34632,8 @@ "supports_native_structured_output": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "high", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "global.anthropic.claude-opus-4-5-20251101-v1:0": { "cache_creation_input_token_cost": 6.25e-06, @@ -34574,7 +34663,8 @@ "supports_native_structured_output": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "high", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "eu.anthropic.claude-opus-4-5-20251101-v1:0": { "cache_creation_input_token_cost": 6.25e-06, @@ -34603,7 +34693,8 @@ "supports_native_structured_output": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "high", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "us.anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -34633,7 +34724,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "bedrock_converse_supports_strict_tools": false + "bedrock_converse_supports_strict_tools": false, + "prompt_cache_min_tokens": 1024 }, "us.deepseek.r1-v1:0": { "input_cost_per_token": 1.35e-06, @@ -36155,7 +36247,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_native_streaming": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-haiku-4-5@20251001": { "cache_creation_input_token_cost": 1.25e-06, @@ -36177,7 +36270,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_native_streaming": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-3-5-sonnet": { "input_cost_per_token": 3e-06, @@ -36332,7 +36426,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-opus-4-1": { "cache_creation_input_token_cost": 1.875e-05, @@ -36395,7 +36490,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-opus-4-5@20251101": { "cache_creation_input_token_cost": 6.25e-06, @@ -36423,7 +36519,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_streaming": true, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-opus-4-6": { "supports_adaptive_thinking": true, @@ -36452,7 +36549,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_output_config": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-opus-4-6@default": { "supports_adaptive_thinking": true, @@ -36481,7 +36579,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_output_config": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-opus-4-7": { "supports_adaptive_thinking": true, @@ -36511,7 +36610,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 2048 }, "vertex_ai/claude-opus-4-7@default": { "supports_adaptive_thinking": true, @@ -36541,7 +36641,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 2048 }, "vertex_ai/claude-fable-5": { "cache_creation_input_token_cost": 1.25e-05, @@ -36631,7 +36732,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-opus-4-8@default": { "supports_adaptive_thinking": true, @@ -36661,7 +36763,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-4-5": { "cache_creation_input_token_cost": 3.75e-06, @@ -36688,7 +36791,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-5": { "cache_creation_input_token_cost": 2.5e-06, @@ -36718,7 +36822,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -36747,7 +36852,8 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-4-5@20250929": { "cache_creation_input_token_cost": 3.75e-06, @@ -36775,7 +36881,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_native_streaming": true + "supports_native_streaming": true, + "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-opus-4@20250514": { "cache_creation_input_token_cost": 1.875e-05, @@ -36801,7 +36908,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-4": { "cache_creation_input_token_cost": 3.75e-06, @@ -36831,7 +36939,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-4@20250514": { "cache_creation_input_token_cost": 3.75e-06, @@ -36861,7 +36970,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "vertex_ai/mistralai/codestral-2@001": { "input_cost_per_token": 3e-07, @@ -44275,7 +44385,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-4-6@default": { "supports_adaptive_thinking": true, @@ -44304,7 +44415,8 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 1024 }, "duckduckgo/search": { "litellm_provider": "duckduckgo", @@ -44800,7 +44912,8 @@ "supports_vision": true, "supports_native_structured_output": true, "supports_pdf_input": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "bedrock/us-gov-west-1/anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.5e-06, @@ -44824,7 +44937,8 @@ "supports_vision": true, "supports_native_structured_output": true, "supports_pdf_input": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "snowflake/claude-sonnet-4-5": { "max_tokens": 16384, diff --git a/tests/test_litellm/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py b/tests/test_litellm/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py new file mode 100644 index 00000000000..6752d76847f --- /dev/null +++ b/tests/test_litellm/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py @@ -0,0 +1,207 @@ +import os +import sys +from typing import List, cast + +import pytest + +sys.path.insert(0, os.path.abspath("../..")) + +import litellm +from litellm.caching.dual_cache import DualCache +from litellm.constants import DEFAULT_MINIMUM_PROMPT_CACHE_TOKEN_COUNT +from litellm.router_utils.pre_call_checks.prompt_caching_deployment_check import ( + PromptCachingDeploymentCheck, + _get_min_token_count_for_deployments, +) +from litellm.router_utils.prompt_caching_cache import PromptCachingCache +from litellm.types.llms.openai import AllMessageValues +from litellm.utils import get_prompt_cache_min_tokens, is_prompt_caching_valid_prompt, token_counter + +MODEL_GROUP_ALIAS = "my-claude-group" +OPUS_4_6_MIN_TOKENS = 4096 + + +@pytest.fixture(autouse=True) +def local_model_cost_map(monkeypatch): + """ + The remote cost map does not carry `prompt_cache_min_tokens` yet, so a test that reads the + default map would pass here and flake in CI. Force the in-repo map. + + `get_model_info` is lru_cached, so swapping `model_cost` is not enough on its own: an earlier + test that resolved these models against the remote map leaves entries with no + `prompt_cache_min_tokens`, and the stale hit resolves to the default. Clear on the way out too, + so the entries these tests warm against the local map do not leak into later tests. + """ + original_model_cost = litellm.model_cost + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.get_model_info.cache_clear() + try: + yield + finally: + litellm.model_cost = original_model_cost + litellm.get_model_info.cache_clear() + + +def _deployments(*models: str) -> List[dict]: + return [ + { + "model_name": MODEL_GROUP_ALIAS, + "litellm_params": {"model": model}, + "model_info": {"id": f"dep-{index}"}, + } + for index, model in enumerate(models, start=1) + ] + + +def _messages(word_count: int) -> List[AllMessageValues]: + return cast( + List[AllMessageValues], + [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "word " * word_count, + "cache_control": {"type": "ephemeral"}, + } + ], + } + ], + ) + + +def test_get_min_token_count_for_deployments_takes_min_across_mixed_group(): + """ + A group may legally mix models whose real minimums differ, and one gate decides for every + member. The threshold must be the lowest minimum in the group. This gate only decides whether + the cache lookup happens, so taking the highest would skip the lookup for a prefix the Sonnet + 4.5 deployment genuinely cached and lose a hit it had earned. + """ + assert get_prompt_cache_min_tokens(model="anthropic/claude-opus-4-5") == 4096 + assert get_prompt_cache_min_tokens(model="anthropic/claude-sonnet-4-5") == 1024 + + deployments = _deployments("anthropic/claude-opus-4-5", "anthropic/claude-sonnet-4-5") + + assert _get_min_token_count_for_deployments(deployments) == 1024 + + +def test_write_gate_is_what_prevents_a_pin_below_the_model_minimum(): + """ + The invariant the read gate relies on. A deployment can only be pinned when the cache already + holds an entry for the prefix, and `async_log_success_event` writes entries against the real + deployment model. Opus 4.5 never records an entry for a prefix it will not cache, so no read + threshold is what keeps it from being pinned. + """ + messages = _messages(word_count=1400) + + token_count = token_counter(messages=messages, model="anthropic/claude-opus-4-5", use_default_image_token_count=True) + assert 1024 < token_count < 4096 + + assert is_prompt_caching_valid_prompt(model="anthropic/claude-opus-4-5", messages=messages) is False + assert is_prompt_caching_valid_prompt(model="anthropic/claude-sonnet-4-5", messages=messages) is True + + +def test_get_min_token_count_for_deployments_falls_back_to_default_for_empty_group(): + """An empty group has no member minimum to read, so it must fall back rather than crash.""" + assert _get_min_token_count_for_deployments([]) == DEFAULT_MINIMUM_PROMPT_CACHE_TOKEN_COUNT + + +@pytest.mark.asyncio +async def test_async_filter_deployments_does_not_narrow_prompt_below_model_minimum(): + """ + The regression. Opus 4.6 will not cache a prefix under 4096 tokens, so a ~1400-token prompt is + not cacheable and routing must stay free across the whole group. Previously the check resolved + its threshold from `model`, which is the operator's group alias and matches nothing in the cost + map, silently fell back to 1024, judged this prompt cacheable, and pinned every request to one + deployment for a cache hit the provider was never going to serve. + """ + cache = DualCache() + check = PromptCachingDeploymentCheck(cache=cache) + deployments = _deployments("anthropic/claude-opus-4-6", "anthropic/claude-opus-4-6") + messages = _messages(word_count=1400) + + token_count = token_counter(messages=messages, model="anthropic/claude-opus-4-6", use_default_image_token_count=True) + assert DEFAULT_MINIMUM_PROMPT_CACHE_TOKEN_COUNT < token_count < OPUS_4_6_MIN_TOKENS + + await PromptCachingCache(cache=cache).async_add_model_id(model_id="dep-2", messages=messages, tools=None) + + filtered = await check.async_filter_deployments( + model=MODEL_GROUP_ALIAS, + healthy_deployments=deployments, + messages=messages, + ) + + assert filtered == deployments + + +@pytest.mark.asyncio +async def test_async_filter_deployments_narrows_prompt_above_model_minimum(): + """ + The positive control for the regression above: once the same group's prompt clears Opus 4.6's + real 4096-token minimum the prefix is genuinely cacheable, so the check must still pin the + deployment that served it. Proves the fix tightened the gate rather than disabling the feature. + """ + cache = DualCache() + check = PromptCachingDeploymentCheck(cache=cache) + deployments = _deployments("anthropic/claude-opus-4-6", "anthropic/claude-opus-4-6") + messages = _messages(word_count=5000) + + token_count = token_counter(messages=messages, model="anthropic/claude-opus-4-6", use_default_image_token_count=True) + assert token_count > OPUS_4_6_MIN_TOKENS + + await PromptCachingCache(cache=cache).async_add_model_id(model_id="dep-2", messages=messages, tools=None) + + filtered = await check.async_filter_deployments( + model=MODEL_GROUP_ALIAS, + healthy_deployments=deployments, + messages=messages, + ) + + assert filtered == [deployments[1]] + + +@pytest.mark.asyncio +async def test_async_filter_deployments_narrows_for_group_whose_model_minimum_is_lower(): + """ + Same ~1400-token prompt that must not pin an Opus 4.6 group, on an Opus 4.8 group whose real + minimum is 1024. Here the prefix is cacheable and the check must pin. Proves the threshold is + resolved per-model from the deployments rather than tightened for everyone. + """ + cache = DualCache() + check = PromptCachingDeploymentCheck(cache=cache) + deployments = _deployments("anthropic/claude-opus-4-8", "anthropic/claude-opus-4-8") + messages = _messages(word_count=1400) + + assert get_prompt_cache_min_tokens(model="anthropic/claude-opus-4-8") == DEFAULT_MINIMUM_PROMPT_CACHE_TOKEN_COUNT + + await PromptCachingCache(cache=cache).async_add_model_id(model_id="dep-2", messages=messages, tools=None) + + filtered = await check.async_filter_deployments( + model=MODEL_GROUP_ALIAS, + healthy_deployments=deployments, + messages=messages, + ) + + assert filtered == [deployments[1]] + + +@pytest.mark.asyncio +async def test_wildcard_route_resolves_underlying_model_minimum(local_model_cost_map): + from litellm import Router + + router = Router( + model_list=[ + { + "model_name": "anthropic/*", + "litellm_params": {"model": "anthropic/*", "api_key": "sk-fake"}, + "model_info": {"id": "wild-1"}, + } + ] + ) + + deployments = await router.async_get_healthy_deployments(model="anthropic/claude-opus-4-6", request_kwargs={}) + + assert deployments[0]["litellm_params"]["model"] == "anthropic/claude-opus-4-6" + assert _get_min_token_count_for_deployments(deployments) == 4096 diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 6d515ecdc73..073ff17991e 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -26,7 +26,9 @@ from litellm.utils import ( _is_streaming_request, get_llm_provider, get_optional_params_image_gen, + get_prompt_cache_min_tokens, is_cached_message, + is_prompt_caching_valid_prompt, ) # Adds the parent directory to the system path @@ -842,6 +844,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "supports_parallel_function_calling": {"type": "boolean"}, "supports_parallel_tool_use_config": {"type": "boolean"}, "supports_pdf_input": {"type": "boolean"}, + "prompt_cache_min_tokens": {"type": "number"}, "supports_prompt_caching": {"type": "boolean"}, "supports_response_schema": {"type": "boolean"}, "supports_system_messages": {"type": "boolean"}, @@ -4741,3 +4744,73 @@ def test_gemini_image_models_do_not_support_reasoning( f"{model} incorrectly classified as reasoning-capable. " "Add 'supports_reasoning: false' to its model_cost entry." ) + + +PROMPT_CACHE_MESSAGES = [{"role": "user", "content": "the quick brown fox jumps over the lazy dog " * 155}] + + +@pytest.mark.parametrize( + "model, expected_min_tokens", + [ + ("claude-opus-4-6", 4096), + ("claude-opus-4-7", 2048), + ("claude-opus-4-8", 1024), + ("claude-fable-5", 512), + ], +) +def test_get_prompt_cache_min_tokens_resolves_per_model( + model: str, expected_min_tokens: int, local_model_cost_map: None +) -> None: + """The smallest cacheable prefix is a per-model property, read from the cost map's + prompt_cache_min_tokens. Anthropic's minimum spans 512..4096 across models and moves in both + directions across releases, so a single global constant is wrong for every model but one.""" + assert get_prompt_cache_min_tokens(model=model) == expected_min_tokens + + +def test_get_prompt_cache_min_tokens_differs_per_platform_for_same_model(local_model_cost_map: None) -> None: + """The same model can carry a different minimum per platform, so the threshold must come from + the platform's own cost-map entry rather than being derived from the model family name.""" + assert get_prompt_cache_min_tokens(model="claude-fable-5") == 512 + assert get_prompt_cache_min_tokens(model="anthropic.claude-fable-5") == 1024 + assert get_prompt_cache_min_tokens(model="claude-fable-5") != get_prompt_cache_min_tokens( + model="anthropic.claude-fable-5" + ) + + +def test_get_prompt_cache_min_tokens_unmapped_model_falls_back_to_default(local_model_cost_map: None) -> None: + """get_model_info raises for a model it has no entry for. The resolver must swallow that and + fall back to the default, otherwise the raise reaches callers that would read it as + "not cacheable" -- turning an unknown model into a silently uncacheable one.""" + assert get_prompt_cache_min_tokens(model="totally-unknown-model-xyz") == 1024 + + +def test_is_prompt_caching_valid_prompt_uses_per_model_minimum(local_model_cost_map: None) -> None: + """Regression: a prompt between two models' minimums is cacheable on one and not the other. + A 1403-token prompt clears claude-opus-4-8's 1024 minimum but not claude-opus-4-6's 4096, so + the flat-1024 check reported claude-opus-4-6 as cacheable and the cache write was rejected + upstream. Both assertions must live together: is_prompt_caching_valid_prompt returns False on + any internal error, so the True case is what proves the False case isn't a swallowed exception.""" + token_count = litellm.token_counter( + model="claude-opus-4-6", messages=PROMPT_CACHE_MESSAGES, use_default_image_token_count=True + ) + assert 1024 <= token_count < 4096, ( + f"prompt drifted to {token_count} tokens; it must sit between claude-opus-4-8's 1024 minimum " + "and claude-opus-4-6's 4096 minimum for this test to distinguish them" + ) + + assert is_prompt_caching_valid_prompt(model="claude-opus-4-6", messages=PROMPT_CACHE_MESSAGES) is False + assert is_prompt_caching_valid_prompt(model="claude-opus-4-8", messages=PROMPT_CACHE_MESSAGES) is True + + +def test_is_prompt_caching_valid_prompt_explicit_min_token_count_overrides_model(local_model_cost_map: None) -> None: + """An explicit min_token_count wins over the model-resolved value in both directions. Callers + holding only a model-group alias resolve the threshold themselves and pass it, because an alias + resolves to nothing here and would silently fall back to the default.""" + assert ( + is_prompt_caching_valid_prompt(model="claude-opus-4-6", messages=PROMPT_CACHE_MESSAGES, min_token_count=512) + is True + ) + assert ( + is_prompt_caching_valid_prompt(model="claude-opus-4-8", messages=PROMPT_CACHE_MESSAGES, min_token_count=8192) + is False + )