From ba70189e328a5376700e9535d0629118857395e7 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Thu, 16 Jul 2026 19:10:23 -0700 Subject: [PATCH 1/3] fix(router): resolve prompt cache minimum per model instead of a flat 1024 MINIMUM_PROMPT_CACHE_TOKEN_COUNT was a flat 1024 described as "minimum number of tokens to cache a prompt by Anthropic". Anthropic's minimum cacheable prefix is per-model and ranges from 512 to 4096, and it can differ per platform for the same model, so one constant is wrong in both directions is_prompt_caching_valid_prompt gates PromptCachingDeploymentCheck, which is what optional_pre_call_checks: ["prompt_caching"] turns on. When it believes a prompt is cacheable, async_filter_deployments pins routing to whichever deployment previously served that prefix. For a prompt between 1024 and 4096 tokens on Opus 4.6, Opus 4.5 or Haiku 4.5, litellm judged it cacheable and constrained routing while the provider never cached it, so the pin cost load balancing for nothing. In the other direction Fable 5 caches from 512 tokens, so a 512 to 1024 token prefix was refused a pin it had earned The minimum now resolves from prompt_cache_min_tokens in the model cost map, which keeps it current with new models and lets the Bedrock override for Fable 5 fall out of the existing per-entry keys with no special casing. MINIMUM_PROMPT_CACHE_TOKEN_COUNT stays as a global escape hatch when explicitly set, and as the fallback for models the cost map has no entry for async_filter_deployments only ever receives the model group alias, never a model name, so it resolves the threshold from healthy_deployments instead. A group may mix models with different minimums, so it takes the max: a prompt is only treated as cacheable when it clears every member's minimum, because an unnecessary pin is the defect being fixed while a missed pin only forfeits an optimization Gemini context caching shares this gate and has the same defect; its entries are left unset so they keep today's behavior, tracked separately in LIT-4525 --- litellm/constants.py | 17 +- litellm/litellm_core_utils/env_utils.py | 16 + ...odel_prices_and_context_window_backup.json | 342 ++++++++++++------ .../prompt_caching_deployment_check.py | 27 +- litellm/types/utils.py | 3 + litellm/utils.py | 40 +- model_prices_and_context_window.json | 342 ++++++++++++------ .../test_prompt_caching_deployment_check.py | 158 ++++++++ tests/test_litellm/test_utils.py | 73 ++++ 9 files changed, 781 insertions(+), 237 deletions(-) create mode 100644 tests/test_litellm/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py diff --git a/litellm/constants.py b/litellm/constants.py index 8e0a5cfe50f..e104c937a9b 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -2,7 +2,7 @@ import os import sys from typing import List, Literal, Optional -from litellm.litellm_core_utils.env_utils import get_env_int +from litellm.litellm_core_utils.env_utils import get_env_int, get_env_int_or_none DEFAULT_HEALTH_CHECK_PROMPT = str(os.getenv("DEFAULT_HEALTH_CHECK_PROMPT", "test from litellm")) AZURE_DEFAULT_RESPONSES_API_VERSION = str(os.getenv("AZURE_DEFAULT_RESPONSES_API_VERSION", "preview")) @@ -269,9 +269,18 @@ TOOL_POLICY_CACHE_TTL_SECONDS = int(os.getenv("TOOL_POLICY_CACHE_TTL_SECONDS", 6 MAX_SIZE_IN_MEMORY_QUEUE = int(os.getenv("MAX_SIZE_IN_MEMORY_QUEUE", int(LITELLM_ASYNCIO_QUEUE_MAXSIZE * 0.8))) MAX_IN_MEMORY_QUEUE_FLUSH_COUNT = int(os.getenv("MAX_IN_MEMORY_QUEUE_FLUSH_COUNT", 1000)) ############################################################################################### -MINIMUM_PROMPT_CACHE_TOKEN_COUNT = int( - os.getenv("MINIMUM_PROMPT_CACHE_TOKEN_COUNT", 1024) -) # minimum number of tokens to cache a prompt by Anthropic +# Providers will not cache a prefix below a minimum size. That minimum is per-model, not global: +# Anthropic's ranges from 512 to 4096 depending on the model, and can differ per platform for the +# same model. The real minimum is resolved from `prompt_cache_min_tokens` in the model cost map; +# this value is only the fallback for models the cost map has no entry for, and doubles as a global +# escape hatch when `MINIMUM_PROMPT_CACHE_TOKEN_COUNT` is explicitly set. +MINIMUM_PROMPT_CACHE_TOKEN_COUNT_OVERRIDE: int | None = get_env_int_or_none("MINIMUM_PROMPT_CACHE_TOKEN_COUNT") +DEFAULT_MINIMUM_PROMPT_CACHE_TOKEN_COUNT = 1024 +MINIMUM_PROMPT_CACHE_TOKEN_COUNT = ( + MINIMUM_PROMPT_CACHE_TOKEN_COUNT_OVERRIDE + if MINIMUM_PROMPT_CACHE_TOKEN_COUNT_OVERRIDE is not None + else DEFAULT_MINIMUM_PROMPT_CACHE_TOKEN_COUNT +) DEFAULT_TRIM_RATIO = float( os.getenv("DEFAULT_TRIM_RATIO", 0.75) ) # default ratio of tokens to trim from the end of a prompt diff --git a/litellm/litellm_core_utils/env_utils.py b/litellm/litellm_core_utils/env_utils.py index 34c65275331..3a64f44fb25 100644 --- a/litellm/litellm_core_utils/env_utils.py +++ b/litellm/litellm_core_utils/env_utils.py @@ -19,3 +19,19 @@ def get_env_int(env_var: str, default: int) -> int: return int(raw) except (ValueError, TypeError): return default + + +def get_env_int_or_none(env_var: str) -> int | None: + """Parse an environment variable as an integer, returning None when it is unset or unusable. + + Use this instead of `get_env_int` when callers must distinguish "explicitly configured" + from "left at the default", for example when an override should take precedence over a + value resolved from somewhere else. + """ + raw = os.getenv(env_var) + if raw is None: + return None + try: + return int(raw.strip()) + except (ValueError, TypeError): + return None diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index dedb9bbf40a..1a24088396f 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -721,7 +721,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "prompt_cache_min_tokens": 2048 }, "anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.25e-06, @@ -745,7 +746,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "anthropic.claude-haiku-4-5@20251001": { "cache_creation_input_token_cost": 1.25e-06, @@ -770,7 +772,8 @@ "supports_vision": true, "supports_native_streaming": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "anthropic.claude-3-5-sonnet-20240620-v1:0": { "input_cost_per_token": 3e-06, @@ -935,7 +938,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "anthropic.claude-opus-4-20250514-v1:0": { "cache_creation_input_token_cost": 1.875e-05, @@ -960,7 +964,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "anthropic.claude-opus-4-5-20251101-v1:0": { "cache_creation_input_token_cost": 6.25e-06, @@ -990,7 +995,8 @@ "supports_native_structured_output": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "high", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, @@ -1022,7 +1028,8 @@ "supports_output_config": true, "supports_max_reasoning_effort": true, "bedrock_output_config_effort_ceiling": "max", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "global.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, @@ -1054,7 +1061,8 @@ "supports_output_config": true, "supports_max_reasoning_effort": true, "bedrock_output_config_effort_ceiling": "max", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "us.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, @@ -1086,7 +1094,8 @@ "supports_output_config": true, "supports_max_reasoning_effort": true, "bedrock_output_config_effort_ceiling": "max", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "eu.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, @@ -1118,7 +1127,8 @@ "supports_output_config": true, "supports_max_reasoning_effort": true, "bedrock_output_config_effort_ceiling": "max", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "au.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, @@ -1150,7 +1160,8 @@ "supports_output_config": true, "supports_max_reasoning_effort": true, "bedrock_output_config_effort_ceiling": "max", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, @@ -1185,7 +1196,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 2048 }, "anthropic.claude-mythos-preview": { "input_cost_per_token": 0, @@ -1235,7 +1247,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 2048 }, "us.anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, @@ -1270,7 +1283,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 2048 }, "eu.anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, @@ -1305,7 +1319,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 2048 }, "au.anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, @@ -1340,7 +1355,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 2048 }, "anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.25e-05, @@ -1375,7 +1391,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "global.anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.25e-05, @@ -1410,7 +1427,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "us.anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.375e-05, @@ -1445,7 +1463,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "eu.anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.375e-05, @@ -1480,7 +1499,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -1516,7 +1536,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "global.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -1552,7 +1573,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "us.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -1588,7 +1610,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "eu.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -1624,7 +1647,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "au.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -1660,7 +1684,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "jp.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -1696,7 +1721,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "jp.anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, @@ -1729,7 +1755,8 @@ "supports_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_minimal_reasoning_effort": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 2048 }, "anthropic.claude-sonnet-5": { "cache_creation_input_token_cost": 2.5e-06, @@ -1764,7 +1791,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "global.anthropic.claude-sonnet-5": { "cache_creation_input_token_cost": 2.5e-06, @@ -1799,7 +1827,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "us.anthropic.claude-sonnet-5": { "cache_creation_input_token_cost": 2.75e-06, @@ -1834,7 +1863,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "eu.anthropic.claude-sonnet-5": { "cache_creation_input_token_cost": 2.75e-06, @@ -1869,7 +1899,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "au.anthropic.claude-sonnet-5": { "cache_creation_input_token_cost": 2.75e-06, @@ -1904,7 +1935,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "jp.anthropic.claude-sonnet-5": { "cache_creation_input_token_cost": 2.75e-06, @@ -1939,7 +1971,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -1970,7 +2003,8 @@ "supports_vision": true, "supports_native_structured_output": true, "supports_output_config": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "global.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -2001,7 +2035,8 @@ "supports_vision": true, "supports_native_structured_output": true, "supports_output_config": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "us.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -2032,7 +2067,8 @@ "supports_vision": true, "supports_native_structured_output": true, "supports_output_config": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "eu.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -2063,7 +2099,8 @@ "supports_vision": true, "supports_native_structured_output": true, "supports_output_config": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "au.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -2094,7 +2131,8 @@ "supports_vision": true, "supports_native_structured_output": true, "supports_output_config": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "jp.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -2125,7 +2163,8 @@ "supports_vision": true, "supports_native_structured_output": true, "supports_output_config": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -2155,7 +2194,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "bedrock_converse_supports_strict_tools": false + "bedrock_converse_supports_strict_tools": false, + "prompt_cache_min_tokens": 1024 }, "anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -2188,7 +2228,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "anthropic.claude-v1": { "input_cost_per_token": 8e-06, @@ -2439,7 +2480,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "apac.anthropic.claude-3-sonnet-20240229-v1:0": { "input_cost_per_token": 3e-06, @@ -2485,7 +2527,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "bedrock_converse_supports_strict_tools": false + "bedrock_converse_supports_strict_tools": false, + "prompt_cache_min_tokens": 1024 }, "assemblyai/best": { "input_cost_per_second": 3.333e-05, @@ -2530,7 +2573,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "azure/ada": { "input_cost_per_token": 1e-07, @@ -10490,7 +10534,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "bedrock/us-gov-east-1/claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.5e-06, @@ -10513,7 +10558,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "bedrock/us-gov-east-1/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 2.65e-06, @@ -10667,7 +10713,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "bedrock/us-gov-west-1/claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.5e-06, @@ -10690,7 +10737,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "bedrock/us-gov-west-1/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 2.65e-06, @@ -10940,7 +10988,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "prompt_cache_min_tokens": 2048 }, "black_forest_labs/flux-kontext-pro": { "litellm_provider": "black_forest_labs", @@ -11160,7 +11209,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 4096 }, "claude-haiku-4-5": { "cache_creation_input_token_cost": 1.25e-06, @@ -11181,7 +11231,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 4096 }, "claude-3-7-sonnet-20250219": { "cache_creation_input_token_cost": 3.75e-06, @@ -11271,7 +11322,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "claude-4-sonnet-20250514": { "cache_creation_input_token_cost": 3.75e-06, @@ -11301,7 +11353,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "prompt_cache_min_tokens": 1024 }, "claude-sonnet-4-5": { "cache_creation_input_token_cost": 3.75e-06, @@ -11333,7 +11386,8 @@ "supports_response_schema": true, "supports_native_structured_output": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "claude-sonnet-4-5-20250929": { "cache_creation_input_token_cost": 3.75e-06, @@ -11366,7 +11420,8 @@ "supports_native_structured_output": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "prompt_cache_min_tokens": 1024 }, "claude-sonnet-5": { "cache_creation_input_token_cost": 2.5e-06, @@ -11400,7 +11455,8 @@ "provider_specific_entry": { "us": 1.1 }, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 1024 }, "claude-sonnet-4-6": { "cache_creation_input_token_cost": 3.75e-06, @@ -11430,7 +11486,8 @@ "supports_max_reasoning_effort": true, "supports_tool_choice": true, "supports_vision": true, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 1024 }, "claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -11457,7 +11514,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "claude-opus-4-1": { "cache_creation_input_token_cost": 1.875e-05, @@ -11484,7 +11542,8 @@ "supports_response_schema": true, "supports_native_structured_output": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "claude-opus-4-1-20250805": { "cache_creation_input_token_cost": 1.875e-05, @@ -11512,7 +11571,8 @@ "supports_response_schema": true, "supports_native_structured_output": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "claude-opus-4-20250514": { "cache_creation_input_token_cost": 1.875e-05, @@ -11539,7 +11599,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "claude-opus-4-5-20251101": { "cache_creation_input_token_cost": 6.25e-06, @@ -11567,7 +11628,8 @@ "supports_native_structured_output": true, "supports_tool_choice": true, "supports_vision": true, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 4096 }, "claude-opus-4-5": { "cache_creation_input_token_cost": 6.25e-06, @@ -11595,7 +11657,8 @@ "supports_native_structured_output": true, "supports_tool_choice": true, "supports_vision": true, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 4096 }, "claude-opus-4-6": { "cache_creation_input_token_cost": 6.25e-06, @@ -11630,7 +11693,8 @@ }, "supports_output_config": true, "supports_max_reasoning_effort": true, - "supports_speed": true + "supports_speed": true, + "prompt_cache_min_tokens": 4096 }, "claude-opus-4-6-20260205": { "cache_creation_input_token_cost": 6.25e-06, @@ -11665,7 +11729,8 @@ }, "supports_max_reasoning_effort": true, "supports_output_config": true, - "supports_speed": true + "supports_speed": true, + "prompt_cache_min_tokens": 4096 }, "claude-opus-4-7": { "cache_creation_input_token_cost": 6.25e-06, @@ -11702,7 +11767,8 @@ "fast": 6.0 }, "supports_output_config": true, - "supports_speed": true + "supports_speed": true, + "prompt_cache_min_tokens": 2048 }, "claude-opus-4-7-20260416": { "cache_creation_input_token_cost": 6.25e-06, @@ -11739,7 +11805,8 @@ "fast": 6.0 }, "supports_output_config": true, - "supports_speed": true + "supports_speed": true, + "prompt_cache_min_tokens": 2048 }, "claude-fable-5": { "cache_creation_input_token_cost": 1.25e-05, @@ -11773,7 +11840,8 @@ "provider_specific_entry": { "us": 1.1 }, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 512 }, "claude-opus-4-8": { "cache_creation_input_token_cost": 6.25e-06, @@ -11810,7 +11878,8 @@ "fast": 2.0 }, "supports_output_config": true, - "supports_speed": true + "supports_speed": true, + "prompt_cache_min_tokens": 1024 }, "claude-sonnet-4-20250514": { "deprecation_date": "2026-05-14", @@ -11841,7 +11910,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "cloudflare/@cf/meta/llama-2-7b-chat-fp16": { "input_cost_per_token": 1.923e-06, @@ -15514,7 +15584,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "cache_read_input_token_cost": 2.5e-08, - "cache_creation_input_token_cost": 3.125e-07 + "cache_creation_input_token_cost": 3.125e-07, + "prompt_cache_min_tokens": 2048 }, "eu.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, @@ -15539,7 +15610,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "eu.anthropic.claude-3-5-sonnet-20240620-v1:0": { "input_cost_per_token": 3e-06, @@ -15666,7 +15738,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "eu.anthropic.claude-opus-4-20250514-v1:0": { "cache_creation_input_token_cost": 1.875e-05, @@ -15691,7 +15764,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "eu.anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -15721,7 +15795,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "bedrock_converse_supports_strict_tools": false + "bedrock_converse_supports_strict_tools": false, + "prompt_cache_min_tokens": 1024 }, "eu.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.125e-06, @@ -15754,7 +15829,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "eu.meta.llama3-2-1b-instruct-v1:0": { "input_cost_per_token": 1.3e-07, @@ -21105,7 +21181,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "global.anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -21135,7 +21212,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "bedrock_converse_supports_strict_tools": false + "bedrock_converse_supports_strict_tools": false, + "prompt_cache_min_tokens": 1024 }, "global.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.25e-06, @@ -21159,7 +21237,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "global.amazon.nova-2-lite-v1:0": { "cache_read_input_token_cost": 7.5e-08, @@ -25511,7 +25590,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "jp.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, @@ -25535,7 +25615,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "crusoe/deepseek-ai/DeepSeek-R1-0528": { "input_cost_per_token": 3e-06, @@ -34163,7 +34244,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "prompt_cache_min_tokens": 2048 }, "us.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, @@ -34187,7 +34269,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "us.anthropic.claude-3-5-sonnet-20240620-v1:0": { "input_cost_per_token": 3e-06, @@ -34314,7 +34397,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "us.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.125e-06, @@ -34347,7 +34431,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "us-gov.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.5e-06, @@ -34375,7 +34460,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "au.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, @@ -34398,7 +34484,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "us.anthropic.claude-opus-4-20250514-v1:0": { "cache_creation_input_token_cost": 1.875e-05, @@ -34423,7 +34510,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "us.anthropic.claude-opus-4-5-20251101-v1:0": { "cache_creation_input_token_cost": 6.875e-06, @@ -34453,7 +34541,8 @@ "supports_native_structured_output": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "high", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "global.anthropic.claude-opus-4-5-20251101-v1:0": { "cache_creation_input_token_cost": 6.25e-06, @@ -34483,7 +34572,8 @@ "supports_native_structured_output": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "high", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "eu.anthropic.claude-opus-4-5-20251101-v1:0": { "cache_creation_input_token_cost": 6.25e-06, @@ -34512,7 +34602,8 @@ "supports_native_structured_output": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "high", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "us.anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -34542,7 +34633,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "bedrock_converse_supports_strict_tools": false + "bedrock_converse_supports_strict_tools": false, + "prompt_cache_min_tokens": 1024 }, "us.deepseek.r1-v1:0": { "input_cost_per_token": 1.35e-06, @@ -36064,7 +36156,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_native_streaming": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-haiku-4-5@20251001": { "cache_creation_input_token_cost": 1.25e-06, @@ -36086,7 +36179,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_native_streaming": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-3-5-sonnet": { "input_cost_per_token": 3e-06, @@ -36241,7 +36335,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-opus-4-1": { "cache_creation_input_token_cost": 1.875e-05, @@ -36304,7 +36399,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-opus-4-5@20251101": { "cache_creation_input_token_cost": 6.25e-06, @@ -36332,7 +36428,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_streaming": true, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-opus-4-6": { "supports_adaptive_thinking": true, @@ -36361,7 +36458,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_output_config": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-opus-4-6@default": { "supports_adaptive_thinking": true, @@ -36390,7 +36488,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_output_config": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-opus-4-7": { "supports_adaptive_thinking": true, @@ -36420,7 +36519,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 2048 }, "vertex_ai/claude-opus-4-7@default": { "supports_adaptive_thinking": true, @@ -36450,7 +36550,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 2048 }, "vertex_ai/claude-fable-5": { "cache_creation_input_token_cost": 1.25e-05, @@ -36540,7 +36641,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-opus-4-8@default": { "supports_adaptive_thinking": true, @@ -36570,7 +36672,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-4-5": { "cache_creation_input_token_cost": 3.75e-06, @@ -36597,7 +36700,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-5": { "cache_creation_input_token_cost": 2.5e-06, @@ -36627,7 +36731,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -36656,7 +36761,8 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-4-5@20250929": { "cache_creation_input_token_cost": 3.75e-06, @@ -36684,7 +36790,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_native_streaming": true + "supports_native_streaming": true, + "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-opus-4@20250514": { "cache_creation_input_token_cost": 1.875e-05, @@ -36710,7 +36817,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-4": { "cache_creation_input_token_cost": 3.75e-06, @@ -36740,7 +36848,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-4@20250514": { "cache_creation_input_token_cost": 3.75e-06, @@ -36770,7 +36879,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "vertex_ai/mistralai/codestral-2@001": { "input_cost_per_token": 3e-07, @@ -44154,7 +44264,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-4-6@default": { "supports_adaptive_thinking": true, @@ -44183,7 +44294,8 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 1024 }, "duckduckgo/search": { "litellm_provider": "duckduckgo", @@ -44679,7 +44791,8 @@ "supports_vision": true, "supports_native_structured_output": true, "supports_pdf_input": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "bedrock/us-gov-west-1/anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.5e-06, @@ -44703,7 +44816,8 @@ "supports_vision": true, "supports_native_structured_output": true, "supports_pdf_input": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "snowflake/claude-sonnet-4-5": { "max_tokens": 16384, diff --git a/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py b/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py index 581783980fd..1d121d79ea3 100644 --- a/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py +++ b/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py @@ -8,14 +8,36 @@ from typing import List, Optional, cast from litellm import verbose_logger from litellm.caching.dual_cache import DualCache +from litellm.constants import DEFAULT_MINIMUM_PROMPT_CACHE_TOKEN_COUNT from litellm.integrations.custom_logger import CustomLogger, Span from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import CallTypes, StandardLoggingPayload -from litellm.utils import is_prompt_caching_valid_prompt +from litellm.utils import get_prompt_cache_min_tokens, is_prompt_caching_valid_prompt from ..prompt_caching_cache import PromptCachingCache +def _get_min_token_count_for_deployments(healthy_deployments: list[dict]) -> int: + """ + Returns the highest minimum cacheable prefix across a model group. + + `model` here is the model-group alias the operator chose, not a model name, so the threshold + has to come from the deployments themselves. A group may mix models with different minimums, + and one gate decides for all of them, so take the max: a prompt is only treated as cacheable + when it clears every member's minimum. The errors are not symmetric. Pinning a deployment for + a prefix its provider will not cache costs load balancing for nothing, which is the bug this + guards against, while declining to pin only forfeits a cache hit. + """ + return max( + ( + get_prompt_cache_min_tokens(model=deployment["litellm_params"]["model"]) + for deployment in healthy_deployments + if deployment.get("litellm_params", {}).get("model") + ), + default=DEFAULT_MINIMUM_PROMPT_CACHE_TOKEN_COUNT, + ) + + class PromptCachingDeploymentCheck(CustomLogger): def __init__(self, cache: DualCache): self.cache = cache @@ -31,7 +53,8 @@ class PromptCachingDeploymentCheck(CustomLogger): if messages is not None and is_prompt_caching_valid_prompt( messages=messages, model=model, - ): # prompt > 1024 tokens + min_token_count=_get_min_token_count_for_deployments(healthy_deployments), + ): prompt_cache = PromptCachingCache( cache=self.cache, ) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 88b3a39844f..01825f0c02c 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -197,6 +197,9 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): cache_read_input_token_cost_above_272k_tokens: Optional[float] cache_read_input_token_cost_above_272k_tokens_priority: Optional[float] cache_read_input_token_cost_above_512k_tokens: Optional[float] + # Smallest prefix this model will actually cache, whatever caching mechanism its provider uses. + # Absent means the provider-agnostic default applies; see MINIMUM_PROMPT_CACHE_TOKEN_COUNT. + prompt_cache_min_tokens: Optional[int] input_cost_per_character: Optional[float] # only for vertex ai models input_cost_per_audio_token: Optional[float] input_cost_per_token_above_128k_tokens: Optional[float] # only for vertex ai models diff --git a/litellm/utils.py b/litellm/utils.py index 0636d3683b7..e19d2b36a52 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -73,7 +73,8 @@ from litellm.constants import ( JITTER, MAX_RETRY_DELAY, MAX_TOKEN_TRIMMING_ATTEMPTS, - MINIMUM_PROMPT_CACHE_TOKEN_COUNT, + DEFAULT_MINIMUM_PROMPT_CACHE_TOKEN_COUNT, + MINIMUM_PROMPT_CACHE_TOKEN_COUNT_OVERRIDE, OPENAI_EMBEDDING_PARAMS, TOOL_CHOICE_OBJECT_TOKEN_COUNT, ) @@ -5402,6 +5403,7 @@ def _get_model_info_helper( "cache_creation_input_token_cost_above_200k_tokens", None ), cache_read_input_token_cost=_model_info.get("cache_read_input_token_cost", None), + prompt_cache_min_tokens=_model_info.get("prompt_cache_min_tokens", None), cache_read_input_token_cost_above_200k_tokens=_model_info.get( "cache_read_input_token_cost_above_200k_tokens", None ), @@ -9039,16 +9041,46 @@ def should_use_cohere_v1_client(api_base: Optional[str], present_version_params: return api_base.endswith("/v1/rerank") or (uses_v1_params and not api_base.endswith("/v2/rerank")) +def get_prompt_cache_min_tokens(model: str) -> int: + """ + Returns the smallest prefix `model` will actually cache. + + Resolution order is an explicitly configured `MINIMUM_PROMPT_CACHE_TOKEN_COUNT`, then the + model's `prompt_cache_min_tokens` in the cost map, then the provider-agnostic default. The + cost map is the source of truth because the real minimum is per-model and per-platform: + Anthropic's ranges from 512 to 4096 and moves in both directions across releases, and the + same model can differ by platform. + + Never raises. An unresolvable model falls back to the default rather than propagating, so a + caller cannot mistake "no entry for this model" for "this prompt is not cacheable". + """ + if MINIMUM_PROMPT_CACHE_TOKEN_COUNT_OVERRIDE is not None: + return MINIMUM_PROMPT_CACHE_TOKEN_COUNT_OVERRIDE + try: + min_tokens = get_model_info(model=model).get("prompt_cache_min_tokens") + except Exception: + return DEFAULT_MINIMUM_PROMPT_CACHE_TOKEN_COUNT + if min_tokens is None: + return DEFAULT_MINIMUM_PROMPT_CACHE_TOKEN_COUNT + return min_tokens + + def is_prompt_caching_valid_prompt( model: str, messages: Optional[List[AllMessageValues]], tools: Optional[List[ChatCompletionToolParam]] = None, custom_llm_provider: Optional[str] = None, + min_token_count: int | None = None, ) -> bool: """ Returns true if the prompt is valid for prompt caching. - OpenAI + Anthropic providers have a minimum token count of 1024 for prompt caching. + The minimum cacheable prefix is per-model, so it is resolved from `model` unless the caller + passes `min_token_count`. Callers that only hold a model-group alias (the router's deployment + checks) must resolve the threshold themselves and pass it, because an alias resolves to + nothing here and would silently fall back to the default. + + OpenAI's minimum is a flat 1024 across models, which the default already covers. """ try: if messages is None and tools is None: @@ -9061,7 +9093,9 @@ def is_prompt_caching_valid_prompt( model=model, use_default_image_token_count=True, ) - return token_count >= MINIMUM_PROMPT_CACHE_TOKEN_COUNT + if min_token_count is None: + min_token_count = get_prompt_cache_min_tokens(model=model) + return token_count >= min_token_count except Exception as e: verbose_logger.error(f"Error in is_prompt_caching_valid_prompt: {e}") return False diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index e10dde793d1..ffbc0dcd098 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -721,7 +721,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "prompt_cache_min_tokens": 2048 }, "anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.25e-06, @@ -745,7 +746,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "anthropic.claude-haiku-4-5@20251001": { "cache_creation_input_token_cost": 1.25e-06, @@ -770,7 +772,8 @@ "supports_vision": true, "supports_native_streaming": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "anthropic.claude-3-5-sonnet-20240620-v1:0": { "input_cost_per_token": 3e-06, @@ -935,7 +938,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "anthropic.claude-opus-4-20250514-v1:0": { "cache_creation_input_token_cost": 1.875e-05, @@ -960,7 +964,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "anthropic.claude-opus-4-5-20251101-v1:0": { "cache_creation_input_token_cost": 6.25e-06, @@ -990,7 +995,8 @@ "supports_native_structured_output": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "high", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, @@ -1022,7 +1028,8 @@ "supports_output_config": true, "supports_max_reasoning_effort": true, "bedrock_output_config_effort_ceiling": "max", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "global.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, @@ -1054,7 +1061,8 @@ "supports_output_config": true, "supports_max_reasoning_effort": true, "bedrock_output_config_effort_ceiling": "max", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "us.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, @@ -1086,7 +1094,8 @@ "supports_output_config": true, "supports_max_reasoning_effort": true, "bedrock_output_config_effort_ceiling": "max", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "eu.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, @@ -1118,7 +1127,8 @@ "supports_output_config": true, "supports_max_reasoning_effort": true, "bedrock_output_config_effort_ceiling": "max", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "au.anthropic.claude-opus-4-6-v1": { "supports_adaptive_thinking": true, @@ -1150,7 +1160,8 @@ "supports_output_config": true, "supports_max_reasoning_effort": true, "bedrock_output_config_effort_ceiling": "max", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, @@ -1185,7 +1196,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 2048 }, "anthropic.claude-mythos-preview": { "input_cost_per_token": 0, @@ -1235,7 +1247,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 2048 }, "us.anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, @@ -1270,7 +1283,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 2048 }, "eu.anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, @@ -1305,7 +1319,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 2048 }, "au.anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, @@ -1340,7 +1355,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 2048 }, "anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.25e-05, @@ -1375,7 +1391,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "global.anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.25e-05, @@ -1410,7 +1427,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "us.anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.375e-05, @@ -1445,7 +1463,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "eu.anthropic.claude-fable-5": { "cache_creation_input_token_cost": 1.375e-05, @@ -1480,7 +1499,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -1516,7 +1536,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "global.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -1552,7 +1573,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "us.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -1588,7 +1610,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "eu.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -1624,7 +1647,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "au.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -1660,7 +1684,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "jp.anthropic.claude-opus-4-8": { "bedrock_converse_supports_strict_tools": false, @@ -1696,7 +1721,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "jp.anthropic.claude-opus-4-7": { "bedrock_converse_supports_strict_tools": false, @@ -1729,7 +1755,8 @@ "supports_native_structured_output": true, "supports_max_reasoning_effort": true, "supports_minimal_reasoning_effort": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 2048 }, "anthropic.claude-sonnet-5": { "cache_creation_input_token_cost": 2.5e-06, @@ -1764,7 +1791,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "global.anthropic.claude-sonnet-5": { "cache_creation_input_token_cost": 2.5e-06, @@ -1799,7 +1827,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "us.anthropic.claude-sonnet-5": { "cache_creation_input_token_cost": 2.75e-06, @@ -1834,7 +1863,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "eu.anthropic.claude-sonnet-5": { "cache_creation_input_token_cost": 2.75e-06, @@ -1869,7 +1899,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "au.anthropic.claude-sonnet-5": { "cache_creation_input_token_cost": 2.75e-06, @@ -1904,7 +1935,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "jp.anthropic.claude-sonnet-5": { "cache_creation_input_token_cost": 2.75e-06, @@ -1939,7 +1971,8 @@ "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -1970,7 +2003,8 @@ "supports_vision": true, "supports_native_structured_output": true, "supports_output_config": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "global.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -2001,7 +2035,8 @@ "supports_vision": true, "supports_native_structured_output": true, "supports_output_config": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "us.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -2032,7 +2067,8 @@ "supports_vision": true, "supports_native_structured_output": true, "supports_output_config": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "eu.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -2063,7 +2099,8 @@ "supports_vision": true, "supports_native_structured_output": true, "supports_output_config": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "au.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -2094,7 +2131,8 @@ "supports_vision": true, "supports_native_structured_output": true, "supports_output_config": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "jp.anthropic.claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -2125,7 +2163,8 @@ "supports_vision": true, "supports_native_structured_output": true, "supports_output_config": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -2155,7 +2194,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "bedrock_converse_supports_strict_tools": false + "bedrock_converse_supports_strict_tools": false, + "prompt_cache_min_tokens": 1024 }, "anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -2188,7 +2228,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "anthropic.claude-v1": { "input_cost_per_token": 8e-06, @@ -2439,7 +2480,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "apac.anthropic.claude-3-sonnet-20240229-v1:0": { "input_cost_per_token": 3e-06, @@ -2485,7 +2527,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "bedrock_converse_supports_strict_tools": false + "bedrock_converse_supports_strict_tools": false, + "prompt_cache_min_tokens": 1024 }, "assemblyai/best": { "input_cost_per_second": 3.333e-05, @@ -2530,7 +2573,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "azure/ada": { "input_cost_per_token": 1e-07, @@ -10490,7 +10534,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "bedrock/us-gov-east-1/claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.5e-06, @@ -10513,7 +10558,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "bedrock/us-gov-east-1/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 2.65e-06, @@ -10667,7 +10713,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "bedrock/us-gov-west-1/claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.5e-06, @@ -10690,7 +10737,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "bedrock/us-gov-west-1/meta.llama3-70b-instruct-v1:0": { "input_cost_per_token": 2.65e-06, @@ -10940,7 +10988,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "prompt_cache_min_tokens": 2048 }, "black_forest_labs/flux-kontext-pro": { "litellm_provider": "black_forest_labs", @@ -11160,7 +11209,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 4096 }, "claude-haiku-4-5": { "cache_creation_input_token_cost": 1.25e-06, @@ -11181,7 +11231,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 4096 }, "claude-3-7-sonnet-20250219": { "cache_creation_input_token_cost": 3.75e-06, @@ -11271,7 +11322,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "claude-4-sonnet-20250514": { "cache_creation_input_token_cost": 3.75e-06, @@ -11301,7 +11353,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "prompt_cache_min_tokens": 1024 }, "claude-sonnet-4-5": { "cache_creation_input_token_cost": 3.75e-06, @@ -11333,7 +11386,8 @@ "supports_response_schema": true, "supports_native_structured_output": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "claude-sonnet-4-5-20250929": { "cache_creation_input_token_cost": 3.75e-06, @@ -11366,7 +11420,8 @@ "supports_native_structured_output": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "prompt_cache_min_tokens": 1024 }, "claude-sonnet-5": { "cache_creation_input_token_cost": 2.5e-06, @@ -11400,7 +11455,8 @@ "provider_specific_entry": { "us": 1.1 }, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 1024 }, "claude-sonnet-4-6": { "cache_creation_input_token_cost": 3.75e-06, @@ -11430,7 +11486,8 @@ "supports_max_reasoning_effort": true, "supports_tool_choice": true, "supports_vision": true, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 1024 }, "claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -11457,7 +11514,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "claude-opus-4-1": { "cache_creation_input_token_cost": 1.875e-05, @@ -11484,7 +11542,8 @@ "supports_response_schema": true, "supports_native_structured_output": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "claude-opus-4-1-20250805": { "cache_creation_input_token_cost": 1.875e-05, @@ -11512,7 +11571,8 @@ "supports_response_schema": true, "supports_native_structured_output": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "claude-opus-4-20250514": { "cache_creation_input_token_cost": 1.875e-05, @@ -11539,7 +11599,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "claude-opus-4-5-20251101": { "cache_creation_input_token_cost": 6.25e-06, @@ -11567,7 +11628,8 @@ "supports_native_structured_output": true, "supports_tool_choice": true, "supports_vision": true, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 4096 }, "claude-opus-4-5": { "cache_creation_input_token_cost": 6.25e-06, @@ -11595,7 +11657,8 @@ "supports_native_structured_output": true, "supports_tool_choice": true, "supports_vision": true, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 4096 }, "claude-opus-4-6": { "cache_creation_input_token_cost": 6.25e-06, @@ -11630,7 +11693,8 @@ }, "supports_output_config": true, "supports_max_reasoning_effort": true, - "supports_speed": true + "supports_speed": true, + "prompt_cache_min_tokens": 4096 }, "claude-opus-4-6-20260205": { "cache_creation_input_token_cost": 6.25e-06, @@ -11665,7 +11729,8 @@ }, "supports_max_reasoning_effort": true, "supports_output_config": true, - "supports_speed": true + "supports_speed": true, + "prompt_cache_min_tokens": 4096 }, "claude-opus-4-7": { "cache_creation_input_token_cost": 6.25e-06, @@ -11702,7 +11767,8 @@ "fast": 6.0 }, "supports_output_config": true, - "supports_speed": true + "supports_speed": true, + "prompt_cache_min_tokens": 2048 }, "claude-opus-4-7-20260416": { "cache_creation_input_token_cost": 6.25e-06, @@ -11739,7 +11805,8 @@ "fast": 6.0 }, "supports_output_config": true, - "supports_speed": true + "supports_speed": true, + "prompt_cache_min_tokens": 2048 }, "claude-fable-5": { "cache_creation_input_token_cost": 1.25e-05, @@ -11773,7 +11840,8 @@ "provider_specific_entry": { "us": 1.1 }, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 512 }, "claude-opus-4-8": { "cache_creation_input_token_cost": 6.25e-06, @@ -11810,7 +11878,8 @@ "fast": 2.0 }, "supports_output_config": true, - "supports_speed": true + "supports_speed": true, + "prompt_cache_min_tokens": 1024 }, "claude-sonnet-4-20250514": { "deprecation_date": "2026-05-14", @@ -11841,7 +11910,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "cloudflare/@cf/meta/llama-2-7b-chat-fp16": { "input_cost_per_token": 1.923e-06, @@ -15514,7 +15584,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "cache_read_input_token_cost": 2.5e-08, - "cache_creation_input_token_cost": 3.125e-07 + "cache_creation_input_token_cost": 3.125e-07, + "prompt_cache_min_tokens": 2048 }, "eu.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, @@ -15539,7 +15610,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "eu.anthropic.claude-3-5-sonnet-20240620-v1:0": { "input_cost_per_token": 3e-06, @@ -15666,7 +15738,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "eu.anthropic.claude-opus-4-20250514-v1:0": { "cache_creation_input_token_cost": 1.875e-05, @@ -15691,7 +15764,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "eu.anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -15721,7 +15795,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "bedrock_converse_supports_strict_tools": false + "bedrock_converse_supports_strict_tools": false, + "prompt_cache_min_tokens": 1024 }, "eu.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.125e-06, @@ -15754,7 +15829,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "eu.meta.llama3-2-1b-instruct-v1:0": { "input_cost_per_token": 1.3e-07, @@ -21180,7 +21256,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "global.anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -21210,7 +21287,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "bedrock_converse_supports_strict_tools": false + "bedrock_converse_supports_strict_tools": false, + "prompt_cache_min_tokens": 1024 }, "global.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.25e-06, @@ -21234,7 +21312,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "global.amazon.nova-2-lite-v1:0": { "cache_read_input_token_cost": 7.5e-08, @@ -25586,7 +25665,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "jp.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, @@ -25610,7 +25690,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "crusoe/deepseek-ai/DeepSeek-R1-0528": { "input_cost_per_token": 3e-06, @@ -34254,7 +34335,8 @@ "supports_pdf_input": true, "supports_prompt_caching": true, "supports_response_schema": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "prompt_cache_min_tokens": 2048 }, "us.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, @@ -34278,7 +34360,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "us.anthropic.claude-3-5-sonnet-20240620-v1:0": { "input_cost_per_token": 3e-06, @@ -34405,7 +34488,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "us.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.125e-06, @@ -34438,7 +34522,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "us-gov.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.5e-06, @@ -34466,7 +34551,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 1024 }, "au.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, @@ -34489,7 +34575,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_structured_output": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "us.anthropic.claude-opus-4-20250514-v1:0": { "cache_creation_input_token_cost": 1.875e-05, @@ -34514,7 +34601,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "us.anthropic.claude-opus-4-5-20251101-v1:0": { "cache_creation_input_token_cost": 6.875e-06, @@ -34544,7 +34632,8 @@ "supports_native_structured_output": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "high", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "global.anthropic.claude-opus-4-5-20251101-v1:0": { "cache_creation_input_token_cost": 6.25e-06, @@ -34574,7 +34663,8 @@ "supports_native_structured_output": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "high", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "eu.anthropic.claude-opus-4-5-20251101-v1:0": { "cache_creation_input_token_cost": 6.25e-06, @@ -34603,7 +34693,8 @@ "supports_native_structured_output": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "high", - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "us.anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, @@ -34633,7 +34724,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "bedrock_converse_supports_strict_tools": false + "bedrock_converse_supports_strict_tools": false, + "prompt_cache_min_tokens": 1024 }, "us.deepseek.r1-v1:0": { "input_cost_per_token": 1.35e-06, @@ -36155,7 +36247,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_native_streaming": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-haiku-4-5@20251001": { "cache_creation_input_token_cost": 1.25e-06, @@ -36177,7 +36270,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_native_streaming": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-3-5-sonnet": { "input_cost_per_token": 3e-06, @@ -36332,7 +36426,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-opus-4-1": { "cache_creation_input_token_cost": 1.875e-05, @@ -36395,7 +36490,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-opus-4-5@20251101": { "cache_creation_input_token_cost": 6.25e-06, @@ -36423,7 +36519,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_native_streaming": true, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-opus-4-6": { "supports_adaptive_thinking": true, @@ -36452,7 +36549,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_output_config": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-opus-4-6@default": { "supports_adaptive_thinking": true, @@ -36481,7 +36579,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_output_config": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 4096 }, "vertex_ai/claude-opus-4-7": { "supports_adaptive_thinking": true, @@ -36511,7 +36610,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 2048 }, "vertex_ai/claude-opus-4-7@default": { "supports_adaptive_thinking": true, @@ -36541,7 +36641,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 2048 }, "vertex_ai/claude-fable-5": { "cache_creation_input_token_cost": 1.25e-05, @@ -36631,7 +36732,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-opus-4-8@default": { "supports_adaptive_thinking": true, @@ -36661,7 +36763,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-4-5": { "cache_creation_input_token_cost": 3.75e-06, @@ -36688,7 +36791,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-5": { "cache_creation_input_token_cost": 2.5e-06, @@ -36718,7 +36822,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-4-6": { "supports_adaptive_thinking": true, @@ -36747,7 +36852,8 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-4-5@20250929": { "cache_creation_input_token_cost": 3.75e-06, @@ -36775,7 +36881,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_native_streaming": true + "supports_native_streaming": true, + "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-opus-4@20250514": { "cache_creation_input_token_cost": 1.875e-05, @@ -36801,7 +36908,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-4": { "cache_creation_input_token_cost": 3.75e-06, @@ -36831,7 +36939,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-4@20250514": { "cache_creation_input_token_cost": 3.75e-06, @@ -36861,7 +36970,8 @@ "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "prompt_cache_min_tokens": 1024 }, "vertex_ai/mistralai/codestral-2@001": { "input_cost_per_token": 3e-07, @@ -44275,7 +44385,8 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_max_reasoning_effort": true + "supports_max_reasoning_effort": true, + "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-4-6@default": { "supports_adaptive_thinking": true, @@ -44304,7 +44415,8 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, - "supports_output_config": true + "supports_output_config": true, + "prompt_cache_min_tokens": 1024 }, "duckduckgo/search": { "litellm_provider": "duckduckgo", @@ -44800,7 +44912,8 @@ "supports_vision": true, "supports_native_structured_output": true, "supports_pdf_input": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "bedrock/us-gov-west-1/anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.5e-06, @@ -44824,7 +44937,8 @@ "supports_vision": true, "supports_native_structured_output": true, "supports_pdf_input": true, - "supports_parallel_tool_use_config": true + "supports_parallel_tool_use_config": true, + "prompt_cache_min_tokens": 4096 }, "snowflake/claude-sonnet-4-5": { "max_tokens": 16384, diff --git a/tests/test_litellm/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py b/tests/test_litellm/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py new file mode 100644 index 00000000000..1ad6caca2a3 --- /dev/null +++ b/tests/test_litellm/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py @@ -0,0 +1,158 @@ +import os +import sys +from typing import List, cast + +import pytest + +sys.path.insert(0, os.path.abspath("../..")) + +import litellm +from litellm.caching.dual_cache import DualCache +from litellm.constants import DEFAULT_MINIMUM_PROMPT_CACHE_TOKEN_COUNT +from litellm.router_utils.pre_call_checks.prompt_caching_deployment_check import ( + PromptCachingDeploymentCheck, + _get_min_token_count_for_deployments, +) +from litellm.router_utils.prompt_caching_cache import PromptCachingCache +from litellm.types.llms.openai import AllMessageValues +from litellm.utils import get_prompt_cache_min_tokens, token_counter + +MODEL_GROUP_ALIAS = "my-claude-group" +OPUS_4_6_MIN_TOKENS = 4096 + + +@pytest.fixture(autouse=True) +def local_model_cost_map(monkeypatch): + """ + The remote cost map does not carry `prompt_cache_min_tokens` yet, so a test that reads the + default map would pass here and flake in CI. Force the in-repo map. + """ + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + + +def _deployments(*models: str) -> List[dict]: + return [ + { + "model_name": MODEL_GROUP_ALIAS, + "litellm_params": {"model": model}, + "model_info": {"id": f"dep-{index}"}, + } + for index, model in enumerate(models, start=1) + ] + + +def _messages(word_count: int) -> List[AllMessageValues]: + return cast( + List[AllMessageValues], + [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "word " * word_count, + "cache_control": {"type": "ephemeral"}, + } + ], + } + ], + ) + + +def test_get_min_token_count_for_deployments_takes_max_across_mixed_group(): + """ + A group may legally mix models whose real minimums differ, and one boolean gate decides for + every member. The threshold must be the highest minimum in the group: taking the lowest would + let a 1024-token prompt pin the Opus 4.5 deployment for a prefix Anthropic will never cache. + """ + assert get_prompt_cache_min_tokens(model="anthropic/claude-opus-4-5") == 4096 + assert get_prompt_cache_min_tokens(model="anthropic/claude-sonnet-4-5") == 1024 + + deployments = _deployments("anthropic/claude-opus-4-5", "anthropic/claude-sonnet-4-5") + + assert _get_min_token_count_for_deployments(deployments) == 4096 + + +def test_get_min_token_count_for_deployments_falls_back_to_default_for_empty_group(): + """An empty group has no member minimum to read, so it must fall back rather than crash.""" + assert _get_min_token_count_for_deployments([]) == DEFAULT_MINIMUM_PROMPT_CACHE_TOKEN_COUNT + + +@pytest.mark.asyncio +async def test_async_filter_deployments_does_not_narrow_prompt_below_model_minimum(): + """ + The regression. Opus 4.6 will not cache a prefix under 4096 tokens, so a ~1400-token prompt is + not cacheable and routing must stay free across the whole group. Previously the check resolved + its threshold from `model`, which is the operator's group alias and matches nothing in the cost + map, silently fell back to 1024, judged this prompt cacheable, and pinned every request to one + deployment for a cache hit the provider was never going to serve. + """ + cache = DualCache() + check = PromptCachingDeploymentCheck(cache=cache) + deployments = _deployments("anthropic/claude-opus-4-6", "anthropic/claude-opus-4-6") + messages = _messages(word_count=1400) + + token_count = token_counter(messages=messages, model="anthropic/claude-opus-4-6", use_default_image_token_count=True) + assert DEFAULT_MINIMUM_PROMPT_CACHE_TOKEN_COUNT < token_count < OPUS_4_6_MIN_TOKENS + + await PromptCachingCache(cache=cache).async_add_model_id(model_id="dep-2", messages=messages, tools=None) + + filtered = await check.async_filter_deployments( + model=MODEL_GROUP_ALIAS, + healthy_deployments=deployments, + messages=messages, + ) + + assert filtered == deployments + + +@pytest.mark.asyncio +async def test_async_filter_deployments_narrows_prompt_above_model_minimum(): + """ + The positive control for the regression above: once the same group's prompt clears Opus 4.6's + real 4096-token minimum the prefix is genuinely cacheable, so the check must still pin the + deployment that served it. Proves the fix tightened the gate rather than disabling the feature. + """ + cache = DualCache() + check = PromptCachingDeploymentCheck(cache=cache) + deployments = _deployments("anthropic/claude-opus-4-6", "anthropic/claude-opus-4-6") + messages = _messages(word_count=5000) + + token_count = token_counter(messages=messages, model="anthropic/claude-opus-4-6", use_default_image_token_count=True) + assert token_count > OPUS_4_6_MIN_TOKENS + + await PromptCachingCache(cache=cache).async_add_model_id(model_id="dep-2", messages=messages, tools=None) + + filtered = await check.async_filter_deployments( + model=MODEL_GROUP_ALIAS, + healthy_deployments=deployments, + messages=messages, + ) + + assert filtered == [deployments[1]] + + +@pytest.mark.asyncio +async def test_async_filter_deployments_narrows_for_group_whose_model_minimum_is_lower(): + """ + Same ~1400-token prompt that must not pin an Opus 4.6 group, on an Opus 4.8 group whose real + minimum is 1024. Here the prefix is cacheable and the check must pin. Proves the threshold is + resolved per-model from the deployments rather than tightened for everyone. + """ + cache = DualCache() + check = PromptCachingDeploymentCheck(cache=cache) + deployments = _deployments("anthropic/claude-opus-4-8", "anthropic/claude-opus-4-8") + messages = _messages(word_count=1400) + + assert get_prompt_cache_min_tokens(model="anthropic/claude-opus-4-8") == DEFAULT_MINIMUM_PROMPT_CACHE_TOKEN_COUNT + + await PromptCachingCache(cache=cache).async_add_model_id(model_id="dep-2", messages=messages, tools=None) + + filtered = await check.async_filter_deployments( + model=MODEL_GROUP_ALIAS, + healthy_deployments=deployments, + messages=messages, + ) + + assert filtered == [deployments[1]] diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 6d515ecdc73..073ff17991e 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -26,7 +26,9 @@ from litellm.utils import ( _is_streaming_request, get_llm_provider, get_optional_params_image_gen, + get_prompt_cache_min_tokens, is_cached_message, + is_prompt_caching_valid_prompt, ) # Adds the parent directory to the system path @@ -842,6 +844,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "supports_parallel_function_calling": {"type": "boolean"}, "supports_parallel_tool_use_config": {"type": "boolean"}, "supports_pdf_input": {"type": "boolean"}, + "prompt_cache_min_tokens": {"type": "number"}, "supports_prompt_caching": {"type": "boolean"}, "supports_response_schema": {"type": "boolean"}, "supports_system_messages": {"type": "boolean"}, @@ -4741,3 +4744,73 @@ def test_gemini_image_models_do_not_support_reasoning( f"{model} incorrectly classified as reasoning-capable. " "Add 'supports_reasoning: false' to its model_cost entry." ) + + +PROMPT_CACHE_MESSAGES = [{"role": "user", "content": "the quick brown fox jumps over the lazy dog " * 155}] + + +@pytest.mark.parametrize( + "model, expected_min_tokens", + [ + ("claude-opus-4-6", 4096), + ("claude-opus-4-7", 2048), + ("claude-opus-4-8", 1024), + ("claude-fable-5", 512), + ], +) +def test_get_prompt_cache_min_tokens_resolves_per_model( + model: str, expected_min_tokens: int, local_model_cost_map: None +) -> None: + """The smallest cacheable prefix is a per-model property, read from the cost map's + prompt_cache_min_tokens. Anthropic's minimum spans 512..4096 across models and moves in both + directions across releases, so a single global constant is wrong for every model but one.""" + assert get_prompt_cache_min_tokens(model=model) == expected_min_tokens + + +def test_get_prompt_cache_min_tokens_differs_per_platform_for_same_model(local_model_cost_map: None) -> None: + """The same model can carry a different minimum per platform, so the threshold must come from + the platform's own cost-map entry rather than being derived from the model family name.""" + assert get_prompt_cache_min_tokens(model="claude-fable-5") == 512 + assert get_prompt_cache_min_tokens(model="anthropic.claude-fable-5") == 1024 + assert get_prompt_cache_min_tokens(model="claude-fable-5") != get_prompt_cache_min_tokens( + model="anthropic.claude-fable-5" + ) + + +def test_get_prompt_cache_min_tokens_unmapped_model_falls_back_to_default(local_model_cost_map: None) -> None: + """get_model_info raises for a model it has no entry for. The resolver must swallow that and + fall back to the default, otherwise the raise reaches callers that would read it as + "not cacheable" -- turning an unknown model into a silently uncacheable one.""" + assert get_prompt_cache_min_tokens(model="totally-unknown-model-xyz") == 1024 + + +def test_is_prompt_caching_valid_prompt_uses_per_model_minimum(local_model_cost_map: None) -> None: + """Regression: a prompt between two models' minimums is cacheable on one and not the other. + A 1403-token prompt clears claude-opus-4-8's 1024 minimum but not claude-opus-4-6's 4096, so + the flat-1024 check reported claude-opus-4-6 as cacheable and the cache write was rejected + upstream. Both assertions must live together: is_prompt_caching_valid_prompt returns False on + any internal error, so the True case is what proves the False case isn't a swallowed exception.""" + token_count = litellm.token_counter( + model="claude-opus-4-6", messages=PROMPT_CACHE_MESSAGES, use_default_image_token_count=True + ) + assert 1024 <= token_count < 4096, ( + f"prompt drifted to {token_count} tokens; it must sit between claude-opus-4-8's 1024 minimum " + "and claude-opus-4-6's 4096 minimum for this test to distinguish them" + ) + + assert is_prompt_caching_valid_prompt(model="claude-opus-4-6", messages=PROMPT_CACHE_MESSAGES) is False + assert is_prompt_caching_valid_prompt(model="claude-opus-4-8", messages=PROMPT_CACHE_MESSAGES) is True + + +def test_is_prompt_caching_valid_prompt_explicit_min_token_count_overrides_model(local_model_cost_map: None) -> None: + """An explicit min_token_count wins over the model-resolved value in both directions. Callers + holding only a model-group alias resolve the threshold themselves and pass it, because an alias + resolves to nothing here and would silently fall back to the default.""" + assert ( + is_prompt_caching_valid_prompt(model="claude-opus-4-6", messages=PROMPT_CACHE_MESSAGES, min_token_count=512) + is True + ) + assert ( + is_prompt_caching_valid_prompt(model="claude-opus-4-8", messages=PROMPT_CACHE_MESSAGES, min_token_count=8192) + is False + ) From 25b2f83f97407d86c9616df99ee13b1c39f44157 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Thu, 16 Jul 2026 19:26:28 -0700 Subject: [PATCH 2/3] test(router): clear the lru_cache when forcing the local cost map get_model_info is lru_cached, so swapping litellm.model_cost is not enough on its own. An earlier test that resolved these models against the remote map, which does not carry prompt_cache_min_tokens yet, leaves cached entries without it, and the stale hit resolves to the default. The assertions would then pass for the wrong reason or fail depending on execution order Clear on teardown as well, so entries these tests warm against the local map do not leak into later tests, matching the fixture already used in test_utils.py Also pin that a wildcard route resolves the underlying model's minimum. That works only because pattern_match_deployments substitutes the real model name into litellm_params before the deployment reaches the check; without the assertion that claim is unpinned and the threshold would silently fall back to the default --- .../test_prompt_caching_deployment_check.py | 34 ++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/tests/test_litellm/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py b/tests/test_litellm/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py index 1ad6caca2a3..6ad928b9737 100644 --- a/tests/test_litellm/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py +++ b/tests/test_litellm/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py @@ -26,9 +26,21 @@ def local_model_cost_map(monkeypatch): """ The remote cost map does not carry `prompt_cache_min_tokens` yet, so a test that reads the default map would pass here and flake in CI. Force the in-repo map. + + `get_model_info` is lru_cached, so swapping `model_cost` is not enough on its own: an earlier + test that resolved these models against the remote map leaves entries with no + `prompt_cache_min_tokens`, and the stale hit resolves to the default. Clear on the way out too, + so the entries these tests warm against the local map do not leak into later tests. """ + original_model_cost = litellm.model_cost monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") - monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.get_model_info.cache_clear() + try: + yield + finally: + litellm.model_cost = original_model_cost + litellm.get_model_info.cache_clear() def _deployments(*models: str) -> List[dict]: @@ -156,3 +168,23 @@ async def test_async_filter_deployments_narrows_for_group_whose_model_minimum_is ) assert filtered == [deployments[1]] + + +@pytest.mark.asyncio +async def test_wildcard_route_resolves_underlying_model_minimum(local_model_cost_map): + from litellm import Router + + router = Router( + model_list=[ + { + "model_name": "anthropic/*", + "litellm_params": {"model": "anthropic/*", "api_key": "sk-fake"}, + "model_info": {"id": "wild-1"}, + } + ] + ) + + deployments = await router.async_get_healthy_deployments(model="anthropic/claude-opus-4-6", request_kwargs={}) + + assert deployments[0]["litellm_params"]["model"] == "anthropic/claude-opus-4-6" + assert _get_min_token_count_for_deployments(deployments) == 4096 From fc5848174e48c56280a0f2892a0c8a5dc4b03ed8 Mon Sep 17 00:00:00 2001 From: Tin Chi Lo Date: Thu, 16 Jul 2026 19:53:10 -0700 Subject: [PATCH 3/3] fix(router): take the lowest minimum across a model group, not the highest The read gate cannot cause a wrong pin. A deployment is only pinned when the cache already holds an entry for the prefix, and async_log_success_event writes entries against the deployment's real model rather than the group alias, so a model that will not cache a prefix never records one and there is nothing to pin it to That makes this gate purely a cheap short-circuit deciding whether the cache lookup is worth doing, so the threshold must be the lowest minimum in the group. Taking the highest skipped the lookup for a prefix a lower-minimum member had genuinely cached, losing a hit it earned, and protected against nothing. It also broke the Fable 5 direction this ticket is meant to fix: its real minimum is 512, so a group gate stuck at a higher value would skip the lookup for a prefix Fable 5 had actually cached --- .../prompt_caching_deployment_check.py | 18 +++++++----- .../test_prompt_caching_deployment_check.py | 29 +++++++++++++++---- 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py b/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py index 1d121d79ea3..d6412c95da0 100644 --- a/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py +++ b/litellm/router_utils/pre_call_checks/prompt_caching_deployment_check.py @@ -19,16 +19,20 @@ from ..prompt_caching_cache import PromptCachingCache def _get_min_token_count_for_deployments(healthy_deployments: list[dict]) -> int: """ - Returns the highest minimum cacheable prefix across a model group. + Returns the lowest minimum cacheable prefix across a model group. + This gate only decides whether the cache lookup is worth doing. It cannot cause a wrong pin, + because a deployment is only pinned when the cache already holds an entry for the prefix, and + entries are written by `async_log_success_event` against the deployment's real model. A model + that will not cache a prefix never records one, so there is nothing to pin it to. + + That makes the lowest minimum in the group the correct threshold rather than the highest. `model` here is the model-group alias the operator chose, not a model name, so the threshold - has to come from the deployments themselves. A group may mix models with different minimums, - and one gate decides for all of them, so take the max: a prompt is only treated as cacheable - when it clears every member's minimum. The errors are not symmetric. Pinning a deployment for - a prefix its provider will not cache costs load balancing for nothing, which is the bug this - guards against, while declining to pin only forfeits a cache hit. + has to come from the deployments themselves, and a group may mix models whose minimums differ. + Taking the highest would skip the lookup for a prefix a lower-minimum member genuinely cached, + losing a cache hit it had earned. The lowest can only cost a lookup that finds nothing. """ - return max( + return min( ( get_prompt_cache_min_tokens(model=deployment["litellm_params"]["model"]) for deployment in healthy_deployments diff --git a/tests/test_litellm/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py b/tests/test_litellm/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py index 6ad928b9737..6752d76847f 100644 --- a/tests/test_litellm/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py +++ b/tests/test_litellm/router_utils/pre_call_checks/test_prompt_caching_deployment_check.py @@ -15,7 +15,7 @@ from litellm.router_utils.pre_call_checks.prompt_caching_deployment_check import ) from litellm.router_utils.prompt_caching_cache import PromptCachingCache from litellm.types.llms.openai import AllMessageValues -from litellm.utils import get_prompt_cache_min_tokens, token_counter +from litellm.utils import get_prompt_cache_min_tokens, is_prompt_caching_valid_prompt, token_counter MODEL_GROUP_ALIAS = "my-claude-group" OPUS_4_6_MIN_TOKENS = 4096 @@ -72,18 +72,35 @@ def _messages(word_count: int) -> List[AllMessageValues]: ) -def test_get_min_token_count_for_deployments_takes_max_across_mixed_group(): +def test_get_min_token_count_for_deployments_takes_min_across_mixed_group(): """ - A group may legally mix models whose real minimums differ, and one boolean gate decides for - every member. The threshold must be the highest minimum in the group: taking the lowest would - let a 1024-token prompt pin the Opus 4.5 deployment for a prefix Anthropic will never cache. + A group may legally mix models whose real minimums differ, and one gate decides for every + member. The threshold must be the lowest minimum in the group. This gate only decides whether + the cache lookup happens, so taking the highest would skip the lookup for a prefix the Sonnet + 4.5 deployment genuinely cached and lose a hit it had earned. """ assert get_prompt_cache_min_tokens(model="anthropic/claude-opus-4-5") == 4096 assert get_prompt_cache_min_tokens(model="anthropic/claude-sonnet-4-5") == 1024 deployments = _deployments("anthropic/claude-opus-4-5", "anthropic/claude-sonnet-4-5") - assert _get_min_token_count_for_deployments(deployments) == 4096 + assert _get_min_token_count_for_deployments(deployments) == 1024 + + +def test_write_gate_is_what_prevents_a_pin_below_the_model_minimum(): + """ + The invariant the read gate relies on. A deployment can only be pinned when the cache already + holds an entry for the prefix, and `async_log_success_event` writes entries against the real + deployment model. Opus 4.5 never records an entry for a prefix it will not cache, so no read + threshold is what keeps it from being pinned. + """ + messages = _messages(word_count=1400) + + token_count = token_counter(messages=messages, model="anthropic/claude-opus-4-5", use_default_image_token_count=True) + assert 1024 < token_count < 4096 + + assert is_prompt_caching_valid_prompt(model="anthropic/claude-opus-4-5", messages=messages) is False + assert is_prompt_caching_valid_prompt(model="anthropic/claude-sonnet-4-5", messages=messages) is True def test_get_min_token_count_for_deployments_falls_back_to_default_for_empty_group():