diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index b135a116753..5a8ada45651 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -76,6 +76,7 @@ from litellm.utils import ( from ..common_utils import ( BedrockError, BedrockModelInfo, + bedrock_converse_supports_parallel_tool_use_config, get_anthropic_beta_from_headers, get_bedrock_tool_name, is_claude_4_5_on_bedrock, @@ -1106,18 +1107,28 @@ class AmazonConverseConfig(BaseConfig): if cache_control is None: return None - cache_point = CachePointBlock(type="default") - if isinstance(cache_control, dict) and "ttl" in cache_control: - ttl = cache_control["ttl"] - if ttl in ["5m", "1h"] and model is not None: - if is_claude_4_5_on_bedrock(model): - cache_point["ttl"] = ttl + cache_point = self._build_cache_point_block(cache_control, model) if block_type == "system": return SystemContentBlock(cachePoint=cache_point) else: return ContentBlock(cachePoint=cache_point) + @staticmethod + def _build_cache_point_block(control: Optional[dict], model: Optional[str] = None) -> CachePointBlock: + """Build a Bedrock ``cachePoint`` block from an OpenAI-style ``cache_control``/``control`` dict. + + ``type`` is always ``"default"`` (the only value Bedrock's Converse API + accepts). ``ttl`` is only honored for models that support extended TTL + caching (Claude 4.5 family on Bedrock). + """ + cache_point = CachePointBlock(type="default") + if isinstance(control, dict) and "ttl" in control: + ttl = control["ttl"] + if ttl in ["5m", "1h"] and model is not None and is_claude_4_5_on_bedrock(model): + cache_point["ttl"] = ttl + return cache_point + def _transform_system_message( self, messages: List[AllMessageValues], model: Optional[str] = None ) -> Tuple[List[AllMessageValues], List[SystemContentBlock]]: @@ -1241,7 +1252,7 @@ class AmazonConverseConfig(BaseConfig): # Handle parallel_tool_calls configuration parallel_tool_use_config = additional_request_params.pop("_parallel_tool_use_config", None) - if parallel_tool_use_config is not None and is_claude_4_5_on_bedrock(model): + if parallel_tool_use_config is not None and bedrock_converse_supports_parallel_tool_use_config(model): for key, value in parallel_tool_use_config.items(): if ( key in additional_request_params @@ -1526,7 +1537,8 @@ class AmazonConverseConfig(BaseConfig): if cache_injection_points and len(bedrock_tools) > 0: for point in cache_injection_points: if point.get("location") == "tool_config": - bedrock_tools.append({"cachePoint": {"type": "default"}}) + cache_point = self._build_cache_point_block(point.get("control"), model) + bedrock_tools.append(ToolBlock(cachePoint=cache_point)) break bedrock_tool_config: Optional[ToolConfigBlock] = None diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index df432a4d7e3..5114677ffc0 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -685,39 +685,27 @@ def get_bedrock_base_model(model: str) -> str: return model +def bedrock_converse_supports_parallel_tool_use_config(model: str) -> bool: + return any( + (litellm.model_cost.get(candidate) or {}).get("supports_parallel_tool_use_config") is True + for candidate in (model, get_bedrock_base_model(model)) + ) + + def is_claude_4_5_on_bedrock(model: str) -> bool: """ - Check if the model is a Claude 4.5 model on Bedrock. - Claude 4.5 models support prompt caching with '5m' and '1h' TTL on Bedrock. + Check if the model supports Bedrock prompt caching with an extended '1h' TTL + (in addition to the default 5m TTL). + + Backed by the ``cache_creation_input_token_cost_above_1hr`` field in + ``model_prices_and_context_window.json`` instead of a hardcoded list of + model-name patterns, so newly released models pick up support as soon as + their pricing entry ships, with no code change required here. """ - model_lower = model.lower() - claude_4_5_patterns = [ - "sonnet-4.5", - "sonnet_4.5", - "sonnet-4-5", - "sonnet_4_5", - "haiku-4.5", - "haiku_4.5", - "haiku-4-5", - "haiku_4_5", - "opus-4.5", - "opus_4.5", - "opus-4-5", - "opus_4_5", - "sonnet-4.6", - "sonnet_4.6", - "sonnet-4-6", - "sonnet_4_6", - "opus-4.6", - "opus_4.6", - "opus-4-6", - "opus_4_6", - "opus-4.7", - "opus_4.7", - "opus-4-7", - "opus_4_7", - ] - return any(pattern in model_lower for pattern in claude_4_5_patterns) + return any( + (litellm.model_cost.get(candidate) or {}).get("cache_creation_input_token_cost_above_1hr") is not None + for candidate in (model, get_bedrock_base_model(model)) + ) _BEDROCK_MODEL_VERSION_SUFFIX_RE = re.compile(r"-v\d+(?::\d+)?$") diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index a8d59b6dbac..fef4301017a 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -724,6 +724,7 @@ "supports_tool_choice": true }, "anthropic.claude-haiku-4-5-20251001-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_read_input_token_cost": 1e-07, @@ -747,6 +748,7 @@ "supports_native_structured_output": true }, "anthropic.claude-haiku-4-5@20251001": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_read_input_token_cost": 1e-07, @@ -787,8 +789,6 @@ "output_cost_per_token_above_200k_tokens": 3e-05, "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, "cache_read_input_token_cost_above_200k_tokens": 6e-07, - "cache_creation_input_token_cost_above_1hr": 7.5e-06, - "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.5e-05, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07 }, @@ -813,9 +813,7 @@ "input_cost_per_token_above_200k_tokens": 6e-06, "output_cost_per_token_above_200k_tokens": 3e-05, "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, - "cache_read_input_token_cost_above_200k_tokens": 6e-07, - "cache_creation_input_token_cost_above_1hr": 7.5e-06, - "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.5e-05 + "cache_read_input_token_cost_above_200k_tokens": 6e-07 }, "anthropic.claude-3-7-sonnet-20240620-v1:0": { "cache_creation_input_token_cost": 4.5e-06, @@ -965,6 +963,7 @@ "supports_vision": true }, "anthropic.claude-opus-4-5-20251101-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -994,6 +993,7 @@ "bedrock_output_config_effort_ceiling": "high" }, "anthropic.claude-opus-4-6-v1": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -1025,6 +1025,7 @@ "bedrock_output_config_effort_ceiling": "max" }, "global.anthropic.claude-opus-4-6-v1": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -1056,6 +1057,7 @@ "bedrock_output_config_effort_ceiling": "max" }, "us.anthropic.claude-opus-4-6-v1": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, @@ -1087,6 +1089,7 @@ "bedrock_output_config_effort_ceiling": "max" }, "eu.anthropic.claude-opus-4-6-v1": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, @@ -1118,6 +1121,7 @@ "bedrock_output_config_effort_ceiling": "max" }, "au.anthropic.claude-opus-4-6-v1": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, @@ -1149,6 +1153,7 @@ "bedrock_output_config_effort_ceiling": "max" }, "anthropic.claude-opus-4-7": { + "supports_parallel_tool_use_config": true, "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -1198,6 +1203,7 @@ "supports_output_config": true }, "global.anthropic.claude-opus-4-7": { + "supports_parallel_tool_use_config": true, "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -1232,6 +1238,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "us.anthropic.claude-opus-4-7": { + "supports_parallel_tool_use_config": true, "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, @@ -1266,6 +1273,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "eu.anthropic.claude-opus-4-7": { + "supports_parallel_tool_use_config": true, "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, @@ -1300,6 +1308,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "au.anthropic.claude-opus-4-7": { + "supports_parallel_tool_use_config": true, "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, @@ -1334,6 +1343,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "anthropic.claude-fable-5": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, "cache_read_input_token_cost": 1e-06, @@ -1367,6 +1377,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "global.anthropic.claude-fable-5": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, "cache_read_input_token_cost": 1e-06, @@ -1400,6 +1411,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "us.anthropic.claude-fable-5": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 1.375e-05, "cache_creation_input_token_cost_above_1hr": 2.2e-05, "cache_read_input_token_cost": 1.1e-06, @@ -1433,6 +1445,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "eu.anthropic.claude-fable-5": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 1.375e-05, "cache_creation_input_token_cost_above_1hr": 2.2e-05, "cache_read_input_token_cost": 1.1e-06, @@ -1466,6 +1479,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "anthropic.claude-opus-4-8": { + "supports_parallel_tool_use_config": true, "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -1500,6 +1514,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "global.anthropic.claude-opus-4-8": { + "supports_parallel_tool_use_config": true, "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -1534,6 +1549,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "us.anthropic.claude-opus-4-8": { + "supports_parallel_tool_use_config": true, "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, @@ -1568,6 +1584,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "eu.anthropic.claude-opus-4-8": { + "supports_parallel_tool_use_config": true, "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, @@ -1602,6 +1619,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "au.anthropic.claude-opus-4-8": { + "supports_parallel_tool_use_config": true, "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, @@ -1636,6 +1654,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "jp.anthropic.claude-opus-4-7": { + "supports_parallel_tool_use_config": true, "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 6.875e-06, "cache_read_input_token_cost": 5.5e-07, @@ -1668,6 +1687,7 @@ "supports_minimal_reasoning_effort": true }, "anthropic.claude-sonnet-4-6": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -1698,6 +1718,7 @@ "supports_output_config": true }, "global.anthropic.claude-sonnet-4-6": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -1728,6 +1749,7 @@ "supports_output_config": true }, "us.anthropic.claude-sonnet-4-6": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 4.125e-06, "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, @@ -1758,6 +1780,7 @@ "supports_output_config": true }, "eu.anthropic.claude-sonnet-4-6": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 4.125e-06, "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, @@ -1788,6 +1811,7 @@ "supports_output_config": true }, "au.anthropic.claude-sonnet-4-6": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 4.125e-06, "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, @@ -1818,6 +1842,7 @@ "supports_output_config": true }, "jp.anthropic.claude-sonnet-4-6": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 4.125e-06, "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, @@ -1877,6 +1902,7 @@ "supports_vision": true }, "anthropic.claude-sonnet-4-5-20250929-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -2137,6 +2163,7 @@ "cache_creation_input_token_cost": 3.125e-07 }, "apac.anthropic.claude-haiku-4-5-20251001-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 1.375e-06, "cache_read_input_token_cost": 1.1e-07, "input_cost_per_token": 1.1e-06, @@ -2216,6 +2243,7 @@ "output_cost_per_second": 0.0 }, "au.anthropic.claude-sonnet-4-5-20250929-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 4.125e-06, "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, @@ -9338,6 +9366,7 @@ "cache_creation_input_token_cost": 3.75e-07 }, "bedrock/us-gov-east-1/anthropic.claude-sonnet-4-5-20250929-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 4.5e-06, "cache_creation_input_token_cost_above_1hr": 7.2e-06, "cache_read_input_token_cost": 3.6e-07, @@ -9360,6 +9389,7 @@ "supports_native_structured_output": true }, "bedrock/us-gov-east-1/claude-sonnet-4-5-20250929-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 4.5e-06, "cache_creation_input_token_cost_above_1hr": 7.2e-06, "cache_read_input_token_cost": 3.6e-07, @@ -9513,6 +9543,7 @@ "cache_creation_input_token_cost": 3.75e-07 }, "bedrock/us-gov-west-1/anthropic.claude-sonnet-4-5-20250929-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 4.5e-06, "cache_creation_input_token_cost_above_1hr": 7.2e-06, "cache_read_input_token_cost": 3.6e-07, @@ -9535,6 +9566,7 @@ "supports_native_structured_output": true }, "bedrock/us-gov-west-1/claude-sonnet-4-5-20250929-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 4.5e-06, "cache_creation_input_token_cost_above_1hr": 7.2e-06, "cache_read_input_token_cost": 3.6e-07, @@ -10261,6 +10293,7 @@ "supports_output_config": true }, "claude-sonnet-4-5-20250929-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.2e-05, @@ -14335,6 +14368,7 @@ "cache_creation_input_token_cost": 3.125e-07 }, "eu.anthropic.claude-haiku-4-5-20251001-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 1.375e-06, "cache_creation_input_token_cost_above_1hr": 2.2e-06, "cache_read_input_token_cost": 1.1e-07, @@ -14540,6 +14574,7 @@ "supports_vision": true }, "eu.anthropic.claude-sonnet-4-5-20250929-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 4.125e-06, "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, @@ -19736,6 +19771,7 @@ "mode": "search" }, "global.anthropic.claude-sonnet-4-5-20250929-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -19797,6 +19833,7 @@ "supports_vision": true }, "global.anthropic.claude-haiku-4-5-20251001-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_read_input_token_cost": 1e-07, @@ -23857,6 +23894,7 @@ "output_cost_per_token": 1.8e-08 }, "jp.anthropic.claude-sonnet-4-5-20250929-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 4.125e-06, "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, @@ -23889,6 +23927,7 @@ "supports_native_structured_output": true }, "jp.anthropic.claude-haiku-4-5-20251001-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 1.375e-06, "cache_creation_input_token_cost_above_1hr": 2.2e-06, "cache_read_input_token_cost": 1.1e-07, @@ -32505,6 +32544,7 @@ "supports_tool_choice": true }, "us.anthropic.claude-haiku-4-5-20251001-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 1.375e-06, "cache_creation_input_token_cost_above_1hr": 2.2e-06, "cache_read_input_token_cost": 1.1e-07, @@ -32655,6 +32695,7 @@ "supports_vision": true }, "us.anthropic.claude-sonnet-4-5-20250929-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 4.125e-06, "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, @@ -32687,6 +32728,7 @@ "supports_native_structured_output": true }, "us-gov.anthropic.claude-sonnet-4-5-20250929-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 4.5e-06, "cache_creation_input_token_cost_above_1hr": 7.2e-06, "cache_read_input_token_cost": 3.6e-07, @@ -32714,6 +32756,7 @@ "supports_native_structured_output": true }, "au.anthropic.claude-haiku-4-5-20251001-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 1.375e-06, "cache_creation_input_token_cost_above_1hr": 2.2e-06, "cache_read_input_token_cost": 1.1e-07, @@ -32761,6 +32804,7 @@ "supports_vision": true }, "us.anthropic.claude-opus-4-5-20251101-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, @@ -32790,6 +32834,7 @@ "bedrock_output_config_effort_ceiling": "high" }, "global.anthropic.claude-opus-4-5-20251101-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -32819,6 +32864,7 @@ "bedrock_output_config_effort_ceiling": "high" }, "eu.anthropic.claude-opus-4-5-20251101-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 6.25e-06, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, @@ -42770,6 +42816,7 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 1.5e-06, "cache_creation_input_token_cost_above_1hr": 2.4e-06, "cache_read_input_token_cost": 1.2e-07, @@ -42793,6 +42840,7 @@ "supports_pdf_input": true }, "bedrock/us-gov-west-1/anthropic.claude-haiku-4-5-20251001-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 1.5e-06, "cache_creation_input_token_cost_above_1hr": 2.4e-06, "cache_read_input_token_cost": 1.2e-07, diff --git a/litellm/types/integrations/anthropic_cache_control_hook.py b/litellm/types/integrations/anthropic_cache_control_hook.py index 3e2d0d688ac..601978bb04f 100644 --- a/litellm/types/integrations/anthropic_cache_control_hook.py +++ b/litellm/types/integrations/anthropic_cache_control_hook.py @@ -18,6 +18,7 @@ class CacheControlToolConfigInjectionPoint(TypedDict): """Type for tool_config-level injection points (Bedrock).""" location: Literal["tool_config"] + control: Optional[ChatCompletionCachedContent] CacheControlInjectionPoint = Union[ diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 2c5ca600840..15fd1052d91 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -724,6 +724,7 @@ "supports_tool_choice": true }, "anthropic.claude-haiku-4-5-20251001-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_read_input_token_cost": 1e-07, @@ -747,6 +748,7 @@ "supports_native_structured_output": true }, "anthropic.claude-haiku-4-5@20251001": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_read_input_token_cost": 1e-07, @@ -787,8 +789,6 @@ "output_cost_per_token_above_200k_tokens": 3e-05, "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, "cache_read_input_token_cost_above_200k_tokens": 6e-07, - "cache_creation_input_token_cost_above_1hr": 7.5e-06, - "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.5e-05, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07 }, @@ -813,9 +813,7 @@ "input_cost_per_token_above_200k_tokens": 6e-06, "output_cost_per_token_above_200k_tokens": 3e-05, "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06, - "cache_read_input_token_cost_above_200k_tokens": 6e-07, - "cache_creation_input_token_cost_above_1hr": 7.5e-06, - "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.5e-05 + "cache_read_input_token_cost_above_200k_tokens": 6e-07 }, "anthropic.claude-3-7-sonnet-20240620-v1:0": { "cache_creation_input_token_cost": 4.5e-06, @@ -965,6 +963,7 @@ "supports_vision": true }, "anthropic.claude-opus-4-5-20251101-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -994,6 +993,7 @@ "bedrock_output_config_effort_ceiling": "high" }, "anthropic.claude-opus-4-6-v1": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -1025,6 +1025,7 @@ "bedrock_output_config_effort_ceiling": "max" }, "global.anthropic.claude-opus-4-6-v1": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -1056,6 +1057,7 @@ "bedrock_output_config_effort_ceiling": "max" }, "us.anthropic.claude-opus-4-6-v1": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, @@ -1087,6 +1089,7 @@ "bedrock_output_config_effort_ceiling": "max" }, "eu.anthropic.claude-opus-4-6-v1": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, @@ -1118,6 +1121,7 @@ "bedrock_output_config_effort_ceiling": "max" }, "au.anthropic.claude-opus-4-6-v1": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, @@ -1149,6 +1153,7 @@ "bedrock_output_config_effort_ceiling": "max" }, "anthropic.claude-opus-4-7": { + "supports_parallel_tool_use_config": true, "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -1198,6 +1203,7 @@ "supports_output_config": true }, "global.anthropic.claude-opus-4-7": { + "supports_parallel_tool_use_config": true, "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -1232,6 +1238,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "us.anthropic.claude-opus-4-7": { + "supports_parallel_tool_use_config": true, "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, @@ -1266,6 +1273,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "eu.anthropic.claude-opus-4-7": { + "supports_parallel_tool_use_config": true, "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, @@ -1300,6 +1308,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "au.anthropic.claude-opus-4-7": { + "supports_parallel_tool_use_config": true, "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, @@ -1334,6 +1343,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "anthropic.claude-fable-5": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, "cache_read_input_token_cost": 1e-06, @@ -1367,6 +1377,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "global.anthropic.claude-fable-5": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, "cache_read_input_token_cost": 1e-06, @@ -1400,6 +1411,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "us.anthropic.claude-fable-5": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 1.375e-05, "cache_creation_input_token_cost_above_1hr": 2.2e-05, "cache_read_input_token_cost": 1.1e-06, @@ -1433,6 +1445,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "eu.anthropic.claude-fable-5": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 1.375e-05, "cache_creation_input_token_cost_above_1hr": 2.2e-05, "cache_read_input_token_cost": 1.1e-06, @@ -1466,6 +1479,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "anthropic.claude-opus-4-8": { + "supports_parallel_tool_use_config": true, "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -1500,6 +1514,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "global.anthropic.claude-opus-4-8": { + "supports_parallel_tool_use_config": true, "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -1534,6 +1549,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "us.anthropic.claude-opus-4-8": { + "supports_parallel_tool_use_config": true, "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, @@ -1568,6 +1584,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "eu.anthropic.claude-opus-4-8": { + "supports_parallel_tool_use_config": true, "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, @@ -1602,6 +1619,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "au.anthropic.claude-opus-4-8": { + "supports_parallel_tool_use_config": true, "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, @@ -1636,6 +1654,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "jp.anthropic.claude-opus-4-7": { + "supports_parallel_tool_use_config": true, "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 6.875e-06, "cache_read_input_token_cost": 5.5e-07, @@ -1668,6 +1687,7 @@ "supports_minimal_reasoning_effort": true }, "anthropic.claude-sonnet-4-6": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -1698,6 +1718,7 @@ "supports_output_config": true }, "global.anthropic.claude-sonnet-4-6": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -1728,6 +1749,7 @@ "supports_output_config": true }, "us.anthropic.claude-sonnet-4-6": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 4.125e-06, "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, @@ -1758,6 +1780,7 @@ "supports_output_config": true }, "eu.anthropic.claude-sonnet-4-6": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 4.125e-06, "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, @@ -1788,6 +1811,7 @@ "supports_output_config": true }, "au.anthropic.claude-sonnet-4-6": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 4.125e-06, "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, @@ -1818,6 +1842,7 @@ "supports_output_config": true }, "jp.anthropic.claude-sonnet-4-6": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 4.125e-06, "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, @@ -1877,6 +1902,7 @@ "supports_vision": true }, "anthropic.claude-sonnet-4-5-20250929-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -2137,6 +2163,7 @@ "cache_creation_input_token_cost": 3.125e-07 }, "apac.anthropic.claude-haiku-4-5-20251001-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 1.375e-06, "cache_read_input_token_cost": 1.1e-07, "input_cost_per_token": 1.1e-06, @@ -2216,6 +2243,7 @@ "output_cost_per_second": 0.0 }, "au.anthropic.claude-sonnet-4-5-20250929-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 4.125e-06, "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, @@ -9338,6 +9366,7 @@ "cache_creation_input_token_cost": 3.75e-07 }, "bedrock/us-gov-east-1/anthropic.claude-sonnet-4-5-20250929-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 4.5e-06, "cache_creation_input_token_cost_above_1hr": 7.2e-06, "cache_read_input_token_cost": 3.6e-07, @@ -9360,6 +9389,7 @@ "supports_native_structured_output": true }, "bedrock/us-gov-east-1/claude-sonnet-4-5-20250929-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 4.5e-06, "cache_creation_input_token_cost_above_1hr": 7.2e-06, "cache_read_input_token_cost": 3.6e-07, @@ -9513,6 +9543,7 @@ "cache_creation_input_token_cost": 3.75e-07 }, "bedrock/us-gov-west-1/anthropic.claude-sonnet-4-5-20250929-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 4.5e-06, "cache_creation_input_token_cost_above_1hr": 7.2e-06, "cache_read_input_token_cost": 3.6e-07, @@ -9535,6 +9566,7 @@ "supports_native_structured_output": true }, "bedrock/us-gov-west-1/claude-sonnet-4-5-20250929-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 4.5e-06, "cache_creation_input_token_cost_above_1hr": 7.2e-06, "cache_read_input_token_cost": 3.6e-07, @@ -10261,6 +10293,7 @@ "supports_output_config": true }, "claude-sonnet-4-5-20250929-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.2e-05, @@ -14335,6 +14368,7 @@ "cache_creation_input_token_cost": 3.125e-07 }, "eu.anthropic.claude-haiku-4-5-20251001-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 1.375e-06, "cache_creation_input_token_cost_above_1hr": 2.2e-06, "cache_read_input_token_cost": 1.1e-07, @@ -14540,6 +14574,7 @@ "supports_vision": true }, "eu.anthropic.claude-sonnet-4-5-20250929-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 4.125e-06, "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, @@ -19897,6 +19932,7 @@ "mode": "search" }, "global.anthropic.claude-sonnet-4-5-20250929-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, @@ -19958,6 +19994,7 @@ "supports_vision": true }, "global.anthropic.claude-haiku-4-5-20251001-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_read_input_token_cost": 1e-07, @@ -24018,6 +24055,7 @@ "output_cost_per_token": 1.8e-08 }, "jp.anthropic.claude-sonnet-4-5-20250929-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 4.125e-06, "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, @@ -24050,6 +24088,7 @@ "supports_native_structured_output": true }, "jp.anthropic.claude-haiku-4-5-20251001-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 1.375e-06, "cache_creation_input_token_cost_above_1hr": 2.2e-06, "cache_read_input_token_cost": 1.1e-07, @@ -32682,6 +32721,7 @@ "supports_tool_choice": true }, "us.anthropic.claude-haiku-4-5-20251001-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 1.375e-06, "cache_creation_input_token_cost_above_1hr": 2.2e-06, "cache_read_input_token_cost": 1.1e-07, @@ -32832,6 +32872,7 @@ "supports_vision": true }, "us.anthropic.claude-sonnet-4-5-20250929-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 4.125e-06, "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, @@ -32864,6 +32905,7 @@ "supports_native_structured_output": true }, "us-gov.anthropic.claude-sonnet-4-5-20250929-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 4.5e-06, "cache_creation_input_token_cost_above_1hr": 7.2e-06, "cache_read_input_token_cost": 3.6e-07, @@ -32891,6 +32933,7 @@ "supports_native_structured_output": true }, "au.anthropic.claude-haiku-4-5-20251001-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 1.375e-06, "cache_creation_input_token_cost_above_1hr": 2.2e-06, "cache_read_input_token_cost": 1.1e-07, @@ -32938,6 +32981,7 @@ "supports_vision": true }, "us.anthropic.claude-opus-4-5-20251101-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, @@ -32967,6 +33011,7 @@ "bedrock_output_config_effort_ceiling": "high" }, "global.anthropic.claude-opus-4-5-20251101-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -32996,6 +33041,7 @@ "bedrock_output_config_effort_ceiling": "high" }, "eu.anthropic.claude-opus-4-5-20251101-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 6.25e-06, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, @@ -43005,6 +43051,7 @@ "source": "https://aws.amazon.com/bedrock/pricing/" }, "bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 1.5e-06, "cache_creation_input_token_cost_above_1hr": 2.4e-06, "cache_read_input_token_cost": 1.2e-07, @@ -43028,6 +43075,7 @@ "supports_pdf_input": true }, "bedrock/us-gov-west-1/anthropic.claude-haiku-4-5-20251001-v1:0": { + "supports_parallel_tool_use_config": true, "cache_creation_input_token_cost": 1.5e-06, "cache_creation_input_token_cost_above_1hr": 2.4e-06, "cache_read_input_token_cost": 1.2e-07, diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index ed2dfc9440e..69587471c7c 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -1,5 +1,6 @@ import base64 import json +import os from unittest.mock import MagicMock, patch import pytest @@ -2608,102 +2609,132 @@ def test_add_cache_point_tool_block_passes_ttl_for_claude_4_5(): TTL ordering constraint (tools -> system -> messages). Ref: https://github.com/BerriAI/litellm/issues/XXXXX + + Forces the bundled local cost map so ttl eligibility (driven by + `cache_creation_input_token_cost_above_1hr` in litellm.model_cost) reads + this branch's pricing data rather than the network-fetched `main` copy, + which lacks the fix until merge. """ from litellm.litellm_core_utils.prompt_templates.factory import ( add_cache_point_tool_block, ) - tool_with_1h = { - "type": "function", - "function": {"name": "get_weather", "parameters": {"type": "object"}}, - "cache_control": {"type": "ephemeral", "ttl": "1h"}, - } + old_env = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP") + old_cost = litellm.model_cost + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + try: + tool_with_1h = { + "type": "function", + "function": {"name": "get_weather", "parameters": {"type": "object"}}, + "cache_control": {"type": "ephemeral", "ttl": "1h"}, + } - # Claude 4.5 model: ttl should be preserved - result = add_cache_point_tool_block( - tool_with_1h, model="us.anthropic.claude-sonnet-4-5-20250514-v1:0" - ) - assert result is not None - assert result["cachePoint"]["type"] == "default" - assert result["cachePoint"]["ttl"] == "1h" + # Claude 4.5 model: ttl should be preserved + result = add_cache_point_tool_block( + tool_with_1h, model="jp.anthropic.claude-opus-4-7" + ) + assert result is not None + assert result["cachePoint"]["type"] == "default" + assert result["cachePoint"]["ttl"] == "1h" - # Claude 4.5 model with 5m ttl: also preserved - tool_with_5m = { - "cache_control": {"type": "ephemeral", "ttl": "5m"}, - } - result_5m = add_cache_point_tool_block( - tool_with_5m, model="us.anthropic.claude-sonnet-4-5-20250514-v1:0" - ) - assert result_5m is not None - assert result_5m["cachePoint"]["ttl"] == "5m" + # Claude 4.5 model with 5m ttl: also preserved + tool_with_5m = { + "cache_control": {"type": "ephemeral", "ttl": "5m"}, + } + result_5m = add_cache_point_tool_block( + tool_with_5m, model="jp.anthropic.claude-opus-4-7" + ) + assert result_5m is not None + assert result_5m["cachePoint"]["ttl"] == "5m" - # Older model: ttl should be stripped - result_old = add_cache_point_tool_block( - tool_with_1h, model="anthropic.claude-3-5-sonnet-20241022-v2:0" - ) - assert result_old is not None - assert result_old["cachePoint"]["type"] == "default" - assert "ttl" not in result_old["cachePoint"] + # Older model: ttl should be stripped + result_old = add_cache_point_tool_block( + tool_with_1h, model="anthropic.claude-3-5-sonnet-20241022-v2:0" + ) + assert result_old is not None + assert result_old["cachePoint"]["type"] == "default" + assert "ttl" not in result_old["cachePoint"] - # No model provided: ttl should be stripped (safe default) - result_no_model = add_cache_point_tool_block(tool_with_1h, model=None) - assert result_no_model is not None - assert "ttl" not in result_no_model["cachePoint"] + # No model provided: ttl should be stripped (safe default) + result_no_model = add_cache_point_tool_block(tool_with_1h, model=None) + assert result_no_model is not None + assert "ttl" not in result_no_model["cachePoint"] - # No cache_control: returns None (unchanged behavior) - tool_no_cache = { - "type": "function", - "function": {"name": "get_weather", "parameters": {"type": "object"}}, - } - assert add_cache_point_tool_block(tool_no_cache) is None + # No cache_control: returns None (unchanged behavior) + tool_no_cache = { + "type": "function", + "function": {"name": "get_weather", "parameters": {"type": "object"}}, + } + assert add_cache_point_tool_block(tool_no_cache) is None - # cache_control without ttl: returns default cachePoint (unchanged behavior) - tool_no_ttl = {"cache_control": {"type": "ephemeral"}} - result_no_ttl = add_cache_point_tool_block( - tool_no_ttl, model="us.anthropic.claude-sonnet-4-5-20250514-v1:0" - ) - assert result_no_ttl is not None - assert result_no_ttl["cachePoint"]["type"] == "default" - assert "ttl" not in result_no_ttl["cachePoint"] + # cache_control without ttl: returns default cachePoint (unchanged behavior) + tool_no_ttl = {"cache_control": {"type": "ephemeral"}} + result_no_ttl = add_cache_point_tool_block( + tool_no_ttl, model="us.anthropic.claude-sonnet-4-5-20250929-v1:0" + ) + assert result_no_ttl is not None + assert result_no_ttl["cachePoint"]["type"] == "default" + assert "ttl" not in result_no_ttl["cachePoint"] + finally: + litellm.model_cost = old_cost + if old_env is None: + os.environ.pop("LITELLM_LOCAL_MODEL_COST_MAP", None) + else: + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = old_env def test_bedrock_tools_pt_passes_ttl_for_claude_4_5(): """ End-to-end: _bedrock_tools_pt should produce cachePoint blocks with ttl for Claude 4.5+ models when tools have cache_control with ttl. + + Forces the bundled local cost map so ttl eligibility (driven by + `cache_creation_input_token_cost_above_1hr` in litellm.model_cost) reads + this branch's pricing data rather than the network-fetched `main` copy, + which lacks the fix until merge. """ from litellm.litellm_core_utils.prompt_templates.factory import _bedrock_tools_pt - tools = [ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get weather", - "parameters": { - "type": "object", - "properties": {"city": {"type": "string"}}, + old_env = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP") + old_cost = litellm.model_cost + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + try: + tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + }, }, - }, - "cache_control": {"type": "ephemeral", "ttl": "1h"}, - } - ] + "cache_control": {"type": "ephemeral", "ttl": "1h"}, + } + ] - # Claude 4.5: cachePoint should have ttl - result = _bedrock_tools_pt( - tools, model="us.anthropic.claude-sonnet-4-5-20250514-v1:0" - ) - cache_blocks = [b for b in result if "cachePoint" in b] - assert len(cache_blocks) == 1 - assert cache_blocks[0]["cachePoint"]["ttl"] == "1h" + # Claude 4.5: cachePoint should have ttl + result = _bedrock_tools_pt(tools, model="jp.anthropic.claude-opus-4-7") + cache_blocks = [b for b in result if "cachePoint" in b] + assert len(cache_blocks) == 1 + assert cache_blocks[0]["cachePoint"]["ttl"] == "1h" - # Older model: cachePoint should not have ttl - result_old = _bedrock_tools_pt( - tools, model="anthropic.claude-3-5-sonnet-20241022-v2:0" - ) - cache_blocks_old = [b for b in result_old if "cachePoint" in b] - assert len(cache_blocks_old) == 1 - assert "ttl" not in cache_blocks_old[0]["cachePoint"] + # Older model: cachePoint should not have ttl + result_old = _bedrock_tools_pt( + tools, model="anthropic.claude-3-5-sonnet-20241022-v2:0" + ) + cache_blocks_old = [b for b in result_old if "cachePoint" in b] + assert len(cache_blocks_old) == 1 + assert "ttl" not in cache_blocks_old[0]["cachePoint"] + finally: + litellm.model_cost = old_cost + if old_env is None: + os.environ.pop("LITELLM_LOCAL_MODEL_COST_MAP", None) + else: + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = old_env def test_convert_to_anthropic_tool_result_openai_file_pdf_becomes_document(): diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index d940f9f47a6..687eeccdc98 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -611,6 +611,65 @@ def test_transform_request_helper_includes_anthropic_beta_and_tools(): assert fields["tools"][0]["type"] == "computer_20250124" +def test_parallel_tool_calls_config_kept_for_sonnet_4_6(): + old_env = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP") + old_cost = litellm.model_cost + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + try: + config = AmazonConverseConfig() + optional_params = config.map_openai_params( + model="anthropic.claude-sonnet-4-6", + non_default_params={"parallel_tool_calls": False}, + optional_params={}, + drop_params=False, + ) + + data = config._transform_request_helper( + model="anthropic.claude-sonnet-4-6", + system_content_blocks=[], + optional_params=optional_params, + messages=None, + ) + + assert data["additionalModelRequestFields"]["tool_choice"] == { + "disable_parallel_tool_use": True + } + finally: + litellm.model_cost = old_cost + if old_env is None: + os.environ.pop("LITELLM_LOCAL_MODEL_COST_MAP", None) + else: + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = old_env + + +def test_parallel_tool_calls_config_dropped_for_ttl_only_model( + monkeypatch: pytest.MonkeyPatch, +): + model = "anthropic.claude-fable-5" + monkeypatch.setitem( + litellm.model_cost, + model, + {"cache_creation_input_token_cost_above_1hr": 2e-05}, + ) + config = AmazonConverseConfig() + optional_params = config.map_openai_params( + model=model, + non_default_params={"parallel_tool_calls": False}, + optional_params={}, + drop_params=False, + ) + + data = config._transform_request_helper( + model=model, + system_content_blocks=[], + optional_params=optional_params, + messages=None, + ) + + assert "tool_choice" not in data.get("additionalModelRequestFields", {}) + + def test_transform_response_with_computer_use_tool(): """Test response transformation with computer use tool call.""" import httpx @@ -4130,6 +4189,42 @@ def test_parallel_tool_calls_newer_model_adds_disable_flag(): assert "parallel_tool_calls" not in request_data["additionalModelRequestFields"] +def test_parallel_tool_calls_flag_decoupled_from_ttl_pricing(monkeypatch): + """ + The disable_parallel_tool_use gate must read supports_parallel_tool_use_config, + not the 1h-TTL pricing field: a model carrying only the former still gets the flag. + """ + from litellm.llms.bedrock.common_utils import is_claude_4_5_on_bedrock + + config = AmazonConverseConfig() + model = "anthropic.claude-parallel-tool-use-only" + monkeypatch.setitem(litellm.model_cost, model, {"supports_parallel_tool_use_config": True}) + assert is_claude_4_5_on_bedrock(model) is False + messages = [{"role": "user", "content": "What's the weather in SF and NYC?"}] + + optional_params = config.map_openai_params( + non_default_params={"parallel_tool_calls": False, "tools": _TOOL_PARAM}, + optional_params={}, + model=model, + drop_params=False, + ) + + request_data = config.transform_request( + model=model, + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert ( + request_data["additionalModelRequestFields"]["tool_choice"][ + "disable_parallel_tool_use" + ] + is True + ) + + def test_parallel_tool_calls_older_model_drops_disable_flag(): """Older Claude models (pre-4.5) must NOT receive disable_parallel_tool_use — Bedrock rejects it.""" config = AmazonConverseConfig() @@ -4554,6 +4649,154 @@ def test_cache_control_injection_tool_config_not_added_without_injection_point() assert all("cachePoint" not in tool for tool in tools) +def test_cache_control_injection_tool_config_honors_ttl_for_supported_model(): + """ + Regression test: cache_control_injection_points with location=tool_config + must honor the requested `control.ttl`, mirroring the message/system + cache_control behavior, instead of always emitting a bare + {"type": "default"} cachePoint with no ttl. + + Forces the bundled local cost map so `is_claude_4_5_on_bedrock` (which + reads `cache_creation_input_token_cost_above_1hr` from litellm.model_cost) + sees this branch's pricing data rather than the network-fetched `main` + copy, which lacks it until merge. + """ + old_env = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP") + old_cost = litellm.model_cost + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + try: + config = AmazonConverseConfig() + messages = [ + {"role": "user", "content": "What is the weather?"}, + ] + optional_params = { + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather for a location", + "parameters": { + "type": "object", + "properties": {"location": {"type": "string"}}, + "required": ["location"], + }, + }, + } + ], + "cache_control_injection_points": [ + {"location": "tool_config", "control": {"type": "ephemeral", "ttl": "1h"}}, + ], + } + result = config._transform_request( + model="us.anthropic.claude-sonnet-4-5-20250929-v1:0", + messages=messages, + optional_params=optional_params, + litellm_params={}, + ) + tools = result["toolConfig"]["tools"] + assert tools[-1] == {"cachePoint": {"type": "default", "ttl": "1h"}} + finally: + litellm.model_cost = old_cost + if old_env is None: + os.environ.pop("LITELLM_LOCAL_MODEL_COST_MAP", None) + else: + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = old_env + + +def test_cache_control_injection_tool_config_honors_ttl_for_regional_model_lacking_own_pricing(): + """ + Regression test: a regional pricing entry that omits + `cache_creation_input_token_cost_above_1hr` (e.g. `jp.anthropic.claude-opus-4-7`) + must not shadow the base model entry that carries it; the requested ttl + survives through the base-model fallback. + """ + old_env = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP") + old_cost = litellm.model_cost + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + try: + assert "cache_creation_input_token_cost_above_1hr" not in litellm.model_cost["jp.anthropic.claude-opus-4-7"] + assert "cache_creation_input_token_cost_above_1hr" in litellm.model_cost["anthropic.claude-opus-4-7"] + config = AmazonConverseConfig() + messages = [ + {"role": "user", "content": "What is the weather?"}, + ] + optional_params = { + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather for a location", + "parameters": { + "type": "object", + "properties": {"location": {"type": "string"}}, + "required": ["location"], + }, + }, + } + ], + "cache_control_injection_points": [ + {"location": "tool_config", "control": {"type": "ephemeral", "ttl": "1h"}}, + ], + } + result = config._transform_request( + model="jp.anthropic.claude-opus-4-7", + messages=messages, + optional_params=optional_params, + litellm_params={}, + ) + tools = result["toolConfig"]["tools"] + assert tools[-1] == {"cachePoint": {"type": "default", "ttl": "1h"}} + finally: + litellm.model_cost = old_cost + if old_env is None: + os.environ.pop("LITELLM_LOCAL_MODEL_COST_MAP", None) + else: + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = old_env + + +def test_cache_control_injection_tool_config_drops_ttl_for_unsupported_model(): + """ + Models that don't support extended TTL caching (only Claude 4.5+ on + Bedrock does) must fall back to the default cachePoint with no ttl, + even if the caller requested one, matching message/system behavior. + """ + config = AmazonConverseConfig() + messages = [ + {"role": "user", "content": "What is the weather?"}, + ] + optional_params = { + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather for a location", + "parameters": { + "type": "object", + "properties": {"location": {"type": "string"}}, + "required": ["location"], + }, + }, + } + ], + "cache_control_injection_points": [ + {"location": "tool_config", "control": {"type": "ephemeral", "ttl": "1h"}}, + ], + } + result = config._transform_request( + model="anthropic.claude-3-5-haiku-20241022-v1:0", + messages=messages, + optional_params=optional_params, + litellm_params={}, + ) + tools = result["toolConfig"]["tools"] + assert tools[-1] == {"cachePoint": {"type": "default"}} + + def test_translate_response_format_json_schema_still_injects_tool(): """ response_format with an explicit json_schema should still use the diff --git a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py index c3191f5b5cb..03d0d87a58c 100644 --- a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py +++ b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py @@ -492,7 +492,7 @@ def test_bedrock_invoke_messages_transform_converts_custom_tool_schema_type_to_o assert result["tools"][0]["type"] == "custom" -def test_remove_ttl_from_cache_control_processes_tools(): +def test_remove_ttl_from_cache_control_processes_tools(local_model_cost_map): """ Ensure _remove_ttl_from_cache_control also sanitizes cache_control on tools. @@ -538,7 +538,7 @@ def test_remove_ttl_from_cache_control_processes_tools(): assert "ttl" not in request["system"][0]["cache_control"] -def test_remove_ttl_from_cache_control_preserves_tools_ttl_for_claude_4_5(): +def test_remove_ttl_from_cache_control_preserves_tools_ttl_for_claude_4_5(local_model_cost_map): """ For Claude 4.5+ models, ttl in ["5m", "1h"] should be preserved on tools, just like it is for system and messages. @@ -564,7 +564,7 @@ def test_remove_ttl_from_cache_control_preserves_tools_ttl_for_claude_4_5(): } cfg._remove_ttl_from_cache_control( - request, model="us.anthropic.claude-sonnet-4-5-20250514-v1:0" + request, model="us.anthropic.claude-sonnet-4-5-20250929-v1:0" ) # Both tools and system should preserve ttl for Claude 4.5 diff --git a/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py b/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py index 301bcba99f4..8cc6e4ff25d 100644 --- a/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py +++ b/tests/test_litellm/llms/bedrock/test_bedrock_common_utils.py @@ -445,3 +445,31 @@ def test_explicit_invoke_route_does_not_match_async_invoke(): BedrockModelInfo._explicit_async_invoke_route(f"bedrock/{async_invoke_model}") is True ) + + +def test_capability_lookups_fall_back_to_base_model_when_regional_entry_lacks_field(monkeypatch): + """ + Regression test: a regional model_cost entry without the capability field + must not shadow a base entry that has it (`get(model) or get(base)` used to + short-circuit on the truthy regional dict and drop the capability). + """ + import litellm + from litellm.llms.bedrock.common_utils import ( + bedrock_converse_supports_parallel_tool_use_config, + is_claude_4_5_on_bedrock, + ) + + base = "anthropic.claude-fallback-test" + regional = f"eu.{base}" + monkeypatch.setitem(litellm.model_cost, regional, {"input_cost_per_token": 1e-06}) + monkeypatch.setitem( + litellm.model_cost, + base, + { + "cache_creation_input_token_cost_above_1hr": 1e-05, + "supports_parallel_tool_use_config": True, + }, + ) + + assert is_claude_4_5_on_bedrock(regional) is True + assert bedrock_converse_supports_parallel_tool_use_config(regional) is True diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 832d2b394bf..9ceb94ef9bf 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -817,6 +817,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "supports_image_input": {"type": "boolean"}, "supports_nova_canvas_image_edit": {"type": "boolean"}, "supports_parallel_function_calling": {"type": "boolean"}, + "supports_parallel_tool_use_config": {"type": "boolean"}, "supports_pdf_input": {"type": "boolean"}, "supports_prompt_caching": {"type": "boolean"}, "supports_response_schema": {"type": "boolean"},