From 6a5353196da40c87ccb99ed544b995eca12bbddc Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 2 Jul 2026 21:07:31 -0700 Subject: [PATCH] fix(bedrock/converse): drop toolSpec.strict for Opus 4.7/4.8 (#31582) (#31923) Backport of #31923 to stable/1.90.x. Cherry-picked from 85f924148a299903fadf568e40b53f8b888016ae (litellm_internal_staging). --- .../prompt_templates/factory.py | 9 +- litellm/llms/bedrock/common_utils.py | 51 +++++++ ...odel_prices_and_context_window_backup.json | 11 ++ litellm/types/utils.py | 1 + litellm/utils.py | 3 + model_prices_and_context_window.json | 11 ++ ...edrock_converse_strict_tools_opus_47_48.py | 124 ++++++++++++++++++ tests/test_litellm/test_utils.py | 1 + 8 files changed, 208 insertions(+), 3 deletions(-) create mode 100644 tests/test_litellm/litellm_core_utils/prompt_templates/test_bedrock_converse_strict_tools_opus_47_48.py diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index b95b73398ac..3d73e2a442d 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -5547,7 +5547,7 @@ def _bedrock_tools_pt( ] """ from litellm.llms.bedrock.common_utils import ( - get_bedrock_base_model, + bedrock_converse_supports_strict_tools, normalize_json_schema_custom_types_to_object, ) from litellm.litellm_core_utils.prompt_templates.common_utils import unpack_defs @@ -5556,9 +5556,12 @@ def _bedrock_tools_pt( ("array", "boolean", "integer", "null", "number", "object", "string") ) # Only Claude on Bedrock honours strict tool schemas; other families - # (Nova, Llama, GPT-OSS) reject the strict field outright. + # (Nova, Llama, GPT-OSS) reject the strict field outright. Opus 4.7/4.8 + # also reject `strict` on Bedrock Converse (see #31582) — their validator + # maps toolSpec to the native Anthropic tool shape, which has no strict + # field, even though Anthropic's native API accepts it as a top-level key. supports_strict_tools = bool( - model and get_bedrock_base_model(model).startswith("anthropic") + model and bedrock_converse_supports_strict_tools(model) ) tool_block_list: List[BedrockToolBlock] = [] for tool_idx, tool in enumerate(tools): diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index bdc5da321c6..16ee4905233 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -4,9 +4,11 @@ from __future__ import annotations Common utilities used across bedrock chat/embedding/image generation """ +import contextlib import functools import json import os +import re from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union if TYPE_CHECKING: @@ -685,6 +687,55 @@ def is_claude_4_5_on_bedrock(model: str) -> bool: return any(pattern in model_lower for pattern in claude_4_5_patterns) +_BEDROCK_MODEL_VERSION_SUFFIX_RE = re.compile(r"-v\d+(?::\d+)?$") + + +def bedrock_converse_supports_strict_tools(model: str) -> bool: + """ + Whether ``toolSpec.strict`` can be forwarded to Bedrock Converse for ``model``. + + Non-Anthropic Bedrock families (Nova, Llama, GPT-OSS) reject the field + outright. Anthropic models forward it unless their entry in + ``model_prices_and_context_window.json`` sets + ``bedrock_converse_supports_strict_tools: false`` — Bedrock routes those + (Opus 4.7/4.8, see #31582) through a stricter validator that rejects the + ``strict`` key on ``toolSpec`` even though Anthropic's native API accepts + it as a top-level tool field. + """ + base = get_bedrock_base_model(model) + if not base.startswith("anthropic"): + return False + flag = _get_bedrock_converse_strict_tools_flag(base) + return flag if flag is not None else True + + +def _get_bedrock_converse_strict_tools_flag(base_model: str) -> Optional[bool]: + candidates = dict.fromkeys( + (base_model, _BEDROCK_MODEL_VERSION_SUFFIX_RE.sub("", base_model)) + ) + for candidate in candidates: + with contextlib.suppress(Exception): + model_info = get_cached_model_info()( + model=candidate, + custom_llm_provider="bedrock", + ) + + flag = model_info.get("bedrock_converse_supports_strict_tools") + if isinstance(flag, bool): + return flag + + model_cost_key = model_info.get("key") + if isinstance(model_cost_key, str): + local_flag = ( + _get_local_model_cost_map() + .get(model_cost_key, {}) + .get("bedrock_converse_supports_strict_tools") + ) + if isinstance(local_flag, bool): + return local_flag + return None + + def normalize_bedrock_opus_output_config_effort(model: str, output_config: Any) -> None: """ Normalize Anthropic ``output_config.effort`` values for Bedrock Opus ids. diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 7a5f8b9e1e3..5556a770658 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1134,6 +1134,7 @@ "bedrock_output_config_effort_ceiling": "max" }, "anthropic.claude-opus-4-7": { + "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -1181,6 +1182,7 @@ "supports_output_config": true }, "global.anthropic.claude-opus-4-7": { + "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -1213,6 +1215,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "us.anthropic.claude-opus-4-7": { + "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, @@ -1245,6 +1248,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "eu.anthropic.claude-opus-4-7": { + "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, @@ -1277,6 +1281,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "au.anthropic.claude-opus-4-7": { + "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, @@ -1441,6 +1446,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "anthropic.claude-opus-4-8": { + "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -1474,6 +1480,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "global.anthropic.claude-opus-4-8": { + "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -1507,6 +1514,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "us.anthropic.claude-opus-4-8": { + "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, @@ -1540,6 +1548,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "eu.anthropic.claude-opus-4-8": { + "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, @@ -1573,6 +1582,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "au.anthropic.claude-opus-4-8": { + "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, @@ -1606,6 +1616,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "jp.anthropic.claude-opus-4-7": { + "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 6.875e-06, "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 5.5e-06, diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 32bfc8835fe..7d155d228b0 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -151,6 +151,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False): bedrock_output_config_effort_ceiling: Optional[ Literal["low", "medium", "high", "max", "xhigh"] ] + bedrock_converse_supports_strict_tools: Optional[bool] class SearchContextCostPerQuery(TypedDict, total=False): diff --git a/litellm/utils.py b/litellm/utils.py index 29f703104da..9a850e624c9 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6221,6 +6221,9 @@ def _get_model_info_helper( bedrock_output_config_effort_ceiling=_model_info.get( "bedrock_output_config_effort_ceiling", None ), + bedrock_converse_supports_strict_tools=_model_info.get( + "bedrock_converse_supports_strict_tools", None + ), supports_computer_use=_model_info.get("supports_computer_use", None), search_context_cost_per_query=_model_info.get( "search_context_cost_per_query", None diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 47b7190185e..9e163bbbd99 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1134,6 +1134,7 @@ "bedrock_output_config_effort_ceiling": "max" }, "anthropic.claude-opus-4-7": { + "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -1181,6 +1182,7 @@ "supports_output_config": true }, "global.anthropic.claude-opus-4-7": { + "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -1213,6 +1215,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "us.anthropic.claude-opus-4-7": { + "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, @@ -1245,6 +1248,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "eu.anthropic.claude-opus-4-7": { + "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, @@ -1277,6 +1281,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "au.anthropic.claude-opus-4-7": { + "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, @@ -1441,6 +1446,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "anthropic.claude-opus-4-8": { + "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -1474,6 +1480,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "global.anthropic.claude-opus-4-8": { + "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -1507,6 +1514,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "us.anthropic.claude-opus-4-8": { + "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, @@ -1540,6 +1548,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "eu.anthropic.claude-opus-4-8": { + "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, @@ -1573,6 +1582,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "au.anthropic.claude-opus-4-8": { + "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, @@ -1606,6 +1616,7 @@ "bedrock_output_config_effort_ceiling": "xhigh" }, "jp.anthropic.claude-opus-4-7": { + "bedrock_converse_supports_strict_tools": false, "cache_creation_input_token_cost": 6.875e-06, "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 5.5e-06, diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_bedrock_converse_strict_tools_opus_47_48.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_bedrock_converse_strict_tools_opus_47_48.py new file mode 100644 index 00000000000..678ca6ef6d5 --- /dev/null +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_bedrock_converse_strict_tools_opus_47_48.py @@ -0,0 +1,124 @@ +"""Regression tests for Bedrock Converse ``toolSpec.strict`` forwarding. + +Bedrock Converse routes Claude Opus 4.7/4.8 through an Anthropic-compatible +validator that rejects ``toolSpec.strict`` even though Anthropic's native API +accepts ``strict`` as a top-level tool field for the same models. See +BerriAI/litellm#31582. +""" + +import pytest + +from litellm.litellm_core_utils.prompt_templates.factory import _bedrock_tools_pt +from litellm.llms.bedrock.common_utils import bedrock_converse_supports_strict_tools + +_STRICT_TOOL = [ + { + "type": "function", + "function": { + "name": "get_weather", + "strict": True, + "description": "Get the weather for a city", + "parameters": { + "type": "object", + "properties": { + "city": {"type": "string"}, + "unit": {"type": "string", "enum": ["celsius"]}, + }, + "required": ["city", "unit"], + "additionalProperties": False, + }, + }, + } +] + + +@pytest.mark.parametrize( + "model_id", + [ + "bedrock/us.anthropic.claude-opus-4-7", + "bedrock/us.anthropic.claude-opus-4-8", + "anthropic.claude-opus-4-7", + "anthropic.claude-opus-4-8", + "anthropic.claude-opus-4-7-v1:0", + "bedrock/eu.anthropic.claude-opus-4-8-v1:0", + "bedrock/global.anthropic.claude-opus-4-7", + ], +) +def test_bedrock_tools_pt_strict_dropped_for_opus_47_48(model_id: str) -> None: + """Opus 4.7/4.8 on Bedrock Converse reject toolSpec.strict — must be dropped.""" + result = _bedrock_tools_pt(_STRICT_TOOL, model=model_id) + assert ( + "strict" not in result[0]["toolSpec"] + ), f"strict leaked into toolSpec for {model_id}: {result[0]['toolSpec']}" + + +@pytest.mark.parametrize( + "model_id", + [ + "anthropic.claude-sonnet-4-5-20250929-v1:0", + "bedrock/us.anthropic.claude-sonnet-4-6", + "bedrock/us.anthropic.claude-opus-4-6", + "bedrock/us.anthropic.claude-opus-4-5", + ], +) +def test_bedrock_tools_pt_strict_kept_for_other_anthropic(model_id: str) -> None: + """Sonnet 4.5/4.6 and Opus <=4.6 accept toolSpec.strict — keep forwarding it.""" + result = _bedrock_tools_pt(_STRICT_TOOL, model=model_id) + assert ( + result[0]["toolSpec"]["strict"] is True + ), f"strict missing for {model_id}: {result[0]['toolSpec']}" + + +@pytest.mark.parametrize( + "model_id", + [ + "us.amazon.nova-micro-v1:0", + "meta.llama3-2-11b-instruct-v1:0", + ], +) +def test_bedrock_tools_pt_strict_dropped_for_non_anthropic(model_id: str) -> None: + """Non-Anthropic Bedrock families reject toolSpec.strict — must be dropped.""" + result = _bedrock_tools_pt(_STRICT_TOOL, model=model_id) + assert "strict" not in result[0]["toolSpec"] + + +def test_bedrock_converse_supports_strict_tools_helper() -> None: + """Direct check for the gate helper used by factory.py.""" + assert ( + bedrock_converse_supports_strict_tools("bedrock/us.anthropic.claude-opus-4-7") + is False + ) + assert ( + bedrock_converse_supports_strict_tools("bedrock/us.anthropic.claude-opus-4-8") + is False + ) + assert ( + bedrock_converse_supports_strict_tools( + "anthropic.claude-sonnet-4-5-20250929-v1:0" + ) + is True + ) + assert ( + bedrock_converse_supports_strict_tools("bedrock/us.anthropic.claude-opus-4-6") + is True + ) + assert bedrock_converse_supports_strict_tools("us.amazon.nova-micro-v1:0") is False + assert bedrock_converse_supports_strict_tools("") is False + + +@pytest.mark.parametrize( + "cost_map_key", + [ + "anthropic.claude-opus-4-7", + "us.anthropic.claude-opus-4-7", + "anthropic.claude-opus-4-8", + "us.anthropic.claude-opus-4-8", + ], +) +def test_strict_tools_flag_set_in_model_cost_map(cost_map_key: str) -> None: + """The gate is driven by ``bedrock_converse_supports_strict_tools: false`` in + ``model_prices_and_context_window.json``, not hardcoded model patterns.""" + from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap + + cost_map = GetModelCostMap.load_local_model_cost_map() + assert cost_map[cost_map_key]["bedrock_converse_supports_strict_tools"] is False diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 44e0b55ee3b..6e33ad38d3e 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -869,6 +869,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "type": "string", "enum": ["low", "medium", "high", "max", "xhigh"], }, + "bedrock_converse_supports_strict_tools": {"type": "boolean"}, "tpm": {"type": "number"}, "provider_specific_entry": {"type": "object"}, "supported_endpoints": {