Merge commit '2523b717db2349889bdeedd8a892b8c0c6176f6b' into litellm_bedrock_strict_tools_retry

litellm/llms/bedrock/chat/native_output_fallback.py

@@ -1,11 +1,14 @@
 from collections.abc import Callable, Coroutine
+from functools import partial
 from typing import Final, TypeAlias
 
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.llms.bedrock.common_utils import (
     NativeOutputFallback,
     is_bedrock_native_output_rejection,
+    is_bedrock_strict_tools_rejection,
     warn_bedrock_native_output_rejected,
+    warn_bedrock_strict_tools_dropped,
 )
 from litellm.types.utils import ModelResponse
 from litellm.utils import CustomStreamWrapper
@@ -14,12 +17,13 @@ NativeOutputResponse: TypeAlias = ModelResponse | CustomStreamWrapper
 NativeOutputAttemptResult: TypeAlias = NativeOutputResponse | Coroutine[object, object, NativeOutputResponse]
 NativeOutputAttempt: TypeAlias = Callable[[NativeOutputFallback | None], NativeOutputAttemptResult]
 NativeOutputFallbackFactory: TypeAlias = Callable[[], NativeOutputFallback | None]
+_RetryFor: TypeAlias = Callable[[BaseLLMException], NativeOutputFallback | None]
 
 
-def _fallback_for(
-    error: BaseLLMException, fallback: NativeOutputFallbackFactory, model: str
+def _native_output_retry(
+    error: BaseLLMException, fallback: NativeOutputFallbackFactory | None, model: str
 ) -> NativeOutputFallback | None:
-    if not is_bedrock_native_output_rejection(error.status_code, error.message):
+    if fallback is None or not is_bedrock_native_output_rejection(error.status_code, error.message):
         return None
     retry: Final = fallback()
     if retry is not None:
@@ -27,16 +31,38 @@ def _fallback_for(
     return retry
 
 
+def _strict_tools_retry(
+    error: BaseLLMException, fallback: NativeOutputFallbackFactory | None, model: str
+) -> NativeOutputFallback | None:
+    if fallback is None or not is_bedrock_strict_tools_rejection(error.status_code, error.message):
+        return None
+    retry: Final = fallback()
+    if retry is not None:
+        warn_bedrock_strict_tools_dropped(model)
+    return retry
+
+
+def _fallback_for(
+    error: BaseLLMException,
+    fallback: NativeOutputFallbackFactory | None,
+    strict_tools_fallback: NativeOutputFallbackFactory | None,
+    model: str,
+) -> NativeOutputFallback | None:
+    native_output_retry: Final = _native_output_retry(error, fallback, model)
+    if native_output_retry is not None:
+        return native_output_retry
+    return _strict_tools_retry(error, strict_tools_fallback, model)
+
+
 async def _await_with_native_output_fallback(
     first_attempt: Coroutine[object, object, NativeOutputResponse],
     attempt: NativeOutputAttempt,
-    fallback: NativeOutputFallbackFactory,
-    model: str,
+    retry_for: _RetryFor,
 ) -> NativeOutputResponse:
     try:
         return await first_attempt
     except BaseLLMException as error:
-        retry: Final = _fallback_for(error, fallback, model)
+        retry: Final = retry_for(error)
         if retry is None:
             raise
         second_attempt: Final = attempt(retry)
@@ -47,16 +73,20 @@ def call_with_native_output_fallback(
     attempt: NativeOutputAttempt,
     fallback: NativeOutputFallbackFactory | None,
     model: str,
+    strict_tools_fallback: NativeOutputFallbackFactory | None = None,
 ) -> NativeOutputAttemptResult:
-    if fallback is None:
+    if fallback is None and strict_tools_fallback is None:
         return attempt(None)
+    retry_for: Final = partial(
+        _fallback_for, fallback=fallback, strict_tools_fallback=strict_tools_fallback, model=model
+    )
     try:
         first_attempt: Final = attempt(None)
     except BaseLLMException as error:
-        retry: Final = _fallback_for(error, fallback, model)
+        retry: Final = retry_for(error)
         if retry is None:
             raise
         return attempt(retry)
     if isinstance(first_attempt, Coroutine):
-        return _await_with_native_output_fallback(first_attempt, attempt, fallback, model)
+        return _await_with_native_output_fallback(first_attempt, attempt, retry_for)
     return first_attempt

litellm/llms/bedrock/common_utils.py

@@ -359,6 +359,44 @@ class NativeOutputFallback:
     litellm_params: Mapping[str, object]
 
 
+_BEDROCK_STRICT_TOOLS_REJECTION: Final = re.compile(
+    r"strict tools|grammar|compilation|tools\.\d+\.custom(?:\.strict)?: "
+)
+_TOOLS_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, object], ...])
+_TOOL_FUNCTION_ADAPTER: Final = TypeAdapter(Mapping[str, object])
+
+
+def is_bedrock_strict_tools_rejection(status_code: int, error_text: str) -> bool:
+    return status_code == 400 and _BEDROCK_STRICT_TOOLS_REJECTION.search(error_text) is not None
+
+
+def warn_bedrock_strict_tools_dropped(model: str) -> None:
+    verbose_logger.warning(
+        "Bedrock rejected the strict tool schemas for model=%s, so litellm sent the request again without `strict` "
+        "because drop_params is set.",
+        model,
+    )
+
+
+def _without_strict(tool: Mapping[str, object]) -> Mapping[str, object]:
+    if "function" not in tool:
+        return tool
+    function: Final = _TOOL_FUNCTION_ADAPTER.validate_python(tool["function"])
+    return {**tool, "function": {key: value for key, value in function.items() if key != "strict"}}
+
+
+def bedrock_strict_tools_fallback(
+    optional_params: Mapping[str, object], litellm_params: Mapping[str, object]
+) -> NativeOutputFallback | None:
+    tools: Final = _TOOLS_ADAPTER.validate_python(optional_params.get("tools", ()))
+    tools_without_strict: Final = tuple(_without_strict(tool) for tool in tools)
+    if tools_without_strict == tools:
+        return None
+    return NativeOutputFallback(
+        optional_params={**optional_params, "tools": list(tools_without_strict)}, litellm_params=litellm_params
+    )
+
+
 def requests_output_config_format(optional_params: Mapping[str, object]) -> bool:
     match optional_params:
         case {"output_config": {"format": object()}}:

litellm/main.py

@@ -121,6 +121,7 @@ from litellm.llms.bedrock.common_utils import (
     BedrockModelInfo,
     NativeOutputFallback,
     bedrock_route_for_request,
+    bedrock_strict_tools_fallback,
     deployment_model_info,
     without_bedrock_native_output_opt_in,
     without_bedrock_route_prefix,
@@ -4353,6 +4354,11 @@ def _complete_bedrock(ctx: CompletionDispatchContext) -> _CompletionDispatchResu
                 else None
             ),
             converse_model,
+            strict_tools_fallback=(
+                partial(bedrock_strict_tools_fallback, converse_params, dict(litellm_params))
+                if litellm.drop_params is True or normalize_drop_params(litellm_params.get("drop_params")) is True
+                else None
+            ),
         )
     elif bedrock_route == "converse_like":
         model = model.replace("converse_like/", "")

tests/unit/llms/bedrock/chat/test_native_output_fallback.py

@@ -121,3 +121,86 @@ def test_a_rejection_logs_the_retry_and_advises_opting_out_only_when_native_outp
 
     assert _MODEL in caplog.text
     assert ("supports_bedrock_native_structured_output: false" in caplog.text) is advises_opting_out
+
+
+_STRICT_FALLBACK: Final = NativeOutputFallback(optional_params={"tools": "without_strict"}, litellm_params={})
+# Bedrock's 400s for strict tools, seen live on us.anthropic.claude-sonnet-4-6 on 2026-10-20 and quoted in issue #34388,
+# each answered 200 once the request dropped strict
+_STRICT_TOOLS_REJECTIONS: Final = tuple(
+    BedrockError(status_code=400, message=f'{{"message":"The model returned the following errors: {text}"}}')
+    for text in (
+        "Too many strict tools (21). The maximum number of strict tools supported is 20.",
+        "Schemas contains too many optional parameters (27), which would make grammar compilation inefficient.",
+        "Schemas contains too many parameters with union types (18 parameters with type arrays or anyOf). "
+        "This causes exponential compilation cost.",
+        "The compiled grammar is too large, which would cause performance issues.",
+        "tools.0.custom: Invalid schema: Schema type is missing for schema: {'description': 'tag'}",
+        "tools.3.custom: For 'integer' type, property 'minimum' is not supported",
+        "Compiled grammar size (410.4MB) exceeds maximum allowed size (300MB)",
+        "tools.1.custom.strict: Extra inputs are not permitted",
+    )
+)
+_TOO_MANY_STRICT_TOOLS: Final = _STRICT_TOOLS_REJECTIONS[0]
+
+
+def _call_with_strict_tools_fallback(
+    bedrock: ScriptedBedrock, fallback: NativeOutputFallback | None, strict_tools_fallback: NativeOutputFallback | None
+) -> object:
+    return _resolve(
+        call_with_native_output_fallback(
+            bedrock,
+            None if fallback is None else lambda: fallback,
+            _MODEL,
+            strict_tools_fallback=None if strict_tools_fallback is None else lambda: strict_tools_fallback,
+        )
+    )
+
+
+@_MODES
+@pytest.mark.parametrize("rejection", _STRICT_TOOLS_REJECTIONS, ids=lambda error: error.message[60:100])
+def test_a_strict_tools_rejection_is_sent_again_once_with_the_strict_tools_fallback(
+    asynchronous: bool, rejection: BedrockError
+) -> None:
+    bedrock: Final = ScriptedBedrock(errors=(rejection, None), asynchronous=asynchronous)
+
+    response: Final = _call_with_strict_tools_fallback(bedrock, None, _STRICT_FALLBACK)
+
+    assert isinstance(response, ModelResponse) and response.id == "answer-2"
+    assert bedrock.sent == (None, _STRICT_FALLBACK)
+
+
+@_MODES
+@pytest.mark.parametrize(
+    ("error", "strict_tools_fallback"),
+    [
+        (_OTHER_400, _STRICT_FALLBACK),
+        (BedrockError(status_code=500, message=_TOO_MANY_STRICT_TOOLS.message), _STRICT_FALLBACK),
+        (_TOO_MANY_STRICT_TOOLS, None),
+    ],
+    ids=["other_400", "server_error", "no_strict_tools_fallback"],
+)
+def test_no_second_request_without_both_a_strict_tools_rejection_and_a_strict_tools_fallback(
+    asynchronous: bool, error: BedrockError, strict_tools_fallback: NativeOutputFallback | None
+) -> None:
+    bedrock: Final = ScriptedBedrock(errors=(error,), asynchronous=asynchronous)
+
+    with pytest.raises(BedrockError) as raised:
+        _call_with_strict_tools_fallback(bedrock, None, strict_tools_fallback)
+
+    assert raised.value is error
+    assert bedrock.sent == (None,)
+
+
+@pytest.mark.parametrize(
+    ("error", "retried_with"),
+    [(_REJECTION, _FALLBACK), (_TOO_MANY_STRICT_TOOLS, _STRICT_FALLBACK)],
+    ids=["native_output_rejection", "strict_tools_rejection"],
+)
+def test_each_rejection_is_sent_again_with_its_own_fallback_when_both_are_set(
+    error: BedrockError, retried_with: NativeOutputFallback
+) -> None:
+    bedrock: Final = ScriptedBedrock(errors=(error, None), asynchronous=False)
+
+    _call_with_strict_tools_fallback(bedrock, _FALLBACK, _STRICT_FALLBACK)
+
+    assert bedrock.sent == (None, retried_with)

tests/unit/llms/bedrock/test_bedrock_common_utils.py

@@ -6,7 +6,9 @@ from typing import Final
 from litellm.llms.bedrock.common_utils import(
     AmazonBedrockGlobalConfig,
     BedrockModelInfo,
+    NativeOutputFallback,
     bedrock_native_output_is_opt_in,
+    bedrock_strict_tools_fallback,
     deployment_model_info,
     extract_model_name_from_bedrock_arn,
     get_bedrock_base_model,
@@ -2150,3 +2152,41 @@ def test_bedrock_response_stream_shape_is_unavailable_when_botocore_describes_th
     assert shape is None
     (logged,) = warning.call_args_list
     assert "only-in-the-description" not in logged.args[0] % logged.args[1:]
+
+
+def test_bedrock_strict_tools_fallback_drops_only_strict_and_keeps_everything_else():
+    strict_tool: Final = {
+        "type": "function",
+        "cache_control": {"type": "ephemeral"},
+        "function": {"name": "lookup", "description": "Look up", "strict": True, "parameters": {"type": "object"}},
+    }
+    plain_tool: Final = {"type": "function", "function": {"name": "plain", "parameters": {"type": "object"}}}
+    anthropic_tool: Final = {"name": "native", "input_schema": {"type": "object"}}
+    litellm_params: Final = {"drop_params": True}
+
+    fallback: Final = bedrock_strict_tools_fallback(
+        {"tools": [strict_tool, plain_tool, anthropic_tool], "maxTokens": 16, "stream": True}, litellm_params
+    )
+
+    assert fallback == NativeOutputFallback(
+        optional_params={
+            "tools": [
+                {**strict_tool, "function": {"name": "lookup", "description": "Look up", "parameters": {"type": "object"}}},
+                plain_tool,
+                anthropic_tool,
+            ],
+            "maxTokens": 16,
+            "stream": True,
+        },
+        litellm_params=litellm_params,
+    )
+    assert strict_tool["function"]["strict"] is True
+
+
+@pytest.mark.parametrize(
+    "optional_params",
+    [{"maxTokens": 16}, {"tools": [{"type": "function", "function": {"name": "plain", "parameters": {}}}]}],
+    ids=["no_tools", "no_strict_tool"],
+)
+def test_bedrock_strict_tools_fallback_is_none_without_a_strict_tool(optional_params):
+    assert bedrock_strict_tools_fallback(optional_params, {}) is None