fix(bedrock): retry without strict when Bedrock rejects strict tool schemas under drop_params (#48733)

* fix(bedrock): retry without strict when Bedrock rejects strict tool schemas under drop_params

* test(bedrock): cover the strict tools 400 retry under drop_params

* fix(bedrock): anchor the strict tools rejection match and strip toolSpec strict

* test(bedrock): return captured requests from the strict tools test helper

* fix(bedrock): retry without strict only when a tool sent strict to Bedrock

A strict-limit 400 on a request whose tools carry strict: false, or on a model litellm never sends strict to, was sent again with an identical body. The fallback now takes the model and returns nothing unless a tool puts strict on the wire. Also types the fallback test parameter and records the mock transport's requests on the recorder instead of a mutated list

* fix(bedrock): decide the strict retry on the profile's base model and retry grammar timeouts

An application inference profile with base_model sends strict on the wire, so the retry check now resolves the
capability model the way the request transformation does. Bedrock's "Grammar compilation timed out" 400 is a
strict-grammar rejection too, so it takes the same single retry under drop_params.

* test(bedrock): annotate the strict-tools retry test helper's values as Final

---------

Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
m6atlas 2026-10-28 04:04:39 -07:00 • committed by GitHub
parent e2d78b69cd
commit a0c490ea54
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 412 additions and 12 deletions

View file

@ -1,11 +1,14 @@
from collections.abc import Callable, Coroutine
from functools import partial
from typing import Final, TypeAlias
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.bedrock.common_utils import (
NativeOutputFallback,
is_bedrock_native_output_rejection,
is_bedrock_strict_tools_rejection,
warn_bedrock_native_output_rejected,
warn_bedrock_strict_tools_dropped,
)
from litellm.types.utils import ModelResponse
from litellm.utils import CustomStreamWrapper
@ -14,12 +17,13 @@ NativeOutputResponse: TypeAlias = ModelResponse | CustomStreamWrapper
NativeOutputAttemptResult: TypeAlias = NativeOutputResponse | Coroutine[object, object, NativeOutputResponse]
NativeOutputAttempt: TypeAlias = Callable[[NativeOutputFallback | None], NativeOutputAttemptResult]
NativeOutputFallbackFactory: TypeAlias = Callable[[], NativeOutputFallback | None]
_RetryFor: TypeAlias = Callable[[BaseLLMException], NativeOutputFallback | None]
def _fallback_for(
error: BaseLLMException, fallback: NativeOutputFallbackFactory, model: str
def _native_output_retry(
error: BaseLLMException, fallback: NativeOutputFallbackFactory | None, model: str
) -> NativeOutputFallback | None:
if not is_bedrock_native_output_rejection(error.status_code, error.message):
if fallback is None or not is_bedrock_native_output_rejection(error.status_code, error.message):
return None
retry: Final = fallback()
if retry is not None:
@ -27,16 +31,38 @@ def _fallback_for(
return retry
def _strict_tools_retry(
error: BaseLLMException, fallback: NativeOutputFallbackFactory | None, model: str
) -> NativeOutputFallback | None:
if fallback is None or not is_bedrock_strict_tools_rejection(error.status_code, error.message):
return None
retry: Final = fallback()
if retry is not None:
warn_bedrock_strict_tools_dropped(model)
return retry
def _fallback_for(
error: BaseLLMException,
fallback: NativeOutputFallbackFactory | None,
strict_tools_fallback: NativeOutputFallbackFactory | None,
model: str,
) -> NativeOutputFallback | None:
native_output_retry: Final = _native_output_retry(error, fallback, model)
if native_output_retry is not None:
return native_output_retry
return _strict_tools_retry(error, strict_tools_fallback, model)
async def _await_with_native_output_fallback(
first_attempt: Coroutine[object, object, NativeOutputResponse],
attempt: NativeOutputAttempt,
fallback: NativeOutputFallbackFactory,
model: str,
retry_for: _RetryFor,
) -> NativeOutputResponse:
try:
return await first_attempt
except BaseLLMException as error:
retry: Final = _fallback_for(error, fallback, model)
retry: Final = retry_for(error)
if retry is None:
raise
second_attempt: Final = attempt(retry)
@ -47,16 +73,20 @@ def call_with_native_output_fallback(
attempt: NativeOutputAttempt,
fallback: NativeOutputFallbackFactory | None,
model: str,
strict_tools_fallback: NativeOutputFallbackFactory | None = None,
) -> NativeOutputAttemptResult:
if fallback is None:
if fallback is None and strict_tools_fallback is None:
return attempt(None)
retry_for: Final = partial(
_fallback_for, fallback=fallback, strict_tools_fallback=strict_tools_fallback, model=model
)
try:
first_attempt: Final = attempt(None)
except BaseLLMException as error:
retry: Final = _fallback_for(error, fallback, model)
retry: Final = retry_for(error)
if retry is None:
raise
return attempt(retry)
if isinstance(first_attempt, Coroutine):
return _await_with_native_output_fallback(first_attempt, attempt, fallback, model)
return _await_with_native_output_fallback(first_attempt, attempt, retry_for)
return first_attempt

View file

@ -362,6 +362,60 @@ class NativeOutputFallback:
litellm_params: Mapping[str, object]
_BEDROCK_STRICT_TOOLS_REJECTION: Final = re.compile(
r"Too many strict tools \(\d+\)"
r"|Schemas contains too many (?:optional parameters|parameters with union types)"
r"|[Cc]ompiled grammar (?:is too large|size)"
r"|tools\.\d+\.custom(?:\.strict)?: "
r"|Grammar compilation timed out"
)
_TOOLS_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, object], ...])
_TOOL_SPEC_ADAPTER: Final = TypeAdapter(Mapping[str, object])
def is_bedrock_strict_tools_rejection(status_code: int, error_text: str) -> bool:
return status_code == 400 and _BEDROCK_STRICT_TOOLS_REJECTION.search(error_text) is not None
def warn_bedrock_strict_tools_dropped(model: str) -> None:
verbose_logger.warning(
"Bedrock rejected the strict tool schemas for model=%s, so litellm sent the request again without `strict` "
"because drop_params is set.",
model,
)
def _without_strict(tool: Mapping[str, object]) -> Mapping[str, object]:
spec_key: Final = next((key for key in ("function", "toolSpec") if key in tool), None)
if spec_key is None:
return tool
spec: Final = _TOOL_SPEC_ADAPTER.validate_python(tool[spec_key])
return {**tool, spec_key: {key: value for key, value in spec.items() if key != "strict"}}
def _sends_strict(tool: Mapping[str, object], supports_strict_tools: bool) -> bool:
if "toolSpec" in tool:
return "strict" in _TOOL_SPEC_ADAPTER.validate_python(tool["toolSpec"])
if "function" not in tool:
return False
return supports_strict_tools and bool(_TOOL_SPEC_ADAPTER.validate_python(tool["function"]).get("strict"))
def bedrock_strict_tools_fallback(
model: str, optional_params: Mapping[str, object], litellm_params: Mapping[str, object]
) -> NativeOutputFallback | None:
tools: Final = _TOOLS_ADAPTER.validate_python(optional_params.get("tools", ()))
supports_strict_tools: Final = bedrock_converse_supports_strict_tools(
bedrock_capability_model(model, litellm_params.get("base_model"))
)
if not any(_sends_strict(tool, supports_strict_tools) for tool in tools):
return None
return NativeOutputFallback(
optional_params={**optional_params, "tools": [_without_strict(tool) for tool in tools]},
litellm_params=litellm_params,
)
def requests_output_config_format(optional_params: Mapping[str, object]) -> bool:
match optional_params:
case {"output_config": {"format": object()}}:

View file

@ -123,6 +123,7 @@ from litellm.llms.bedrock.common_utils import (
BedrockModelInfo,
NativeOutputFallback,
bedrock_route_for_request,
bedrock_strict_tools_fallback,
deployment_model_info,
without_bedrock_native_output_opt_in,
without_bedrock_route_prefix,
@ -206,7 +207,12 @@ from litellm.utils import (
from ._logging import verbose_logger
from .caching.caching import disable_cache, enable_cache, update_cache
from .litellm_core_utils.core_helpers import as_str_mapping, normalize_drop_params, safe_deep_copy
from .litellm_core_utils.core_helpers import (
as_str_mapping,
drops_unsupported_params,
normalize_drop_params,
safe_deep_copy,
)
from .litellm_core_utils.fallback_utils import (
async_completion_with_fallbacks,
completion_with_fallbacks,
@ -4378,6 +4384,11 @@ def _complete_bedrock(ctx: CompletionDispatchContext) -> _CompletionDispatchResu
else None
),
converse_model,
strict_tools_fallback=(
partial(bedrock_strict_tools_fallback, converse_model, converse_params, dict(litellm_params))
if drops_unsupported_params(litellm_params.get("drop_params"))
else None
),
)
elif bedrock_route == "converse_like":
model = model.replace("converse_like/", "")

View file

@ -121,3 +121,105 @@ def test_a_rejection_logs_the_retry_and_advises_opting_out_only_when_native_outp
assert _MODEL in caplog.text
assert ("supports_bedrock_native_structured_output: false" in caplog.text) is advises_opting_out
_STRICT_FALLBACK: Final = NativeOutputFallback(optional_params={"tools": "without_strict"}, litellm_params={})
# Bedrock's 400s for strict tools, seen live on us.anthropic.claude-sonnet-4-6 from 2026-10-20 to 2026-10-28 and quoted
# in issue #34388, each answered 200 once the request dropped strict
_STRICT_TOOLS_REJECTIONS: Final = tuple(
BedrockError(status_code=400, message=f'{{"message":"The model returned the following errors: {text}"}}')
for text in (
"Too many strict tools (21). The maximum number of strict tools supported is 20.",
"Schemas contains too many optional parameters (27), which would make grammar compilation inefficient.",
"Schemas contains too many parameters with union types (18 parameters with type arrays or anyOf). "
"This causes exponential compilation cost.",
"The compiled grammar is too large, which would cause performance issues.",
"tools.0.custom: Invalid schema: Schema type is missing for schema: {'description': 'tag'}",
"tools.3.custom: For 'integer' type, property 'minimum' is not supported",
"Compiled grammar size (410.4MB) exceeds maximum allowed size (300MB)",
"tools.1.custom.strict: Extra inputs are not permitted",
"Grammar compilation timed out.",
)
)
_TOO_MANY_STRICT_TOOLS: Final = _STRICT_TOOLS_REJECTIONS[0]
# Bedrock's 400s for requests with a strict tool that fail for other reasons, seen live on
# us.anthropic.claude-sonnet-4-6 on 2026-10-21, which echo tool names and tool call ids
_OTHER_400S_WITH_A_STRICT_TOOL: Final = tuple(
BedrockError(status_code=400, message=f'{{"message":"{text}"}}')
for text in (
"The tool check_grammar is already defined at toolConfig.tools.1.",
"Expected toolResult blocks at messages.0.content for the following Ids: call_grammar_9",
"tool_use ids were found without tool_result blocks immediately after: call_grammar_1",
)
)
def _call_with_strict_tools_fallback(
bedrock: ScriptedBedrock, fallback: NativeOutputFallback | None, strict_tools_fallback: NativeOutputFallback | None
) -> object:
return _resolve(
call_with_native_output_fallback(
bedrock,
None if fallback is None else lambda: fallback,
_MODEL,
strict_tools_fallback=None if strict_tools_fallback is None else lambda: strict_tools_fallback,
)
)
@_MODES
@pytest.mark.parametrize("rejection", _STRICT_TOOLS_REJECTIONS, ids=lambda error: error.message[60:100])
def test_a_strict_tools_rejection_is_sent_again_once_with_the_strict_tools_fallback(
asynchronous: bool, rejection: BedrockError
) -> None:
bedrock: Final = ScriptedBedrock(errors=(rejection, None), asynchronous=asynchronous)
response: Final = _call_with_strict_tools_fallback(bedrock, None, _STRICT_FALLBACK)
assert isinstance(response, ModelResponse) and response.id == "answer-2"
assert bedrock.sent == (None, _STRICT_FALLBACK)
@_MODES
@pytest.mark.parametrize(
("error", "strict_tools_fallback"),
[
(_OTHER_400, _STRICT_FALLBACK),
*((error, _STRICT_FALLBACK) for error in _OTHER_400S_WITH_A_STRICT_TOOL),
(BedrockError(status_code=500, message=_TOO_MANY_STRICT_TOOLS.message), _STRICT_FALLBACK),
(_TOO_MANY_STRICT_TOOLS, None),
],
ids=[
"other_400",
"duplicate_tool_name",
"missing_tool_result",
"orphan_tool_use",
"server_error",
"no_strict_tools_fallback",
],
)
def test_no_second_request_without_both_a_strict_tools_rejection_and_a_strict_tools_fallback(
asynchronous: bool, error: BedrockError, strict_tools_fallback: NativeOutputFallback | None
) -> None:
bedrock: Final = ScriptedBedrock(errors=(error,), asynchronous=asynchronous)
with pytest.raises(BedrockError) as raised:
_call_with_strict_tools_fallback(bedrock, None, strict_tools_fallback)
assert raised.value is error
assert bedrock.sent == (None,)
@pytest.mark.parametrize(
("error", "retried_with"),
[(_REJECTION, _FALLBACK), (_TOO_MANY_STRICT_TOOLS, _STRICT_FALLBACK)],
ids=["native_output_rejection", "strict_tools_rejection"],
)
def test_each_rejection_is_sent_again_with_its_own_fallback_when_both_are_set(
error: BedrockError, retried_with: NativeOutputFallback
) -> None:
bedrock: Final = ScriptedBedrock(errors=(error, None), asynchronous=False)
_call_with_strict_tools_fallback(bedrock, _FALLBACK, _STRICT_FALLBACK)
assert bedrock.sent == (None, retried_with)

View file

@ -6,7 +6,9 @@ from typing import Final
from litellm.llms.bedrock.common_utils import(
AmazonBedrockGlobalConfig,
BedrockModelInfo,
NativeOutputFallback,
bedrock_native_output_is_opt_in,
bedrock_strict_tools_fallback,
deployment_model_info,
extract_model_name_from_bedrock_arn,
get_bedrock_base_model,
@ -2150,3 +2152,80 @@ def test_bedrock_response_stream_shape_is_unavailable_when_botocore_describes_th
assert shape is None
(logged,) = warning.call_args_list
assert "only-in-the-description" not in logged.args[0] % logged.args[1:]
def test_bedrock_strict_tools_fallback_drops_only_strict_and_keeps_everything_else():
strict_tool: Final = {
"type": "function",
"cache_control": {"type": "ephemeral"},
"function": {"name": "lookup", "description": "Look up", "strict": True, "parameters": {"type": "object"}},
}
plain_tool: Final = {"type": "function", "function": {"name": "plain", "parameters": {"type": "object"}}}
anthropic_tool: Final = {"name": "native", "input_schema": {"type": "object"}}
bedrock_tool: Final = {"toolSpec": {"name": "spec", "strict": True, "inputSchema": {"json": {"type": "object"}}}}
litellm_params: Final = {"drop_params": True}
fallback: Final = bedrock_strict_tools_fallback(
"us.anthropic.claude-sonnet-4-6",
{"tools": [strict_tool, plain_tool, anthropic_tool, bedrock_tool], "maxTokens": 16, "stream": True},
litellm_params,
)
assert fallback == NativeOutputFallback(
optional_params={
"tools": [
{**strict_tool, "function": {"name": "lookup", "description": "Look up", "parameters": {"type": "object"}}},
plain_tool,
anthropic_tool,
{"toolSpec": {"name": "spec", "inputSchema": {"json": {"type": "object"}}}},
],
"maxTokens": 16,
"stream": True,
},
litellm_params=litellm_params,
)
assert strict_tool["function"]["strict"] is True
assert bedrock_tool["toolSpec"]["strict"] is True
@pytest.mark.parametrize(
"optional_params",
[
{"maxTokens": 16},
{"tools": [{"type": "function", "function": {"name": "plain", "parameters": {}}}]},
{"tools": [{"toolSpec": {"name": "spec", "inputSchema": {"json": {}}}}]},
],
ids=["no_tools", "no_strict_tool", "no_strict_tool_spec"],
)
def test_bedrock_strict_tools_fallback_is_none_without_a_strict_tool(optional_params: dict[str, object]) -> None:
assert bedrock_strict_tools_fallback("us.anthropic.claude-sonnet-4-6", optional_params, {}) is None
_CLAUDE_PROFILE_ARN: Final = "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/claude"
@pytest.mark.parametrize(
("model", "litellm_params"),
[
("us.anthropic.claude-sonnet-4-6", {}),
("us.amazon.nova-pro-v1:0", {}),
(_CLAUDE_PROFILE_ARN, {"base_model": "bedrock/us.anthropic.claude-sonnet-4-6"}),
(_CLAUDE_PROFILE_ARN, {}),
],
ids=["claude", "nova", "claude_profile_with_base_model", "profile_without_base_model"],
)
@pytest.mark.parametrize("strict", [True, False])
def test_bedrock_strict_tools_fallback_retries_only_when_the_request_sent_strict(
model: str, litellm_params: dict[str, object], strict: bool
) -> None:
from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig
tools: Final = [{"type": "function", "function": {"name": "lookup", "strict": strict, "parameters": {}}}]
request: Final = AmazonConverseConfig().transform_request(
model, [{"role": "user", "content": "hi"}], {"tools": tools}, {**litellm_params}, {}
)
sent_strict: Final = any("strict" in tool["toolSpec"] for tool in request["toolConfig"]["tools"])
fallback: Final = bedrock_strict_tools_fallback(model, {"tools": tools}, litellm_params)
assert (fallback is not None) == sent_strict

View file

@ -14,7 +14,7 @@ from dataclasses import dataclass
from datetime import datetime
from importlib import import_module
from pathlib import Path
from typing import Final
from typing import Final, Literal
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
@ -44,12 +44,13 @@ from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kw
from litellm.litellm_core_utils.get_litellm_params import stored_control_options
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt
from litellm.llms.custom_httpx.http_handler import HTTPHandler
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.types.litellm_params import ControlOptions
from litellm.types.llms.openai import AllMessageValues, HttpxBinaryResponseContent
from litellm.types.prompts.init_prompts import PromptSpec
from litellm.types.utils import Delta, ModelResponseStream, StandardCallbackDynamicParams, StreamingChoices, Usage
from tests.unit.llms.bedrock.slow_upstream import (
CONVERSE_ANSWER,
STREAM_TIMEOUT_SECONDS,
UPSTREAM_FIRST_BYTE_SECONDS,
slow_upstream_sync_client,
@ -9950,3 +9951,126 @@ async def test_converse_fallback_prices_the_current_attempt(
else:
client.close()
configuration.reset_rust_configuration()
_LOOKUP_FUNCTION: Final = {
"name": "lookup_record",
"description": "Look up a record",
"parameters": {
"type": "object",
"properties": {"query": {"type": "string"}},
"required": ["query"],
"additionalProperties": False,
},
}
_STRICT_LOOKUP_TOOL: Final = {"type": "function", "function": {**_LOOKUP_FUNCTION, "strict": True}}
# Bedrock's 400 for a request past Claude's strict tool limits, seen live on us.anthropic.claude-sonnet-4-6 on 2026-10-20
_TOO_MANY_STRICT_TOOLS: Final = (
"The model returned the following errors: Too many strict tools (21). The maximum number of strict tools "
"supported is 20. Try reducing the number of tools marked as strict."
)
@dataclass(slots=True)
class _StrictToolsLimitRejectionFirst:
stream: Literal["off", "fake", "real"]
sent: tuple[tuple[str, dict[str, object]], ...] = ()
def __call__(self, request: httpx.Request) -> httpx.Response:
self.sent = (*self.sent, (request.url.path.rsplit("/", 1)[-1], json.loads(request.content)))
if len(self.sent) == 1:
return httpx.Response(400, json={"message": _TOO_MANY_STRICT_TOOLS})
return httpx.Response(200, content=b"" if self.stream == "real" else CONVERSE_ANSWER)
async def _converse_with_a_strict_tools_limit_rejection_first(
asynchronous: bool,
stream: Literal["off", "fake", "real"],
drop_params: bool | None,
tool: Mapping[str, object] = _STRICT_LOOKUP_TOOL,
) -> tuple[tuple[tuple[str, dict[str, object]], ...], litellm.BadRequestError | None]:
respond: Final = _StrictToolsLimitRejectionFirst(stream)
kwargs: Final = {
"model": "bedrock/us.anthropic.claude-sonnet-4-6",
"messages": [{"role": "user", "content": "Find invoice 42"}],
"tools": [tool],
"aws_access_key_id": "AKIASTRICTTOOLS",
"aws_secret_access_key": "strict-tools-secret",
"aws_region_name": "us-east-1",
"num_retries": 0,
"drop_params": drop_params,
"stream": stream != "off",
"fake_stream": stream == "fake",
}
try:
chunks: Final = (
await _acompletion_chunks(AsyncHTTPHandler(transport=httpx.MockTransport(respond)), kwargs, stream)
if asynchronous
else _completion_chunks(
HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))), kwargs, stream
)
)
except litellm.BadRequestError as error:
return respond.sent, error
assert chunks
return respond.sent, None
async def _acompletion_chunks(
client: AsyncHTTPHandler, kwargs: Mapping[str, object], stream: Literal["off", "fake", "real"]
) -> tuple[object, ...]:
response: Final = await litellm.acompletion(client=client, **kwargs)
return tuple([chunk async for chunk in response]) if stream == "fake" else (response,)
def _completion_chunks(
client: HTTPHandler, kwargs: Mapping[str, object], stream: Literal["off", "fake", "real"]
) -> tuple[object, ...]:
response: Final = litellm.completion(client=client, **kwargs)
return tuple(response) if stream == "fake" else (response,)
@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"])
@pytest.mark.parametrize("stream", ["off", "fake", "real"], ids=["completion", "fake_stream", "stream"])
async def test_bedrock_strict_tools_limit_rejection_is_sent_again_without_strict_under_drop_params(
asynchronous: bool, stream: Literal["off", "fake", "real"], local_model_cost_map: None
) -> None:
sent, error = await _converse_with_a_strict_tools_limit_rejection_first(asynchronous, stream, drop_params=True)
assert error is None
(first_path, first_body), (second_path, second_body) = sent
first_tool_spec: Final = first_body["toolConfig"]["tools"][0]["toolSpec"]
assert first_tool_spec["strict"] is True
assert first_path == second_path == ("converse-stream" if stream == "real" else "converse")
assert second_body == {
**first_body,
"toolConfig": {
**first_body["toolConfig"],
"tools": [{"toolSpec": {key: value for key, value in first_tool_spec.items() if key != "strict"}}],
},
}
@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"])
async def test_bedrock_strict_tools_limit_rejection_reaches_the_caller_without_drop_params(
asynchronous: bool, local_model_cost_map: None
) -> None:
sent, error = await _converse_with_a_strict_tools_limit_rejection_first(asynchronous, stream="off", drop_params=None)
assert error is not None and "Too many strict tools" in str(error)
assert len(sent) == 1
@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"])
async def test_bedrock_strict_tools_limit_rejection_is_not_sent_again_when_no_tool_sent_strict(
asynchronous: bool, local_model_cost_map: None
) -> None:
sent, error = await _converse_with_a_strict_tools_limit_rejection_first(
asynchronous,
stream="off",
drop_params=True,
tool={"type": "function", "function": {**_LOOKUP_FUNCTION, "strict": False}},
)
assert error is not None and "Too many strict tools" in str(error)
assert len(sent) == 1