This commit is contained in:
Mateo Wang 2026-10-27 16:03:25 -07:00 • committed by GitHub
commit 324273daa7
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 340 additions and 11 deletions

View file

@ -1,11 +1,14 @@
from collections.abc import Callable, Coroutine
from functools import partial
from typing import Final, TypeAlias
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.bedrock.common_utils import (
NativeOutputFallback,
is_bedrock_native_output_rejection,
is_bedrock_strict_tools_rejection,
warn_bedrock_native_output_rejected,
warn_bedrock_strict_tools_dropped,
)
from litellm.types.utils import ModelResponse
from litellm.utils import CustomStreamWrapper
@ -14,12 +17,13 @@ NativeOutputResponse: TypeAlias = ModelResponse | CustomStreamWrapper
NativeOutputAttemptResult: TypeAlias = NativeOutputResponse | Coroutine[object, object, NativeOutputResponse]
NativeOutputAttempt: TypeAlias = Callable[[NativeOutputFallback | None], NativeOutputAttemptResult]
NativeOutputFallbackFactory: TypeAlias = Callable[[], NativeOutputFallback | None]
_RetryFor: TypeAlias = Callable[[BaseLLMException], NativeOutputFallback | None]
def _fallback_for(
error: BaseLLMException, fallback: NativeOutputFallbackFactory, model: str
def _native_output_retry(
error: BaseLLMException, fallback: NativeOutputFallbackFactory | None, model: str
) -> NativeOutputFallback | None:
if not is_bedrock_native_output_rejection(error.status_code, error.message):
if fallback is None or not is_bedrock_native_output_rejection(error.status_code, error.message):
return None
retry: Final = fallback()
if retry is not None:
@ -27,16 +31,38 @@ def _fallback_for(
return retry
def _strict_tools_retry(
error: BaseLLMException, fallback: NativeOutputFallbackFactory | None, model: str
) -> NativeOutputFallback | None:
if fallback is None or not is_bedrock_strict_tools_rejection(error.status_code, error.message):
return None
retry: Final = fallback()
if retry is not None:
warn_bedrock_strict_tools_dropped(model)
return retry
def _fallback_for(
error: BaseLLMException,
fallback: NativeOutputFallbackFactory | None,
strict_tools_fallback: NativeOutputFallbackFactory | None,
model: str,
) -> NativeOutputFallback | None:
native_output_retry: Final = _native_output_retry(error, fallback, model)
if native_output_retry is not None:
return native_output_retry
return _strict_tools_retry(error, strict_tools_fallback, model)
async def _await_with_native_output_fallback(
first_attempt: Coroutine[object, object, NativeOutputResponse],
attempt: NativeOutputAttempt,
fallback: NativeOutputFallbackFactory,
model: str,
retry_for: _RetryFor,
) -> NativeOutputResponse:
try:
return await first_attempt
except BaseLLMException as error:
retry: Final = _fallback_for(error, fallback, model)
retry: Final = retry_for(error)
if retry is None:
raise
second_attempt: Final = attempt(retry)
@ -47,16 +73,20 @@ def call_with_native_output_fallback(
attempt: NativeOutputAttempt,
fallback: NativeOutputFallbackFactory | None,
model: str,
strict_tools_fallback: NativeOutputFallbackFactory | None = None,
) -> NativeOutputAttemptResult:
if fallback is None:
if fallback is None and strict_tools_fallback is None:
return attempt(None)
retry_for: Final = partial(
_fallback_for, fallback=fallback, strict_tools_fallback=strict_tools_fallback, model=model
)
try:
first_attempt: Final = attempt(None)
except BaseLLMException as error:
retry: Final = _fallback_for(error, fallback, model)
retry: Final = retry_for(error)
if retry is None:
raise
return attempt(retry)
if isinstance(first_attempt, Coroutine):
return _await_with_native_output_fallback(first_attempt, attempt, fallback, model)
return _await_with_native_output_fallback(first_attempt, attempt, retry_for)
return first_attempt

View file

@ -362,6 +362,48 @@ class NativeOutputFallback:
litellm_params: Mapping[str, object]
_BEDROCK_STRICT_TOOLS_REJECTION: Final = re.compile(
r"Too many strict tools \(\d+\)"
r"|Schemas contains too many (?:optional parameters|parameters with union types)"
r"|[Cc]ompiled grammar (?:is too large|size)"
r"|tools\.\d+\.custom(?:\.strict)?: "
)
_TOOLS_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, object], ...])
_TOOL_SPEC_ADAPTER: Final = TypeAdapter(Mapping[str, object])
def is_bedrock_strict_tools_rejection(status_code: int, error_text: str) -> bool:
return status_code == 400 and _BEDROCK_STRICT_TOOLS_REJECTION.search(error_text) is not None
def warn_bedrock_strict_tools_dropped(model: str) -> None:
verbose_logger.warning(
"Bedrock rejected the strict tool schemas for model=%s, so litellm sent the request again without `strict` "
"because drop_params is set.",
model,
)
def _without_strict(tool: Mapping[str, object]) -> Mapping[str, object]:
spec_key: Final = next((key for key in ("function", "toolSpec") if key in tool), None)
if spec_key is None:
return tool
spec: Final = _TOOL_SPEC_ADAPTER.validate_python(tool[spec_key])
return {**tool, spec_key: {key: value for key, value in spec.items() if key != "strict"}}
def bedrock_strict_tools_fallback(
optional_params: Mapping[str, object], litellm_params: Mapping[str, object]
) -> NativeOutputFallback | None:
tools: Final = _TOOLS_ADAPTER.validate_python(optional_params.get("tools", ()))
tools_without_strict: Final = tuple(_without_strict(tool) for tool in tools)
if tools_without_strict == tools:
return None
return NativeOutputFallback(
optional_params={**optional_params, "tools": list(tools_without_strict)}, litellm_params=litellm_params
)
def requests_output_config_format(optional_params: Mapping[str, object]) -> bool:
match optional_params:
case {"output_config": {"format": object()}}:

View file

@ -123,6 +123,7 @@ from litellm.llms.bedrock.common_utils import (
BedrockModelInfo,
NativeOutputFallback,
bedrock_route_for_request,
bedrock_strict_tools_fallback,
deployment_model_info,
without_bedrock_native_output_opt_in,
without_bedrock_route_prefix,
@ -206,7 +207,12 @@ from litellm.utils import (
from ._logging import verbose_logger
from .caching.caching import disable_cache, enable_cache, update_cache
from .litellm_core_utils.core_helpers import as_str_mapping, normalize_drop_params, safe_deep_copy
from .litellm_core_utils.core_helpers import (
as_str_mapping,
drops_unsupported_params,
normalize_drop_params,
safe_deep_copy,
)
from .litellm_core_utils.fallback_utils import (
async_completion_with_fallbacks,
completion_with_fallbacks,
@ -4378,6 +4384,11 @@ def _complete_bedrock(ctx: CompletionDispatchContext) -> _CompletionDispatchResu
else None
),
converse_model,
strict_tools_fallback=(
partial(bedrock_strict_tools_fallback, converse_params, dict(litellm_params))
if drops_unsupported_params(litellm_params.get("drop_params"))
else None
),
)
elif bedrock_route == "converse_like":
model = model.replace("converse_like/", "")

View file

@ -121,3 +121,104 @@ def test_a_rejection_logs_the_retry_and_advises_opting_out_only_when_native_outp
assert _MODEL in caplog.text
assert ("supports_bedrock_native_structured_output: false" in caplog.text) is advises_opting_out
_STRICT_FALLBACK: Final = NativeOutputFallback(optional_params={"tools": "without_strict"}, litellm_params={})
# Bedrock's 400s for strict tools, seen live on us.anthropic.claude-sonnet-4-6 on 2026-10-20 and quoted in issue #34388,
# each answered 200 once the request dropped strict
_STRICT_TOOLS_REJECTIONS: Final = tuple(
BedrockError(status_code=400, message=f'{{"message":"The model returned the following errors: {text}"}}')
for text in (
"Too many strict tools (21). The maximum number of strict tools supported is 20.",
"Schemas contains too many optional parameters (27), which would make grammar compilation inefficient.",
"Schemas contains too many parameters with union types (18 parameters with type arrays or anyOf). "
"This causes exponential compilation cost.",
"The compiled grammar is too large, which would cause performance issues.",
"tools.0.custom: Invalid schema: Schema type is missing for schema: {'description': 'tag'}",
"tools.3.custom: For 'integer' type, property 'minimum' is not supported",
"Compiled grammar size (410.4MB) exceeds maximum allowed size (300MB)",
"tools.1.custom.strict: Extra inputs are not permitted",
)
)
_TOO_MANY_STRICT_TOOLS: Final = _STRICT_TOOLS_REJECTIONS[0]
# Bedrock's 400s for requests with a strict tool that fail for other reasons, seen live on
# us.anthropic.claude-sonnet-4-6 on 2026-10-21, which echo tool names and tool call ids
_OTHER_400S_WITH_A_STRICT_TOOL: Final = tuple(
BedrockError(status_code=400, message=f'{{"message":"{text}"}}')
for text in (
"The tool check_grammar is already defined at toolConfig.tools.1.",
"Expected toolResult blocks at messages.0.content for the following Ids: call_grammar_9",
"tool_use ids were found without tool_result blocks immediately after: call_grammar_1",
)
)
def _call_with_strict_tools_fallback(
bedrock: ScriptedBedrock, fallback: NativeOutputFallback | None, strict_tools_fallback: NativeOutputFallback | None
) -> object:
return _resolve(
call_with_native_output_fallback(
bedrock,
None if fallback is None else lambda: fallback,
_MODEL,
strict_tools_fallback=None if strict_tools_fallback is None else lambda: strict_tools_fallback,
)
)
@_MODES
@pytest.mark.parametrize("rejection", _STRICT_TOOLS_REJECTIONS, ids=lambda error: error.message[60:100])
def test_a_strict_tools_rejection_is_sent_again_once_with_the_strict_tools_fallback(
asynchronous: bool, rejection: BedrockError
) -> None:
bedrock: Final = ScriptedBedrock(errors=(rejection, None), asynchronous=asynchronous)
response: Final = _call_with_strict_tools_fallback(bedrock, None, _STRICT_FALLBACK)
assert isinstance(response, ModelResponse) and response.id == "answer-2"
assert bedrock.sent == (None, _STRICT_FALLBACK)
@_MODES
@pytest.mark.parametrize(
("error", "strict_tools_fallback"),
[
(_OTHER_400, _STRICT_FALLBACK),
*((error, _STRICT_FALLBACK) for error in _OTHER_400S_WITH_A_STRICT_TOOL),
(BedrockError(status_code=500, message=_TOO_MANY_STRICT_TOOLS.message), _STRICT_FALLBACK),
(_TOO_MANY_STRICT_TOOLS, None),
],
ids=[
"other_400",
"duplicate_tool_name",
"missing_tool_result",
"orphan_tool_use",
"server_error",
"no_strict_tools_fallback",
],
)
def test_no_second_request_without_both_a_strict_tools_rejection_and_a_strict_tools_fallback(
asynchronous: bool, error: BedrockError, strict_tools_fallback: NativeOutputFallback | None
) -> None:
bedrock: Final = ScriptedBedrock(errors=(error,), asynchronous=asynchronous)
with pytest.raises(BedrockError) as raised:
_call_with_strict_tools_fallback(bedrock, None, strict_tools_fallback)
assert raised.value is error
assert bedrock.sent == (None,)
@pytest.mark.parametrize(
("error", "retried_with"),
[(_REJECTION, _FALLBACK), (_TOO_MANY_STRICT_TOOLS, _STRICT_FALLBACK)],
ids=["native_output_rejection", "strict_tools_rejection"],
)
def test_each_rejection_is_sent_again_with_its_own_fallback_when_both_are_set(
error: BedrockError, retried_with: NativeOutputFallback
) -> None:
bedrock: Final = ScriptedBedrock(errors=(error, None), asynchronous=False)
_call_with_strict_tools_fallback(bedrock, _FALLBACK, _STRICT_FALLBACK)
assert bedrock.sent == (None, retried_with)

View file

@ -6,7 +6,9 @@ from typing import Final
from litellm.llms.bedrock.common_utils import(
AmazonBedrockGlobalConfig,
BedrockModelInfo,
NativeOutputFallback,
bedrock_native_output_is_opt_in,
bedrock_strict_tools_fallback,
deployment_model_info,
extract_model_name_from_bedrock_arn,
get_bedrock_base_model,
@ -2150,3 +2152,49 @@ def test_bedrock_response_stream_shape_is_unavailable_when_botocore_describes_th
assert shape is None
(logged,) = warning.call_args_list
assert "only-in-the-description" not in logged.args[0] % logged.args[1:]
def test_bedrock_strict_tools_fallback_drops_only_strict_and_keeps_everything_else():
strict_tool: Final = {
"type": "function",
"cache_control": {"type": "ephemeral"},
"function": {"name": "lookup", "description": "Look up", "strict": True, "parameters": {"type": "object"}},
}
plain_tool: Final = {"type": "function", "function": {"name": "plain", "parameters": {"type": "object"}}}
anthropic_tool: Final = {"name": "native", "input_schema": {"type": "object"}}
bedrock_tool: Final = {"toolSpec": {"name": "spec", "strict": True, "inputSchema": {"json": {"type": "object"}}}}
litellm_params: Final = {"drop_params": True}
fallback: Final = bedrock_strict_tools_fallback(
{"tools": [strict_tool, plain_tool, anthropic_tool, bedrock_tool], "maxTokens": 16, "stream": True},
litellm_params,
)
assert fallback == NativeOutputFallback(
optional_params={
"tools": [
{**strict_tool, "function": {"name": "lookup", "description": "Look up", "parameters": {"type": "object"}}},
plain_tool,
anthropic_tool,
{"toolSpec": {"name": "spec", "inputSchema": {"json": {"type": "object"}}}},
],
"maxTokens": 16,
"stream": True,
},
litellm_params=litellm_params,
)
assert strict_tool["function"]["strict"] is True
assert bedrock_tool["toolSpec"]["strict"] is True
@pytest.mark.parametrize(
"optional_params",
[
{"maxTokens": 16},
{"tools": [{"type": "function", "function": {"name": "plain", "parameters": {}}}]},
{"tools": [{"toolSpec": {"name": "spec", "inputSchema": {"json": {}}}}]},
],
ids=["no_tools", "no_strict_tool", "no_strict_tool_spec"],
)
def test_bedrock_strict_tools_fallback_is_none_without_a_strict_tool(optional_params):
assert bedrock_strict_tools_fallback(optional_params, {}) is None

View file

@ -14,7 +14,7 @@ from dataclasses import dataclass
from datetime import datetime
from importlib import import_module
from pathlib import Path
from typing import Final
from typing import Final, Literal
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
@ -9950,3 +9950,100 @@ async def test_converse_fallback_prices_the_current_attempt(
else:
client.close()
configuration.reset_rust_configuration()
_STRICT_LOOKUP_TOOL: Final = {
"type": "function",
"function": {
"name": "lookup_record",
"description": "Look up a record",
"strict": True,
"parameters": {
"type": "object",
"properties": {"query": {"type": "string"}},
"required": ["query"],
"additionalProperties": False,
},
},
}
# Bedrock's 400 for a request past Claude's strict tool limits, seen live on us.anthropic.claude-sonnet-4-6 on 2026-10-20
_TOO_MANY_STRICT_TOOLS: Final = (
"The model returned the following errors: Too many strict tools (21). The maximum number of strict tools "
"supported is 20. Try reducing the number of tools marked as strict."
)
async def _converse_with_a_strict_tools_limit_rejection_first(
asynchronous: bool,
stream: Literal["off", "fake", "real"],
drop_params: bool | None,
) -> tuple[tuple[tuple[str, dict[str, object]], ...], litellm.BadRequestError | None]:
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from tests.unit.llms.bedrock.slow_upstream import CONVERSE_ANSWER
sent: Final[list[tuple[str, dict[str, object]]]] = []
def respond(request: httpx.Request) -> httpx.Response:
sent.append((request.url.path.rsplit("/", 1)[-1], json.loads(request.content)))
if len(sent) == 1:
return httpx.Response(400, json={"message": _TOO_MANY_STRICT_TOOLS})
return httpx.Response(200, content=b"" if stream == "real" else CONVERSE_ANSWER)
kwargs: Final = {
"model": "bedrock/us.anthropic.claude-sonnet-4-6",
"messages": [{"role": "user", "content": "Find invoice 42"}],
"tools": [_STRICT_LOOKUP_TOOL],
"aws_access_key_id": "AKIASTRICTTOOLS",
"aws_secret_access_key": "strict-tools-secret",
"aws_region_name": "us-east-1",
"num_retries": 0,
"drop_params": drop_params,
"stream": stream != "off",
"fake_stream": stream == "fake",
}
try:
if asynchronous:
response = await litellm.acompletion(
client=AsyncHTTPHandler(transport=httpx.MockTransport(respond)), **kwargs
)
chunks = [chunk async for chunk in response] if stream == "fake" else [response]
else:
response = litellm.completion(
client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))), **kwargs
)
chunks = list(response) if stream == "fake" else [response]
except litellm.BadRequestError as error:
return tuple(sent), error
assert chunks
return tuple(sent), None
@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"])
@pytest.mark.parametrize("stream", ["off", "fake", "real"], ids=["completion", "fake_stream", "stream"])
async def test_bedrock_strict_tools_limit_rejection_is_sent_again_without_strict_under_drop_params(
asynchronous: bool, stream: Literal["off", "fake", "real"], local_model_cost_map: None
) -> None:
sent, error = await _converse_with_a_strict_tools_limit_rejection_first(asynchronous, stream, drop_params=True)
assert error is None
(first_path, first_body), (second_path, second_body) = sent
first_tool_spec: Final = first_body["toolConfig"]["tools"][0]["toolSpec"]
assert first_tool_spec["strict"] is True
assert first_path == second_path == ("converse-stream" if stream == "real" else "converse")
assert second_body == {
**first_body,
"toolConfig": {
**first_body["toolConfig"],
"tools": [{"toolSpec": {key: value for key, value in first_tool_spec.items() if key != "strict"}}],
},
}
@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"])
async def test_bedrock_strict_tools_limit_rejection_reaches_the_caller_without_drop_params(
asynchronous: bool, local_model_cost_map: None
) -> None:
sent, error = await _converse_with_a_strict_tools_limit_rejection_first(asynchronous, stream="off", drop_params=None)
assert error is not None and "Too many strict tools" in str(error)
assert len(sent) == 1