diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index c508514373c..7d83a37bdae 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -360,10 +360,13 @@ class NativeOutputFallback: _BEDROCK_STRICT_TOOLS_REJECTION: Final = re.compile( - r"strict tools|grammar|compilation|tools\.\d+\.custom(?:\.strict)?: " + r"Too many strict tools \(\d+\)" + r"|Schemas contains too many (?:optional parameters|parameters with union types)" + r"|[Cc]ompiled grammar (?:is too large|size)" + r"|tools\.\d+\.custom(?:\.strict)?: " ) _TOOLS_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, object], ...]) -_TOOL_FUNCTION_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_TOOL_SPEC_ADAPTER: Final = TypeAdapter(Mapping[str, object]) def is_bedrock_strict_tools_rejection(status_code: int, error_text: str) -> bool: @@ -379,10 +382,11 @@ def warn_bedrock_strict_tools_dropped(model: str) -> None: def _without_strict(tool: Mapping[str, object]) -> Mapping[str, object]: - if "function" not in tool: + spec_key: Final = next((key for key in ("function", "toolSpec") if key in tool), None) + if spec_key is None: return tool - function: Final = _TOOL_FUNCTION_ADAPTER.validate_python(tool["function"]) - return {**tool, "function": {key: value for key, value in function.items() if key != "strict"}} + spec: Final = _TOOL_SPEC_ADAPTER.validate_python(tool[spec_key]) + return {**tool, spec_key: {key: value for key, value in spec.items() if key != "strict"}} def bedrock_strict_tools_fallback( diff --git a/litellm/main.py b/litellm/main.py index f6c2bec40f6..a41a092f18f 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -204,7 +204,12 @@ from litellm.utils import ( from ._logging import verbose_logger from .caching.caching import disable_cache, enable_cache, update_cache -from .litellm_core_utils.core_helpers import as_str_mapping, normalize_drop_params, safe_deep_copy +from .litellm_core_utils.core_helpers import ( + as_str_mapping, + drops_unsupported_params, + normalize_drop_params, + safe_deep_copy, +) from .litellm_core_utils.fallback_utils import ( async_completion_with_fallbacks, completion_with_fallbacks, @@ -4356,7 +4361,7 @@ def _complete_bedrock(ctx: CompletionDispatchContext) -> _CompletionDispatchResu converse_model, strict_tools_fallback=( partial(bedrock_strict_tools_fallback, converse_params, dict(litellm_params)) - if litellm.drop_params is True or normalize_drop_params(litellm_params.get("drop_params")) is True + if drops_unsupported_params(litellm_params.get("drop_params")) else None ), ) diff --git a/tests/unit/llms/bedrock/chat/test_native_output_fallback.py b/tests/unit/llms/bedrock/chat/test_native_output_fallback.py index 4e2bd9069b8..06f88b84e9d 100644 --- a/tests/unit/llms/bedrock/chat/test_native_output_fallback.py +++ b/tests/unit/llms/bedrock/chat/test_native_output_fallback.py @@ -141,6 +141,16 @@ _STRICT_TOOLS_REJECTIONS: Final = tuple( ) ) _TOO_MANY_STRICT_TOOLS: Final = _STRICT_TOOLS_REJECTIONS[0] +# Bedrock's 400s for requests with a strict tool that fail for other reasons, seen live on +# us.anthropic.claude-sonnet-4-6 on 2026-10-21, which echo tool names and tool call ids +_OTHER_400S_WITH_A_STRICT_TOOL: Final = tuple( + BedrockError(status_code=400, message=f'{{"message":"{text}"}}') + for text in ( + "The tool check_grammar is already defined at toolConfig.tools.1.", + "Expected toolResult blocks at messages.0.content for the following Ids: call_grammar_9", + "tool_use ids were found without tool_result blocks immediately after: call_grammar_1", + ) +) def _call_with_strict_tools_fallback( @@ -174,10 +184,18 @@ def test_a_strict_tools_rejection_is_sent_again_once_with_the_strict_tools_fallb ("error", "strict_tools_fallback"), [ (_OTHER_400, _STRICT_FALLBACK), + *((error, _STRICT_FALLBACK) for error in _OTHER_400S_WITH_A_STRICT_TOOL), (BedrockError(status_code=500, message=_TOO_MANY_STRICT_TOOLS.message), _STRICT_FALLBACK), (_TOO_MANY_STRICT_TOOLS, None), ], - ids=["other_400", "server_error", "no_strict_tools_fallback"], + ids=[ + "other_400", + "duplicate_tool_name", + "missing_tool_result", + "orphan_tool_use", + "server_error", + "no_strict_tools_fallback", + ], ) def test_no_second_request_without_both_a_strict_tools_rejection_and_a_strict_tools_fallback( asynchronous: bool, error: BedrockError, strict_tools_fallback: NativeOutputFallback | None diff --git a/tests/unit/llms/bedrock/test_bedrock_common_utils.py b/tests/unit/llms/bedrock/test_bedrock_common_utils.py index 4bc450b3ad9..a60f3207404 100644 --- a/tests/unit/llms/bedrock/test_bedrock_common_utils.py +++ b/tests/unit/llms/bedrock/test_bedrock_common_utils.py @@ -2162,10 +2162,12 @@ def test_bedrock_strict_tools_fallback_drops_only_strict_and_keeps_everything_el } plain_tool: Final = {"type": "function", "function": {"name": "plain", "parameters": {"type": "object"}}} anthropic_tool: Final = {"name": "native", "input_schema": {"type": "object"}} + bedrock_tool: Final = {"toolSpec": {"name": "spec", "strict": True, "inputSchema": {"json": {"type": "object"}}}} litellm_params: Final = {"drop_params": True} fallback: Final = bedrock_strict_tools_fallback( - {"tools": [strict_tool, plain_tool, anthropic_tool], "maxTokens": 16, "stream": True}, litellm_params + {"tools": [strict_tool, plain_tool, anthropic_tool, bedrock_tool], "maxTokens": 16, "stream": True}, + litellm_params, ) assert fallback == NativeOutputFallback( @@ -2174,6 +2176,7 @@ def test_bedrock_strict_tools_fallback_drops_only_strict_and_keeps_everything_el {**strict_tool, "function": {"name": "lookup", "description": "Look up", "parameters": {"type": "object"}}}, plain_tool, anthropic_tool, + {"toolSpec": {"name": "spec", "inputSchema": {"json": {"type": "object"}}}}, ], "maxTokens": 16, "stream": True, @@ -2181,12 +2184,17 @@ def test_bedrock_strict_tools_fallback_drops_only_strict_and_keeps_everything_el litellm_params=litellm_params, ) assert strict_tool["function"]["strict"] is True + assert bedrock_tool["toolSpec"]["strict"] is True @pytest.mark.parametrize( "optional_params", - [{"maxTokens": 16}, {"tools": [{"type": "function", "function": {"name": "plain", "parameters": {}}}]}], - ids=["no_tools", "no_strict_tool"], + [ + {"maxTokens": 16}, + {"tools": [{"type": "function", "function": {"name": "plain", "parameters": {}}}]}, + {"tools": [{"toolSpec": {"name": "spec", "inputSchema": {"json": {}}}}]}, + ], + ids=["no_tools", "no_strict_tool", "no_strict_tool_spec"], ) def test_bedrock_strict_tools_fallback_is_none_without_a_strict_tool(optional_params): assert bedrock_strict_tools_fallback(optional_params, {}) is None diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 10e398cb157..064d8625ade 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -14,7 +14,7 @@ from dataclasses import dataclass from datetime import datetime from importlib import import_module from pathlib import Path -from typing import Final +from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -9838,16 +9838,19 @@ _TOO_MANY_STRICT_TOOLS: Final = ( async def _converse_with_a_strict_tools_limit_rejection_first( - sent: list[dict[str, object]], asynchronous: bool, stream: bool, drop_params: bool | None + sent: list[tuple[str, dict[str, object]]], + asynchronous: bool, + stream: Literal["off", "fake", "real"], + drop_params: bool | None, ) -> None: from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from tests.unit.llms.bedrock.slow_upstream import CONVERSE_ANSWER def respond(request: httpx.Request) -> httpx.Response: - sent.append(json.loads(request.content)) + sent.append((request.url.path.rsplit("/", 1)[-1], json.loads(request.content))) if len(sent) == 1: return httpx.Response(400, json={"message": _TOO_MANY_STRICT_TOOLS}) - return httpx.Response(200, content=CONVERSE_ANSWER) + return httpx.Response(200, content=b"" if stream == "real" else CONVERSE_ANSWER) kwargs: Final = { "model": "bedrock/us.anthropic.claude-sonnet-4-6", @@ -9858,37 +9861,39 @@ async def _converse_with_a_strict_tools_limit_rejection_first( "aws_region_name": "us-east-1", "num_retries": 0, "drop_params": drop_params, - "stream": stream, - "fake_stream": stream, + "stream": stream != "off", + "fake_stream": stream == "fake", } if asynchronous: response = await litellm.acompletion( client=AsyncHTTPHandler(transport=httpx.MockTransport(respond)), **kwargs ) - chunks = [chunk async for chunk in response] if stream else [response] + chunks = [chunk async for chunk in response] if stream == "fake" else [response] else: response = litellm.completion( client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))), **kwargs ) - chunks = list(response) if stream else [response] + chunks = list(response) if stream == "fake" else [response] assert chunks @pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) -@pytest.mark.parametrize("stream", [False, True], ids=["completion", "stream"]) +@pytest.mark.parametrize("stream", ["off", "fake", "real"], ids=["completion", "fake_stream", "stream"]) async def test_bedrock_strict_tools_limit_rejection_is_sent_again_without_strict_under_drop_params( - asynchronous: bool, stream: bool, local_model_cost_map: None + asynchronous: bool, stream: Literal["off", "fake", "real"], local_model_cost_map: None ) -> None: - sent: Final[list[dict[str, object]]] = [] + sent: Final[list[tuple[str, dict[str, object]]]] = [] await _converse_with_a_strict_tools_limit_rejection_first(sent, asynchronous, stream, drop_params=True) - first_tool_spec: Final = sent[0]["toolConfig"]["tools"][0]["toolSpec"] + (first_path, first_body), (second_path, second_body) = sent + first_tool_spec: Final = first_body["toolConfig"]["tools"][0]["toolSpec"] assert first_tool_spec["strict"] is True - assert sent[1] == { - **sent[0], + assert first_path == second_path == ("converse-stream" if stream == "real" else "converse") + assert second_body == { + **first_body, "toolConfig": { - **sent[0]["toolConfig"], + **first_body["toolConfig"], "tools": [{"toolSpec": {key: value for key, value in first_tool_spec.items() if key != "strict"}}], }, } @@ -9898,9 +9903,9 @@ async def test_bedrock_strict_tools_limit_rejection_is_sent_again_without_strict async def test_bedrock_strict_tools_limit_rejection_reaches_the_caller_without_drop_params( asynchronous: bool, local_model_cost_map: None ) -> None: - sent: Final[list[dict[str, object]]] = [] + sent: Final[list[tuple[str, dict[str, object]]]] = [] with pytest.raises(litellm.BadRequestError, match="Too many strict tools"): - await _converse_with_a_strict_tools_limit_rejection_first(sent, asynchronous, stream=False, drop_params=None) + await _converse_with_a_strict_tools_limit_rejection_first(sent, asynchronous, stream="off", drop_params=None) assert len(sent) == 1