From 036cfb951b9a413b2b34b313420be7dee23ed212 Mon Sep 17 00:00:00 2001 From: kimnamu Date: Wed, 23 Sep 2026 15:07:50 +0900 Subject: [PATCH] fix(bedrock): return JSON for Claude Opus 5.5 response_format when tool_choice can't be forced Opus 5.5 and Fable 5.1 reject a forced tool_choice on Bedrock, and litellm also skips the forced tool_choice whenever thinking is on, so the synthetic json_tool_call tool goes out unforced on those requests. With the default description the model often skips the tool and answers in plain text. When thinking is on or the model rejects forced tool use, give the tool a directive description, with the same condition in the Converse and Invoke routes. Other requests are unchanged --- litellm/constants.py | 3 ++ .../bedrock/chat/converse_transformation.py | 11 +++-- .../anthropic_claude3_transformation.py | 9 +++- .../chat/test_converse_transformation.py | 49 ++++++++++++++++++- ...ations_anthropic_claude3_transformation.py | 49 ++++++++++++++++++- 5 files changed, 113 insertions(+), 8 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index e5b662bd515..1207f2e931a 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1508,6 +1508,9 @@ OPENAI_FINISH_REASONS: Final = [ ] HUMANLOOP_PROMPT_CACHE_TTL_SECONDS: Final = int(os.getenv("HUMANLOOP_PROMPT_CACHE_TTL_SECONDS", 60)) # 1 minute RESPONSE_FORMAT_TOOL_NAME = "json_tool_call" # default tool name used when converting response format to tool call +RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION: Final = ( + "Always respond by calling this tool. Its input is your final answer; do not reply in plain text." +) ########################### Logging Callback Constants ########################### AZURE_STORAGE_MSFT_VERSION: Final = "2019-07-07" diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 497020c2836..8a3b00e6f0e 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -17,6 +17,7 @@ from litellm._logging import verbose_logger from litellm.constants import ( BEDROCK_MIN_THINKING_BUDGET_TOKENS, RESPONSE_FORMAT_TOOL_NAME, + RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION, ) from litellm.litellm_core_utils.core_helpers import ( filter_exceptions_from_params, @@ -1233,16 +1234,20 @@ class AmazonConverseConfig(BaseConfig): elif json_schema is not None: # Fallback: translate to a synthetic tool call # https://docs.anthropic.com/en/docs/build-with-claude/tool-use#json-mode + tool_choice_blocked: Final = is_thinking_enabled or AnthropicModelInfo.forced_tool_use_unsupported(model) _tool: Final = self._create_json_tool_call_for_response_format( json_schema=json_schema, - description=description, + description=( + " ".join(filter(None, (RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION, description))) + if tool_choice_blocked + else description + ), ) optional_params = self._add_tools_to_optional_params(optional_params=optional_params, tools=[_tool]) if ( litellm.utils.supports_tool_choice(model=model, custom_llm_provider=self.custom_llm_provider) - and not is_thinking_enabled - and not AnthropicModelInfo.forced_tool_use_unsupported(model) + and not tool_choice_blocked ): optional_params["tool_choice"] = ToolChoiceValuesBlock( tool=SpecificToolChoiceBlock(name=RESPONSE_FORMAT_TOOL_NAME) diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py index a8b94fb5703..d585a943eef 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py @@ -3,7 +3,7 @@ from typing import TYPE_CHECKING, Any, Final import httpx from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers -from litellm.constants import RESPONSE_FORMAT_TOOL_NAME +from litellm.constants import RESPONSE_FORMAT_TOOL_NAME, RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION from litellm.litellm_core_utils.prompt_templates.factory import ( convert_to_anthropic_image_obj, ) @@ -124,6 +124,13 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): and AnthropicModelInfo.forced_tool_use_unsupported(original_model) ): optional_params.pop("tool_choice") + if "response_format" in non_default_params and ( + self.is_thinking_enabled(non_default_params) + or AnthropicModelInfo.forced_tool_use_unsupported(original_model) + ): + for tool in optional_params.get("tools", ()): + if tool.get("name") == RESPONSE_FORMAT_TOOL_NAME: + tool["description"] = RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION return optional_params diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index ad77f9d4d1b..8acf30df5f7 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -8,6 +8,7 @@ import pytest import litellm from litellm import ModelResponse +from litellm.constants import RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig from litellm.types.llms.bedrock import ConverseTokenUsageBlock @@ -7394,7 +7395,12 @@ def test_unforced_tool_choice_unaffected_on_fable_5_1_converse(local_model_cost_ @pytest.mark.parametrize( "model", - ["anthropic.claude-fable-5-1", "us.anthropic.claude-fable-5-1"], + [ + "anthropic.claude-fable-5-1", + "us.anthropic.claude-fable-5-1", + "anthropic.claude-opus-5-5", + "us.anthropic.claude-opus-5-5", + ], ) def test_response_format_avoids_native_and_forced_tool_choice_on_fable_5_1_converse( local_model_cost_map, model @@ -7420,11 +7426,50 @@ def test_response_format_avoids_native_and_forced_tool_choice_on_fable_5_1_conve ) assert "outputConfig" not in result - assert "tools" in result + assert result["tools"][0]["function"]["description"] == RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION assert "tool_choice" not in result assert result.get("json_mode") is True +@pytest.mark.parametrize( + ("extra_params", "expected_tool_choice", "expected_description"), + [ + ({}, {"tool": {"name": "json_tool_call"}}, "A city"), + ({"reasoning_effort": "low"}, None, f"{RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION} A city"), + ( + { + "tools": [{"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}}], + "tool_choice": "auto", + }, + {"auto": {}}, + "A city", + ), + ], +) +def test_response_format_tool_description_asks_for_the_tool_only_when_it_cannot_be_forced( + local_model_cost_map, extra_params, expected_tool_choice, expected_description +): + result = AmazonConverseConfig().map_openai_params( + non_default_params={ + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "City", + "description": "A city", + "schema": {"type": "object", "properties": {"name": {"type": "string"}}}, + }, + }, + **extra_params, + }, + optional_params={}, + model="us.anthropic.claude-opus-5", + drop_params=False, + ) + + assert result.get("tool_choice") == expected_tool_choice + assert result["tools"][0]["function"]["description"] == expected_description + + def test_forced_tool_choice_forwarded_on_converse_models_that_support_it( local_model_cost_map, monkeypatch ): diff --git a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py index 3846a94c9fe..4feaadb04e6 100644 --- a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py +++ b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py @@ -13,6 +13,7 @@ import pytest # tests are executed from any working directory. import litellm +from litellm.constants import RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import ( AmazonAnthropicClaudeConfig, ) @@ -751,7 +752,12 @@ def test_bedrock_chat_invoke_drop_params_still_inlines_for_non_native(local_mode @pytest.mark.parametrize( "model", - ["us.anthropic.claude-fable-5-1", "anthropic.claude-fable-5-1"], + [ + "us.anthropic.claude-fable-5-1", + "anthropic.claude-fable-5-1", + "us.anthropic.claude-opus-5-5", + "anthropic.claude-opus-5-5", + ], ) def test_bedrock_chat_invoke_fable_5_1_response_format_avoids_forced_tool_choice(local_model_cost_map, model): """Regression: Bedrock rejects both native ``output_config.format`` and forced @@ -773,10 +779,49 @@ def test_bedrock_chat_invoke_fable_5_1_response_format_avoids_forced_tool_choice ) assert "output_format" not in result - assert "tools" in result + assert result["tools"][0]["description"] == RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION assert "tool_choice" not in result +@pytest.mark.parametrize( + ("extra_params", "expected_tool_choice", "expected_description"), + [ + ({}, {"name": "json_tool_call", "type": "tool"}, None), + ({"reasoning_effort": "low"}, None, RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION), + ( + { + "tools": [{"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}}], + "tool_choice": "auto", + }, + {"type": "auto"}, + None, + ), + ], +) +def test_bedrock_chat_invoke_response_format_tool_description_asks_for_the_tool_only_when_it_cannot_be_forced( + local_model_cost_map, extra_params, expected_tool_choice, expected_description +): + result = AmazonAnthropicClaudeConfig().map_openai_params( + non_default_params={ + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "City", + "schema": {"type": "object", "properties": {"name": {"type": "string"}}}, + }, + }, + "max_tokens": 4096, + **extra_params, + }, + optional_params={}, + model="us.anthropic.claude-opus-5", + drop_params=False, + ) + + assert result.get("tool_choice") == expected_tool_choice + assert result["tools"][0].get("description") == expected_description + + @pytest.mark.parametrize("model", ["us.anthropic.claude-sonnet-5", "us.anthropic.claude-fable-5-1"]) def test_bedrock_chat_invoke_tool_based_response_format_still_upgrades_legacy_thinking(local_model_cost_map, model): result = AmazonAnthropicClaudeConfig().map_openai_params(