From 036cfb951b9a413b2b34b313420be7dee23ed212 Mon Sep 17 00:00:00 2001 From: kimnamu Date: Wed, 23 Sep 2026 15:07:50 +0900 Subject: [PATCH 1/4] fix(bedrock): return JSON for Claude Opus 5.5 response_format when tool_choice can't be forced Opus 5.5 and Fable 5.1 reject a forced tool_choice on Bedrock, and litellm also skips the forced tool_choice whenever thinking is on, so the synthetic json_tool_call tool goes out unforced on those requests. With the default description the model often skips the tool and answers in plain text. When thinking is on or the model rejects forced tool use, give the tool a directive description, with the same condition in the Converse and Invoke routes. Other requests are unchanged --- litellm/constants.py | 3 ++ .../bedrock/chat/converse_transformation.py | 11 +++-- .../anthropic_claude3_transformation.py | 9 +++- .../chat/test_converse_transformation.py | 49 ++++++++++++++++++- ...ations_anthropic_claude3_transformation.py | 49 ++++++++++++++++++- 5 files changed, 113 insertions(+), 8 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index e5b662bd515..1207f2e931a 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1508,6 +1508,9 @@ OPENAI_FINISH_REASONS: Final = [ ] HUMANLOOP_PROMPT_CACHE_TTL_SECONDS: Final = int(os.getenv("HUMANLOOP_PROMPT_CACHE_TTL_SECONDS", 60)) # 1 minute RESPONSE_FORMAT_TOOL_NAME = "json_tool_call" # default tool name used when converting response format to tool call +RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION: Final = ( + "Always respond by calling this tool. Its input is your final answer; do not reply in plain text." +) ########################### Logging Callback Constants ########################### AZURE_STORAGE_MSFT_VERSION: Final = "2019-07-07" diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 497020c2836..8a3b00e6f0e 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -17,6 +17,7 @@ from litellm._logging import verbose_logger from litellm.constants import ( BEDROCK_MIN_THINKING_BUDGET_TOKENS, RESPONSE_FORMAT_TOOL_NAME, + RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION, ) from litellm.litellm_core_utils.core_helpers import ( filter_exceptions_from_params, @@ -1233,16 +1234,20 @@ class AmazonConverseConfig(BaseConfig): elif json_schema is not None: # Fallback: translate to a synthetic tool call # https://docs.anthropic.com/en/docs/build-with-claude/tool-use#json-mode + tool_choice_blocked: Final = is_thinking_enabled or AnthropicModelInfo.forced_tool_use_unsupported(model) _tool: Final = self._create_json_tool_call_for_response_format( json_schema=json_schema, - description=description, + description=( + " ".join(filter(None, (RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION, description))) + if tool_choice_blocked + else description + ), ) optional_params = self._add_tools_to_optional_params(optional_params=optional_params, tools=[_tool]) if ( litellm.utils.supports_tool_choice(model=model, custom_llm_provider=self.custom_llm_provider) - and not is_thinking_enabled - and not AnthropicModelInfo.forced_tool_use_unsupported(model) + and not tool_choice_blocked ): optional_params["tool_choice"] = ToolChoiceValuesBlock( tool=SpecificToolChoiceBlock(name=RESPONSE_FORMAT_TOOL_NAME) diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py index a8b94fb5703..d585a943eef 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py @@ -3,7 +3,7 @@ from typing import TYPE_CHECKING, Any, Final import httpx from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers -from litellm.constants import RESPONSE_FORMAT_TOOL_NAME +from litellm.constants import RESPONSE_FORMAT_TOOL_NAME, RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION from litellm.litellm_core_utils.prompt_templates.factory import ( convert_to_anthropic_image_obj, ) @@ -124,6 +124,13 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): and AnthropicModelInfo.forced_tool_use_unsupported(original_model) ): optional_params.pop("tool_choice") + if "response_format" in non_default_params and ( + self.is_thinking_enabled(non_default_params) + or AnthropicModelInfo.forced_tool_use_unsupported(original_model) + ): + for tool in optional_params.get("tools", ()): + if tool.get("name") == RESPONSE_FORMAT_TOOL_NAME: + tool["description"] = RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION return optional_params diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index ad77f9d4d1b..8acf30df5f7 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -8,6 +8,7 @@ import pytest import litellm from litellm import ModelResponse +from litellm.constants import RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig from litellm.types.llms.bedrock import ConverseTokenUsageBlock @@ -7394,7 +7395,12 @@ def test_unforced_tool_choice_unaffected_on_fable_5_1_converse(local_model_cost_ @pytest.mark.parametrize( "model", - ["anthropic.claude-fable-5-1", "us.anthropic.claude-fable-5-1"], + [ + "anthropic.claude-fable-5-1", + "us.anthropic.claude-fable-5-1", + "anthropic.claude-opus-5-5", + "us.anthropic.claude-opus-5-5", + ], ) def test_response_format_avoids_native_and_forced_tool_choice_on_fable_5_1_converse( local_model_cost_map, model @@ -7420,11 +7426,50 @@ def test_response_format_avoids_native_and_forced_tool_choice_on_fable_5_1_conve ) assert "outputConfig" not in result - assert "tools" in result + assert result["tools"][0]["function"]["description"] == RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION assert "tool_choice" not in result assert result.get("json_mode") is True +@pytest.mark.parametrize( + ("extra_params", "expected_tool_choice", "expected_description"), + [ + ({}, {"tool": {"name": "json_tool_call"}}, "A city"), + ({"reasoning_effort": "low"}, None, f"{RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION} A city"), + ( + { + "tools": [{"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}}], + "tool_choice": "auto", + }, + {"auto": {}}, + "A city", + ), + ], +) +def test_response_format_tool_description_asks_for_the_tool_only_when_it_cannot_be_forced( + local_model_cost_map, extra_params, expected_tool_choice, expected_description +): + result = AmazonConverseConfig().map_openai_params( + non_default_params={ + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "City", + "description": "A city", + "schema": {"type": "object", "properties": {"name": {"type": "string"}}}, + }, + }, + **extra_params, + }, + optional_params={}, + model="us.anthropic.claude-opus-5", + drop_params=False, + ) + + assert result.get("tool_choice") == expected_tool_choice + assert result["tools"][0]["function"]["description"] == expected_description + + def test_forced_tool_choice_forwarded_on_converse_models_that_support_it( local_model_cost_map, monkeypatch ): diff --git a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py index 3846a94c9fe..4feaadb04e6 100644 --- a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py +++ b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py @@ -13,6 +13,7 @@ import pytest # tests are executed from any working directory. import litellm +from litellm.constants import RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import ( AmazonAnthropicClaudeConfig, ) @@ -751,7 +752,12 @@ def test_bedrock_chat_invoke_drop_params_still_inlines_for_non_native(local_mode @pytest.mark.parametrize( "model", - ["us.anthropic.claude-fable-5-1", "anthropic.claude-fable-5-1"], + [ + "us.anthropic.claude-fable-5-1", + "anthropic.claude-fable-5-1", + "us.anthropic.claude-opus-5-5", + "anthropic.claude-opus-5-5", + ], ) def test_bedrock_chat_invoke_fable_5_1_response_format_avoids_forced_tool_choice(local_model_cost_map, model): """Regression: Bedrock rejects both native ``output_config.format`` and forced @@ -773,10 +779,49 @@ def test_bedrock_chat_invoke_fable_5_1_response_format_avoids_forced_tool_choice ) assert "output_format" not in result - assert "tools" in result + assert result["tools"][0]["description"] == RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION assert "tool_choice" not in result +@pytest.mark.parametrize( + ("extra_params", "expected_tool_choice", "expected_description"), + [ + ({}, {"name": "json_tool_call", "type": "tool"}, None), + ({"reasoning_effort": "low"}, None, RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION), + ( + { + "tools": [{"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}}], + "tool_choice": "auto", + }, + {"type": "auto"}, + None, + ), + ], +) +def test_bedrock_chat_invoke_response_format_tool_description_asks_for_the_tool_only_when_it_cannot_be_forced( + local_model_cost_map, extra_params, expected_tool_choice, expected_description +): + result = AmazonAnthropicClaudeConfig().map_openai_params( + non_default_params={ + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "City", + "schema": {"type": "object", "properties": {"name": {"type": "string"}}}, + }, + }, + "max_tokens": 4096, + **extra_params, + }, + optional_params={}, + model="us.anthropic.claude-opus-5", + drop_params=False, + ) + + assert result.get("tool_choice") == expected_tool_choice + assert result["tools"][0].get("description") == expected_description + + @pytest.mark.parametrize("model", ["us.anthropic.claude-sonnet-5", "us.anthropic.claude-fable-5-1"]) def test_bedrock_chat_invoke_tool_based_response_format_still_upgrades_legacy_thinking(local_model_cost_map, model): result = AmazonAnthropicClaudeConfig().map_openai_params( From 6bd579c84e3b5681e97ed4facbe08e2ccad75192 Mon Sep 17 00:00:00 2001 From: kimnamu Date: Thu, 24 Sep 2026 16:30:53 +0900 Subject: [PATCH 2/4] fix(bedrock): rebuild the unforced json_tool_call tool instead of writing into it Greptile flagged the Invoke directive for mutating tool dicts in place. Anthropic-format tools reach map_openai_params as the caller's own dicts, so a caller tool that reused the reserved json_tool_call name would have carried the directive back into its later requests, and AGENTS.md rules out in-place mutation. Build a new dict for the json_tool_call tool and store the rebuilt list on optional_params, the way _add_tools_to_optional_params already does. Request bodies are unchanged. --- .../anthropic_claude3_transformation.py | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py index d585a943eef..dc3061009b5 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py @@ -124,13 +124,22 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): and AnthropicModelInfo.forced_tool_use_unsupported(original_model) ): optional_params.pop("tool_choice") - if "response_format" in non_default_params and ( - self.is_thinking_enabled(non_default_params) - or AnthropicModelInfo.forced_tool_use_unsupported(original_model) + if ( + "response_format" in non_default_params + and "tools" in optional_params + and ( + self.is_thinking_enabled(non_default_params) + or AnthropicModelInfo.forced_tool_use_unsupported(original_model) + ) ): - for tool in optional_params.get("tools", ()): - if tool.get("name") == RESPONSE_FORMAT_TOOL_NAME: - tool["description"] = RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION + # Rebuild the tool instead of writing into it: Anthropic-format tools reach + # here as the caller's own dicts, and the repo rules out in-place mutation + optional_params["tools"] = [ # rebind-ok: out-param store like siblings + {**tool, "description": RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION} + if tool.get("name") == RESPONSE_FORMAT_TOOL_NAME + else tool + for tool in optional_params["tools"] + ] return optional_params From 79eef9aa8d5477b5bdbd341ab735806f24d04c62 Mon Sep 17 00:00:00 2001 From: kimnamu Date: Thu, 24 Sep 2026 18:26:29 +0900 Subject: [PATCH 3/4] ci: retrigger From 0a9d0a1e088dfc1be8dc6fef5dd22375a8f2f24c Mon Sep 17 00:00:00 2001 From: kimnamu Date: Mon, 5 Oct 2026 19:21:14 +0900 Subject: [PATCH 4/4] ci: retrigger