diff --git a/litellm/constants.py b/litellm/constants.py index 69ff3cf7a5e..d3f09b30fbe 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1540,6 +1540,9 @@ OPENAI_FINISH_REASONS: Final = [ ] HUMANLOOP_PROMPT_CACHE_TTL_SECONDS: Final = int(os.getenv("HUMANLOOP_PROMPT_CACHE_TTL_SECONDS", 60)) # 1 minute RESPONSE_FORMAT_TOOL_NAME = "json_tool_call" # default tool name used when converting response format to tool call +RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION: Final = ( + "Always respond by calling this tool. Its input is your final answer; do not reply in plain text." +) ########################### Logging Callback Constants ########################### AZURE_STORAGE_MSFT_VERSION: Final = "2019-07-07" diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 5600e25750f..e8c9e3fc2ba 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -19,6 +19,7 @@ from litellm._logging import verbose_logger from litellm.constants import ( BEDROCK_MIN_THINKING_BUDGET_TOKENS, RESPONSE_FORMAT_TOOL_NAME, + RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION, ) from litellm.litellm_core_utils.core_helpers import ( filter_exceptions_from_params, @@ -1277,16 +1278,20 @@ class AmazonConverseConfig(BaseConfig): elif json_schema is not None: # Fallback: translate to a synthetic tool call # https://docs.anthropic.com/en/docs/build-with-claude/tool-use#json-mode + tool_choice_blocked: Final = is_thinking_enabled or AnthropicModelInfo.forced_tool_use_unsupported(model) _tool: Final = self._create_json_tool_call_for_response_format( json_schema=json_schema, - description=description, + description=( + " ".join(filter(None, (RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION, description))) + if tool_choice_blocked + else description + ), ) optional_params = self._add_tools_to_optional_params(optional_params=optional_params, tools=[_tool]) if ( litellm.utils.supports_tool_choice(model=model, custom_llm_provider=self.custom_llm_provider) - and not is_thinking_enabled - and not AnthropicModelInfo.forced_tool_use_unsupported(model) + and not tool_choice_blocked ): optional_params["tool_choice"] = ToolChoiceValuesBlock( tool=SpecificToolChoiceBlock(name=RESPONSE_FORMAT_TOOL_NAME) diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py index 0be087d2dd4..5cf3aaabb4d 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py @@ -3,7 +3,7 @@ from typing import TYPE_CHECKING, Any, Final import httpx from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers -from litellm.constants import RESPONSE_FORMAT_TOOL_NAME +from litellm.constants import RESPONSE_FORMAT_TOOL_NAME, RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION from litellm.litellm_core_utils.prompt_templates.factory import ( convert_to_anthropic_image_obj, ) @@ -135,6 +135,22 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): and AnthropicModelInfo.forced_tool_use_unsupported(original_model) ): optional_params.pop("tool_choice") + if ( + "response_format" in non_default_params + and "tools" in optional_params + and ( + self.is_thinking_enabled(non_default_params) + or AnthropicModelInfo.forced_tool_use_unsupported(original_model) + ) + ): + # Rebuild the tool instead of writing into it: Anthropic-format tools reach + # here as the caller's own dicts, and the repo rules out in-place mutation + optional_params["tools"] = [ # rebind-ok: out-param store like siblings + {**tool, "description": RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION} + if tool.get("name") == RESPONSE_FORMAT_TOOL_NAME + else tool + for tool in optional_params["tools"] + ] return optional_params diff --git a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py index 2c82ba1c5b8..61d0eb266e7 100644 --- a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py +++ b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py @@ -12,6 +12,7 @@ import pytest # Ensure the project root is on the import path so `litellm` can be imported when # tests are executed from any working directory. import litellm +from litellm.constants import RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import ( AmazonAnthropicClaudeConfig, ) @@ -709,7 +710,12 @@ def test_bedrock_chat_invoke_drop_params_still_inlines_for_non_native(local_mode @pytest.mark.parametrize( "model", - ["us.anthropic.claude-fable-5-1", "anthropic.claude-fable-5-1"], + [ + "us.anthropic.claude-fable-5-1", + "anthropic.claude-fable-5-1", + "us.anthropic.claude-opus-5-5", + "anthropic.claude-opus-5-5", + ], ) def test_bedrock_chat_invoke_fable_5_1_response_format_avoids_forced_tool_choice(local_model_cost_map, model): """Regression: Bedrock rejects both native ``output_config.format`` and forced @@ -731,10 +737,49 @@ def test_bedrock_chat_invoke_fable_5_1_response_format_avoids_forced_tool_choice ) assert "output_format" not in result - assert "tools" in result + assert result["tools"][0]["description"] == RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION assert "tool_choice" not in result +@pytest.mark.parametrize( + ("extra_params", "expected_tool_choice", "expected_description"), + [ + ({}, {"name": "json_tool_call", "type": "tool"}, None), + ({"reasoning_effort": "low"}, None, RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION), + ( + { + "tools": [{"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}}], + "tool_choice": "auto", + }, + {"type": "auto"}, + None, + ), + ], +) +def test_bedrock_chat_invoke_response_format_tool_description_asks_for_the_tool_only_when_it_cannot_be_forced( + local_model_cost_map, extra_params, expected_tool_choice, expected_description +): + result = AmazonAnthropicClaudeConfig().map_openai_params( + non_default_params={ + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "City", + "schema": {"type": "object", "properties": {"name": {"type": "string"}}}, + }, + }, + "max_tokens": 4096, + **extra_params, + }, + optional_params={}, + model="us.anthropic.claude-opus-5", + drop_params=False, + ) + + assert result.get("tool_choice") == expected_tool_choice + assert result["tools"][0].get("description") == expected_description + + @pytest.mark.parametrize("model", ["us.anthropic.claude-sonnet-5", "us.anthropic.claude-fable-5-1"]) def test_bedrock_chat_invoke_tool_based_response_format_still_upgrades_legacy_thinking(local_model_cost_map, model): result = AmazonAnthropicClaudeConfig().map_openai_params( diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 6db3836f68e..10ba73aad58 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -9,6 +9,7 @@ import pytest import litellm from litellm import ModelResponse +from litellm.constants import RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import CONVERTED_SYSTEM_NOTE from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig from litellm.types.llms.bedrock import ConverseTokenUsageBlock @@ -7642,7 +7643,12 @@ def test_unforced_tool_choice_unaffected_on_fable_5_1_converse(local_model_cost_ @pytest.mark.parametrize( "model", - ["anthropic.claude-fable-5-1", "us.anthropic.claude-fable-5-1"], + [ + "anthropic.claude-fable-5-1", + "us.anthropic.claude-fable-5-1", + "anthropic.claude-opus-5-5", + "us.anthropic.claude-opus-5-5", + ], ) def test_response_format_avoids_native_and_forced_tool_choice_on_fable_5_1_converse( local_model_cost_map, model @@ -7668,11 +7674,50 @@ def test_response_format_avoids_native_and_forced_tool_choice_on_fable_5_1_conve ) assert "outputConfig" not in result - assert "tools" in result + assert result["tools"][0]["function"]["description"] == RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION assert "tool_choice" not in result assert result.get("json_mode") is True +@pytest.mark.parametrize( + ("extra_params", "expected_tool_choice", "expected_description"), + [ + ({}, {"tool": {"name": "json_tool_call"}}, "A city"), + ({"reasoning_effort": "low"}, None, f"{RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION} A city"), + ( + { + "tools": [{"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}}], + "tool_choice": "auto", + }, + {"auto": {}}, + "A city", + ), + ], +) +def test_response_format_tool_description_asks_for_the_tool_only_when_it_cannot_be_forced( + local_model_cost_map, extra_params, expected_tool_choice, expected_description +): + result = AmazonConverseConfig().map_openai_params( + non_default_params={ + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "City", + "description": "A city", + "schema": {"type": "object", "properties": {"name": {"type": "string"}}}, + }, + }, + **extra_params, + }, + optional_params={}, + model="us.anthropic.claude-opus-5", + drop_params=False, + ) + + assert result.get("tool_choice") == expected_tool_choice + assert result["tools"][0]["function"]["description"] == expected_description + + def test_forced_tool_choice_forwarded_on_converse_models_that_support_it( local_model_cost_map, monkeypatch ):