diff --git a/litellm/constants.py b/litellm/constants.py index e5b662bd515..1207f2e931a 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1508,6 +1508,9 @@ OPENAI_FINISH_REASONS: Final = [ ] HUMANLOOP_PROMPT_CACHE_TTL_SECONDS: Final = int(os.getenv("HUMANLOOP_PROMPT_CACHE_TTL_SECONDS", 60)) # 1 minute RESPONSE_FORMAT_TOOL_NAME = "json_tool_call" # default tool name used when converting response format to tool call +RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION: Final = ( + "Always respond by calling this tool. Its input is your final answer; do not reply in plain text." +) ########################### Logging Callback Constants ########################### AZURE_STORAGE_MSFT_VERSION: Final = "2019-07-07" diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 497020c2836..8a3b00e6f0e 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -17,6 +17,7 @@ from litellm._logging import verbose_logger from litellm.constants import ( BEDROCK_MIN_THINKING_BUDGET_TOKENS, RESPONSE_FORMAT_TOOL_NAME, + RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION, ) from litellm.litellm_core_utils.core_helpers import ( filter_exceptions_from_params, @@ -1233,16 +1234,20 @@ class AmazonConverseConfig(BaseConfig): elif json_schema is not None: # Fallback: translate to a synthetic tool call # https://docs.anthropic.com/en/docs/build-with-claude/tool-use#json-mode + tool_choice_blocked: Final = is_thinking_enabled or AnthropicModelInfo.forced_tool_use_unsupported(model) _tool: Final = self._create_json_tool_call_for_response_format( json_schema=json_schema, - description=description, + description=( + " ".join(filter(None, (RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION, description))) + if tool_choice_blocked + else description + ), ) optional_params = self._add_tools_to_optional_params(optional_params=optional_params, tools=[_tool]) if ( litellm.utils.supports_tool_choice(model=model, custom_llm_provider=self.custom_llm_provider) - and not is_thinking_enabled - and not AnthropicModelInfo.forced_tool_use_unsupported(model) + and not tool_choice_blocked ): optional_params["tool_choice"] = ToolChoiceValuesBlock( tool=SpecificToolChoiceBlock(name=RESPONSE_FORMAT_TOOL_NAME) diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py index a8b94fb5703..d585a943eef 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py @@ -3,7 +3,7 @@ from typing import TYPE_CHECKING, Any, Final import httpx from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers -from litellm.constants import RESPONSE_FORMAT_TOOL_NAME +from litellm.constants import RESPONSE_FORMAT_TOOL_NAME, RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION from litellm.litellm_core_utils.prompt_templates.factory import ( convert_to_anthropic_image_obj, ) @@ -124,6 +124,13 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): and AnthropicModelInfo.forced_tool_use_unsupported(original_model) ): optional_params.pop("tool_choice") + if "response_format" in non_default_params and ( + self.is_thinking_enabled(non_default_params) + or AnthropicModelInfo.forced_tool_use_unsupported(original_model) + ): + for tool in optional_params.get("tools", ()): + if tool.get("name") == RESPONSE_FORMAT_TOOL_NAME: + tool["description"] = RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION return optional_params diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index ad77f9d4d1b..8acf30df5f7 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -8,6 +8,7 @@ import pytest import litellm from litellm import ModelResponse +from litellm.constants import RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig from litellm.types.llms.bedrock import ConverseTokenUsageBlock @@ -7394,7 +7395,12 @@ def test_unforced_tool_choice_unaffected_on_fable_5_1_converse(local_model_cost_ @pytest.mark.parametrize( "model", - ["anthropic.claude-fable-5-1", "us.anthropic.claude-fable-5-1"], + [ + "anthropic.claude-fable-5-1", + "us.anthropic.claude-fable-5-1", + "anthropic.claude-opus-5-5", + "us.anthropic.claude-opus-5-5", + ], ) def test_response_format_avoids_native_and_forced_tool_choice_on_fable_5_1_converse( local_model_cost_map, model @@ -7420,11 +7426,50 @@ def test_response_format_avoids_native_and_forced_tool_choice_on_fable_5_1_conve ) assert "outputConfig" not in result - assert "tools" in result + assert result["tools"][0]["function"]["description"] == RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION assert "tool_choice" not in result assert result.get("json_mode") is True +@pytest.mark.parametrize( + ("extra_params", "expected_tool_choice", "expected_description"), + [ + ({}, {"tool": {"name": "json_tool_call"}}, "A city"), + ({"reasoning_effort": "low"}, None, f"{RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION} A city"), + ( + { + "tools": [{"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}}], + "tool_choice": "auto", + }, + {"auto": {}}, + "A city", + ), + ], +) +def test_response_format_tool_description_asks_for_the_tool_only_when_it_cannot_be_forced( + local_model_cost_map, extra_params, expected_tool_choice, expected_description +): + result = AmazonConverseConfig().map_openai_params( + non_default_params={ + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "City", + "description": "A city", + "schema": {"type": "object", "properties": {"name": {"type": "string"}}}, + }, + }, + **extra_params, + }, + optional_params={}, + model="us.anthropic.claude-opus-5", + drop_params=False, + ) + + assert result.get("tool_choice") == expected_tool_choice + assert result["tools"][0]["function"]["description"] == expected_description + + def test_forced_tool_choice_forwarded_on_converse_models_that_support_it( local_model_cost_map, monkeypatch ): diff --git a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py index 3846a94c9fe..4feaadb04e6 100644 --- a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py +++ b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py @@ -13,6 +13,7 @@ import pytest # tests are executed from any working directory. import litellm +from litellm.constants import RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import ( AmazonAnthropicClaudeConfig, ) @@ -751,7 +752,12 @@ def test_bedrock_chat_invoke_drop_params_still_inlines_for_non_native(local_mode @pytest.mark.parametrize( "model", - ["us.anthropic.claude-fable-5-1", "anthropic.claude-fable-5-1"], + [ + "us.anthropic.claude-fable-5-1", + "anthropic.claude-fable-5-1", + "us.anthropic.claude-opus-5-5", + "anthropic.claude-opus-5-5", + ], ) def test_bedrock_chat_invoke_fable_5_1_response_format_avoids_forced_tool_choice(local_model_cost_map, model): """Regression: Bedrock rejects both native ``output_config.format`` and forced @@ -773,10 +779,49 @@ def test_bedrock_chat_invoke_fable_5_1_response_format_avoids_forced_tool_choice ) assert "output_format" not in result - assert "tools" in result + assert result["tools"][0]["description"] == RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION assert "tool_choice" not in result +@pytest.mark.parametrize( + ("extra_params", "expected_tool_choice", "expected_description"), + [ + ({}, {"name": "json_tool_call", "type": "tool"}, None), + ({"reasoning_effort": "low"}, None, RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION), + ( + { + "tools": [{"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}}], + "tool_choice": "auto", + }, + {"type": "auto"}, + None, + ), + ], +) +def test_bedrock_chat_invoke_response_format_tool_description_asks_for_the_tool_only_when_it_cannot_be_forced( + local_model_cost_map, extra_params, expected_tool_choice, expected_description +): + result = AmazonAnthropicClaudeConfig().map_openai_params( + non_default_params={ + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "City", + "schema": {"type": "object", "properties": {"name": {"type": "string"}}}, + }, + }, + "max_tokens": 4096, + **extra_params, + }, + optional_params={}, + model="us.anthropic.claude-opus-5", + drop_params=False, + ) + + assert result.get("tool_choice") == expected_tool_choice + assert result["tools"][0].get("description") == expected_description + + @pytest.mark.parametrize("model", ["us.anthropic.claude-sonnet-5", "us.anthropic.claude-fable-5-1"]) def test_bedrock_chat_invoke_tool_based_response_format_still_upgrades_legacy_thinking(local_model_cost_map, model): result = AmazonAnthropicClaudeConfig().map_openai_params(