scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-12 03:48:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 119

fix(bedrock): return JSON for Claude Opus 5.5 response_format when tool_choice can't be forced

Browse source 
Opus 5.5 and Fable 5.1 reject a forced tool_choice on Bedrock, and litellm
also skips the forced tool_choice whenever thinking is on, so the synthetic
json_tool_call tool goes out unforced on those requests. With the default
description the model often skips the tool and answers in plain text. When
thinking is on or the model rejects forced tool use, give the tool a
directive description, with the same condition in the Converse and Invoke
routes. Other requests are unchanged
This commit is contained in:
kimnamu 2026-09-23 15:07:50 +09:00
parent 59db39626f
commit 036cfb951b
5 changed files with 113 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
litellm/constants.py
View file

@ -1508,6 +1508,9 @@ OPENAI_FINISH_REASONS: Final = [
]
HUMANLOOP_PROMPT_CACHE_TTL_SECONDS: Final = int(os.getenv("HUMANLOOP_PROMPT_CACHE_TTL_SECONDS", 60)) # 1 minute
RESPONSE_FORMAT_TOOL_NAME = "json_tool_call" # default tool name used when converting response format to tool call
RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION: Final = (
"Always respond by calling this tool. Its input is your final answer; do not reply in plain text."
)
########################### Logging Callback Constants ###########################
AZURE_STORAGE_MSFT_VERSION: Final = "2019-07-07"

11
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -17,6 +17,7 @@ from litellm._logging import verbose_logger
from litellm.constants import (
BEDROCK_MIN_THINKING_BUDGET_TOKENS,
RESPONSE_FORMAT_TOOL_NAME,
RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION,
)
from litellm.litellm_core_utils.core_helpers import (
filter_exceptions_from_params,
@ -1233,16 +1234,20 @@ class AmazonConverseConfig(BaseConfig):
elif json_schema is not None:
# Fallback: translate to a synthetic tool call
# https://docs.anthropic.com/en/docs/build-with-claude/tool-use#json-mode
tool_choice_blocked: Final = is_thinking_enabled or AnthropicModelInfo.forced_tool_use_unsupported(model)
_tool: Final = self._create_json_tool_call_for_response_format(
json_schema=json_schema,
description=description,
description=(
" ".join(filter(None, (RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION, description)))
if tool_choice_blocked
else description
),
)
optional_params = self._add_tools_to_optional_params(optional_params=optional_params, tools=[_tool])
if (
litellm.utils.supports_tool_choice(model=model, custom_llm_provider=self.custom_llm_provider)
and not is_thinking_enabled
and not AnthropicModelInfo.forced_tool_use_unsupported(model)
and not tool_choice_blocked
):
optional_params["tool_choice"] = ToolChoiceValuesBlock(
tool=SpecificToolChoiceBlock(name=RESPONSE_FORMAT_TOOL_NAME)

9
litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py
View file

@ -3,7 +3,7 @@ from typing import TYPE_CHECKING, Any, Final
import httpx
from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers
from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
from litellm.constants import RESPONSE_FORMAT_TOOL_NAME, RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION
from litellm.litellm_core_utils.prompt_templates.factory import (
convert_to_anthropic_image_obj,
)
@ -124,6 +124,13 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
and AnthropicModelInfo.forced_tool_use_unsupported(original_model)
):
optional_params.pop("tool_choice")
if "response_format" in non_default_params and (
self.is_thinking_enabled(non_default_params)
or AnthropicModelInfo.forced_tool_use_unsupported(original_model)
):
for tool in optional_params.get("tools", ()):
if tool.get("name") == RESPONSE_FORMAT_TOOL_NAME:
tool["description"] = RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION
return optional_params

49
tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py
View file

@ -8,6 +8,7 @@ import pytest
import litellm
from litellm import ModelResponse
from litellm.constants import RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION
from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig
from litellm.types.llms.bedrock import ConverseTokenUsageBlock
@ -7394,7 +7395,12 @@ def test_unforced_tool_choice_unaffected_on_fable_5_1_converse(local_model_cost_
@pytest.mark.parametrize(
"model",
["anthropic.claude-fable-5-1", "us.anthropic.claude-fable-5-1"],
[
"anthropic.claude-fable-5-1",
"us.anthropic.claude-fable-5-1",
"anthropic.claude-opus-5-5",
"us.anthropic.claude-opus-5-5",
],
)
def test_response_format_avoids_native_and_forced_tool_choice_on_fable_5_1_converse(
local_model_cost_map, model
@ -7420,11 +7426,50 @@ def test_response_format_avoids_native_and_forced_tool_choice_on_fable_5_1_conve
)
assert "outputConfig" not in result
assert "tools" in result
assert result["tools"][0]["function"]["description"] == RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION
assert "tool_choice" not in result
assert result.get("json_mode") is True
@pytest.mark.parametrize(
("extra_params", "expected_tool_choice", "expected_description"),
[
({}, {"tool": {"name": "json_tool_call"}}, "A city"),
({"reasoning_effort": "low"}, None, f"{RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION} A city"),
(
{
"tools": [{"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}}],
"tool_choice": "auto",
},
{"auto": {}},
"A city",
),
],
)
def test_response_format_tool_description_asks_for_the_tool_only_when_it_cannot_be_forced(
local_model_cost_map, extra_params, expected_tool_choice, expected_description
):
result = AmazonConverseConfig().map_openai_params(
non_default_params={
"response_format": {
"type": "json_schema",
"json_schema": {
"name": "City",
"description": "A city",
"schema": {"type": "object", "properties": {"name": {"type": "string"}}},
},
},
**extra_params,
},
optional_params={},
model="us.anthropic.claude-opus-5",
drop_params=False,
)
assert result.get("tool_choice") == expected_tool_choice
assert result["tools"][0]["function"]["description"] == expected_description
def test_forced_tool_choice_forwarded_on_converse_models_that_support_it(
local_model_cost_map, monkeypatch
):

49
tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py
View file

@ -13,6 +13,7 @@ import pytest
# tests are executed from any working directory.
import litellm
from litellm.constants import RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION
from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import (
AmazonAnthropicClaudeConfig,
)
@ -751,7 +752,12 @@ def test_bedrock_chat_invoke_drop_params_still_inlines_for_non_native(local_mode
@pytest.mark.parametrize(
"model",
["us.anthropic.claude-fable-5-1", "anthropic.claude-fable-5-1"],
[
"us.anthropic.claude-fable-5-1",
"anthropic.claude-fable-5-1",
"us.anthropic.claude-opus-5-5",
"anthropic.claude-opus-5-5",
],
)
def test_bedrock_chat_invoke_fable_5_1_response_format_avoids_forced_tool_choice(local_model_cost_map, model):
"""Regression: Bedrock rejects both native ``output_config.format`` and forced
@ -773,10 +779,49 @@ def test_bedrock_chat_invoke_fable_5_1_response_format_avoids_forced_tool_choice
)
assert "output_format" not in result
assert "tools" in result
assert result["tools"][0]["description"] == RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION
assert "tool_choice" not in result
@pytest.mark.parametrize(
("extra_params", "expected_tool_choice", "expected_description"),
[
({}, {"name": "json_tool_call", "type": "tool"}, None),
({"reasoning_effort": "low"}, None, RESPONSE_FORMAT_UNFORCED_TOOL_DESCRIPTION),
(
{
"tools": [{"type": "function", "function": {"name": "get_weather", "parameters": {"type": "object"}}}],
"tool_choice": "auto",
},
{"type": "auto"},
None,
),
],
)
def test_bedrock_chat_invoke_response_format_tool_description_asks_for_the_tool_only_when_it_cannot_be_forced(
local_model_cost_map, extra_params, expected_tool_choice, expected_description
):
result = AmazonAnthropicClaudeConfig().map_openai_params(
non_default_params={
"response_format": {
"type": "json_schema",
"json_schema": {
"name": "City",
"schema": {"type": "object", "properties": {"name": {"type": "string"}}},
},
},
"max_tokens": 4096,
**extra_params,
},
optional_params={},
model="us.anthropic.claude-opus-5",
drop_params=False,
)
assert result.get("tool_choice") == expected_tool_choice
assert result["tools"][0].get("description") == expected_description
@pytest.mark.parametrize("model", ["us.anthropic.claude-sonnet-5", "us.anthropic.claude-fable-5-1"])
def test_bedrock_chat_invoke_tool_based_response_format_still_upgrades_legacy_thinking(local_model_cost_map, model):
result = AmazonAnthropicClaudeConfig().map_openai_params(