scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-07 02:59:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 103

Merge pull request #39184 from BerriAI/litellm_fable_5_1_structured_output

Browse source
This commit is contained in:
Mateo Wang 2026-09-01 13:41:22 -07:00 • committed by GitHub
commit 8ce02c0666
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 165 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
litellm/llms/anthropic/chat/transformation.py
View file

@ -1516,7 +1516,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
_tool = self.map_response_format_to_anthropic_tool(value, optional_params, is_thinking_enabled)
if _tool is None:
continue
if not is_thinking_enabled:
if not is_thinking_enabled and not AnthropicModelInfo.forced_tool_use_unsupported(model):
_tool_choice = {
"name": RESPONSE_FORMAT_TOOL_NAME,
"type": "tool",

6
litellm/llms/anthropic/common_utils.py
View file

@ -325,13 +325,17 @@ class AnthropicModelInfo(BaseLLMModelInfo):
status_code=400,
)
@staticmethod
def forced_tool_use_unsupported(model: str) -> bool:
return AnthropicModelInfo._get_model_capability(model, "supports_forced_tool_use") is False
@staticmethod
def forced_tool_use_downgraded(model: str, drop_params: bool) -> bool:
"""True when the model map flags the model with
``supports_forced_tool_use: false`` (Fable 5.1 / Mythos 5.1 400 on
``any``/``tool``) and ``drop_params`` asks for the ``auto`` downgrade;
raises a clean client-side 400 for such models without ``drop_params``."""
if AnthropicModelInfo._get_model_capability(model, "supports_forced_tool_use") is not False:
if not AnthropicModelInfo.forced_tool_use_unsupported(model):
return False
if not (litellm.drop_params or drop_params):
raise litellm.utils.UnsupportedParamsError(

1
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -1073,6 +1073,7 @@ class AmazonConverseConfig(BaseConfig):
if (
litellm.utils.supports_tool_choice(model=model, custom_llm_provider=self.custom_llm_provider)
and not is_thinking_enabled
and not AnthropicModelInfo.forced_tool_use_unsupported(model)
):
optional_params["tool_choice"] = ToolChoiceValuesBlock(
tool=SpecificToolChoiceBlock(name=RESPONSE_FORMAT_TOOL_NAME)

22
litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py
View file

@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Any, Final
import httpx
from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers
from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
from litellm.litellm_core_utils.prompt_templates.factory import (
convert_to_anthropic_image_obj,
)
@ -11,6 +12,7 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import (
convert_url_to_base64,
)
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
AmazonInvokeConfig,
)
@ -74,10 +76,14 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
drop_params: bool,
) -> dict:
# Force tool-based structured outputs for Bedrock Invoke
# (similar to VertexAI fix in #19201)
# Bedrock Invoke doesn't support output_format parameter
# (similar to VertexAI fix in #19201) unless the model map advertises
# native structured output
from litellm.utils import supports_native_structured_output
original_model: Final = model
if "response_format" in non_default_params:
if "response_format" in non_default_params and not supports_native_structured_output(
model=model, custom_llm_provider="bedrock"
):
# Use a model name that forces tool-based approach
model = "claude-3-sonnet-20240229"
@ -101,6 +107,16 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig):
# Restore original model name
model = original_model
# The stub model hides the original model from the parent's forced-tool-use backstop
response_format_tool_choice: Final = optional_params.get("tool_choice")
if (
"response_format" in non_default_params
and isinstance(response_format_tool_choice, dict)
and response_format_tool_choice.get("name") == RESPONSE_FORMAT_TOOL_NAME
and AnthropicModelInfo.forced_tool_use_unsupported(original_model)
):
optional_params.pop("tool_choice")
return optional_params
@staticmethod

15
litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py
View file

@ -153,14 +153,17 @@ class VertexAIAnthropicConfig(AnthropicConfig):
drop_params: bool,
) -> dict:
"""
Override parent method to ensure VertexAI always uses tool-based structured outputs.
VertexAI doesn't support the output_format parameter, so we force all models
to use the tool-based approach for structured outputs.
Override parent method so VertexAI uses tool-based structured outputs
unless the vertex map entry advertises native structured output
(``output_format``, which Vertex AI Claude forwards for those models).
"""
# Temporarily override model name to force tool-based approach
# This ensures Claude Sonnet 4.5 uses tools instead of output_format
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
original_model: Final = model
if "response_format" in non_default_params:
native_structured_output: Final = AnthropicModelInfo._get_provider_resolved_capability(
model, "supports_native_structured_output", "vertex_ai"
)
if "response_format" in non_default_params and native_structured_output is not True:
model = "claude-3-sonnet-20240229" # Use a model that will use tool-based approach
# Call parent method with potentially modified model name

11
litellm/model_prices_and_context_window_backup.json
View file

@ -1482,7 +1482,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": true,
"supports_native_structured_output": false,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -1557,7 +1557,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": true,
"supports_native_structured_output": false,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -1632,7 +1632,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": true,
"supports_native_structured_output": false,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -1707,7 +1707,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": true,
"supports_native_structured_output": false,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -3254,6 +3254,7 @@
"supports_computer_use": true,
"supports_forced_tool_use": false,
"supports_function_calling": true,
"supports_native_structured_output": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
@ -44132,6 +44133,7 @@
"supports_computer_use": true,
"supports_forced_tool_use": false,
"supports_function_calling": true,
"supports_native_structured_output": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
@ -44202,6 +44204,7 @@
"supports_computer_use": true,
"supports_forced_tool_use": false,
"supports_function_calling": true,
"supports_native_structured_output": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,

11
model_prices_and_context_window.json
View file

@ -1482,7 +1482,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": true,
"supports_native_structured_output": false,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -1557,7 +1557,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": true,
"supports_native_structured_output": false,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -1632,7 +1632,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": true,
"supports_native_structured_output": false,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -1707,7 +1707,7 @@
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": true,
"supports_native_structured_output": false,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
@ -3254,6 +3254,7 @@
"supports_computer_use": true,
"supports_forced_tool_use": false,
"supports_function_calling": true,
"supports_native_structured_output": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
@ -44132,6 +44133,7 @@
"supports_computer_use": true,
"supports_forced_tool_use": false,
"supports_function_calling": true,
"supports_native_structured_output": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
@ -44202,6 +44204,7 @@
"supports_computer_use": true,
"supports_forced_tool_use": false,
"supports_function_calling": true,
"supports_native_structured_output": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,

30
tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py
View file

@ -6354,3 +6354,33 @@ def test_anthropic_drop_params_reduces_mixed_output_config_to_format(monkeypatch
)
assert result.get("output_config") == {"format": schema_format}
def test_response_format_tool_path_skips_forced_tool_choice_when_unsupported(local_model_cost_map, monkeypatch):
"""Backstop: on the tool-based structured-output path, a model flagged
``supports_forced_tool_use: false`` must not get the forced response-format
tool_choice the provider would 400 on."""
monkeypatch.setitem(
litellm.model_cost,
"claude-test-no-forced-tools",
{"litellm_provider": "anthropic", "mode": "chat", "supports_forced_tool_use": False},
)
config = AnthropicConfig()
result = config.map_openai_params(
non_default_params={
"response_format": {
"type": "json_schema",
"json_schema": {
"name": "test_schema",
"schema": {"type": "object", "properties": {"result": {"type": "string"}}},
},
}
},
optional_params={},
model="claude-test-no-forced-tools",
drop_params=False,
)
assert "tools" in result
assert "tool_choice" not in result

28
tests/test_litellm/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py
View file

@ -643,3 +643,31 @@ def test_bedrock_chat_invoke_drop_params_still_inlines_for_non_native(local_mode
assert "output_config" not in result
last_content = result["messages"][-1]["content"]
assert json.loads(last_content[-1]["text"]) == schema
@pytest.mark.parametrize(
"model",
["us.anthropic.claude-fable-5-1", "anthropic.claude-fable-5-1"],
)
def test_bedrock_chat_invoke_fable_5_1_response_format_avoids_forced_tool_choice(local_model_cost_map, model):
"""Regression: Bedrock rejects both native ``output_config.format`` and forced
tool_choice for Fable 5.1, so invoke must use the tool-based path without a
forced ``tool_choice``."""
result = AmazonAnthropicClaudeConfig().map_openai_params(
non_default_params={
"response_format": {
"type": "json_schema",
"json_schema": {
"name": "test_schema",
"schema": {"type": "object", "properties": {"result": {"type": "string"}}},
},
}
},
optional_params={},
model=model,
drop_params=False,
)
assert "output_format" not in result
assert "tools" in result
assert "tool_choice" not in result

33
tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py
View file

@ -6497,6 +6497,39 @@ def test_unforced_tool_choice_unaffected_on_fable_5_1_converse(local_model_cost_
assert result == ({"auto": {}} if tool_choice == "auto" else None)
@pytest.mark.parametrize(
"model",
["anthropic.claude-fable-5-1", "us.anthropic.claude-fable-5-1"],
)
def test_response_format_avoids_native_and_forced_tool_choice_on_fable_5_1_converse(
local_model_cost_map, model
):
"""Regression: Bedrock rejects both ``outputConfig`` structured output and forced
tool_choice for Fable 5.1, so response_format must map to a tool without a forced
tool_choice."""
config = AmazonConverseConfig()
result = config.map_openai_params(
non_default_params={
"response_format": {
"type": "json_schema",
"json_schema": {
"name": "test_schema",
"schema": {"type": "object", "properties": {"result": {"type": "string"}}},
},
}
},
optional_params={},
model=model,
drop_params=False,
)
assert "outputConfig" not in result
assert "tools" in result
assert "tool_choice" not in result
assert result.get("json_mode") is True
def test_forced_tool_choice_forwarded_on_converse_models_that_support_it(
local_model_cost_map, monkeypatch
):

25
tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py
View file

@ -727,3 +727,28 @@ def test_sanitize_strips_effort_for_haiku_45():
data = {"output_config": {"effort": "high"}}
sanitize_vertex_anthropic_output_params(data, "vertex_ai/claude-opus-4-6")
assert data["output_config"] == {"effort": "high"}
def test_vertex_ai_fable_5_1_response_format_uses_native_output_format(local_model_cost_map):
"""Regression: Fable 5.1 rejects forced tool use, so the vertex map entry
advertises native structured output and ``response_format`` must map to
``output_format`` instead of the tool-based path's forced tool_choice."""
config = VertexAIAnthropicConfig()
response_format = {
"type": "json_schema",
"json_schema": {
"name": "test_schema",
"schema": {"type": "object", "properties": {"result": {"type": "string"}}},
},
}
result_params = config.map_openai_params(
non_default_params={"response_format": response_format},
optional_params={},
model="claude-fable-5-1",
drop_params=False,
)
assert "output_format" in result_params
assert "tool_choice" not in result_params
assert "tools" not in result_params