mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-24 00:52:24 +00:00
fix(bedrock): send json_schema as a forced tool on Claude Opus 4.7 and 4.8 Converse (#42644)
* fix(bedrock): send json_schema as a forced tool on Claude Opus 4.7 and 4.8 Converse Bedrock rejects outputConfig.textFormat on Opus 4.7 and 4.8 with "output_config.format: Extra inputs are not permitted", and the AWS model cards list structured outputs as not supported for both, so their cost-map entries no longer claim supports_native_structured_output and json_schema requests fall back to the json_tool_call tool. Fixes #27846 * test(bedrock): assert Opus 4.7 and 4.8 inline the schema on Invoke, move the native case to Sonnet 4.6 --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot]
GitHub
parent
975bd28549
commit
5c24802fbd
4 changed files with 138 additions and 38 deletions
|
|
@ -1255,7 +1255,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
|
|
@ -1309,7 +1309,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
|
|
@ -1347,7 +1347,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
|
|
@ -1385,7 +1385,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
|
|
@ -1422,7 +1422,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
|
|
@ -2231,7 +2231,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
|
|
@ -2270,7 +2270,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
|
|
@ -2309,7 +2309,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
|
|
@ -2348,7 +2348,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
|
|
@ -2386,7 +2386,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
|
|
@ -2424,7 +2424,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
|
|
@ -2460,7 +2460,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_minimal_reasoning_effort": true,
|
||||
"supports_parallel_tool_use_config": true,
|
||||
|
|
@ -44760,7 +44760,7 @@
|
|||
"supports_function_calling": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_mid_conversation_system": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_output_config": true,
|
||||
"supports_parallel_tool_use_config": true,
|
||||
"supports_pdf_input": true,
|
||||
|
|
@ -61334,7 +61334,7 @@
|
|||
"supports_function_calling": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_mid_conversation_system": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_output_config": true,
|
||||
"supports_parallel_tool_use_config": true,
|
||||
"supports_pdf_input": true,
|
||||
|
|
@ -61588,7 +61588,7 @@
|
|||
"supports_function_calling": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_mid_conversation_system": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_output_config": true,
|
||||
"supports_parallel_tool_use_config": true,
|
||||
"supports_pdf_input": true,
|
||||
|
|
|
|||
|
|
@ -1255,7 +1255,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
|
|
@ -1309,7 +1309,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
|
|
@ -1347,7 +1347,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
|
|
@ -1385,7 +1385,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
|
|
@ -1422,7 +1422,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
|
|
@ -2231,7 +2231,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
|
|
@ -2270,7 +2270,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
|
|
@ -2309,7 +2309,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
|
|
@ -2348,7 +2348,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
|
|
@ -2386,7 +2386,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
|
|
@ -2424,7 +2424,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
|
|
@ -2460,7 +2460,7 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_minimal_reasoning_effort": true,
|
||||
"supports_parallel_tool_use_config": true,
|
||||
|
|
@ -44760,7 +44760,7 @@
|
|||
"supports_function_calling": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_mid_conversation_system": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_output_config": true,
|
||||
"supports_parallel_tool_use_config": true,
|
||||
"supports_pdf_input": true,
|
||||
|
|
@ -61334,7 +61334,7 @@
|
|||
"supports_function_calling": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_mid_conversation_system": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_output_config": true,
|
||||
"supports_parallel_tool_use_config": true,
|
||||
"supports_pdf_input": true,
|
||||
|
|
@ -61588,7 +61588,7 @@
|
|||
"supports_function_calling": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_mid_conversation_system": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_output_config": true,
|
||||
"supports_parallel_tool_use_config": true,
|
||||
"supports_pdf_input": true,
|
||||
|
|
|
|||
|
|
@ -801,13 +801,13 @@ def test_output_config_format_translated_to_native_output_config_converse():
|
|||
}
|
||||
|
||||
result = config._transform_request(
|
||||
model="bedrock/converse/us.anthropic.claude-opus-4-7",
|
||||
model="bedrock/converse/us.anthropic.claude-sonnet-4-6",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
optional_params={
|
||||
"maxTokens": 256,
|
||||
"thinking": {"type": "adaptive"},
|
||||
"output_config": {
|
||||
"effort": "xhigh",
|
||||
"effort": "max",
|
||||
"format": {"type": "json_schema", "schema": schema},
|
||||
},
|
||||
},
|
||||
|
|
@ -816,7 +816,7 @@ def test_output_config_format_translated_to_native_output_config_converse():
|
|||
)
|
||||
|
||||
additional = result.get("additionalModelRequestFields", {})
|
||||
assert additional.get("output_config") == {"effort": "xhigh"}
|
||||
assert additional.get("output_config") == {"effort": "max"}
|
||||
assert "format" not in additional["output_config"]
|
||||
assert result["outputConfig"]["textFormat"]["type"] == "json_schema"
|
||||
parsed_schema = json.loads(
|
||||
|
|
@ -4291,6 +4291,84 @@ def test_translate_response_format_native_output_config(monkeypatch):
|
|||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", old_env)
|
||||
|
||||
|
||||
BEDROCK_OPUS_4_7_AND_4_8_MODELS: Final = (
|
||||
"anthropic.claude-opus-4-7",
|
||||
"global.anthropic.claude-opus-4-7",
|
||||
"us.anthropic.claude-opus-4-7",
|
||||
"eu.anthropic.claude-opus-4-7",
|
||||
"au.anthropic.claude-opus-4-7",
|
||||
"jp.anthropic.claude-opus-4-7",
|
||||
"anthropic.claude-opus-4-8",
|
||||
"global.anthropic.claude-opus-4-8",
|
||||
"us.anthropic.claude-opus-4-8",
|
||||
"eu.anthropic.claude-opus-4-8",
|
||||
"au.anthropic.claude-opus-4-8",
|
||||
"jp.anthropic.claude-opus-4-8",
|
||||
"us-gov.anthropic.claude-opus-4-8",
|
||||
"us-gov-west-1/anthropic.claude-opus-4-8",
|
||||
"us-gov-east-1/anthropic.claude-opus-4-8",
|
||||
)
|
||||
|
||||
CAPITAL_RESPONSE_FORMAT: Final = {
|
||||
"type": "json_schema",
|
||||
"json_schema": {
|
||||
"name": "capital",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}, "country": {"type": "string"}},
|
||||
"required": ["city", "country"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _converse_request_for_json_schema(model: str, stream: bool) -> tuple[dict, dict]:
|
||||
config = AmazonConverseConfig()
|
||||
optional_params = config.map_openai_params(
|
||||
non_default_params={"response_format": CAPITAL_RESPONSE_FORMAT, "stream": stream},
|
||||
optional_params={},
|
||||
model=model,
|
||||
drop_params=False,
|
||||
)
|
||||
request = config._transform_request(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "Name the capital of France."}],
|
||||
optional_params=optional_params,
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
return optional_params, request
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", BEDROCK_OPUS_4_7_AND_4_8_MODELS)
|
||||
@pytest.mark.parametrize("stream", [False, True])
|
||||
def test_opus_4_7_and_4_8_json_schema_sent_as_forced_tool_not_output_config(monkeypatch, model, stream):
|
||||
"""Regression for issue #27846: Bedrock rejects outputConfig on Opus 4.7 and 4.8
|
||||
(``output_config.format: Extra inputs are not permitted``), so json_schema has to
|
||||
go out as the forced json_tool_call tool, streamed through fake_stream."""
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
|
||||
|
||||
optional_params, request = _converse_request_for_json_schema(model=model, stream=stream)
|
||||
|
||||
assert "outputConfig" not in request
|
||||
assert [tool["toolSpec"]["name"] for tool in request["toolConfig"]["tools"]] == ["json_tool_call"]
|
||||
assert request["toolConfig"]["toolChoice"] == {"tool": {"name": "json_tool_call"}}
|
||||
assert optional_params.get("fake_stream", False) is stream
|
||||
|
||||
|
||||
def test_sonnet_4_6_json_schema_still_uses_native_output_config(monkeypatch):
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
|
||||
|
||||
optional_params, request = _converse_request_for_json_schema(model="us.anthropic.claude-sonnet-4-6", stream=True)
|
||||
|
||||
assert request["outputConfig"]["textFormat"]["structure"]["jsonSchema"]["name"] == "capital"
|
||||
assert "toolConfig" not in request
|
||||
assert "fake_stream" not in optional_params
|
||||
|
||||
|
||||
def test_translate_response_format_fallback_tool_call():
|
||||
"""For unsupported models, should fall back to tool-call approach."""
|
||||
config = AmazonConverseConfig()
|
||||
|
|
|
|||
|
|
@ -541,26 +541,48 @@ def test_output_config_format_converted_for_bedrock_chat_invoke_request():
|
|||
assert json.loads(last_content[-1]["text"]) == schema
|
||||
|
||||
|
||||
def test_output_config_format_forwarded_for_bedrock_chat_invoke_request():
|
||||
@pytest.mark.parametrize("model", ["anthropic.claude-opus-4-7", "us.anthropic.claude-opus-4-8"])
|
||||
def test_output_config_format_inlined_for_bedrock_chat_invoke_opus_4_7_and_4_8(local_model_cost_map, model):
|
||||
"""Bedrock rejects ``output_config.format`` on Claude Opus 4.7 and 4.8, so the
|
||||
Invoke chat path inlines the schema into the last user message and keeps effort,
|
||||
driven by the cost map alone (no capability stub)."""
|
||||
schema = {"type": "object", "properties": {"answer": {"type": "string"}}}
|
||||
|
||||
result = AmazonAnthropicClaudeConfig().transform_request(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "test"}],
|
||||
optional_params={
|
||||
"max_tokens": 100,
|
||||
"output_config": {"effort": "xhigh", "format": {"type": "json_schema", "schema": schema}},
|
||||
},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert result.get("output_config") == {"effort": "xhigh"}
|
||||
assert json.loads(result["messages"][-1]["content"][-1]["text"]) == schema
|
||||
|
||||
|
||||
def test_output_config_format_forwarded_for_bedrock_chat_invoke_request(local_model_cost_map):
|
||||
"""Bedrock Invoke chat path forwards ``output_config.format`` alongside effort
|
||||
for models with native structured-output support (Claude Opus 4.7)."""
|
||||
for models with native structured-output support (Claude Sonnet 4.6)."""
|
||||
schema_format = {
|
||||
"type": "json_schema",
|
||||
"schema": {"type": "object", "properties": {"answer": {"type": "string"}}},
|
||||
}
|
||||
|
||||
result = AmazonAnthropicClaudeConfig().transform_request(
|
||||
model="anthropic.claude-opus-4-7",
|
||||
model="us.anthropic.claude-sonnet-4-6",
|
||||
messages=[{"role": "user", "content": "test"}],
|
||||
optional_params={
|
||||
"max_tokens": 100,
|
||||
"output_config": {"effort": "xhigh", "format": schema_format},
|
||||
"output_config": {"effort": "max", "format": schema_format},
|
||||
},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert result.get("output_config") == {"effort": "xhigh", "format": schema_format}
|
||||
assert result.get("output_config") == {"effort": "max", "format": schema_format}
|
||||
assert "answer" not in json.dumps(result["messages"])
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue