From 5c24802fbdbe282fdf5d2415b1d155f553160bf0 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 20:09:21 -0700 Subject: [PATCH] fix(bedrock): send json_schema as a forced tool on Claude Opus 4.7 and 4.8 Converse (#42644) * fix(bedrock): send json_schema as a forced tool on Claude Opus 4.7 and 4.8 Converse Bedrock rejects outputConfig.textFormat on Opus 4.7 and 4.8 with "output_config.format: Extra inputs are not permitted", and the AWS model cards list structured outputs as not supported for both, so their cost-map entries no longer claim supports_native_structured_output and json_schema requests fall back to the json_tool_call tool. Fixes #27846 * test(bedrock): assert Opus 4.7 and 4.8 inline the schema on Invoke, move the native case to Sonnet 4.6 --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 30 +++---- model_prices_and_context_window.json | 30 +++---- .../chat/test_converse_transformation.py | 84 ++++++++++++++++++- ...ations_anthropic_claude3_transformation.py | 32 +++++-- 4 files changed, 138 insertions(+), 38 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 0e8d43ae4d8..7e0e246d4b2 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1255,7 +1255,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -1309,7 +1309,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -1347,7 +1347,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -1385,7 +1385,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -1422,7 +1422,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -2231,7 +2231,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -2270,7 +2270,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -2309,7 +2309,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -2348,7 +2348,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -2386,7 +2386,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -2424,7 +2424,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -2460,7 +2460,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_minimal_reasoning_effort": true, "supports_parallel_tool_use_config": true, @@ -44760,7 +44760,7 @@ "supports_function_calling": true, "supports_max_reasoning_effort": true, "supports_mid_conversation_system": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_output_config": true, "supports_parallel_tool_use_config": true, "supports_pdf_input": true, @@ -61334,7 +61334,7 @@ "supports_function_calling": true, "supports_max_reasoning_effort": true, "supports_mid_conversation_system": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_output_config": true, "supports_parallel_tool_use_config": true, "supports_pdf_input": true, @@ -61588,7 +61588,7 @@ "supports_function_calling": true, "supports_max_reasoning_effort": true, "supports_mid_conversation_system": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_output_config": true, "supports_parallel_tool_use_config": true, "supports_pdf_input": true, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 0e8d43ae4d8..7e0e246d4b2 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1255,7 +1255,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -1309,7 +1309,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -1347,7 +1347,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -1385,7 +1385,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -1422,7 +1422,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -2231,7 +2231,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -2270,7 +2270,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -2309,7 +2309,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -2348,7 +2348,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -2386,7 +2386,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -2424,7 +2424,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh", @@ -2460,7 +2460,7 @@ "supports_tool_choice": true, "supports_vision": true, "supports_xhigh_reasoning_effort": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_max_reasoning_effort": true, "supports_minimal_reasoning_effort": true, "supports_parallel_tool_use_config": true, @@ -44760,7 +44760,7 @@ "supports_function_calling": true, "supports_max_reasoning_effort": true, "supports_mid_conversation_system": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_output_config": true, "supports_parallel_tool_use_config": true, "supports_pdf_input": true, @@ -61334,7 +61334,7 @@ "supports_function_calling": true, "supports_max_reasoning_effort": true, "supports_mid_conversation_system": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_output_config": true, "supports_parallel_tool_use_config": true, "supports_pdf_input": true, @@ -61588,7 +61588,7 @@ "supports_function_calling": true, "supports_max_reasoning_effort": true, "supports_mid_conversation_system": true, - "supports_native_structured_output": true, + "supports_native_structured_output": false, "supports_output_config": true, "supports_parallel_tool_use_config": true, "supports_pdf_input": true, diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index 799962eea3a..9d012cce2d4 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -801,13 +801,13 @@ def test_output_config_format_translated_to_native_output_config_converse(): } result = config._transform_request( - model="bedrock/converse/us.anthropic.claude-opus-4-7", + model="bedrock/converse/us.anthropic.claude-sonnet-4-6", messages=[{"role": "user", "content": "hi"}], optional_params={ "maxTokens": 256, "thinking": {"type": "adaptive"}, "output_config": { - "effort": "xhigh", + "effort": "max", "format": {"type": "json_schema", "schema": schema}, }, }, @@ -816,7 +816,7 @@ def test_output_config_format_translated_to_native_output_config_converse(): ) additional = result.get("additionalModelRequestFields", {}) - assert additional.get("output_config") == {"effort": "xhigh"} + assert additional.get("output_config") == {"effort": "max"} assert "format" not in additional["output_config"] assert result["outputConfig"]["textFormat"]["type"] == "json_schema" parsed_schema = json.loads( @@ -4291,6 +4291,84 @@ def test_translate_response_format_native_output_config(monkeypatch): monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", old_env) +BEDROCK_OPUS_4_7_AND_4_8_MODELS: Final = ( + "anthropic.claude-opus-4-7", + "global.anthropic.claude-opus-4-7", + "us.anthropic.claude-opus-4-7", + "eu.anthropic.claude-opus-4-7", + "au.anthropic.claude-opus-4-7", + "jp.anthropic.claude-opus-4-7", + "anthropic.claude-opus-4-8", + "global.anthropic.claude-opus-4-8", + "us.anthropic.claude-opus-4-8", + "eu.anthropic.claude-opus-4-8", + "au.anthropic.claude-opus-4-8", + "jp.anthropic.claude-opus-4-8", + "us-gov.anthropic.claude-opus-4-8", + "us-gov-west-1/anthropic.claude-opus-4-8", + "us-gov-east-1/anthropic.claude-opus-4-8", +) + +CAPITAL_RESPONSE_FORMAT: Final = { + "type": "json_schema", + "json_schema": { + "name": "capital", + "schema": { + "type": "object", + "properties": {"city": {"type": "string"}, "country": {"type": "string"}}, + "required": ["city", "country"], + "additionalProperties": False, + }, + }, +} + + +def _converse_request_for_json_schema(model: str, stream: bool) -> tuple[dict, dict]: + config = AmazonConverseConfig() + optional_params = config.map_openai_params( + non_default_params={"response_format": CAPITAL_RESPONSE_FORMAT, "stream": stream}, + optional_params={}, + model=model, + drop_params=False, + ) + request = config._transform_request( + model=model, + messages=[{"role": "user", "content": "Name the capital of France."}], + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + return optional_params, request + + +@pytest.mark.parametrize("model", BEDROCK_OPUS_4_7_AND_4_8_MODELS) +@pytest.mark.parametrize("stream", [False, True]) +def test_opus_4_7_and_4_8_json_schema_sent_as_forced_tool_not_output_config(monkeypatch, model, stream): + """Regression for issue #27846: Bedrock rejects outputConfig on Opus 4.7 and 4.8 + (``output_config.format: Extra inputs are not permitted``), so json_schema has to + go out as the forced json_tool_call tool, streamed through fake_stream.""" + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + + optional_params, request = _converse_request_for_json_schema(model=model, stream=stream) + + assert "outputConfig" not in request + assert [tool["toolSpec"]["name"] for tool in request["toolConfig"]["tools"]] == ["json_tool_call"] + assert request["toolConfig"]["toolChoice"] == {"tool": {"name": "json_tool_call"}} + assert optional_params.get("fake_stream", False) is stream + + +def test_sonnet_4_6_json_schema_still_uses_native_output_config(monkeypatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + + optional_params, request = _converse_request_for_json_schema(model="us.anthropic.claude-sonnet-4-6", stream=True) + + assert request["outputConfig"]["textFormat"]["structure"]["jsonSchema"]["name"] == "capital" + assert "toolConfig" not in request + assert "fake_stream" not in optional_params + + def test_translate_response_format_fallback_tool_call(): """For unsupported models, should fall back to tool-call approach.""" config = AmazonConverseConfig() diff --git a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py index cf2fd78a896..3846a94c9fe 100644 --- a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py +++ b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py @@ -541,26 +541,48 @@ def test_output_config_format_converted_for_bedrock_chat_invoke_request(): assert json.loads(last_content[-1]["text"]) == schema -def test_output_config_format_forwarded_for_bedrock_chat_invoke_request(): +@pytest.mark.parametrize("model", ["anthropic.claude-opus-4-7", "us.anthropic.claude-opus-4-8"]) +def test_output_config_format_inlined_for_bedrock_chat_invoke_opus_4_7_and_4_8(local_model_cost_map, model): + """Bedrock rejects ``output_config.format`` on Claude Opus 4.7 and 4.8, so the + Invoke chat path inlines the schema into the last user message and keeps effort, + driven by the cost map alone (no capability stub).""" + schema = {"type": "object", "properties": {"answer": {"type": "string"}}} + + result = AmazonAnthropicClaudeConfig().transform_request( + model=model, + messages=[{"role": "user", "content": "test"}], + optional_params={ + "max_tokens": 100, + "output_config": {"effort": "xhigh", "format": {"type": "json_schema", "schema": schema}}, + }, + litellm_params={}, + headers={}, + ) + + assert result.get("output_config") == {"effort": "xhigh"} + assert json.loads(result["messages"][-1]["content"][-1]["text"]) == schema + + +def test_output_config_format_forwarded_for_bedrock_chat_invoke_request(local_model_cost_map): """Bedrock Invoke chat path forwards ``output_config.format`` alongside effort - for models with native structured-output support (Claude Opus 4.7).""" + for models with native structured-output support (Claude Sonnet 4.6).""" schema_format = { "type": "json_schema", "schema": {"type": "object", "properties": {"answer": {"type": "string"}}}, } result = AmazonAnthropicClaudeConfig().transform_request( - model="anthropic.claude-opus-4-7", + model="us.anthropic.claude-sonnet-4-6", messages=[{"role": "user", "content": "test"}], optional_params={ "max_tokens": 100, - "output_config": {"effort": "xhigh", "format": schema_format}, + "output_config": {"effort": "max", "format": schema_format}, }, litellm_params={}, headers={}, ) - assert result.get("output_config") == {"effort": "xhigh", "format": schema_format} + assert result.get("output_config") == {"effort": "max", "format": schema_format} assert "answer" not in json.dumps(result["messages"])