From aba027beed9aab1d2923120f7bef8ebc7adc33c3 Mon Sep 17 00:00:00 2001 From: Nicholas Gigliotti Date: Mon, 16 Mar 2026 19:34:17 -0400 Subject: [PATCH] Remove native structured output flag from models broken on Bedrock Integration testing confirmed gemma-3 (4b/12b/27b) ignores the JSON schema and returns free text, and nemotron-nano (9b/12b) errors with "Tool calling is not supported in streaming mode" even on sync calls. Remove the flag so these models fall back to the tool-call approach. Also fix test assertions to match (nemotron-nano-3-30b is supported, gemma-3 and nemotron-nano-12b are not). --- .../model_prices_and_context_window_backup.json | 15 +++++---------- model_prices_and_context_window.json | 15 +++++---------- .../bedrock/chat/test_converse_transformation.py | 8 +++++--- 3 files changed, 15 insertions(+), 23 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 13df25c1425..77ba0954b0d 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -16716,8 +16716,7 @@ "mode": "chat", "output_cost_per_token": 2.9e-07, "supports_system_messages": true, - "supports_vision": true, - "supports_native_structured_output": true + "supports_vision": true }, "google.gemma-3-27b-it": { "input_cost_per_token": 2.3e-07, @@ -16728,8 +16727,7 @@ "mode": "chat", "output_cost_per_token": 3.8e-07, "supports_system_messages": true, - "supports_vision": true, - "supports_native_structured_output": true + "supports_vision": true }, "google.gemma-3-4b-it": { "input_cost_per_token": 4e-08, @@ -16740,8 +16738,7 @@ "mode": "chat", "output_cost_per_token": 8e-08, "supports_system_messages": true, - "supports_vision": true, - "supports_native_structured_output": true + "supports_vision": true }, "google_pse/search": { "input_cost_per_query": 0.005, @@ -23089,8 +23086,7 @@ "mode": "chat", "output_cost_per_token": 6e-07, "supports_system_messages": true, - "supports_vision": true, - "supports_native_structured_output": true + "supports_vision": true }, "nvidia.nemotron-nano-9b-v2": { "input_cost_per_token": 6e-08, @@ -23100,8 +23096,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.3e-07, - "supports_system_messages": true, - "supports_native_structured_output": true + "supports_system_messages": true }, "nvidia.nemotron-nano-3-30b": { "input_cost_per_token": 6e-08, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 13df25c1425..77ba0954b0d 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -16716,8 +16716,7 @@ "mode": "chat", "output_cost_per_token": 2.9e-07, "supports_system_messages": true, - "supports_vision": true, - "supports_native_structured_output": true + "supports_vision": true }, "google.gemma-3-27b-it": { "input_cost_per_token": 2.3e-07, @@ -16728,8 +16727,7 @@ "mode": "chat", "output_cost_per_token": 3.8e-07, "supports_system_messages": true, - "supports_vision": true, - "supports_native_structured_output": true + "supports_vision": true }, "google.gemma-3-4b-it": { "input_cost_per_token": 4e-08, @@ -16740,8 +16738,7 @@ "mode": "chat", "output_cost_per_token": 8e-08, "supports_system_messages": true, - "supports_vision": true, - "supports_native_structured_output": true + "supports_vision": true }, "google_pse/search": { "input_cost_per_query": 0.005, @@ -23089,8 +23086,7 @@ "mode": "chat", "output_cost_per_token": 6e-07, "supports_system_messages": true, - "supports_vision": true, - "supports_native_structured_output": true + "supports_vision": true }, "nvidia.nemotron-nano-9b-v2": { "input_cost_per_token": 6e-08, @@ -23100,8 +23096,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.3e-07, - "supports_system_messages": true, - "supports_native_structured_output": true + "supports_system_messages": true }, "nvidia.nemotron-nano-3-30b": { "input_cost_per_token": 6e-08, diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index bf957567a99..c9fc0c39d67 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -2664,19 +2664,21 @@ def test_supports_native_structured_outputs(): # Non-Anthropic models assert config._supports_native_structured_outputs("qwen.qwen3-235b-a22b-2507-v1:0") assert config._supports_native_structured_outputs("mistral.mistral-large-3-675b-instruct") - assert config._supports_native_structured_outputs("google.gemma-3-27b-it") assert config._supports_native_structured_outputs("minimax.minimax-m2") assert config._supports_native_structured_outputs("moonshot.kimi-k2-thinking") - assert config._supports_native_structured_outputs("nvidia.nemotron-nano-12b-v2") + assert config._supports_native_structured_outputs("nvidia.nemotron-nano-3-30b") # Unsupported models -- should fall back to tool-call approach assert not config._supports_native_structured_outputs("anthropic.claude-3-5-sonnet-20241022-v2:0") assert not config._supports_native_structured_outputs("anthropic.claude-sonnet-4-20250514-v1:0") assert not config._supports_native_structured_outputs("meta.llama3-3-70b-instruct-v1:0") assert not config._supports_native_structured_outputs("amazon.nova-pro-v1:0") - # Excluded despite AWS listing them: broken constrained decoding on Bedrock + # Excluded: broken constrained decoding on Bedrock assert not config._supports_native_structured_outputs("openai.gpt-oss-120b-1:0") assert not config._supports_native_structured_outputs("mistral.magistral-small-2509") + # Excluded: ignores schema or broken on Bedrock + assert not config._supports_native_structured_outputs("google.gemma-3-27b-it") + assert not config._supports_native_structured_outputs("nvidia.nemotron-nano-12b-v2") def test_create_output_config_for_response_format():