scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-10 22:41:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 129

Remove native structured output flag from models broken on Bedrock

Browse source 
Integration testing confirmed gemma-3 (4b/12b/27b) ignores the JSON
schema and returns free text, and nemotron-nano (9b/12b) errors with
"Tool calling is not supported in streaming mode" even on sync calls.
Remove the flag so these models fall back to the tool-call approach.
Also fix test assertions to match (nemotron-nano-3-30b is supported,
gemma-3 and nemotron-nano-12b are not).
This commit is contained in:
Nicholas Gigliotti 2026-03-16 19:34:17 -04:00
parent cb66672017
commit aba027beed
3 changed files with 15 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
litellm/model_prices_and_context_window_backup.json
View file

@ -16716,8 +16716,7 @@
"mode": "chat",
"output_cost_per_token": 2.9e-07,
"supports_system_messages": true,
"supports_vision": true,
"supports_native_structured_output": true
"supports_vision": true
},
"google.gemma-3-27b-it": {
"input_cost_per_token": 2.3e-07,
@ -16728,8 +16727,7 @@
"mode": "chat",
"output_cost_per_token": 3.8e-07,
"supports_system_messages": true,
"supports_vision": true,
"supports_native_structured_output": true
"supports_vision": true
},
"google.gemma-3-4b-it": {
"input_cost_per_token": 4e-08,
@ -16740,8 +16738,7 @@
"mode": "chat",
"output_cost_per_token": 8e-08,
"supports_system_messages": true,
"supports_vision": true,
"supports_native_structured_output": true
"supports_vision": true
},
"google_pse/search": {
"input_cost_per_query": 0.005,
@ -23089,8 +23086,7 @@
"mode": "chat",
"output_cost_per_token": 6e-07,
"supports_system_messages": true,
"supports_vision": true,
"supports_native_structured_output": true
"supports_vision": true
},
"nvidia.nemotron-nano-9b-v2": {
"input_cost_per_token": 6e-08,
@ -23100,8 +23096,7 @@
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 2.3e-07,
"supports_system_messages": true,
"supports_native_structured_output": true
"supports_system_messages": true
},
"nvidia.nemotron-nano-3-30b": {
"input_cost_per_token": 6e-08,

15
model_prices_and_context_window.json
View file

@ -16716,8 +16716,7 @@
"mode": "chat",
"output_cost_per_token": 2.9e-07,
"supports_system_messages": true,
"supports_vision": true,
"supports_native_structured_output": true
"supports_vision": true
},
"google.gemma-3-27b-it": {
"input_cost_per_token": 2.3e-07,
@ -16728,8 +16727,7 @@
"mode": "chat",
"output_cost_per_token": 3.8e-07,
"supports_system_messages": true,
"supports_vision": true,
"supports_native_structured_output": true
"supports_vision": true
},
"google.gemma-3-4b-it": {
"input_cost_per_token": 4e-08,
@ -16740,8 +16738,7 @@
"mode": "chat",
"output_cost_per_token": 8e-08,
"supports_system_messages": true,
"supports_vision": true,
"supports_native_structured_output": true
"supports_vision": true
},
"google_pse/search": {
"input_cost_per_query": 0.005,
@ -23089,8 +23086,7 @@
"mode": "chat",
"output_cost_per_token": 6e-07,
"supports_system_messages": true,
"supports_vision": true,
"supports_native_structured_output": true
"supports_vision": true
},
"nvidia.nemotron-nano-9b-v2": {
"input_cost_per_token": 6e-08,
@ -23100,8 +23096,7 @@
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 2.3e-07,
"supports_system_messages": true,
"supports_native_structured_output": true
"supports_system_messages": true
},
"nvidia.nemotron-nano-3-30b": {
"input_cost_per_token": 6e-08,

8
tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py
View file

@ -2664,19 +2664,21 @@ def test_supports_native_structured_outputs():
# Non-Anthropic models
assert config._supports_native_structured_outputs("qwen.qwen3-235b-a22b-2507-v1:0")
assert config._supports_native_structured_outputs("mistral.mistral-large-3-675b-instruct")
assert config._supports_native_structured_outputs("google.gemma-3-27b-it")
assert config._supports_native_structured_outputs("minimax.minimax-m2")
assert config._supports_native_structured_outputs("moonshot.kimi-k2-thinking")
assert config._supports_native_structured_outputs("nvidia.nemotron-nano-12b-v2")
assert config._supports_native_structured_outputs("nvidia.nemotron-nano-3-30b")
# Unsupported models -- should fall back to tool-call approach
assert not config._supports_native_structured_outputs("anthropic.claude-3-5-sonnet-20241022-v2:0")
assert not config._supports_native_structured_outputs("anthropic.claude-sonnet-4-20250514-v1:0")
assert not config._supports_native_structured_outputs("meta.llama3-3-70b-instruct-v1:0")
assert not config._supports_native_structured_outputs("amazon.nova-pro-v1:0")
# Excluded despite AWS listing them: broken constrained decoding on Bedrock
# Excluded: broken constrained decoding on Bedrock
assert not config._supports_native_structured_outputs("openai.gpt-oss-120b-1:0")
assert not config._supports_native_structured_outputs("mistral.magistral-small-2509")
# Excluded: ignores schema or broken on Bedrock
assert not config._supports_native_structured_outputs("google.gemma-3-27b-it")
assert not config._supports_native_structured_outputs("nvidia.nemotron-nano-12b-v2")
def test_create_output_config_for_response_format():