From 8c0d74eab64fd34493025e11e051e9d79b4d5e45 Mon Sep 17 00:00:00 2001 From: Mohamed EL-Habib Date: Sat, 23 May 2026 18:02:42 +0200 Subject: [PATCH] fix(model-registry): add gpt-5.1-mini/nano and fix azure/gpt-5.4 family reasoning effort flags - Add gpt-5.1-mini and gpt-5.1-nano entries with supports_none_reasoning_effort=true, matching the gpt-5.1 family contract (xhigh=false, minimal=true). These models were absent from the registry causing a spurious 400 UnsupportedParamsError for any temperature != 1 even when reasoning_effort was omitted. - Fix azure/gpt-5.4 and azure/gpt-5.4-2026-03-05: missing all three flags, now set to none=true/xhigh=true/minimal=true (aligned with azure_ai/gpt-5.4). - Fix azure/gpt-5.4-mini, azure/gpt-5.4-mini-2026-03-17, azure/gpt-5.4-nano, azure/gpt-5.4-nano-2026-03-17: had none=false/xhigh=false explicitly, now none=true/xhigh=true/minimal=false (aligned with azure_ai counterparts). - Mirror all changes to model_prices_and_context_window_backup.json. Fixes: https://github.com/BerriAI/litellm/issues/27351 Co-Authored-By: Claude Sonnet 4.6 --- ...odel_prices_and_context_window_backup.json | 108 ++++++++++++++-- model_prices_and_context_window.json | 108 ++++++++++++++-- .../chat/test_azure_gpt5_transformation.py | 121 ++++++++++++++++++ .../llms/openai/test_gpt5_transformation.py | 84 ++++++++++++ 4 files changed, 401 insertions(+), 20 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 6a4a5dd6a03..2788ba69dab 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -4971,7 +4971,10 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_service_tier": true, - "supports_vision": true + "supports_vision": true, + "supports_none_reasoning_effort": true, + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": true }, "azure/gpt-5.4-2026-03-05": { "cache_read_input_token_cost": 2.5e-07, @@ -5013,7 +5016,10 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_service_tier": true, - "supports_vision": true + "supports_vision": true, + "supports_none_reasoning_effort": true, + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": true }, "azure/gpt-5.4-pro": { "cache_read_input_token_cost": 3e-06, @@ -5281,8 +5287,9 @@ "supports_service_tier": true, "supports_vision": true, "supports_web_search": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false + "supports_none_reasoning_effort": true, + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": false }, "azure/gpt-5.4-mini-2026-03-17": { "cache_read_input_token_cost": 7.5e-08, @@ -5317,8 +5324,9 @@ "supports_service_tier": true, "supports_vision": true, "supports_web_search": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false + "supports_none_reasoning_effort": true, + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": false }, "azure/gpt-5.4-nano": { "cache_read_input_token_cost": 2e-08, @@ -5353,8 +5361,9 @@ "supports_service_tier": true, "supports_vision": true, "supports_web_search": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false + "supports_none_reasoning_effort": true, + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": false }, "azure/gpt-5.4-nano-2026-03-17": { "cache_read_input_token_cost": 2e-08, @@ -5389,8 +5398,9 @@ "supports_service_tier": true, "supports_vision": true, "supports_web_search": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false + "supports_none_reasoning_effort": true, + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": false }, "azure/gpt-image-1": { "cache_read_input_image_token_cost": 2.5e-06, @@ -20420,6 +20430,84 @@ "supports_xhigh_reasoning_effort": false, "supports_minimal_reasoning_effort": true }, + "gpt-5.1-mini": { + "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_priority": 2.5e-07, + "input_cost_per_token": 1.25e-06, + "input_cost_per_token_priority": 2.5e-06, + "litellm_provider": "openai", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "output_cost_per_token_priority": 2e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_service_tier": true, + "supports_vision": true, + "supports_web_search": true, + "supports_none_reasoning_effort": true, + "supports_xhigh_reasoning_effort": false, + "supports_minimal_reasoning_effort": true + }, + "gpt-5.1-nano": { + "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_priority": 2.5e-07, + "input_cost_per_token": 1.25e-06, + "input_cost_per_token_priority": 2.5e-06, + "litellm_provider": "openai", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "output_cost_per_token_priority": 2e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_service_tier": true, + "supports_vision": true, + "supports_web_search": true, + "supports_none_reasoning_effort": true, + "supports_xhigh_reasoning_effort": false, + "supports_minimal_reasoning_effort": true + }, "gpt-5.2": { "cache_read_input_token_cost": 1.75e-07, "cache_read_input_token_cost_priority": 3.5e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 2140493ec4a..d47fa9166aa 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -4984,7 +4984,10 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_service_tier": true, - "supports_vision": true + "supports_vision": true, + "supports_none_reasoning_effort": true, + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": true }, "azure/gpt-5.4-2026-03-05": { "cache_read_input_token_cost": 2.5e-07, @@ -5026,7 +5029,10 @@ "supports_system_messages": true, "supports_tool_choice": true, "supports_service_tier": true, - "supports_vision": true + "supports_vision": true, + "supports_none_reasoning_effort": true, + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": true }, "azure/gpt-5.4-pro": { "cache_read_input_token_cost": 3e-06, @@ -5294,8 +5300,9 @@ "supports_service_tier": true, "supports_vision": true, "supports_web_search": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false + "supports_none_reasoning_effort": true, + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": false }, "azure/gpt-5.4-mini-2026-03-17": { "cache_read_input_token_cost": 7.5e-08, @@ -5330,8 +5337,9 @@ "supports_service_tier": true, "supports_vision": true, "supports_web_search": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false + "supports_none_reasoning_effort": true, + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": false }, "azure/gpt-5.4-nano": { "cache_read_input_token_cost": 2e-08, @@ -5366,8 +5374,9 @@ "supports_service_tier": true, "supports_vision": true, "supports_web_search": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false + "supports_none_reasoning_effort": true, + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": false }, "azure/gpt-5.4-nano-2026-03-17": { "cache_read_input_token_cost": 2e-08, @@ -5402,8 +5411,9 @@ "supports_service_tier": true, "supports_vision": true, "supports_web_search": true, - "supports_none_reasoning_effort": false, - "supports_xhigh_reasoning_effort": false + "supports_none_reasoning_effort": true, + "supports_xhigh_reasoning_effort": true, + "supports_minimal_reasoning_effort": false }, "azure/gpt-image-1": { "cache_read_input_image_token_cost": 2.5e-06, @@ -20456,6 +20466,84 @@ "supports_xhigh_reasoning_effort": false, "supports_minimal_reasoning_effort": true }, + "gpt-5.1-mini": { + "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_priority": 2.5e-07, + "input_cost_per_token": 1.25e-06, + "input_cost_per_token_priority": 2.5e-06, + "litellm_provider": "openai", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "output_cost_per_token_priority": 2e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_service_tier": true, + "supports_vision": true, + "supports_web_search": true, + "supports_none_reasoning_effort": true, + "supports_xhigh_reasoning_effort": false, + "supports_minimal_reasoning_effort": true + }, + "gpt-5.1-nano": { + "cache_read_input_token_cost": 1.25e-07, + "cache_read_input_token_cost_priority": 2.5e-07, + "input_cost_per_token": 1.25e-06, + "input_cost_per_token_priority": 2.5e-06, + "litellm_provider": "openai", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 1e-05, + "output_cost_per_token_priority": 2e-05, + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_service_tier": true, + "supports_vision": true, + "supports_web_search": true, + "supports_none_reasoning_effort": true, + "supports_xhigh_reasoning_effort": false, + "supports_minimal_reasoning_effort": true + }, "gpt-5.2": { "cache_read_input_token_cost": 1.75e-07, "cache_read_input_token_cost_priority": 3.5e-07, diff --git a/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py b/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py index 83562331b9a..081a7048e75 100644 --- a/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py +++ b/tests/test_litellm/llms/azure/chat/test_azure_gpt5_transformation.py @@ -1,6 +1,7 @@ import pytest import litellm +from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map from litellm.llms.azure.chat.gpt_5_transformation import AzureOpenAIGPT5Config @@ -9,6 +10,15 @@ def config() -> AzureOpenAIGPT5Config: return AzureOpenAIGPT5Config() +@pytest.fixture(autouse=True) +def use_local_model_cost_map(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr( + litellm, "model_cost", get_model_cost_map(url=litellm.model_cost_map_url) + ) + litellm.add_known_models(model_cost_map=litellm.model_cost) + + def test_azure_gpt5_supports_reasoning_effort(config: AzureOpenAIGPT5Config): assert "reasoning_effort" in config.get_supported_openai_params(model="gpt-5") assert "reasoning_effort" in config.get_supported_openai_params( @@ -299,3 +309,114 @@ def test_azure_gpt5_1_does_not_support_logprobs(config: AzureOpenAIGPT5Config): supported_params = config.get_supported_openai_params(model="gpt-5.1") assert "logprobs" not in supported_params assert "top_logprobs" not in supported_params + + +# --------------------------------------------------------------------------- +# Regression: azure/gpt-5.4, azure/gpt-5.4-mini, azure/gpt-5.4-nano +# temperature support (issue #27351 — Azure variants) +# +# azure/gpt-5.4 was missing supports_none_reasoning_effort entirely. +# azure/gpt-5.4-mini and azure/gpt-5.4-nano had it explicitly set to False. +# All three must allow temperature when reasoning_effort defaults to "none". +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "model", + [ + "gpt-5.4", + "gpt-5.4-mini", + "gpt-5.4-nano", + "azure/gpt-5.4", + "azure/gpt-5.4-mini", + "azure/gpt-5.4-nano", + ], +) +def test_azure_gpt5_4_variants_support_none_reasoning_effort( + config: AzureOpenAIGPT5Config, model: str +): + """azure/gpt-5.4, azure/gpt-5.4-mini, azure/gpt-5.4-nano must all + have supports_none_reasoning_effort=True in the registry.""" + assert config._supports_reasoning_effort_level( + model, "none" + ), f"Expected {model!r} to support reasoning_effort='none'" + + +@pytest.mark.parametrize( + "model", + [ + "gpt-5.4", + "gpt-5.4-mini", + "gpt-5.4-nano", + "azure/gpt-5.4", + "azure/gpt-5.4-mini", + "azure/gpt-5.4-nano", + ], +) +def test_azure_gpt5_4_variants_allow_temperature_without_reasoning_effort( + config: AzureOpenAIGPT5Config, model: str +): + """azure/gpt-5.4, azure/gpt-5.4-mini, azure/gpt-5.4-nano must accept any + temperature when reasoning_effort is omitted (defaults to 'none'). + + Regression for https://github.com/BerriAI/litellm/issues/27351. + """ + for temp in [0.0, 0.7, 1.0, 1.5]: + params = config.map_openai_params( + non_default_params={"temperature": temp}, + optional_params={}, + model=model, + drop_params=False, + api_version="2025-01-01-preview", + ) + assert params["temperature"] == temp, ( + f"{model}: expected temperature={temp} to pass through, got {params}" + ) + + +@pytest.mark.parametrize( + "model", + [ + "gpt-5.4-mini", + "gpt-5.4-nano", + "azure/gpt-5.4-mini", + "azure/gpt-5.4-nano", + ], +) +def test_azure_gpt5_4_mini_nano_reject_reasoning_effort_minimal( + config: AzureOpenAIGPT5Config, model: str +): + """azure/gpt-5.4-mini and azure/gpt-5.4-nano do not support reasoning_effort='minimal'.""" + with pytest.raises(litellm.utils.UnsupportedParamsError): + config.map_openai_params( + non_default_params={"reasoning_effort": "minimal"}, + optional_params={}, + model=model, + drop_params=False, + api_version="2025-01-01-preview", + ) + + +@pytest.mark.parametrize( + "model", + [ + "gpt-5.4", + "gpt-5.4-mini", + "gpt-5.4-nano", + "azure/gpt-5.4", + "azure/gpt-5.4-mini", + "azure/gpt-5.4-nano", + ], +) +def test_azure_gpt5_4_variants_allow_reasoning_effort_xhigh( + config: AzureOpenAIGPT5Config, model: str +): + """azure/gpt-5.4, azure/gpt-5.4-mini, azure/gpt-5.4-nano support xhigh.""" + params = config.map_openai_params( + non_default_params={"reasoning_effort": "xhigh"}, + optional_params={}, + model=model, + drop_params=False, + api_version="2025-01-01-preview", + ) + assert params["reasoning_effort"] == "xhigh" diff --git a/tests/test_litellm/llms/openai/test_gpt5_transformation.py b/tests/test_litellm/llms/openai/test_gpt5_transformation.py index d279b119efe..cb7c39275da 100644 --- a/tests/test_litellm/llms/openai/test_gpt5_transformation.py +++ b/tests/test_litellm/llms/openai/test_gpt5_transformation.py @@ -1309,3 +1309,87 @@ def test_responses_gpt54_allow_temperature_effort_none( drop_params=False, ) assert params["temperature"] == 0.7 + + +# --------------------------------------------------------------------------- +# Regression: gpt-5.1-mini / gpt-5.1-nano temperature support (issue #27351) +# These tests confirm the bug: both models must allow temperature when +# reasoning_effort is omitted (defaults to "none" for the gpt-5.1 family). +# They fail before the registry entries are added and pass afterwards. +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("model", ["gpt-5.1-mini", "gpt-5.1-nano"]) +def test_gpt5_1_mini_nano_support_none_reasoning_effort( + gpt5_config: OpenAIGPT5Config, model: str +): + """gpt-5.1-mini and gpt-5.1-nano must have supports_none_reasoning_effort=True.""" + assert gpt5_config._supports_reasoning_effort_level( + model, "none" + ), f"Expected {model!r} to support reasoning_effort='none'" + + +@pytest.mark.parametrize("model", ["gpt-5.1-mini", "gpt-5.1-nano"]) +def test_gpt5_1_mini_nano_allow_temperature_without_reasoning_effort( + config: OpenAIConfig, model: str +): + """gpt-5.1-mini / gpt-5.1-nano must accept any temperature when reasoning_effort + is not specified (defaults to 'none' for the gpt-5.1 family). + + Regression for https://github.com/BerriAI/litellm/issues/27351 — models were + missing from the registry so supports_none_reasoning_effort resolved to False, + causing a spurious 400 UnsupportedParamsError. + """ + for temp in [0.0, 0.7, 1.0, 1.5]: + params = config.map_openai_params( + non_default_params={"temperature": temp}, + optional_params={}, + model=model, + drop_params=False, + ) + assert params["temperature"] == temp, ( + f"{model}: expected temperature={temp} to pass through, got {params}" + ) + + +@pytest.mark.parametrize("model", ["gpt-5.1-mini", "gpt-5.1-nano"]) +def test_gpt5_1_mini_nano_allow_temperature_with_reasoning_effort_none( + config: OpenAIConfig, model: str +): + """Explicit reasoning_effort='none' must also allow flexible temperature.""" + params = config.map_openai_params( + non_default_params={"temperature": 0.7, "reasoning_effort": "none"}, + optional_params={}, + model=model, + drop_params=False, + ) + assert params["temperature"] == 0.7 + assert params["reasoning_effort"] == "none" + + +@pytest.mark.parametrize("model", ["gpt-5.1-mini", "gpt-5.1-nano"]) +def test_gpt5_1_mini_nano_reject_temperature_with_high_reasoning_effort( + config: OpenAIConfig, model: str +): + """When reasoning_effort is non-none, only temperature=1 is allowed.""" + with pytest.raises(litellm.utils.UnsupportedParamsError): + config.map_openai_params( + non_default_params={"temperature": 0.7, "reasoning_effort": "high"}, + optional_params={}, + model=model, + drop_params=False, + ) + + +@pytest.mark.parametrize("model", ["gpt-5.1-mini", "gpt-5.1-nano"]) +def test_gpt5_1_mini_nano_provider_prefixed_allows_temperature( + config: OpenAIConfig, model: str +): + """openai/ prefix must be normalised before registry lookup.""" + params = config.map_openai_params( + non_default_params={"temperature": 0.7}, + optional_params={}, + model=f"openai/{model}", + drop_params=False, + ) + assert params["temperature"] == 0.7