fix(openai): drop or refuse an o-series reasoning_effort level the model map turns off

This commit is contained in:
mateo-berri 2026-11-07 17:55:59 -08:00
parent 83e2e5bf99
commit e2cf039509
6 changed files with 177 additions and 4 deletions

View file

@ -11,7 +11,7 @@ Translations handled by LiteLLM:
- Logprobs => drop param (if user opts in to dropping param)
"""
from collections.abc import Coroutine, Sequence
from collections.abc import Collection, Coroutine, Sequence
from typing import Final, Literal, cast, overload
import litellm
@ -19,12 +19,18 @@ from litellm import verbose_logger
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.types.llms.openai import AllMessageValues, ChatCompletionUserMessage
from litellm.utils import (
is_explicitly_disabled_factory,
supports_function_calling,
supports_parallel_function_calling,
supports_response_schema,
supports_system_messages,
)
from .gpt_reasoning_transformation import (
get_effort_level, # pyright: ignore[reportUnknownVariableType] # its signature takes the bare reasoning_effort dict
refused_reasoning_effort_warning,
unsupported_reasoning_effort_message,
)
from .gpt_transformation import OpenAIGPTConfig
@ -102,7 +108,35 @@ class OpenAIOSeriesConfig(OpenAIGPTConfig):
optional_params: dict,
model: str,
drop_params: bool,
*,
declared_reasoning_efforts: Collection[str] | None = None,
):
requested_effort: Final = get_effort_level(
non_default_params.get("reasoning_effort") or optional_params.get("reasoning_effort") # pyright: ignore[reportUnknownMemberType] # the request params arrive as a bare dict
)
if (
requested_effort is not None
and requested_effort not in (declared_reasoning_efforts or ())
and is_explicitly_disabled_factory(
model=model, custom_llm_provider=None, key=f"supports_{requested_effort}_reasoning_effort"
)
):
if not (litellm.drop_params or drop_params):
raise litellm.utils.UnsupportedParamsError(
message=unsupported_reasoning_effort_message("reasoning_effort", requested_effort, model),
status_code=400,
)
verbose_logger.warning(
refused_reasoning_effort_warning(
suggest_declared_levels=declared_reasoning_efforts is not None, suggest_base_model=False
),
"reasoning_effort",
requested_effort,
model,
model,
)
non_default_params.pop("reasoning_effort", None) # pyright: ignore[reportUnknownMemberType] # the request params arrive as a bare dict
optional_params.pop("reasoning_effort", None) # pyright: ignore[reportUnknownMemberType] # the request params arrive as a bare dict
if "max_tokens" in non_default_params:
optional_params["max_completion_tokens"] = non_default_params.pop("max_tokens")
if "temperature" in non_default_params:

View file

@ -46038,6 +46038,8 @@
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_minimal_reasoning_effort": false,
"supports_none_reasoning_effort": false,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
@ -46162,6 +46164,8 @@
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_minimal_reasoning_effort": false,
"supports_none_reasoning_effort": false,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,

View file

@ -6072,7 +6072,15 @@ def get_optional_params(
)
elif custom_llm_provider == "openai":
optional_params = (
litellm.OpenAIGPT5Config().map_openai_params(
litellm.openaiOSeriesConfig.map_openai_params( # pyright: ignore[reportUnknownMemberType] # the request params arrive as a bare dict
non_default_params=non_default_params,
optional_params=optional_params,
model=model,
drop_params=bool(drop_params),
declared_reasoning_efforts=_declared_reasoning_efforts(model_info),
)
if litellm.openaiOSeriesConfig.is_model_o_series_model(model=model)
else litellm.OpenAIGPT5Config().map_openai_params(
non_default_params=non_default_params,
optional_params=optional_params,
model=model,
@ -6102,6 +6110,7 @@ def get_optional_params(
optional_params=optional_params,
model=_azure_detection_model,
drop_params=bool(drop_params),
declared_reasoning_efforts=_declared_reasoning_efforts(model_info),
)
elif litellm.AzureOpenAIGPT5Config.is_model_gpt_5_model(model=_azure_detection_model):
optional_params = litellm.AzureOpenAIGPT5Config().map_openai_params(

View file

@ -46038,6 +46038,8 @@
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_minimal_reasoning_effort": false,
"supports_none_reasoning_effort": false,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
@ -46162,6 +46164,8 @@
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_minimal_reasoning_effort": false,
"supports_none_reasoning_effort": false,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,

View file

@ -1,4 +1,5 @@
import copy
from collections.abc import Mapping
from typing import Final
from unittest.mock import AsyncMock, MagicMock, patch
@ -7,6 +8,7 @@ import pytest
import litellm
from litellm import ModelResponse
from litellm.llms.openai.chat.o_series_transformation import OpenAIOSeriesConfig
from litellm.utils import get_optional_params
import os
@ -219,3 +221,123 @@ def test_o_series_request_turns_system_into_user_without_touching_the_callers_me
assert [message["role"] for message in body["messages"]] == ["user", "user"]
assert messages == messages_before
def _o3_row_with(flags: Mapping[str, bool]) -> dict[str, object]:
return {
**{
field: value
for field, value in litellm.model_cost["o3"].items()
if not (field.startswith("supports_") and field.endswith("_reasoning_effort"))
},
**flags,
}
@pytest.mark.parametrize(
("flags", "level", "forwarded"),
[
({}, "minimal", True),
({"supports_minimal_reasoning_effort": False}, "minimal", False),
({"supports_minimal_reasoning_effort": True}, "minimal", True),
({"supports_none_reasoning_effort": False}, "none", False),
({"supports_minimal_reasoning_effort": False}, "low", True),
({"supports_xhigh_reasoning_effort": False}, "xhigh", False),
({}, "xhigh", True),
],
)
def test_the_o_series_row_flags_alone_decide_whether_a_chat_level_is_forwarded(
local_model_cost_map: None,
monkeypatch: pytest.MonkeyPatch,
flags: dict[str, bool],
level: str,
forwarded: bool,
):
monkeypatch.setitem(litellm.model_cost, "o3", _o3_row_with(flags))
config: Final = OpenAIOSeriesConfig()
dropped: Final = config.map_openai_params(
non_default_params={"reasoning_effort": level, "max_tokens": 10},
optional_params={},
model="o3",
drop_params=True,
)
assert dropped.get("reasoning_effort") == (level if forwarded else None)
assert dropped["max_completion_tokens"] == 10
if forwarded:
assert (
config.map_openai_params(
non_default_params={"reasoning_effort": level},
optional_params={},
model="o3",
drop_params=False,
)["reasoning_effort"]
== level
)
else:
with pytest.raises(litellm.utils.UnsupportedParamsError, match=f"reasoning_effort={level} is not supported"):
config.map_openai_params(
non_default_params={"reasoning_effort": level},
optional_params={},
model="o3",
drop_params=False,
)
@pytest.mark.parametrize("model", ["o3", "o3-2025-04-16"])
def test_o3_rows_turn_minimal_and_none_off(local_model_cost_map: None, model: str):
"""Pinned so the row-flag test above cannot go vacuous: OpenAI's chat completions endpoint answered
400 for reasoning_effort minimal and none on o3 and 200 for low, medium, high and xhigh, both through
a local proxy and straight from api.openai.com on 2026-11-07. When OpenAI adds a level, flip the rows
and this assertion together."""
row: Final = litellm.model_cost[model]
assert row["supports_minimal_reasoning_effort"] is False
assert row["supports_none_reasoning_effort"] is False
@pytest.mark.parametrize(
("non_default_params", "optional_params"),
[
({"reasoning_effort": {"effort": "minimal", "summary": "auto"}}, {}),
({}, {"reasoning_effort": "minimal"}),
],
)
def test_o3_judges_every_form_of_a_requested_level(
local_model_cost_map: None, non_default_params: dict[str, object], optional_params: dict[str, object]
):
assert "reasoning_effort" not in OpenAIOSeriesConfig().map_openai_params(
non_default_params=non_default_params,
optional_params=optional_params,
model="o3",
drop_params=True,
)
@pytest.mark.parametrize("custom_llm_provider", ["openai", "azure"])
@pytest.mark.parametrize(
("model_info", "drop_params", "sent"),
[
(None, True, None),
({"reasoning_effort_levels": ["minimal", "low"]}, False, "minimal"),
],
)
def test_o3_chat_request_forwards_minimal_only_when_the_deployment_declares_it(
local_model_cost_map: None,
custom_llm_provider: str,
model_info: Mapping[str, object] | None,
drop_params: bool,
sent: str | None,
):
optional_params: Final = get_optional_params(
model="o3",
custom_llm_provider=custom_llm_provider,
reasoning_effort="minimal",
drop_params=drop_params,
model_info=model_info,
)
assert optional_params.get("reasoning_effort") == sent
@pytest.mark.parametrize("custom_llm_provider", ["openai", "azure"])
def test_o3_chat_request_refuses_minimal_without_drop_params(local_model_cost_map: None, custom_llm_provider: str):
with pytest.raises(litellm.utils.UnsupportedParamsError, match="reasoning_effort=minimal is not supported"):
get_optional_params(model="o3", custom_llm_provider=custom_llm_provider, reasoning_effort="minimal")

View file

@ -160,12 +160,12 @@ class TestNoneLevelPolarity:
def test_none_stays_opt_out_on_an_azure_model_outside_the_gpt_5_family(self):
"""AzureOpenAIGPT5Config is selected by is_model_gpt_5_model, so an azure o-series or
anthropic deployment never reaches the gate that refuses none and must keep the level."""
anthropic deployment never reaches the gate that refuses none unless its row turns none off."""
resolved = resolve_supported_reasoning_efforts(
{
"supports_reasoning": True,
"litellm_provider": "azure",
"key": "azure/o3",
"key": "azure/o-series-deployment",
"supports_max_reasoning_effort": True,
},
deployment_is_mapped=True,