mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-14 09:03:16 +00:00
fix(openai): drop or refuse an o-series reasoning_effort level the model map turns off
This commit is contained in:
parent
83e2e5bf99
commit
e2cf039509
6 changed files with 177 additions and 4 deletions
|
|
@ -11,7 +11,7 @@ Translations handled by LiteLLM:
|
|||
- Logprobs => drop param (if user opts in to dropping param)
|
||||
"""
|
||||
|
||||
from collections.abc import Coroutine, Sequence
|
||||
from collections.abc import Collection, Coroutine, Sequence
|
||||
from typing import Final, Literal, cast, overload
|
||||
|
||||
import litellm
|
||||
|
|
@ -19,12 +19,18 @@ from litellm import verbose_logger
|
|||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
from litellm.types.llms.openai import AllMessageValues, ChatCompletionUserMessage
|
||||
from litellm.utils import (
|
||||
is_explicitly_disabled_factory,
|
||||
supports_function_calling,
|
||||
supports_parallel_function_calling,
|
||||
supports_response_schema,
|
||||
supports_system_messages,
|
||||
)
|
||||
|
||||
from .gpt_reasoning_transformation import (
|
||||
get_effort_level, # pyright: ignore[reportUnknownVariableType] # its signature takes the bare reasoning_effort dict
|
||||
refused_reasoning_effort_warning,
|
||||
unsupported_reasoning_effort_message,
|
||||
)
|
||||
from .gpt_transformation import OpenAIGPTConfig
|
||||
|
||||
|
||||
|
|
@ -102,7 +108,35 @@ class OpenAIOSeriesConfig(OpenAIGPTConfig):
|
|||
optional_params: dict,
|
||||
model: str,
|
||||
drop_params: bool,
|
||||
*,
|
||||
declared_reasoning_efforts: Collection[str] | None = None,
|
||||
):
|
||||
requested_effort: Final = get_effort_level(
|
||||
non_default_params.get("reasoning_effort") or optional_params.get("reasoning_effort") # pyright: ignore[reportUnknownMemberType] # the request params arrive as a bare dict
|
||||
)
|
||||
if (
|
||||
requested_effort is not None
|
||||
and requested_effort not in (declared_reasoning_efforts or ())
|
||||
and is_explicitly_disabled_factory(
|
||||
model=model, custom_llm_provider=None, key=f"supports_{requested_effort}_reasoning_effort"
|
||||
)
|
||||
):
|
||||
if not (litellm.drop_params or drop_params):
|
||||
raise litellm.utils.UnsupportedParamsError(
|
||||
message=unsupported_reasoning_effort_message("reasoning_effort", requested_effort, model),
|
||||
status_code=400,
|
||||
)
|
||||
verbose_logger.warning(
|
||||
refused_reasoning_effort_warning(
|
||||
suggest_declared_levels=declared_reasoning_efforts is not None, suggest_base_model=False
|
||||
),
|
||||
"reasoning_effort",
|
||||
requested_effort,
|
||||
model,
|
||||
model,
|
||||
)
|
||||
non_default_params.pop("reasoning_effort", None) # pyright: ignore[reportUnknownMemberType] # the request params arrive as a bare dict
|
||||
optional_params.pop("reasoning_effort", None) # pyright: ignore[reportUnknownMemberType] # the request params arrive as a bare dict
|
||||
if "max_tokens" in non_default_params:
|
||||
optional_params["max_completion_tokens"] = non_default_params.pop("max_tokens")
|
||||
if "temperature" in non_default_params:
|
||||
|
|
|
|||
|
|
@ -46038,6 +46038,8 @@
|
|||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_minimal_reasoning_effort": false,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
|
|
@ -46162,6 +46164,8 @@
|
|||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_minimal_reasoning_effort": false,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
|
|
|
|||
|
|
@ -6072,7 +6072,15 @@ def get_optional_params(
|
|||
)
|
||||
elif custom_llm_provider == "openai":
|
||||
optional_params = (
|
||||
litellm.OpenAIGPT5Config().map_openai_params(
|
||||
litellm.openaiOSeriesConfig.map_openai_params( # pyright: ignore[reportUnknownMemberType] # the request params arrive as a bare dict
|
||||
non_default_params=non_default_params,
|
||||
optional_params=optional_params,
|
||||
model=model,
|
||||
drop_params=bool(drop_params),
|
||||
declared_reasoning_efforts=_declared_reasoning_efforts(model_info),
|
||||
)
|
||||
if litellm.openaiOSeriesConfig.is_model_o_series_model(model=model)
|
||||
else litellm.OpenAIGPT5Config().map_openai_params(
|
||||
non_default_params=non_default_params,
|
||||
optional_params=optional_params,
|
||||
model=model,
|
||||
|
|
@ -6102,6 +6110,7 @@ def get_optional_params(
|
|||
optional_params=optional_params,
|
||||
model=_azure_detection_model,
|
||||
drop_params=bool(drop_params),
|
||||
declared_reasoning_efforts=_declared_reasoning_efforts(model_info),
|
||||
)
|
||||
elif litellm.AzureOpenAIGPT5Config.is_model_gpt_5_model(model=_azure_detection_model):
|
||||
optional_params = litellm.AzureOpenAIGPT5Config().map_openai_params(
|
||||
|
|
|
|||
|
|
@ -46038,6 +46038,8 @@
|
|||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_minimal_reasoning_effort": false,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
|
|
@ -46162,6 +46164,8 @@
|
|||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_minimal_reasoning_effort": false,
|
||||
"supports_none_reasoning_effort": false,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import copy
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
|
|
@ -7,6 +8,7 @@ import pytest
|
|||
import litellm
|
||||
from litellm import ModelResponse
|
||||
from litellm.llms.openai.chat.o_series_transformation import OpenAIOSeriesConfig
|
||||
from litellm.utils import get_optional_params
|
||||
import os
|
||||
|
||||
|
||||
|
|
@ -219,3 +221,123 @@ def test_o_series_request_turns_system_into_user_without_touching_the_callers_me
|
|||
|
||||
assert [message["role"] for message in body["messages"]] == ["user", "user"]
|
||||
assert messages == messages_before
|
||||
|
||||
|
||||
def _o3_row_with(flags: Mapping[str, bool]) -> dict[str, object]:
|
||||
return {
|
||||
**{
|
||||
field: value
|
||||
for field, value in litellm.model_cost["o3"].items()
|
||||
if not (field.startswith("supports_") and field.endswith("_reasoning_effort"))
|
||||
},
|
||||
**flags,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("flags", "level", "forwarded"),
|
||||
[
|
||||
({}, "minimal", True),
|
||||
({"supports_minimal_reasoning_effort": False}, "minimal", False),
|
||||
({"supports_minimal_reasoning_effort": True}, "minimal", True),
|
||||
({"supports_none_reasoning_effort": False}, "none", False),
|
||||
({"supports_minimal_reasoning_effort": False}, "low", True),
|
||||
({"supports_xhigh_reasoning_effort": False}, "xhigh", False),
|
||||
({}, "xhigh", True),
|
||||
],
|
||||
)
|
||||
def test_the_o_series_row_flags_alone_decide_whether_a_chat_level_is_forwarded(
|
||||
local_model_cost_map: None,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
flags: dict[str, bool],
|
||||
level: str,
|
||||
forwarded: bool,
|
||||
):
|
||||
monkeypatch.setitem(litellm.model_cost, "o3", _o3_row_with(flags))
|
||||
config: Final = OpenAIOSeriesConfig()
|
||||
dropped: Final = config.map_openai_params(
|
||||
non_default_params={"reasoning_effort": level, "max_tokens": 10},
|
||||
optional_params={},
|
||||
model="o3",
|
||||
drop_params=True,
|
||||
)
|
||||
assert dropped.get("reasoning_effort") == (level if forwarded else None)
|
||||
assert dropped["max_completion_tokens"] == 10
|
||||
if forwarded:
|
||||
assert (
|
||||
config.map_openai_params(
|
||||
non_default_params={"reasoning_effort": level},
|
||||
optional_params={},
|
||||
model="o3",
|
||||
drop_params=False,
|
||||
)["reasoning_effort"]
|
||||
== level
|
||||
)
|
||||
else:
|
||||
with pytest.raises(litellm.utils.UnsupportedParamsError, match=f"reasoning_effort={level} is not supported"):
|
||||
config.map_openai_params(
|
||||
non_default_params={"reasoning_effort": level},
|
||||
optional_params={},
|
||||
model="o3",
|
||||
drop_params=False,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["o3", "o3-2025-04-16"])
|
||||
def test_o3_rows_turn_minimal_and_none_off(local_model_cost_map: None, model: str):
|
||||
"""Pinned so the row-flag test above cannot go vacuous: OpenAI's chat completions endpoint answered
|
||||
400 for reasoning_effort minimal and none on o3 and 200 for low, medium, high and xhigh, both through
|
||||
a local proxy and straight from api.openai.com on 2026-11-07. When OpenAI adds a level, flip the rows
|
||||
and this assertion together."""
|
||||
row: Final = litellm.model_cost[model]
|
||||
assert row["supports_minimal_reasoning_effort"] is False
|
||||
assert row["supports_none_reasoning_effort"] is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("non_default_params", "optional_params"),
|
||||
[
|
||||
({"reasoning_effort": {"effort": "minimal", "summary": "auto"}}, {}),
|
||||
({}, {"reasoning_effort": "minimal"}),
|
||||
],
|
||||
)
|
||||
def test_o3_judges_every_form_of_a_requested_level(
|
||||
local_model_cost_map: None, non_default_params: dict[str, object], optional_params: dict[str, object]
|
||||
):
|
||||
assert "reasoning_effort" not in OpenAIOSeriesConfig().map_openai_params(
|
||||
non_default_params=non_default_params,
|
||||
optional_params=optional_params,
|
||||
model="o3",
|
||||
drop_params=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("custom_llm_provider", ["openai", "azure"])
|
||||
@pytest.mark.parametrize(
|
||||
("model_info", "drop_params", "sent"),
|
||||
[
|
||||
(None, True, None),
|
||||
({"reasoning_effort_levels": ["minimal", "low"]}, False, "minimal"),
|
||||
],
|
||||
)
|
||||
def test_o3_chat_request_forwards_minimal_only_when_the_deployment_declares_it(
|
||||
local_model_cost_map: None,
|
||||
custom_llm_provider: str,
|
||||
model_info: Mapping[str, object] | None,
|
||||
drop_params: bool,
|
||||
sent: str | None,
|
||||
):
|
||||
optional_params: Final = get_optional_params(
|
||||
model="o3",
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
reasoning_effort="minimal",
|
||||
drop_params=drop_params,
|
||||
model_info=model_info,
|
||||
)
|
||||
assert optional_params.get("reasoning_effort") == sent
|
||||
|
||||
|
||||
@pytest.mark.parametrize("custom_llm_provider", ["openai", "azure"])
|
||||
def test_o3_chat_request_refuses_minimal_without_drop_params(local_model_cost_map: None, custom_llm_provider: str):
|
||||
with pytest.raises(litellm.utils.UnsupportedParamsError, match="reasoning_effort=minimal is not supported"):
|
||||
get_optional_params(model="o3", custom_llm_provider=custom_llm_provider, reasoning_effort="minimal")
|
||||
|
|
|
|||
|
|
@ -160,12 +160,12 @@ class TestNoneLevelPolarity:
|
|||
|
||||
def test_none_stays_opt_out_on_an_azure_model_outside_the_gpt_5_family(self):
|
||||
"""AzureOpenAIGPT5Config is selected by is_model_gpt_5_model, so an azure o-series or
|
||||
anthropic deployment never reaches the gate that refuses none and must keep the level."""
|
||||
anthropic deployment never reaches the gate that refuses none unless its row turns none off."""
|
||||
resolved = resolve_supported_reasoning_efforts(
|
||||
{
|
||||
"supports_reasoning": True,
|
||||
"litellm_provider": "azure",
|
||||
"key": "azure/o3",
|
||||
"key": "azure/o-series-deployment",
|
||||
"supports_max_reasoning_effort": True,
|
||||
},
|
||||
deployment_is_mapped=True,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue