diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index d1228ac2610..17815976b4a 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -5,7 +5,7 @@ Handler for transforming /chat/completions api requests to litellm.responses req import json import os from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence -from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, Union, cast +from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, Union, cast, get_args from openai.types.responses.custom_tool_param import CustomToolParam from openai.types.responses.response_input_param import ( @@ -35,6 +35,7 @@ from litellm.responses.sse_output_recovery import ( ) from litellm.responses.utils import normalize_responses_api_stream_options from litellm.types.llms.openai import ( + REASONING_EFFORT, ChatCompletionAnnotation, ChatCompletionReasoningItem, ChatCompletionToolCallChunk, @@ -1113,10 +1114,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): litellm.reasoning_auto_summary or os.getenv("LITELLM_REASONING_AUTO_SUMMARY", "false").lower() == "true" ) - # Level-agnostic: providers own effort validation, so an unknown level (max, ultra, future - # ones) passes through instead of being silently dropped here. "default" means no override, - # so it maps to nothing rather than reaching a provider as a literal effort value. - if reasoning_effort and reasoning_effort != "default": + if reasoning_effort in get_args(REASONING_EFFORT): return ( Reasoning(effort=reasoning_effort, summary="detailed") if auto_summary_enabled diff --git a/litellm/llms/openai/chat/gpt_5_transformation.py b/litellm/llms/openai/chat/gpt_5_transformation.py index 3a79e6570e9..3a65e4a9426 100644 --- a/litellm/llms/openai/chat/gpt_5_transformation.py +++ b/litellm/llms/openai/chat/gpt_5_transformation.py @@ -16,7 +16,7 @@ def _normalize_reasoning_effort_for_chat_completion( ) -> str | None: """Convert reasoning_effort to the string format expected by OpenAI chat completion API. - The chat completion API expects a simple effort string ('none' through 'ultra'). + The chat completion API expects an effort string such as 'low' or 'high'. Config/deployments may pass the Responses API format: {'effort': 'high', 'summary': 'detailed'}. """ if value is None: diff --git a/litellm/main.py b/litellm/main.py index a1b41315b85..c2dbf595bf6 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -416,8 +416,7 @@ async def acompletion( logprobs: bool | None = None, top_logprobs: int | None = None, deployment_id=None, - reasoning_effort: Literal["none", "minimal", "low", "medium", "high", "xhigh", "max", "ultra", "default"] - | None = None, + reasoning_effort: Literal["none", "minimal", "low", "medium", "high", "xhigh", "max", "default"] | None = None, verbosity: Literal["low", "medium", "high"] | None = None, safety_identifier: str | None = None, service_tier: str | None = None, @@ -4921,8 +4920,7 @@ def completion( logit_bias: dict | None = None, user: str | None = None, # openai v1.0+ new params - reasoning_effort: Literal["none", "minimal", "low", "medium", "high", "xhigh", "max", "ultra", "default"] - | None = None, + reasoning_effort: Literal["none", "minimal", "low", "medium", "high", "xhigh", "max", "default"] | None = None, verbosity: Literal["low", "medium", "high"] | None = None, response_format: dict | type[BaseModel] | None = None, seed: int | None = None, diff --git a/litellm/router.py b/litellm/router.py index bd5f97b27c4..766dcd67986 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -168,6 +168,7 @@ from litellm.router_utils.pre_call_checks.prompt_caching_deployment_check import PromptCachingDeploymentCheck, ) from litellm.router_utils.reasoning_effort_capability import ( + deployment_is_catalog_mapped, intersect_supported_reasoning_efforts, resolve_supported_reasoning_efforts, ) @@ -9443,6 +9444,8 @@ class Router: except Exception: model_info = None + deployment_is_mapped = deployment_is_catalog_mapped(model_info, model_info_dict) + # get llm provider litellm_model, llm_provider = "", "" try: @@ -9565,7 +9568,7 @@ class Router: model_group_info.supported_reasoning_efforts = intersect_supported_reasoning_efforts( model_group_info.supported_reasoning_efforts, - resolve_supported_reasoning_efforts(model_info), + resolve_supported_reasoning_efforts(model_info, deployment_is_mapped=deployment_is_mapped), ) if _deployment_tpm is not None: diff --git a/litellm/router_utils/reasoning_effort_capability.py b/litellm/router_utils/reasoning_effort_capability.py index d445d611cd6..d20b1151803 100644 --- a/litellm/router_utils/reasoning_effort_capability.py +++ b/litellm/router_utils/reasoning_effort_capability.py @@ -1,91 +1,131 @@ """Resolve which reasoning_effort values a deployment, and by intersection a model group, accepts. -The model-map flags carry different polarity per level, mirroring the provider gates -(gpt_5_transformation.py restricts xhigh to explicit opt-in and treats minimal/low as opt-out): -medium and high are unconditional for any reasoning model, minimal/low are supported unless the map -explicitly says false, and xhigh/max require an explicit true. Shipping the resolved list keeps that -polarity in one place instead of re-encoding it in every consumer. +The model map's supports_*_reasoning_effort flags are the only signal, and each level's polarity +mirrors how a request path reads that same flag. medium and high are unconditional for a reasoning +model. minimal and low are opt-out: openai/chat/gpt_5_transformation.py refuses them only when the +map says false. xhigh and max are opt-in. none is opt-out everywhere except the azure gpt-5 family, +whose config raises UnsupportedParamsError without an explicit true. -Only openai and azure gate xhigh and max on the request path. anthropic/chat/transformation.py -gates them on the output_config path alone, so its reasoning_effort path maps every level to a -thinking budget whatever the map says, and a claude group whose entry omits -supports_xhigh_reasoning_effort stops offering a level litellm would have forwarded. That is the -deliberate trade: an explicit flag is the only signal that the tier is a real one rather than -litellm quietly rounding the level to a budget, and what gets dropped is advisory metadata rather -than a restriction on the request path. +xhigh is gated on the request path by the openai and azure gpt-5 configs. max is not gated there at +all: every entry carrying supports_max_reasoning_effort is Claude-family, and +anthropic/chat/transformation.py gates max on the output_config path while its reasoning_effort +path maps any level to a thinking budget. Making max opt-in is a deliberate trade, then, since an +explicit flag is the only signal that the tier is a real one rather than litellm rounding the level +to a budget, and a missing flag costs advisory metadata rather than a rejected request. -The none level is the one flag whose polarity is provider-dependent. OpenAI never refuses it on the -request path (azure/chat/gpt_5_transformation.py is the only caller that does, and it raises -UnsupportedParamsError unless supports_none_reasoning_effort is explicitly true), so none is opt-out -everywhere except azure, where it is opt-in. Resolving it the other way for azure would advertise a -level litellm itself rejects, which is the failure this module exists to prevent. +A deployment the map describes with no effort flags at all resolves to None rather than to the +opt-out defaults. 689 of the map's 854 reasoning entries carry no flag, and the o-series, xai and +bedrock nova entries among them take neither none nor minimal, so composing a set out of the +defaults alone would advertise levels those providers reject. + +The advertisement order is the REASONING_EFFORT declaration order, which is presentation only. It +is not a strength scale and does not reconcile with bedrock's output_config ceiling order in +llms/bedrock/common_utils.py, which ranks max below xhigh while the thinking-budget constants rank +it above. """ from collections.abc import Mapping, Sequence -from typing import Final +from typing import Final, get_args -REASONING_EFFORT_CAPABILITY_ORDER: Final = ("none", "minimal", "low", "medium", "high", "xhigh", "max", "ultra") +import litellm +from litellm.types.llms.openai import REASONING_EFFORT -_OPT_OUT_FLAGS: Final = ( +REASONING_EFFORT_ADVERTISEMENT_ORDER: Final = get_args(REASONING_EFFORT) + +_EFFORT_FLAGS: Final = ( + ("none", "supports_none_reasoning_effort"), ("minimal", "supports_minimal_reasoning_effort"), ("low", "supports_low_reasoning_effort"), -) -_OPT_IN_FLAGS: Final = ( ("xhigh", "supports_xhigh_reasoning_effort"), ("max", "supports_max_reasoning_effort"), - ("ultra", "supports_ultra_reasoning_effort"), ) +_OPT_OUT_EFFORTS: Final = ("minimal", "low") +_OPT_IN_EFFORTS: Final = ("xhigh", "max") _UNCONDITIONAL_EFFORTS: Final = frozenset(("medium", "high")) -_NONE_FLAG: Final = "supports_none_reasoning_effort" -_NONE_OPT_IN_PROVIDERS: Final = frozenset(("azure",)) -def _supports_none_reasoning_effort(model_info: Mapping[str, object]) -> bool: - """Opt-in on azure, whose gpt-5 config raises on reasoning_effort='none' without an explicit - true; opt-out elsewhere, where no request path refuses the level. A missing azure flag defers to - _supports_factory, the same resolver the azure gate calls, so its bare-model-name fallback - (azure/gpt-5.2 inheriting the flag from gpt-5.2) reaches both sides alike.""" - flag: Final = model_info.get(_NONE_FLAG) - if model_info.get("litellm_provider") not in _NONE_OPT_IN_PROVIDERS: +def _bare_model_entry(model_info: Mapping[str, object]) -> Mapping[str, object]: + """The unprefixed twin of a provider-prefixed map entry, which is where the flags often live: + azure/gpt-5-mini carries none of them while gpt-5-mini carries all three. The request-path + gates resolve through the same twin (_supports_factory, #20885), so reading it here is what + keeps the advertisement and the gate on the same answer.""" + key: Final = model_info.get("key") + provider: Final = model_info.get("litellm_provider") + if not isinstance(key, str) or not isinstance(provider, str) or not key.startswith(f"{provider}/"): + return {} + entry: Final[Mapping[str, object] | None] = litellm.model_cost.get(key.removeprefix(f"{provider}/")) + return entry if entry is not None else {} + + +def _declared_effort_flags(model_info: Mapping[str, object]) -> Mapping[str, object]: + bare: Final = _bare_model_entry(model_info) + return { + effort: model_info.get(flag) if model_info.get(flag) is not None else bare.get(flag) + for effort, flag in _EFFORT_FLAGS + } + + +def _supports_none_reasoning_effort(model_info: Mapping[str, object], flag: object) -> bool: + """Opt-in only where a request path refuses the level. AzureOpenAIGPT5Config raises + UnsupportedParamsError on reasoning_effort='none' without an explicit true, and it is selected + only for the gpt-5 family, so every other azure deployment keeps the opt-out default.""" + if model_info.get("litellm_provider") != "azure": return flag is not False - if flag is not None: - return flag is True - model_key: Final = model_info.get("key") - if not isinstance(model_key, str): + + from litellm.llms.azure.chat.gpt_5_transformation import AzureOpenAIGPT5Config + + key: Final = model_info.get("key") + if not isinstance(key, str) or not AzureOpenAIGPT5Config.is_model_gpt_5_model(key): + return flag is not False + return flag is True + + +def deployment_is_catalog_mapped( + resolved_model_info: Mapping[str, object] | None, + operator_model_info: Mapping[str, object], +) -> bool: + """Whether the model map described this deployment, as opposed to the operator describing it. + + Every deployment is registered in the cost map under its own id, so a mode the operator wrote + on an off-map deployment reads back here exactly like one the catalog supplied. Excluding it is + what stops such a deployment from claiming to be a known non-reasoning model and emptying the + levels its mapped siblings agree on. + """ + if resolved_model_info is None or resolved_model_info.get("mode") is None: return False - from litellm.utils import ( - _supports_factory, # pyright: ignore[reportPrivateUsage] # the resolver the azure gate itself calls; a public wrapper would fork the fallback - ) - - return _supports_factory(model=model_key, custom_llm_provider=None, key=_NONE_FLAG) + return operator_model_info.get("mode") is None -def resolve_supported_reasoning_efforts(model_info: Mapping[str, object]) -> tuple[str, ...] | None: - """None = nothing is known about this deployment, so it must not narrow its group; () = this is a - known model that accepts no effort level, which correctly empties the group. Keeping the two - apart matters because the router registers a deployment absent from the model map under a - synthesized entry, and get_model_info then answers with supports_reasoning None exactly as it - does for a mapped non-reasoning model. Mode is what separates them: every map entry for a - routable model declares one, so an unset flag with no mode is read as unknown and one custom - model in a group no longer wipes the levels its mapped siblings agree on. +def resolve_supported_reasoning_efforts( + model_info: Mapping[str, object], + *, + deployment_is_mapped: bool, +) -> tuple[str, ...] | None: + """None = nothing is known about this deployment, so it must not narrow its group; () = a known + model that accepts no effort level, which correctly empties the group. - The separation is only as good as that signal. An off-map deployment carrying any model_info of - its own is registered under its deployment id with mode defaulting to chat, so it reads as a - known non-reasoning model and does still empty its group, with supports_reasoning on that - deployment as the way out. Telling the two apart for real needs provenance that the flattened - ModelInfo does not carry.""" - if "supports_reasoning" not in model_info: - return None + Telling those apart needs provenance the flattened ModelInfo does not carry. A deployment the + map does not describe arrives with supports_reasoning None, exactly like a mapped non-reasoning + model: 2273 of the map's 3165 entries omit the key rather than setting it false, so reading an + unset flag as () would let one custom deployment empty every level its mapped siblings agree + on. deployment_is_mapped is that provenance, and an operator who wants either answer for an + off-map deployment gets it by setting supports_reasoning explicitly. + """ supports_reasoning: Final = model_info.get("supports_reasoning") - if supports_reasoning is None and model_info.get("mode") is None: - return None if supports_reasoning is not True: - return () - opt_out: Final = frozenset(effort for effort, flag in _OPT_OUT_FLAGS if model_info.get(flag) is not False) - opt_in: Final = frozenset(effort for effort, flag in _OPT_IN_FLAGS if model_info.get(flag) is True) - none_level: Final = frozenset(("none",)) if _supports_none_reasoning_effort(model_info) else frozenset() + return () if supports_reasoning is False or deployment_is_mapped else None + + flags: Final = _declared_effort_flags(model_info) + if all(value is None for value in flags.values()): + return None + + opt_out: Final = frozenset(effort for effort in _OPT_OUT_EFFORTS if flags[effort] is not False) + opt_in: Final = frozenset(effort for effort in _OPT_IN_EFFORTS if flags[effort] is True) + none_level: Final = ( + frozenset(("none",)) if _supports_none_reasoning_effort(model_info, flags["none"]) else frozenset() + ) allowed: Final = opt_out | _UNCONDITIONAL_EFFORTS | opt_in | none_level - return tuple(effort for effort in REASONING_EFFORT_CAPABILITY_ORDER if effort in allowed) + return tuple(effort for effort in REASONING_EFFORT_ADVERTISEMENT_ORDER if effort in allowed) def intersect_supported_reasoning_efforts( @@ -99,4 +139,4 @@ def intersect_supported_reasoning_efforts( if current is None: return tuple(resolved) keep: Final = frozenset(current) & frozenset(resolved) - return tuple(effort for effort in REASONING_EFFORT_CAPABILITY_ORDER if effort in keep) + return tuple(effort for effort in REASONING_EFFORT_ADVERTISEMENT_ORDER if effort in keep) diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index e3d28a0097f..4a6c4a5bbb5 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1840,7 +1840,7 @@ ResponsesAPIStreamingResponse = Annotated[ ] -REASONING_EFFORT = Literal["none", "minimal", "low", "medium", "high", "xhigh", "max", "ultra"] +REASONING_EFFORT = Literal["none", "minimal", "low", "medium", "high", "xhigh", "max"] class OpenAIRealtimeStreamSession(TypedDict, total=False): diff --git a/litellm/types/router.py b/litellm/types/router.py index fd32b70405b..d4c735387a5 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -641,19 +641,6 @@ class ModelGroupInfo(BaseModel): supported_openai_params: list[str] | None = Field(default=[]) configurable_clientside_auth_params: CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS = None - @field_validator("supported_reasoning_efforts", mode="before") - @classmethod - def _accept_only_well_formed_reasoning_efforts(cls, value: object) -> tuple[str, ...] | None: - """Deployment model_info reaches this model through a **kwargs splat, so an operator can put - any shape under this key. Anything that is not a level or a list of levels resolves to None - and lets the computed intersection stand, because raising here fails the whole - /model_group/info response rather than the one group that carries the bad value.""" - if isinstance(value, str): - return (value,) - if isinstance(value, (list, tuple)) and all(isinstance(level, str) for level in value): - return tuple(value) - return None - def __init__(self, **data) -> None: for field_name, field_type in get_type_hints(self.__class__).items(): if field_type is bool and data.get(field_name) is None: diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 6a5e4c24dce..4d59650f410 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -164,7 +164,6 @@ class ProviderSpecificModelInfo(TypedDict, total=False): supports_low_reasoning_effort: bool | None supports_xhigh_reasoning_effort: bool | None supports_max_reasoning_effort: bool | None - supports_ultra_reasoning_effort: bool | None # writable-ok: Pydantic warns on ReadOnly TypedDict fields supports_output_config: bool | None supports_image_size: bool | None bedrock_output_config_effort_ceiling: Literal["low", "medium", "high", "max", "xhigh"] | None diff --git a/litellm/utils.py b/litellm/utils.py index a74cf23a4ae..5b2ef93edb3 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5764,7 +5764,6 @@ def _get_model_info_helper( supports_low_reasoning_effort=_model_info.get("supports_low_reasoning_effort", None), supports_xhigh_reasoning_effort=_model_info.get("supports_xhigh_reasoning_effort", None), supports_max_reasoning_effort=_model_info.get("supports_max_reasoning_effort", None), - supports_ultra_reasoning_effort=_model_info.get("supports_ultra_reasoning_effort", None), bedrock_output_config_effort_ceiling=_model_info.get("bedrock_output_config_effort_ceiling", None), bedrock_converse_supports_strict_tools=_model_info.get("bedrock_converse_supports_strict_tools", None), supports_computer_use=_model_info.get("supports_computer_use", None), diff --git a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index 5c76fe5e7da..b358e6fa28c 100644 --- a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -1585,16 +1585,16 @@ def test_map_reasoning_effort_adds_summary_detailed(monkeypatch): assert result_dict["summary"] == "custom_summary" print("✓ Dict input is passed through without modification") - # Test 5: levels this bridge does not enumerate (max, ultra, future ones) pass through so the - # provider can judge them, instead of being silently dropped before the request is built + # Test 5: every REASONING_EFFORT level reaches the provider, and anything else (a typo, an + # unshipped level, "default") is dropped so the request still succeeds at the provider default from litellm.types.llms.openai import Reasoning - for effort in ("max", "ultra", "unknown_value"): + for effort in ("max", "xhigh", "none"): result_passthrough = handler._map_reasoning_effort(effort) assert result_passthrough == Reasoning(effort=effort) - assert handler._map_reasoning_effort("") is None - assert handler._map_reasoning_effort("default") is None - print("✓ Unenumerated reasoning_effort levels pass through to the provider") + for dropped in ("ultra", "hgih", "unknown_value", "", "default"): + assert handler._map_reasoning_effort(dropped) is None + print("✓ Enumerated levels pass through and unknown ones are dropped") print( "✓ All reasoning_effort behaviors work correctly with flag/env var control" diff --git a/tests/test_litellm/llms/openai/test_gpt5_transformation.py b/tests/test_litellm/llms/openai/test_gpt5_transformation.py index eaae3d8c2c5..a35b75a6106 100644 --- a/tests/test_litellm/llms/openai/test_gpt5_transformation.py +++ b/tests/test_litellm/llms/openai/test_gpt5_transformation.py @@ -1334,7 +1334,7 @@ def test_gpt5_6_never_advertises_reasoning_effort_max(model: str): gpt-5.6 entry asserts supports_max_reasoning_effort and the advertised set stops at xhigh.""" from litellm.router_utils.reasoning_effort_capability import resolve_supported_reasoning_efforts - resolved = resolve_supported_reasoning_efforts(litellm.get_model_info(model)) + resolved = resolve_supported_reasoning_efforts(litellm.get_model_info(model), deployment_is_mapped=True) assert resolved is not None assert "max" not in resolved assert "xhigh" in resolved @@ -1355,28 +1355,16 @@ def test_gpt5_6_keeps_reasoning_effort_max_on_the_responses_api( assert params["reasoning"] == {"effort": "max"} -def test_gpt5_6_never_advertises_reasoning_effort_ultra(): - """ultra is plumbed as an opt-in level but no map entry asserts it: OpenAI's model guidance - documents effort values only up to max, and /v1/responses answers ultra with a 400. A verified - supports_ultra_reasoning_effort flag lights it up with no code change.""" - from litellm.router_utils.reasoning_effort_capability import resolve_supported_reasoning_efforts - - resolved = resolve_supported_reasoning_efforts(litellm.get_model_info("gpt-5.6")) - assert resolved is not None - assert "ultra" not in resolved - - -@pytest.mark.parametrize("effort", ["max", "ultra"]) -def test_gpt5_forwards_levels_the_chat_gate_does_not_own(config: OpenAIConfig, effort: str): - """Only xhigh is gated on this surface. max and ultra reach the provider (or the responses - bridge) and are answered there, which is what happened before per-group capabilities existed.""" +def test_gpt5_forwards_levels_the_chat_gate_does_not_own(config: OpenAIConfig): + """Only xhigh is gated on this surface. max reaches the provider (or the responses bridge) and + is answered there, which is what happened before per-group capabilities existed.""" params = config.map_openai_params( - non_default_params={"reasoning_effort": effort}, + non_default_params={"reasoning_effort": "max"}, optional_params={}, model="gpt-5.1", drop_params=False, ) - assert params["reasoning_effort"] == effort + assert params["reasoning_effort"] == "max" def test_gpt5_rejects_xhigh_for_models_without_the_flag(config: OpenAIConfig): diff --git a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py index a906b0e638f..791d64c6428 100644 --- a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py @@ -1352,7 +1352,7 @@ class TestParseCursorModelVariant: ("gemini-3.0-pro-thinking-low", "gemini-3.0-pro", "low"), ("claude-opus-5-fast", "claude-opus-5", None), ("gpt-5.6-sol", "gpt-5.6-sol", None), - ("gpt-5.6-thinking-ultra-fast", "gpt-5.6", "ultra"), + ("foo-thinking-ultra-fast", "foo-thinking-ultra", None), ("gpt-5.6-thinking-max", "gpt-5.6", "max"), ("foo-thinking-mega-fast", "foo-thinking-mega", None), ("-thinking-high", "-thinking-high", None), diff --git a/tests/test_litellm/router_utils/test_reasoning_effort_capability.py b/tests/test_litellm/router_utils/test_reasoning_effort_capability.py index f3d71d19c47..6cd5b956b05 100644 --- a/tests/test_litellm/router_utils/test_reasoning_effort_capability.py +++ b/tests/test_litellm/router_utils/test_reasoning_effort_capability.py @@ -1,38 +1,51 @@ import pytest from litellm.router_utils.reasoning_effort_capability import ( + deployment_is_catalog_mapped, intersect_supported_reasoning_efforts, resolve_supported_reasoning_efforts, ) +class TestDeploymentIsCatalogMapped: + def test_a_mode_the_catalog_supplied_marks_the_deployment_mapped(self): + assert deployment_is_catalog_mapped({"mode": "chat"}, {}) is True + + def test_a_deployment_the_catalog_never_described_is_not_mapped(self): + assert deployment_is_catalog_mapped(None, {}) is False + assert deployment_is_catalog_mapped({"max_input_tokens": 200000}, {}) is False + + def test_a_mode_the_operator_wrote_does_not_make_the_deployment_mapped(self): + # Every deployment is registered in the cost map under its own id, so an operator-written + # mode reads back identically to one the catalog supplied and would otherwise let an + # off-map deployment empty the levels its mapped siblings agree on. + assert deployment_is_catalog_mapped({"mode": "chat"}, {"mode": "chat", "id": "abc"}) is False + + +class TestProvenanceSeparatesUnknownFromNonReasoning: + def test_an_off_map_deployment_resolves_to_unknown(self): + # get_model_info answers supports_reasoning None both for a deployment the map never + # described and for a mapped non-reasoning model, so reading an unset flag as () would let + # one custom deployment empty every level its mapped siblings agree on. + assert resolve_supported_reasoning_efforts({}, deployment_is_mapped=False) is None + assert resolve_supported_reasoning_efforts({"supports_reasoning": None}, deployment_is_mapped=False) is None + + def test_a_mapped_deployment_the_map_calls_non_reasoning_supports_no_efforts(self): + assert resolve_supported_reasoning_efforts({}, deployment_is_mapped=True) == () + assert resolve_supported_reasoning_efforts({"supports_reasoning": None}, deployment_is_mapped=True) == () + + def test_an_explicit_false_supports_no_efforts_off_the_map_too(self): + # The operator's own escape hatch: saying so on an off-map deployment must still empty the + # group, since nothing else can tell the resolver that model takes no effort level. + assert resolve_supported_reasoning_efforts({"supports_reasoning": False}, deployment_is_mapped=False) == () + + class TestResolveSupportedReasoningEfforts: - def test_no_metadata_resolves_to_unknown(self): - assert resolve_supported_reasoning_efforts({}) is None - - def test_non_reasoning_model_supports_no_efforts(self): - assert resolve_supported_reasoning_efforts({"supports_reasoning": False}) == () - assert resolve_supported_reasoning_efforts({"mode": "chat", "supports_reasoning": None}) == () - - def test_unset_flag_on_an_entry_with_no_mode_resolves_to_unknown(self): - # The router registers a deployment absent from the model map under a synthesized entry, and - # get_model_info then answers with supports_reasoning None just as it does for a mapped - # non-reasoning model. Only the missing mode tells them apart, and reading the synthesized - # one as () would let one custom model empty every level its mapped siblings agree on. - assert resolve_supported_reasoning_efforts({"supports_reasoning": None}) is None - assert resolve_supported_reasoning_efforts({"mode": None, "supports_reasoning": None}) is None - - def test_reasoning_model_with_no_flags_gets_the_opt_out_levels_only(self): - # The kimi shape: supports_reasoning true, zero effort flags. medium/high are unconditional, - # none/minimal/low are opt-out so absence means supported, xhigh/max are opt-in so absence - # means unsupported. - assert resolve_supported_reasoning_efforts({"supports_reasoning": True}) == ( - "none", - "minimal", - "low", - "medium", - "high", - ) + def test_a_reasoning_model_with_no_flags_at_all_resolves_to_unknown(self): + # 689 of the map's 854 reasoning entries carry no effort flag, and the o-series, xai and + # bedrock nova entries among them accept neither none nor minimal, so composing a set out of + # the opt-out defaults alone would advertise levels those providers reject. + assert resolve_supported_reasoning_efforts({"supports_reasoning": True}, deployment_is_mapped=True) is None def test_explicit_false_removes_an_opt_out_level(self): # The gpt-5.5-pro shape from the model map: only medium/high/xhigh are accepted upstream. @@ -43,7 +56,8 @@ class TestResolveSupportedReasoningEfforts: "supports_minimal_reasoning_effort": False, "supports_low_reasoning_effort": False, "supports_xhigh_reasoning_effort": True, - } + }, + deployment_is_mapped=True, ) assert resolved == ("medium", "high", "xhigh") @@ -54,47 +68,100 @@ class TestResolveSupportedReasoningEfforts: "supports_reasoning": True, "supports_xhigh_reasoning_effort": True, "supports_max_reasoning_effort": True, - } + }, + deployment_is_mapped=True, ) assert resolved == ("none", "minimal", "low", "medium", "high", "xhigh", "max") - def test_ultra_is_opt_in(self): - without_flag = resolve_supported_reasoning_efforts({"supports_reasoning": True}) - with_flag = resolve_supported_reasoning_efforts( - {"supports_reasoning": True, "supports_ultra_reasoning_effort": True} - ) - assert without_flag is not None and "ultra" not in without_flag - assert with_flag is not None and with_flag[-1] == "ultra" - def test_opt_in_flag_set_false_stays_excluded(self): resolved = resolve_supported_reasoning_efforts( - {"supports_reasoning": True, "supports_xhigh_reasoning_effort": False} + { + "supports_reasoning": True, + "supports_minimal_reasoning_effort": True, + "supports_xhigh_reasoning_effort": False, + }, + deployment_is_mapped=True, ) - assert resolved is not None - assert "xhigh" not in resolved + assert resolved == ("none", "minimal", "low", "medium", "high") + + +class TestBareModelNameFallback: + def test_a_prefixed_entry_inherits_the_flags_of_its_unprefixed_twin(self): + """azure/gpt-5-mini carries no effort flag while gpt-5-mini carries three, and the request + path resolves capability flags through that same twin (#20885). Reading only the prefixed + entry would answer unknown for a model the map fully describes.""" + from litellm.utils import _get_model_info_helper + + model_info = dict(_get_model_info_helper(model="gpt-5-mini", custom_llm_provider="azure")) + + assert resolve_supported_reasoning_efforts(model_info, deployment_is_mapped=True) == ( + "minimal", + "low", + "medium", + "high", + ) + + def test_the_prefixed_entry_wins_over_its_twin_per_flag(self): + resolved = resolve_supported_reasoning_efforts( + { + "supports_reasoning": True, + "litellm_provider": "azure", + "key": "azure/gpt-5-mini", + "supports_xhigh_reasoning_effort": True, + }, + deployment_is_mapped=True, + ) + assert resolved == ("minimal", "low", "medium", "high", "xhigh") class TestNoneLevelPolarity: def test_none_stays_opt_out_off_azure(self): resolved = resolve_supported_reasoning_efforts( - {"supports_reasoning": True, "litellm_provider": "openai", "key": "gpt-5-mini"} + { + "supports_reasoning": True, + "litellm_provider": "openai", + "key": "openai/some-reasoner", + "supports_max_reasoning_effort": True, + }, + deployment_is_mapped=True, ) assert resolved is not None and "none" in resolved - def test_azure_without_the_flag_does_not_advertise_none(self): - resolved = resolve_supported_reasoning_efforts( - {"supports_reasoning": True, "litellm_provider": "azure", "key": "azure/unmapped-deployment"} - ) - assert resolved == ("minimal", "low", "medium", "high") - - def test_azure_with_the_flag_advertises_none(self): + def test_none_stays_opt_out_on_an_azure_model_outside_the_gpt_5_family(self): + """AzureOpenAIGPT5Config is selected by is_model_gpt_5_model, so an azure o-series or + anthropic deployment never reaches the gate that refuses none and must keep the level.""" resolved = resolve_supported_reasoning_efforts( { "supports_reasoning": True, "litellm_provider": "azure", + "key": "azure/o3", + "supports_max_reasoning_effort": True, + }, + deployment_is_mapped=True, + ) + assert resolved is not None and "none" in resolved + + def test_azure_gpt_5_without_the_flag_does_not_advertise_none(self): + resolved = resolve_supported_reasoning_efforts( + { + "supports_reasoning": True, + "litellm_provider": "azure", + "key": "azure/gpt-5-turbo", + "supports_minimal_reasoning_effort": True, + }, + deployment_is_mapped=True, + ) + assert resolved == ("minimal", "low", "medium", "high") + + def test_azure_gpt_5_with_the_flag_advertises_none(self): + resolved = resolve_supported_reasoning_efforts( + { + "supports_reasoning": True, + "litellm_provider": "azure", + "key": "azure/gpt-5-turbo", "supports_none_reasoning_effort": True, - "key": "azure/unmapped-deployment", - } + }, + deployment_is_mapped=True, ) assert resolved is not None and "none" in resolved @@ -109,7 +176,7 @@ class TestNoneLevelPolarity: from litellm.utils import _get_model_info_helper model_info = dict(_get_model_info_helper(model=model_key.split("/", 1)[1], custom_llm_provider="azure")) - resolved = resolve_supported_reasoning_efforts(model_info) + resolved = resolve_supported_reasoning_efforts(model_info, deployment_is_mapped=True) assert resolved is not None gate_accepts_none = AzureOpenAIGPT5Config._supports_reasoning_effort_level(model_key, "none") diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index b8a5a454613..93b04cc2330 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -8994,6 +8994,7 @@ def test_model_group_info_reasoning_efforts_empty_on_a_mapped_non_reasoning_depl "litellm_provider": "anthropic", "mode": "chat", "supports_reasoning": True, + "supports_max_reasoning_effort": True, } return {"key": model_name, "litellm_provider": "openai", "mode": "chat", "supports_reasoning": None} @@ -9032,6 +9033,7 @@ def test_model_group_info_reasoning_efforts_ignore_a_value_declared_in_model_inf "litellm_provider": "openai", "mode": "chat", "supports_reasoning": True, + "supports_none_reasoning_effort": True, } if model_id == "first-deployment": info["supported_reasoning_efforts"] = ("high",) @@ -9047,26 +9049,39 @@ def test_model_group_info_reasoning_efforts_ignore_a_value_declared_in_model_inf assert result.supported_reasoning_efforts == ("none", "minimal", "low", "medium", "high") -@pytest.mark.parametrize( - "configured, expected", - [ - ("high", ("high",)), - (["low", "high"], ("low", "high")), - (17, None), - ({"effort": "high"}, None), - ([1, 2], None), - ], -) -def test_model_group_info_tolerates_any_configured_reasoning_efforts_shape(configured, expected): - """Deployment model_info is splatted into ModelGroupInfo, so this key arrives with whatever an - operator wrote in the config. A shape pydantic cannot validate used to fail the whole - /model_group/info response, not just the group carrying it.""" - from litellm.types.router import ModelGroupInfo +def test_model_group_info_reasoning_efforts_ignore_a_mode_the_operator_declared(): + """A deployment is registered in the cost map under its own id with whatever model_info the + operator wrote, so a mode they set themselves reads back exactly like one the map supplied. Only + a mode the map supplied marks the deployment as known, or an off-map deployment carrying any + mode empties the group it sits in.""" + from litellm.router_utils.reasoning_effort_capability import resolve_supported_reasoning_efforts - info = ModelGroupInfo( - model_group="g", - providers=["openai"], - supported_reasoning_efforts=configured, + mapped_model = "openai/gpt-5.6-sol" + expected = resolve_supported_reasoning_efforts( + litellm.get_model_info(model=mapped_model), + deployment_is_mapped=True, + ) + assert expected + + router = litellm.Router( + model_list=[ + { + "model_name": "smart-group", + "litellm_params": {"model": mapped_model, "api_key": "sk-fake"}, + "model_info": {"id": "mapped-deployment"}, + }, + { + "model_name": "smart-group", + "litellm_params": {"model": "openai/a-model-the-map-never-heard-of", "api_key": "sk-fake"}, + "model_info": {"id": "off-map-deployment", "mode": "chat"}, + }, + ] ) - assert info.supported_reasoning_efforts == expected + result = router._set_model_group_info( + model_group="smart-group", + user_facing_model_group_name="smart-group", + ) + + assert result is not None + assert result.supported_reasoning_efforts == expected diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index af69d117e04..27cf9067914 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -997,7 +997,6 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "supports_none_reasoning_effort": {"type": "boolean"}, "supports_xhigh_reasoning_effort": {"type": "boolean"}, "supports_max_reasoning_effort": {"type": "boolean"}, - "supports_ultra_reasoning_effort": {"type": "boolean"}, "supports_adaptive_thinking": {"type": "boolean"}, "supports_legacy_thinking": {"type": "boolean"}, "thinking_always_on": {"type": "boolean"}, diff --git a/ui/litellm-dashboard/src/components/add_model/TierModelEffortRows.test.tsx b/ui/litellm-dashboard/src/components/add_model/TierModelEffortRows.test.tsx new file mode 100644 index 00000000000..218a18b805e --- /dev/null +++ b/ui/litellm-dashboard/src/components/add_model/TierModelEffortRows.test.tsx @@ -0,0 +1,78 @@ +import { describe, expect, it } from "vitest"; +import { tierEffortRows } from "./TierModelEffortRows"; + +describe("tierEffortRows", () => { + it("offers the levels the proxy reports for the model", () => { + const rows = tierEffortRows({ + models: ["gpt-5-mini"], + effortOptionsByModel: { "gpt-5-mini": ["low", "medium", "high"] }, + paramsByModel: undefined, + }); + + expect(rows).toEqual([{ model: "gpt-5-mini", effort: undefined, options: ["low", "medium", "high"] }]); + }); + + it("keeps a row whose model reports no level but already has an effort stored, so it can be cleared", () => { + const rows = tierEffortRows({ + models: ["off-map-model"], + effortOptionsByModel: {}, + paramsByModel: { "off-map-model": { reasoning_effort: "high" } }, + }); + + expect(rows).toEqual([{ model: "off-map-model", effort: "high", options: ["high"] }]); + }); + + it("drops a row whose model reports no level and has nothing stored", () => { + const rows = tierEffortRows({ + models: ["plain-chat-model"], + effortOptionsByModel: { "plain-chat-model": [] }, + paramsByModel: { "plain-chat-model": { temperature: 0.5 } }, + }); + + expect(rows).toEqual([]); + }); + + it("lists a stored level the model no longer reports without duplicating the ones it does", () => { + const rows = tierEffortRows({ + models: ["gpt-5-mini"], + effortOptionsByModel: { "gpt-5-mini": ["low", "high"] }, + paramsByModel: { "gpt-5-mini": { reasoning_effort: "xhigh" } }, + }); + + expect(rows).toEqual([{ model: "gpt-5-mini", effort: "xhigh", options: ["low", "high", "xhigh"] }]); + }); + + it("does not repeat a stored level the model already reports", () => { + const rows = tierEffortRows({ + models: ["gpt-5-mini"], + effortOptionsByModel: { "gpt-5-mini": ["low", "high"] }, + paramsByModel: { "gpt-5-mini": { reasoning_effort: "high" } }, + }); + + expect(rows).toEqual([{ model: "gpt-5-mini", effort: "high", options: ["low", "high"] }]); + }); + + it.each([ + ["an unset key", {}], + ["an explicit null", { reasoning_effort: null }], + ["an empty string", { reasoning_effort: "" }], + ])("reads %s as no stored effort", (_label, params) => { + const rows = tierEffortRows({ + models: ["gpt-5-mini"], + effortOptionsByModel: { "gpt-5-mini": ["low"] }, + paramsByModel: { "gpt-5-mini": params }, + }); + + expect(rows[0].effort).toBeUndefined(); + }); + + it("renders a non-string stored value as a string so the select can show and clear it", () => { + const rows = tierEffortRows({ + models: ["gpt-5-mini"], + effortOptionsByModel: { "gpt-5-mini": ["low"] }, + paramsByModel: { "gpt-5-mini": { reasoning_effort: 3 } }, + }); + + expect(rows).toEqual([{ model: "gpt-5-mini", effort: "3", options: ["low", "3"] }]); + }); +}); diff --git a/ui/litellm-dashboard/src/components/add_model/TierModelEffortRows.tsx b/ui/litellm-dashboard/src/components/add_model/TierModelEffortRows.tsx index 0c93b3228dc..ec9705b9451 100644 --- a/ui/litellm-dashboard/src/components/add_model/TierModelEffortRows.tsx +++ b/ui/litellm-dashboard/src/components/add_model/TierModelEffortRows.tsx @@ -8,7 +8,8 @@ const PROVIDER_DEFAULT = "__provider_default__"; const storedEffort = (params: TierModelParams | undefined): ReasoningEffort | undefined => { const stored = params?.reasoning_effort; - return typeof stored === "string" && stored ? stored : undefined; + if (stored === undefined || stored === null || stored === "") return undefined; + return typeof stored === "string" ? stored : String(stored); }; interface TierModelEffortRowsProps { @@ -19,6 +20,31 @@ interface TierModelEffortRowsProps { onEffortChange: (model: string, effort: ReasoningEffort | undefined) => void; } +export interface TierEffortRow { + model: string; + effort: ReasoningEffort | undefined; + options: string[]; +} + +/** + * A stored effort outside the model's supported set (hand-authored, or capabilities changed since + * it was saved) is listed anyway, so the row renders with its value selected and can be cleared. + * Only a model with no supported level and nothing stored drops out. + */ +export const tierEffortRows = ({ + models, + effortOptionsByModel, + paramsByModel, +}: Pick): TierEffortRow[] => + models + .map((model) => { + const effort = storedEffort(paramsByModel?.[model]); + const supported = effortOptionsByModel[model] ?? []; + const listed = effort !== undefined && !supported.includes(effort) ? [...supported, effort] : supported; + return { model, effort, options: Array.from(new Set(listed)) }; + }) + .filter(({ options }) => options.length > 0); + const TierModelEffortRows: React.FC = ({ tierLabel, models, @@ -26,16 +52,7 @@ const TierModelEffortRows: React.FC = ({ paramsByModel, onEffortChange, }) => { - const rows = models - .map((model) => { - const effort = storedEffort(paramsByModel?.[model]); - const supported = effortOptionsByModel[model] ?? []; - // A stored effort outside the supported set (hand-authored, or capabilities changed since it - // was saved) stays listed so it renders and can be cleared. - const options = effort !== undefined && !supported.includes(effort) ? [...supported, effort] : supported; - return { model, effort, options }; - }) - .filter(({ model, options }) => options.length > 0 || Object.keys(paramsByModel?.[model] ?? {}).length > 0); + const rows = tierEffortRows({ models, effortOptionsByModel, paramsByModel }); if (rows.length === 0) return null; return (
diff --git a/ui/litellm-dashboard/src/components/llm_calls/fetch_models.test.tsx b/ui/litellm-dashboard/src/components/llm_calls/fetch_models.test.tsx index bd691c7f629..998924ce36d 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/fetch_models.test.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/fetch_models.test.tsx @@ -1,6 +1,6 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; -import { modelAvailableCall } from "@/components/networking"; -import { fetchAvailableModelsForTeam } from "./fetch_models"; +import { modelAvailableCall, modelHubCall } from "@/components/networking"; +import { fetchAvailableModels, fetchAvailableModelsForTeam } from "./fetch_models"; vi.mock("@/components/networking", () => ({ modelAvailableCall: vi.fn(), @@ -8,6 +8,7 @@ vi.mock("@/components/networking", () => ({ })); const modelAvailableCallMock = vi.mocked(modelAvailableCall); +const modelHubCallMock = vi.mocked(modelHubCall); describe("fetchAvailableModelsForTeam", () => { beforeEach(() => { @@ -31,3 +32,33 @@ describe("fetchAvailableModelsForTeam", () => { expect(await fetchAvailableModelsForTeam("token", "team-123")).toEqual([]); }); }); + +describe("fetchAvailableModels", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("carries the reasoning capabilities the model hub reports for each group", async () => { + modelHubCallMock.mockResolvedValue({ + data: [ + { model_group: "smart", mode: "chat", supports_reasoning: true, supported_reasoning_efforts: ["low", "high"] }, + { model_group: "plain", mode: "chat", supports_reasoning: false }, + ], + }); + + expect(await fetchAvailableModels("token")).toEqual([ + { model_group: "plain", mode: "chat" }, + { model_group: "smart", mode: "chat", supports_reasoning: true, supported_reasoning_efforts: ["low", "high"] }, + ]); + }); + + it.each([ + ["an error payload in place of the list", { data: { error: "no access" } }], + ["a missing data key", {}], + ["no body at all", undefined], + ])("returns an empty list on %s rather than throwing", async (_label, response) => { + modelHubCallMock.mockResolvedValue(response); + + expect(await fetchAvailableModels("token")).toEqual([]); + }); +}); diff --git a/ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx b/ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx index 96b87887d43..1f812cf0377 100644 --- a/ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx +++ b/ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx @@ -44,7 +44,8 @@ export const fetchAvailableModelsForTeam = async (accessToken: string, teamId: s export const fetchAvailableModels = async (accessToken: string): Promise => { try { const fetchedModels = await modelHubCall(accessToken); - const models: ModelGroup[] = (fetchedModels?.data ?? []) + const fetchedData: unknown = fetchedModels?.data; + const models: ModelGroup[] = (Array.isArray(fetchedData) ? fetchedData : []) .map(toModelGroup) .filter((model: ModelGroup) => model.model_group !== "") .sort((a: ModelGroup, b: ModelGroup) => a.model_group.localeCompare(b.model_group));