From b45fae77cd931f9970f1b7b00af600ed3f58b895 Mon Sep 17 00:00:00 2001 From: zoveki <349028329+zoveki@users.noreply.github.com> Date: Mon, 16 Nov 2026 16:40:16 -0800 Subject: [PATCH] fix(azure_ai): send reasoning_effort to Foundry reasoning models (#51775) * fix(azure_ai): send reasoning_effort to Foundry reasoning models Foundry models outside the GPT-5 family that the cost map marks as reasoning models, like DeepSeek V4, now accept reasoning_effort. A level the model row declares is sent as is. A level it does not declare is dropped under drop_params and gets a 400 otherwise. Declares the levels DeepSeek V4 Flash and Pro accept on Foundry * fix(azure_ai): send reasoning_effort only to Foundry rows that declare levels Gate reasoning_effort on the cost map's reasoning_effort_levels instead of supports_reasoning, so undeclared reasoning rows keep the drop_params behavior they had before. Look the levels up case-insensitively, keep a dropped effort out of the body when allowed_openai_params lists it, model the level check as values, and name the proxy drop_params setting in the 400 * fix(azure_ai): read Foundry reasoning levels from the loaded catalog row and honor allowed_openai_params Levels now come only from the model's own azure_ai catalog row, checked for the deployment's base_model first, so a sibling deployment's model_info or a bare openai row no longer decides them. A request that allows reasoning_effort sends any level, as before this PR * fix(azure_ai): honor register_model reasoning levels on Foundry rows The level check read only the loaded cost map, so a model registered with litellm.register_model lost its declared reasoning_effort_levels. Read the loaded map with register_model registrations on top, still without the router's per-deployment registrations, so sibling deployments stay separate * fix(azure_ai): leave default and auto to the model and keep rows without levels unchanged default is stripped even when allowed_openai_params skips the level check, since Foundry refuses it on DeepSeek V4 auto leaves the level to the model, the same way the Fireworks config treats it, so Kimi K3 auto keeps working A model whose catalog row lists no levels gets reasoning_effort exactly as it was sent Updated the Kimi K3 level tests to the seven-level Foundry row, and the catalog-only test to the level check * fix(router): drop a classifier reasoning_effort level the deployment refuses The complexity router's classifier kept reasoning_effort once azure_ai listed it, so a level Foundry's row refuses failed the classifier call and sent every request to the capable tier. The classifier now drops a level the deployment's provider config refuses, unless the deployment declares its own levels or allows reasoning_effort * fix(azure_ai): keep the catalog row's reasoning levels under a registration spelled in another case A price-only register_model call under azure_ai/fw-kimi-k3 hid the azure_ai/FW-Kimi-K3 row, so its levels were lost and reasoning_effort answered 400 or was dropped The level lookup now merges the catalog row under the registration, the way register_model itself merges The Foundry Kimi K3 level test cites the live check behind its seven levels --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../litellm_core_utils/get_model_cost_map.py | 7 +- .../reasoning_effort_capability.py | 2 +- litellm/llms/azure_ai/chat/transformation.py | 231 ++++++++----- litellm/llms/base_llm/chat/transformation.py | 3 + ...odel_prices_and_context_window_backup.json | 19 ++ litellm/router_rust/invoke/shaping.py | 74 ++-- litellm/utils.py | 3 + model_prices_and_context_window.json | 19 ++ .../messages/test_reasoning_effort_fields.py | 15 +- .../chat/test_azure_ai_transformation.py | 318 ++++++++++++++++-- .../test_reasoning_effort_capability.py | 16 +- tests/unit/test_main.py | 9 +- ...st_endpoints_characterization_inference.py | 69 ++++ 13 files changed, 647 insertions(+), 138 deletions(-) diff --git a/litellm/litellm_core_utils/get_model_cost_map.py b/litellm/litellm_core_utils/get_model_cost_map.py index 77f07fa60c5..89fb631eea8 100644 --- a/litellm/litellm_core_utils/get_model_cost_map.py +++ b/litellm/litellm_core_utils/get_model_cost_map.py @@ -113,11 +113,10 @@ class GetModelCostMap: return _loaded_snapshot(model_cost_inputs.provenance.revision).rows @staticmethod - def loaded_model_cost_map_entry(key: str) -> Mapping[str, object] | None: - """The loaded catalog's entry for ``key``, else for the key that differs from it only in case.""" + def loaded_model_cost_map_key(key: str) -> str | None: + """``key`` as the loaded catalog spells it, matched exactly first and then ignoring case.""" snapshot: Final = _loaded_snapshot(model_cost_inputs.provenance.revision) - exact: Final = snapshot.rows.get(key) - return exact if exact is not None else snapshot.rows.get(snapshot.keys_by_casefold.get(key.casefold(), key)) + return key if key in snapshot.rows else snapshot.keys_by_casefold.get(key.casefold()) @staticmethod def loaded_model_cost_map_fields() -> frozenset[str]: diff --git a/litellm/litellm_core_utils/reasoning_effort_capability.py b/litellm/litellm_core_utils/reasoning_effort_capability.py index c315a31b65d..8c72c388b1f 100644 --- a/litellm/litellm_core_utils/reasoning_effort_capability.py +++ b/litellm/litellm_core_utils/reasoning_effort_capability.py @@ -1,7 +1,7 @@ """Resolve which reasoning_effort values a deployment, and by intersection a model group, accepts. An entry that states its levels outright in reasoning_effort_levels wins whole, for a model whose set -the per-level flags cannot express (Kimi K3 takes low, high and max, and no flag can drop medium). +the per-level flags cannot express (DeepSeek V4 on Foundry takes none, high and max, and no flag can drop medium). Every other entry answers through its supports_*_reasoning_effort flags, read with the polarity the chat gate uses: medium and high are unconditional, minimal and low are opt-out (refused only on an explicit false), xhigh is opt-in, and none is opt-out except on the azure gpt-5 family, whose chat diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py index a6bbbb35c8c..e257c3ceb26 100644 --- a/litellm/llms/azure_ai/chat/transformation.py +++ b/litellm/llms/azure_ai/chat/transformation.py @@ -1,13 +1,14 @@ import copy import enum import re -from collections.abc import AsyncIterator, Collection, Iterator, Mapping +from collections.abc import AsyncIterator, Collection, Iterator, Mapping, Sequence from dataclasses import dataclass from typing import TYPE_CHECKING, Final, cast import httpx from httpx import Response from pydantic import TypeAdapter, ValidationError +from typing_extensions import assert_never import litellm from litellm._logging import WarningLogger, verbose_logger @@ -50,7 +51,7 @@ from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues, PromptCacheBreakpoint from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ModelResponse, ModelResponseStream, ProviderField -from litellm.utils import add_path_to_api_base, supports_tool_choice +from litellm.utils import add_path_to_api_base, catalog_with_runtime_registrations, supports_tool_choice if TYPE_CHECKING: from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer @@ -152,76 +153,13 @@ def _deployment_reads_breakpoints(model: str, litellm_params: Mapping[str, objec ) -_DEFAULT_REASONING_EFFORT: Final = "default" _REASONING_CONTENT_MODEL_PREFIXES: Final = (AZURE_FOUNDRY_FIREWORKS_MODEL_ID_PREFIX.casefold(), "kimi-") -_FOUNDRY_REASONING_EFFORTS: Final = (*REASONING_EFFORT_ADVERTISEMENT_ORDER, "auto") - - -@dataclass(frozen=True, slots=True) -class _RefusedReasoningEffort: - requested: object def _capability_models(model: str, base_model: str | None) -> tuple[str, ...]: return tuple(name.removeprefix("azure_ai/") for name in (model, base_model) if name) -def _declares_reasoning_effort(model: str) -> bool: - row: Final = GetModelCostMap.loaded_model_cost_map_entry(f"azure_ai/{model.removeprefix('azure_ai/')}") - return row is not None and bool(declared_reasoning_efforts(row)) - - -def _refused_reasoning_effort_error(refused: _RefusedReasoningEffort, model: str) -> UnsupportedParamsError: - return UnsupportedParamsError( - message=f"azure_ai takes reasoning_effort as one of {', '.join(_FOUNDRY_REASONING_EFFORTS)}, " - f"got {refused.requested!r}. To drop unsupported openai params from the call, set `litellm.drop_params = True`", - llm_provider="azure_ai", - model=model, - status_code=400, - ) - - -def _unsupported_reasoning_effort( - requested: object, model: str, drop_params: bool -) -> Mapping[str, str] | _RefusedReasoningEffort: - if not drop_params and litellm.drop_params is not True: - return _RefusedReasoningEffort(requested) - warn_dropped_reasoning( - "reasoning_effort", requested, model, "Azure AI Foundry takes only a level name.", drop_params_gated=True - ) - return {} - - -def _reasoning_effort_level( - requested: object, model: str, drop_params: bool -) -> Mapping[str, str] | _RefusedReasoningEffort: - fields: Final = as_str_mapping(requested) - level: Final = fields.get("effort") if fields is not None else requested - if level == _DEFAULT_REASONING_EFFORT: - return {} - if not isinstance(level, str) or level not in _FOUNDRY_REASONING_EFFORTS: - return _unsupported_reasoning_effort(requested, model, drop_params) - dropped: Final = {key: value for key, value in fields.items() if key != "effort"} if fields is not None else {} - if dropped: - warn_dropped_reasoning( - "reasoning_effort", - dropped, - model, - f"Azure AI Foundry takes only a level name, so only {level!r} is sent.", - drop_params_gated=False, - ) - return {"reasoning_effort": level} - - -def _declared_reasoning_effort_param( - non_default_params: Mapping[str, object], model: str, base_model: str | None, drop_params: bool -) -> Mapping[str, str] | _RefusedReasoningEffort: - declared: Final = any(_declares_reasoning_effort(name) for name in _capability_models(model, base_model)) - if "reasoning_effort" not in non_default_params or not declared: - return {} - return _reasoning_effort_level(non_default_params["reasoning_effort"], model, drop_params) - - def _message_fields_to_strip(model: str, base_model: str | None) -> tuple[str, ...]: names: Final = _capability_models(model, base_model) if not any(name.casefold().startswith(_REASONING_CONTENT_MODEL_PREFIXES) for name in names): @@ -257,6 +195,130 @@ class AzureAIGPT5Config(OpenAIGPT5Config): azureAIGPT5Config: Final = AzureAIGPT5Config() +_REASONING_EFFORT: Final = "reasoning_effort" +_LEVELS_LEFT_TO_THE_MODEL: Final = frozenset(("default", "auto")) + + +def _catalog_row(key: str) -> Mapping[str, object] | None: + catalog: Final = catalog_with_runtime_registrations() + canonical: Final = GetModelCostMap.loaded_model_cost_map_key(key) + catalog_row: Final = catalog.get(canonical) if canonical is not None else None + registered_row: Final = catalog.get(key) + if catalog_row is None or registered_row is None: + return registered_row if catalog_row is None else catalog_row + return {**catalog_row, **registered_row} + + +def _catalog_levels(model: str) -> tuple[str, ...] | None: + row: Final = _catalog_row(f"azure_ai/{model.removeprefix('azure_ai/')}") + return declared_reasoning_efforts(row) if row is not None else None + + +def _catalog_reasoning_efforts(model: str, base_model: str | None) -> tuple[str, ...] | None: + return next(filter(None, (_catalog_levels(name) for name in (base_model, model) if name)), None) + + +def _with_reasoning_effort( + supported_params: Sequence[str], model: str +) -> list[str]: # mutable-ok: get_supported_openai_params returns a list its callers extend + if _REASONING_EFFORT in supported_params or not _catalog_levels(model): + return list(supported_params) + return [*supported_params, _REASONING_EFFORT] + + +@dataclass(frozen=True, slots=True) +class _SendReasoningEffort: + level: str + dropped_fields: Mapping[str, object] + + +@dataclass(frozen=True, slots=True) +class _LeaveReasoningEffortToModel: + pass + + +@dataclass(frozen=True, slots=True) +class _UnsendableReasoningEffort: + reason: str + + +def _unsendable_reason(level: object, accepted_levels: tuple[str, ...]) -> str: + if isinstance(level, str) and level in REASONING_EFFORT_ADVERTISEMENT_ORDER: + return f"the model catalog lists only {list(accepted_levels)} in its reasoning_effort_levels." + return f"it is not one of the levels {', '.join(REASONING_EFFORT_ADVERTISEMENT_ORDER)}." + + +def _foundry_reasoning_effort( + requested: object, accepted_levels: tuple[str, ...] | None +) -> _SendReasoningEffort | _LeaveReasoningEffortToModel | _UnsendableReasoningEffort: + fields: Final = as_str_mapping(requested) + level: Final = fields.get("effort") if fields is not None else requested + if isinstance(level, str) and level in _LEVELS_LEFT_TO_THE_MODEL: + return _LeaveReasoningEffortToModel() + if isinstance(level, str) and (accepted_levels is None or level in accepted_levels): + return _SendReasoningEffort(level, {key: value for key, value in (fields or {}).items() if key != "effort"}) + return _UnsendableReasoningEffort( + _unsendable_reason(level, accepted_levels or ()) + if isinstance(level, str) + else "it names no level such as 'high'." + ) + + +def _catalog_reasoning_effort_outcome( + requested: object, model: str, *, base_model: str | None, level_check_waived: bool +) -> _SendReasoningEffort | _LeaveReasoningEffortToModel | _UnsendableReasoningEffort | None: + catalog_levels: Final = _catalog_reasoning_efforts(model, base_model) + if catalog_levels is None: + return None + return _foundry_reasoning_effort(requested, None if level_check_waived else catalog_levels) + + +def _reasoning_effort_params( + non_default_params: dict[str, object], # mutable-ok: a drop removes the effort from the caller's params + model: str, + *, + base_model: str | None, + level_check_waived: bool, + drop_params: bool, +) -> Mapping[str, str]: + if _REASONING_EFFORT not in non_default_params: + return {} + requested: Final = non_default_params[_REASONING_EFFORT] + outcome: Final = _catalog_reasoning_effort_outcome( + requested, model, base_model=base_model, level_check_waived=level_check_waived + ) + match outcome: + case None: + return {} + case _SendReasoningEffort(level=level, dropped_fields=dropped_fields): + if dropped_fields: + warn_dropped_reasoning( + _REASONING_EFFORT, + dropped_fields, + model, + f"Azure AI Foundry takes only a level name, so only {level!r} is sent.", + drop_params_gated=False, + ) + return {_REASONING_EFFORT: level} + case _LeaveReasoningEffortToModel(): + non_default_params.pop(_REASONING_EFFORT) + return {} + case _UnsendableReasoningEffort(reason=reason) if drop_params: + warn_dropped_reasoning(_REASONING_EFFORT, requested, model, reason, drop_params_gated=True) + non_default_params.pop(_REASONING_EFFORT) + return {} + case _UnsendableReasoningEffort(reason=reason): + raise UnsupportedParamsError( + message=f"azure_ai cannot send reasoning_effort={requested!r} for model={model} because {reason} " + "To drop it from the call, set `litellm.drop_params=True` or for proxy:\n\n" + "`litellm_settings:\n drop_params: true`\n", + llm_provider="azure_ai", + model=model, + status_code=400, + ) + case _: + assert_never(outcome) + class AzureAIStudioConfig(OpenAIConfig): def get_supported_openai_params(self, model: str) -> list[str]: @@ -266,7 +328,7 @@ class AzureAIStudioConfig(OpenAIConfig): supported_params = ( azureAIGPT5Config.get_supported_openai_params(model) if azureAIGPT5Config.is_model_gpt_5_model(model) - else super().get_supported_openai_params(model) + else _with_reasoning_effort(super().get_supported_openai_params(model), model) ) if not model_supports_tool_choice: filtered_supported_params: Final = [] @@ -279,12 +341,7 @@ class AzureAIStudioConfig(OpenAIConfig): if not self._supports_stop_reason(model): supported_params = [param for param in supported_params if param != "stop"] - declared_extra: Final = ( - ("reasoning_effort",) - if "reasoning_effort" not in supported_params and _declares_reasoning_effort(model) - else () - ) - return [*supported_params, *declared_extra] + return supported_params def map_openai_params( self, @@ -296,6 +353,7 @@ class AzureAIStudioConfig(OpenAIConfig): declared_reasoning_efforts: Collection[str] | None = None, reasoning_drop_logger: WarningLogger = verbose_logger, base_model: str | None = None, + allowed_openai_params: Collection[str] = (), ) -> dict[str, object]: # mutable-ok: OpenAIConfig.map_openai_params signature if azureAIGPT5Config.is_model_gpt_5_model(model): return azureAIGPT5Config.map_openai_params( @@ -306,14 +364,27 @@ class AzureAIStudioConfig(OpenAIConfig): declared_reasoning_efforts=declared_reasoning_efforts, reasoning_drop_logger=reasoning_drop_logger, ) - effort: Final = _declared_reasoning_effort_param(non_default_params, model, base_model, drop_params) - if isinstance(effort, _RefusedReasoningEffort): - raise _refused_reasoning_effort_error(effort, model) - return super().map_openai_params( - non_default_params=non_default_params, - optional_params={**optional_params, **effort}, - model=model, - drop_params=drop_params, + foundry_effort: Final = _reasoning_effort_params( + non_default_params, + model, + base_model=base_model, + level_check_waived=bool(declared_reasoning_efforts) or _REASONING_EFFORT in allowed_openai_params, + drop_params=drop_params or litellm.drop_params is True, + ) + return { + **super().map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model=model, + drop_params=drop_params, + ), + **foundry_effort, + } + + def refuses_reasoning_effort(self, model: str, reasoning_effort: object, *, base_model: str | None) -> bool: + return not azureAIGPT5Config.is_model_gpt_5_model(model) and isinstance( + _catalog_reasoning_effort_outcome(reasoning_effort, model, base_model=base_model, level_check_waived=False), + _UnsendableReasoningEffort, ) def _supports_stop_reason(self, model: str) -> bool: diff --git a/litellm/llms/base_llm/chat/transformation.py b/litellm/llms/base_llm/chat/transformation.py index 08f986de85a..8e2cc03543f 100644 --- a/litellm/llms/base_llm/chat/transformation.py +++ b/litellm/llms/base_llm/chat/transformation.py @@ -515,6 +515,9 @@ class BaseConfig(ABC): def accepts_reasoning_summary_in_reasoning_effort(self) -> bool: return False + def refuses_reasoning_effort(self, model: str, reasoning_effort: object, *, base_model: str | None) -> bool: + return False + def post_stream_processing(self, stream: object) -> object: """Hook for providers to post-process streaming responses. Default: pass-through.""" return stream diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 668cc68f453..4ed07e8c103 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -13275,8 +13275,12 @@ "mode": "chat", "output_cost_per_token": 1.5e-05, "reasoning_effort_levels": [ + "none", + "minimal", "low", + "medium", "high", + "xhigh", "max" ], "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -13736,6 +13740,11 @@ "max_output_tokens": 384000, "mode": "chat", "output_cost_per_token": 3.48e-06, + "reasoning_effort_levels": [ + "none", + "high", + "max" + ], "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_pdf_input": false, @@ -13757,6 +13766,11 @@ "max_output_tokens": 384000, "mode": "chat", "output_cost_per_token": 5.1e-07, + "reasoning_effort_levels": [ + "none", + "high", + "max" + ], "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_pdf_input": false, @@ -13778,6 +13792,11 @@ "max_output_tokens": 384000, "mode": "chat", "output_cost_per_token": 1.32e-06, + "reasoning_effort_levels": [ + "none", + "high", + "max" + ], "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", "supports_function_calling": true, "supports_pdf_input": false, diff --git a/litellm/router_rust/invoke/shaping.py b/litellm/router_rust/invoke/shaping.py index fec11c3d642..3162d8fa32b 100644 --- a/litellm/router_rust/invoke/shaping.py +++ b/litellm/router_rust/invoke/shaping.py @@ -31,6 +31,7 @@ from litellm.litellm_core_utils.completion_timeout import CompletionTimeout from litellm.litellm_core_utils.core_helpers import allowed_openai_param_names, without from litellm.litellm_core_utils.get_litellm_params import SUPPLEMENTAL_KWARGS_KEYS from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider, inferred_provider +from litellm.litellm_core_utils.reasoning_effort_capability import deployment_reasoning_effort_levels from litellm.litellm_core_utils.request_timeout_resolver import get_service_tier_timeout from litellm.litellm_core_utils.routing_common_utils import deployment_model_id_pin, metadata_defaults_under_request from litellm.litellm_core_utils.tool_fields import TOOL_FIELDS, drop_policy @@ -49,7 +50,8 @@ from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import ( from litellm.router_utils.stream_usage_injection import router_config_requests_stream_usage from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD from litellm.types.router import Deployment, GenericLiteLLMParams, LiteLLM_Params, validate_litellm_params_strictly -from litellm.types.utils import AUTOROUTER_CLASSIFIER_CALL_ORIGIN +from litellm.types.utils import AUTOROUTER_CLASSIFIER_CALL_ORIGIN, LlmProviders +from litellm.utils import ProviderConfigManager if TYPE_CHECKING: from litellm.redis.stores.router_usage import RouterUsage @@ -245,38 +247,65 @@ def _declared_param_allowlist(params: Mapping[str, object]) -> frozenset[str]: return frozenset(allowed_openai_param_names(params.get("allowed_openai_params"))) +def _resolved_upstream(params: Mapping[str, object], group: str, host: "PythonHost") -> tuple[str, str]: + model_label: Final = str(params.get("model") or group) + provider_param: Final = params.get("custom_llm_provider") + authenticating: Final = declared_authenticating_provider( + model_label, provider_param if isinstance(provider_param, str) else None + ) + if authenticating is not None: + return model_label.removeprefix(f"{authenticating}/"), authenticating + model, custom_llm_provider, _, _ = host.invoke("get_llm_provider")( + model=model_label, + custom_llm_provider=provider_param, + litellm_params=LiteLLM_Params.model_validate(params), + ) + return model, custom_llm_provider + + +def _deployment_base_model(deployment: Mapping[str, object], params: Mapping[str, object]) -> str | None: + base_model: Final = as_dict(deployment.get("model_info")).get("base_model") or params.get("base_model") + return base_model if isinstance(base_model, str) else None + + def _deployment_supports_param(deployment: Mapping[str, object], group: str, param: str, host: "PythonHost") -> bool: params: Final = as_dict(deployment.get("litellm_params")) if not params: return True - model_info: Final = as_dict(deployment.get("model_info")) - base_model: Final = model_info.get("base_model") or params.get("base_model") try: - model_label: Final = str(params.get("model") or group) - provider_param: Final = params.get("custom_llm_provider") - authenticating: Final = declared_authenticating_provider( - model_label, provider_param if isinstance(provider_param, str) else None - ) - if authenticating is not None: - model, custom_llm_provider = model_label.removeprefix(f"{authenticating}/"), authenticating - else: - model, custom_llm_provider, _, _ = host.invoke( # rebind-ok: rebound as the Python router rebinds it - "get_llm_provider" - )( - model=model_label, - custom_llm_provider=provider_param, - litellm_params=LiteLLM_Params.model_validate(params), - ) + model, custom_llm_provider = _resolved_upstream(params, group, host) supported: Final = host.invoke("get_supported_openai_params")( model=model, custom_llm_provider=custom_llm_provider, - base_model=base_model if isinstance(base_model, str) else None, + base_model=_deployment_base_model(deployment, params), ) except Exception: # noqa: BLE001 # best-effort filter: an unresolvable provider must not narrow the request return True return supported is None or param in supported +def _deployment_refuses_reasoning_effort( + deployment: Mapping[str, object], group: str, reasoning_effort: object, host: "PythonHost" +) -> bool: + params: Final = as_dict(deployment.get("litellm_params")) + if ( + not params + or "reasoning_effort" in _declared_param_allowlist(params) + or deployment_reasoning_effort_levels(deployment.get("model_info")) + ): + return False + try: + model, custom_llm_provider = _resolved_upstream(params, group, host) + chat_config: Final = ProviderConfigManager.get_provider_chat_config( + model=model, provider=LlmProviders(custom_llm_provider) + ) + except Exception: # noqa: BLE001 # best-effort filter: an unresolvable provider must not narrow the request + return False + return chat_config is not None and chat_config.refuses_reasoning_effort( + model, reasoning_effort, base_model=_deployment_base_model(deployment, params) + ) + + def deployment_accepts_param(deployment: Mapping[str, object], group: str, param: str, host: "PythonHost") -> bool: params: Final = as_dict(deployment.get("litellm_params")) if not params: @@ -298,9 +327,12 @@ def drop_unsupported_classifier_reasoning_effort( host: "PythonHost", ) -> None: """``_drop_unsupported_classifier_reasoning_effort``.""" - if kwargs.get("reasoning_effort") is None or not _is_classifier_internal_call(kwargs): + reasoning_effort: Final = kwargs.get("reasoning_effort") + if reasoning_effort is None or not _is_classifier_internal_call(kwargs): return - if deployment_accepts_param(deployment, model, "reasoning_effort", host): + if deployment_accepts_param(deployment, model, "reasoning_effort", host) and not ( + _deployment_refuses_reasoning_effort(deployment, model, reasoning_effort, host) + ): return kwargs.pop("reasoning_effort", None) proxy_server_request: Final = kwargs.get("proxy_server_request") diff --git a/litellm/utils.py b/litellm/utils.py index d963bc3304f..62e778b7766 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5597,6 +5597,7 @@ def _map_params_with_provider_config( model_info: Mapping[str, object] | None, reasoning_drop_logger: WarningLogger, base_model: str | None, + allowed_openai_params: Sequence[str] | None, ) -> dict[str, object]: # mutable-ok: get_optional_params keeps building on the mapped params if custom_llm_provider == "azure_ai" and isinstance(provider_config, litellm.AzureAIStudioConfig): return provider_config.map_openai_params( @@ -5607,6 +5608,7 @@ def _map_params_with_provider_config( declared_reasoning_efforts=_declared_reasoning_efforts(model_info), reasoning_drop_logger=reasoning_drop_logger, base_model=base_model, + allowed_openai_params=allowed_openai_param_names(allowed_openai_params), ) if isinstance(provider_config, (litellm.ChatGPTConfig, litellm.GithubCopilotConfig)): return provider_config.map_openai_params( @@ -6284,6 +6286,7 @@ def get_optional_params( model_info=model_info, reasoning_drop_logger=_litellm_reasoning_drop_logger, base_model=base_model, + allowed_openai_params=allowed_openai_params, ) else: # assume passing in params for openai-like api optional_params = litellm.OpenAILikeChatConfig().map_openai_params( diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 668cc68f453..4ed07e8c103 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -13275,8 +13275,12 @@ "mode": "chat", "output_cost_per_token": 1.5e-05, "reasoning_effort_levels": [ + "none", + "minimal", "low", + "medium", "high", + "xhigh", "max" ], "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -13736,6 +13740,11 @@ "max_output_tokens": 384000, "mode": "chat", "output_cost_per_token": 3.48e-06, + "reasoning_effort_levels": [ + "none", + "high", + "max" + ], "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_pdf_input": false, @@ -13757,6 +13766,11 @@ "max_output_tokens": 384000, "mode": "chat", "output_cost_per_token": 5.1e-07, + "reasoning_effort_levels": [ + "none", + "high", + "max" + ], "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_pdf_input": false, @@ -13778,6 +13792,11 @@ "max_output_tokens": 384000, "mode": "chat", "output_cost_per_token": 1.32e-06, + "reasoning_effort_levels": [ + "none", + "high", + "max" + ], "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", "supports_function_calling": true, "supports_pdf_input": false, diff --git a/tests/unit/llms/anthropic/messages/test_reasoning_effort_fields.py b/tests/unit/llms/anthropic/messages/test_reasoning_effort_fields.py index 7a26f403408..53acf1f4bfa 100644 --- a/tests/unit/llms/anthropic/messages/test_reasoning_effort_fields.py +++ b/tests/unit/llms/anthropic/messages/test_reasoning_effort_fields.py @@ -223,18 +223,18 @@ class TestAdvertisedLevelsAreTheForwardedLevels: "which levels does this deployment take" through two different readers, so the proxy advertised kimi-k3 max while /v1/messages quietly forwarded high. Both now resolve through one owner.""" - KIMI_K3_SPELLINGS = ( + KIMI_K3_PASSTHROUGH_SPELLINGS = ( ("kimi-k3", "moonshot"), ("kimi-k3", "fireworks_ai"), ("kimi-k3-us", "fireworks_ai"), - ("FW-Kimi-K3", "azure_ai"), ) + KIMI_K3_SPELLINGS = (*KIMI_K3_PASSTHROUGH_SPELLINGS, ("FW-Kimi-K3", "azure_ai")) @pytest.mark.parametrize("model, provider", KIMI_K3_SPELLINGS) def test_a_declared_level_is_forwarded_rather_than_degraded(self, local_model_cost_map, model, provider): assert normalize_reasoning_effort_value("max", model, provider) == "max" - @pytest.mark.parametrize("model, provider", KIMI_K3_SPELLINGS) + @pytest.mark.parametrize("model, provider", KIMI_K3_PASSTHROUGH_SPELLINGS) def test_a_level_the_entry_does_not_declare_still_degrades(self, local_model_cost_map, model, provider): """kimi-k3 declares low, high and max, so xhigh and minimal are absent from its set and keep falling through the chain rather than being waved past by the presence of a declaration.""" @@ -263,11 +263,12 @@ class TestAdvertisedLevelsAreTheForwardedLevels: for effort in ("minimal", "xhigh", "max"): assert normalize_reasoning_effort_value(effort, model, provider) in supported - def test_the_wider_perplexity_entry_keeps_the_levels_it_declares(self, local_model_cost_map): - """The entry describing that reseller declares a six-level set, and every one of them is + @pytest.mark.parametrize("model, provider", [("perplexity/kimi-k3", "perplexity"), ("FW-Kimi-K3", "azure_ai")]) + def test_a_wider_entry_keeps_the_levels_it_declares(self, local_model_cost_map, model, provider): + """The entries describing that reseller and Foundry declare wider sets, and every level in them is forwarded, which is what the declared list exists to express.""" - assert normalize_reasoning_effort_value("xhigh", "perplexity/kimi-k3", "perplexity") == "xhigh" - assert normalize_reasoning_effort_value("minimal", "perplexity/kimi-k3", "perplexity") == "minimal" + assert normalize_reasoning_effort_value("xhigh", model, provider) == "xhigh" + assert normalize_reasoning_effort_value("minimal", model, provider) == "minimal" def test_the_minimal_chain_clears_a_deployment_that_refuses_low(self, local_model_cost_map): """gpt-5.5-pro accepts medium, high and xhigh only, so the nearest level to ``minimal`` it diff --git a/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py b/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py index d924b0609e0..229b47bdc40 100644 --- a/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py +++ b/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py @@ -1,7 +1,7 @@ import json import logging import traceback -from collections.abc import Callable +from collections.abc import Callable, Iterator, Mapping from types import MappingProxyType from typing import Final from unittest.mock import MagicMock, patch @@ -12,7 +12,7 @@ from httpx import Response import litellm from litellm import Router -from litellm.llms.custom_httpx.http_handler import HTTPHandler +from litellm.constants import DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET from litellm.litellm_core_utils.get_model_cost_map import ( GetModelCostMap, ModelCostMapReloaded, @@ -23,6 +23,7 @@ from litellm.llms.azure_ai.azure_model_router.transformation import ( AzureModelRouterConfig, ) from litellm.llms.azure_ai.chat.transformation import AzureAIStudioConfig +from litellm.llms.custom_httpx.http_handler import HTTPHandler @pytest.mark.asyncio @@ -340,6 +341,276 @@ def test_foundry_gpt_6_astra_refuses_reasoning_effort_none_before_the_call(_loca ) +_FOUNDRY_HOST: Final = "qa-reasoning.services.ai.azure.com" +_FOUNDRY_MESSAGES: Final = ({"role": "user", "content": "Say hi."},) + + +def _foundry_row(**capabilities: object) -> Mapping[str, object]: + return { + "litellm_provider": "azure_ai", + "mode": "chat", + "input_cost_per_token": 0, + "output_cost_per_token": 0, + **capabilities, + } + + +_DECLARED_LEVELS: Final = ("none", "high", "max") +_FOUNDRY_ROWS: Final = MappingProxyType( + { + "azure_ai/qa-reasoner": _foundry_row(supports_reasoning=True, reasoning_effort_levels=list(_DECLARED_LEVELS)), + "azure_ai/qa-unleveled": _foundry_row(supports_reasoning=True), + "azure_ai/qa-chat-only": _foundry_row(supports_reasoning=False), + "azure_ai/QA-Cased-Up": _foundry_row(supports_reasoning=True, reasoning_effort_levels=list(_DECLARED_LEVELS)), + "azure_ai/qa-cased-down": _foundry_row(supports_reasoning=True, reasoning_effort_levels=list(_DECLARED_LEVELS)), + "qa-bare-only": { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 0, + "output_cost_per_token": 0, + "supports_reasoning": True, + "reasoning_effort_levels": ["low", "high"], + }, + } +) + + +@pytest.fixture +def foundry_catalog(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + monkeypatch.setattr(litellm, "model_cost", litellm.model_cost) + catalog: Final = {**GetModelCostMap.load_local_model_cost_map(), **_FOUNDRY_ROWS} + assert isinstance(install_model_cost_map(json.dumps(catalog).encode()), ModelCostMapReloaded) + litellm.get_model_info.cache_clear() + yield + litellm.get_model_info.cache_clear() + + +def _mock_foundry_chat(respx_mock: respx.MockRouter, model: str) -> respx.Route: + return respx_mock.post(host=_FOUNDRY_HOST, path="/models/chat/completions").mock( + return_value=Response( + 200, + json={ + "id": "chatcmpl-foundry", + "object": "chat.completion", + "created": 1700000000, + "model": model, + "choices": [{"index": 0, "message": {"role": "assistant", "content": "Hi"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}, + }, + ) + ) + + +def _foundry_request_body(respx_mock: respx.MockRouter, model: str, **params: object) -> dict[str, object]: + route: Final = _mock_foundry_chat(respx_mock, model) + litellm.completion( + model=f"azure_ai/{model}", + api_base=f"https://{_FOUNDRY_HOST}", + api_key="azure-ai-test-key", + messages=list(_FOUNDRY_MESSAGES), + **params, + ) + return json.loads(route.calls.last.request.content) + + +@pytest.mark.usefixtures("foundry_catalog") +@pytest.mark.parametrize( + ("requested", "sent"), + [ + pytest.param("max", "max", id="level"), + pytest.param({"effort": "high", "summary": "auto"}, "high", id="effort-with-summary"), + ], +) +def test_azure_ai_sends_reasoning_effort_to_a_foundry_reasoning_model( + respx_mock: respx.MockRouter, requested: object, sent: str +) -> None: + body: Final = _foundry_request_body(respx_mock, "qa-reasoner", reasoning_effort=requested) + + assert body == {"model": "qa-reasoner", "messages": list(_FOUNDRY_MESSAGES), "reasoning_effort": sent} + + +@pytest.mark.usefixtures("foundry_catalog") +@pytest.mark.parametrize( + "model", ["qa-reasoner", "qa-unleveled"], ids=["catalog-declares-other-levels", "catalog-declares-none"] +) +def test_azure_ai_sends_a_level_the_deployment_declares_whatever_the_catalog_says( + respx_mock: respx.MockRouter, model: str +) -> None: + body: Final = _foundry_request_body( + respx_mock, model, reasoning_effort="low", model_info={"reasoning_effort_levels": ["low", "high"]} + ) + + assert body["reasoning_effort"] == "low" + + +@pytest.mark.usefixtures("foundry_catalog") +@pytest.mark.parametrize( + "requested", + [ + pytest.param("auto", id="auto"), + pytest.param("default", id="default"), + pytest.param({"effort": "high", "summary": "auto"}, id="effort-with-summary"), + ], +) +def test_azure_ai_sends_reasoning_effort_unchanged_for_a_model_the_catalog_lists_no_levels_for( + respx_mock: respx.MockRouter, requested: object +) -> None: + body: Final = _foundry_request_body(respx_mock, "o3-mini", reasoning_effort=requested) + + assert body["reasoning_effort"] == requested + + +@pytest.mark.usefixtures("foundry_catalog") +def test_azure_ai_refuses_reasoning_effort_for_a_foundry_model_without_reasoning(respx_mock: respx.MockRouter) -> None: + with pytest.raises(litellm.UnsupportedParamsError): + _foundry_request_body(respx_mock, "qa-chat-only", reasoning_effort="high") + + +@pytest.mark.usefixtures("foundry_catalog") +def test_azure_ai_sends_a_level_the_foundry_model_declares(respx_mock: respx.MockRouter) -> None: + body: Final = _foundry_request_body(respx_mock, "qa-reasoner", reasoning_effort="none", drop_params=True) + + assert body["reasoning_effort"] == "none" + + +@pytest.mark.usefixtures("foundry_catalog") +@pytest.mark.parametrize( + ("model", "requested"), + [ + pytest.param("qa-reasoner", "low", id="undeclared"), + pytest.param("qa-reasoner", {"summary": "auto"}, id="no-level"), + pytest.param("qa-unleveled", "high", id="catalog-declares-no-levels"), + ], +) +@pytest.mark.parametrize("drop_params_global", [False, True], ids=["per-request", "global"]) +def test_azure_ai_drops_a_reasoning_effort_the_foundry_model_cannot_take_under_drop_params( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + model: str, + requested: object, + drop_params_global: bool, +) -> None: + monkeypatch.setattr(litellm, "drop_params", drop_params_global) + drop_params: Final = {} if drop_params_global else {"drop_params": True} + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + body: Final = _foundry_request_body(respx_mock, model, reasoning_effort=requested, **drop_params) + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] + + assert "reasoning_effort" not in body + assert len(drop_warnings) == 1 + assert "reasoning_effort=" in drop_warnings[0] + + +@pytest.mark.usefixtures("foundry_catalog") +@pytest.mark.parametrize( + ("requested", "named"), + [ + pytest.param("low", "['none', 'high', 'max']", id="undeclared"), + pytest.param({"summary": "auto"}, "no level", id="no-level"), + ], +) +def test_azure_ai_refuses_a_reasoning_effort_the_foundry_model_cannot_take_without_drop_params( + respx_mock: respx.MockRouter, requested: object, named: str +) -> None: + with pytest.raises(litellm.UnsupportedParamsError) as refused: + _foundry_request_body(respx_mock, "qa-reasoner", reasoning_effort=requested) + + assert refused.value.status_code == 400 + assert named in str(refused.value) + + +@pytest.mark.usefixtures("foundry_catalog") +@pytest.mark.parametrize("drop_params", [{}, {"drop_params": True}], ids=["strict", "drop-params"]) +def test_azure_ai_sends_any_level_when_the_request_allows_reasoning_effort( + respx_mock: respx.MockRouter, drop_params: dict[str, object] +) -> None: + body: Final = _foundry_request_body( + respx_mock, "qa-reasoner", reasoning_effort="medium", allowed_openai_params=["reasoning_effort"], **drop_params + ) + + assert body["reasoning_effort"] == "medium" + + +@pytest.mark.usefixtures("foundry_catalog") +@pytest.mark.parametrize("base_model", ["azure_ai/qa-reasoner", "qa-reasoner"], ids=["prefixed", "bare"]) +@pytest.mark.parametrize(("requested", "sent"), [("max", "max"), ("low", None)], ids=["declared", "undeclared"]) +def test_azure_ai_checks_levels_against_the_base_model_row( + respx_mock: respx.MockRouter, base_model: str, requested: str, sent: str | None +) -> None: + body: Final = _foundry_request_body( + respx_mock, "qa-custom-deployment", reasoning_effort=requested, drop_params=True, base_model=base_model + ) + + assert body.get("reasoning_effort") == sent + + +@pytest.mark.usefixtures("foundry_catalog") +def test_azure_ai_checks_levels_against_the_catalog_not_a_sibling_deployments_model_info( + respx_mock: respx.MockRouter, +) -> None: + route: Final = _mock_foundry_chat(respx_mock, "qa-reasoner") + deployment: Final = {"model": "azure_ai/qa-reasoner", "api_base": f"https://{_FOUNDRY_HOST}", "api_key": "k"} + router: Final = Router( + model_list=[ + {"model_name": "narrow", "litellm_params": deployment, "model_info": {"reasoning_effort_levels": ["none"]}}, + {"model_name": "plain", "litellm_params": deployment}, + ] + ) + + router.completion(model="plain", messages=list(_FOUNDRY_MESSAGES), reasoning_effort="max") + + assert json.loads(route.calls.last.request.content)["reasoning_effort"] == "max" + + +@pytest.mark.usefixtures("foundry_catalog") +def test_azure_ai_reads_no_levels_from_a_row_that_does_not_describe_the_foundry_model( + respx_mock: respx.MockRouter, +) -> None: + with pytest.raises(litellm.UnsupportedParamsError, match="does not support"): + _foundry_request_body(respx_mock, "qa-bare-only", reasoning_effort="low") + + +@pytest.mark.usefixtures("foundry_catalog") +@pytest.mark.parametrize( + "requested_as", + [pytest.param("qa-cased-up", id="catalog-mixed-case"), pytest.param("QA-Cased-Down", id="request-mixed-case")], +) +def test_azure_ai_checks_declared_levels_whatever_the_model_name_casing( + respx_mock: respx.MockRouter, requested_as: str +) -> None: + with pytest.raises(litellm.UnsupportedParamsError): + _foundry_request_body(respx_mock, requested_as, reasoning_effort="low") + + +@pytest.mark.usefixtures("foundry_catalog") +@pytest.mark.parametrize("model", ["qa-registered", "qa-reasoner"], ids=["new-row", "over-catalog-row"]) +@pytest.mark.parametrize(("requested", "sent"), [("low", "low"), ("max", None)], ids=["declared", "undeclared"]) +def test_azure_ai_checks_levels_against_a_register_model_registration( + respx_mock: respx.MockRouter, model: str, requested: str, sent: str | None +) -> None: + litellm.register_model( + {f"azure_ai/{model}": _foundry_row(supports_reasoning=True, reasoning_effort_levels=["low", "high"])} + ) + + body: Final = _foundry_request_body(respx_mock, model, reasoning_effort=requested, drop_params=True) + + assert body.get("reasoning_effort") == sent + + +@pytest.mark.usefixtures("foundry_catalog") +@pytest.mark.parametrize(("requested", "sent"), [("max", "max"), ("low", None)], ids=["declared", "undeclared"]) +def test_azure_ai_keeps_catalog_levels_under_a_pricing_registration_spelled_in_another_case( + respx_mock: respx.MockRouter, requested: str, sent: str | None +) -> None: + litellm.register_model({"azure_ai/qa-cased-up": _foundry_row(input_cost_per_token=1e-6)}) + + body: Final = _foundry_request_body(respx_mock, "qa-cased-up", reasoning_effort=requested, drop_params=True) + + assert body.get("reasoning_effort") == sent + + FOUNDRY_MODELS_CHAT_URL: Final = "https://example-resource.services.ai.azure.com/models/chat/completions" FOUNDRY_CHAT_COMPLETION: Final = MappingProxyType( { @@ -376,9 +647,6 @@ def foundry_reasoner(monkeypatch: pytest.MonkeyPatch) -> str: ("model", "requested", "sent"), [ pytest.param("FW-Reasoner", "high", "high", id="declared-level"), - pytest.param("FW-Reasoner", "medium", "medium", id="level-the-row-does-not-list-is-sent-as-asked"), - pytest.param("FW-Reasoner", "none", "none", id="none"), - pytest.param("FW-Reasoner", "auto", "auto", id="auto"), pytest.param("FW-Reasoner", {"effort": "low", "summary": "detailed"}, "low", id="effort-with-summary"), pytest.param("fw-reasoner", "high", "high", id="model-named-in-another-case"), ], @@ -447,11 +715,19 @@ def test_a_foundry_reasoner_named_with_its_provider_prefix_advertises_reasoning_ assert "reasoning_effort" in AzureAIStudioConfig().get_supported_openai_params(f"azure_ai/{foundry_reasoner}") -@pytest.mark.parametrize("drop_params", [False, True]) +@pytest.mark.parametrize( + "request_options", + [ + pytest.param(MappingProxyType({}), id="checked"), + pytest.param(MappingProxyType({"drop_params": True}), id="drop-params"), + pytest.param(MappingProxyType({"allowed_openai_params": ["reasoning_effort"]}), id="check-waived"), + ], +) +@pytest.mark.parametrize("requested", ["default", "auto"]) @pytest.mark.usefixtures("foundry_reasoner") @pytest.mark.asyncio -async def test_foundry_reasoning_effort_default_leaves_the_level_to_the_model( - respx_mock: respx.MockRouter, drop_params: bool +async def test_foundry_reasoning_effort_default_or_auto_leaves_the_level_to_the_model( + respx_mock: respx.MockRouter, request_options: Mapping[str, object], requested: str ) -> None: route: Final = respx_mock.post(FOUNDRY_MODELS_CHAT_URL).mock( return_value=Response(200, json=dict(FOUNDRY_CHAT_COMPLETION)) @@ -462,8 +738,8 @@ async def test_foundry_reasoning_effort_default_leaves_the_level_to_the_model( api_key="azure-ai-test-key", api_base="https://example-resource.services.ai.azure.com", messages=[{"role": "user", "content": "What is 17*23?"}], - reasoning_effort="default", - drop_params=drop_params, + reasoning_effort=requested, + **request_options, ) assert "reasoning_effort" not in json.loads(route.calls.last.request.content) @@ -492,15 +768,17 @@ def test_foundry_reasoning_effort_object_sends_its_level_and_warns_about_the_res assert any("summary" in message and "'effort'" not in message for message in warnings) is warns -NOT_A_FOUNDRY_LEVEL: Final = ( +UNSENDABLE_FOUNDRY_EFFORT: Final = ( + pytest.param("medium", id="level-the-row-does-not-list"), + pytest.param("none", id="none-the-row-does-not-list"), pytest.param("High", id="level-in-another-case"), pytest.param("ultra", id="unknown-level"), pytest.param({"summary": "detailed"}, id="object-without-a-level"), ) -@pytest.mark.parametrize("requested", NOT_A_FOUNDRY_LEVEL) -def test_foundry_reasoning_effort_that_is_not_a_level_is_refused_without_drop_params( +@pytest.mark.parametrize("requested", UNSENDABLE_FOUNDRY_EFFORT) +def test_foundry_reasoning_effort_the_row_does_not_take_is_refused_without_drop_params( foundry_reasoner: str, requested: object ) -> None: with pytest.raises(litellm.UnsupportedParamsError): @@ -512,12 +790,12 @@ def test_foundry_reasoning_effort_that_is_not_a_level_is_refused_without_drop_pa ) -@pytest.mark.parametrize("requested", NOT_A_FOUNDRY_LEVEL) +@pytest.mark.parametrize("requested", UNSENDABLE_FOUNDRY_EFFORT) @pytest.mark.parametrize( ("request_drop_params", "global_drop_params"), (pytest.param(True, False, id="request-drop-params"), pytest.param(False, True, id="global-drop-params")), ) -def test_foundry_reasoning_effort_that_is_not_a_level_is_dropped_with_a_warning_under_drop_params( +def test_foundry_reasoning_effort_the_row_does_not_take_is_dropped_with_a_warning_under_drop_params( foundry_reasoner: str, caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch, @@ -555,7 +833,7 @@ async def test_anthropic_messages_thinking_reaches_a_foundry_reasoner_as_reasoni api_key="azure-ai-test-key", api_base="https://example-resource.services.ai.azure.com", max_tokens=1024, - thinking={"type": "enabled", "budget_tokens": 4000}, + thinking={"type": "enabled", "budget_tokens": DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET}, messages=[ {"role": "user", "content": "What is the weather in SF?"}, { @@ -577,7 +855,7 @@ async def test_anthropic_messages_thinking_reaches_a_foundry_reasoner_as_reasoni ) sent: Final = json.loads(route.calls.last.request.content) - assert isinstance(sent["reasoning_effort"], str) + assert sent["reasoning_effort"] == "high" assert sent["messages"][1]["reasoning_content"] == earlier_thinking assert not _find_key_anywhere(sent["messages"], "thinking_blocks") @@ -594,7 +872,7 @@ async def test_anthropic_messages_thinking_reaches_a_foundry_reasoner_as_reasoni "reasoning_effort_levels": ["low", "high"], } }, - id="levels-a-sibling-deployment-registered-on-the-shared-row", + id="levels-a-router-deployment-registered-on-the-shared-row", ), ], ) @@ -602,7 +880,7 @@ async def test_anthropic_messages_thinking_reaches_a_foundry_reasoner_as_reasoni async def test_foundry_model_without_declared_reasoning_efforts_still_refuses_reasoning_effort( respx_mock: respx.MockRouter, foundry_reasoner: str, registered_at_runtime: dict[str, dict[str, object]] ) -> None: - litellm.register_model(registered_at_runtime) + litellm.register_model(registered_at_runtime, persist_across_reloads=False) route: Final = respx_mock.post(FOUNDRY_MODELS_CHAT_URL).mock( return_value=Response(200, json=dict(FOUNDRY_CHAT_COMPLETION)) ) @@ -1704,7 +1982,7 @@ def test_azure_ai_services_handler(api_base, expected_url): def test_azure_ai_services_with_api_version(): - from litellm.llms.custom_httpx.http_handler import HTTPHandler, AsyncHTTPHandler + from litellm.llms.custom_httpx.http_handler import HTTPHandler client = HTTPHandler() diff --git a/tests/unit/router_utils/test_reasoning_effort_capability.py b/tests/unit/router_utils/test_reasoning_effort_capability.py index c0ff141ffab..88d595a921b 100644 --- a/tests/unit/router_utils/test_reasoning_effort_capability.py +++ b/tests/unit/router_utils/test_reasoning_effort_capability.py @@ -322,7 +322,6 @@ class TestDeclaredEffortList: KIMI_K3_PASSTHROUGH_KEYS = ( - "azure_ai/FW-Kimi-K3", "moonshot/kimi-k3", "together_ai/moonshotai/Kimi-K3", "fireworks_ai/kimi-k3", @@ -345,6 +344,21 @@ class TestKimiK3AdvertisesItsDocumentedLevels: assert resolve_supported_reasoning_efforts(entry, deployment_is_mapped=True) == ("low", "high", "max") + def test_the_foundry_entry_advertises_every_level_foundry_takes(self, local_model_cost_map): + """Foundry's Kimi K3 answered 200 for every level from none to max on a live deployment, + checked on 2026-11-14 for PR #51747, so its row declares all seven.""" + entry = dict(litellm.model_cost["azure_ai/FW-Kimi-K3"], key="azure_ai/FW-Kimi-K3") + + assert resolve_supported_reasoning_efforts(entry, deployment_is_mapped=True) == ( + "none", + "minimal", + "low", + "medium", + "high", + "xhigh", + "max", + ) + def test_the_perplexity_entry_advertises_the_wider_set_it_maps_down(self, local_model_cost_map): """Perplexity's Agent API takes a six-value enum and maps it down internally, so this deployment is legitimately wider than a passthrough. One blanket list could not say both.""" diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index a44e14f662d..ed6a03afe5c 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -10810,12 +10810,13 @@ def foundry_reasoner_row(monkeypatch: pytest.MonkeyPatch) -> str: return model -def test_a_model_map_declaration_alone_forwards_reasoning_effort_without_rounding_it( +def test_a_model_map_declaration_alone_checks_reasoning_effort_without_rounding_it( foundry_reasoner_row: str, respx_mock: respx.MockRouter ) -> None: - assert _chat_body_sent(respx_mock, foundry_reasoner_row, "medium", {"id": "reasoner-1"}, drop_params=True)[ - "reasoning_effort" - ] == "medium" + assert _chat_body_sent(respx_mock, foundry_reasoner_row, "low", {"id": "reasoner-1"})["reasoning_effort"] == "low" + assert "reasoning_effort" not in _chat_body_sent( + respx_mock, foundry_reasoner_row, "medium", {"id": "reasoner-1"}, drop_params=True + ) @pytest.mark.parametrize("model_info, expected", [(_KIMI_DEPLOYMENT, "high"), (None, "medium")]) diff --git a/tests/unit/test_router/core/test_endpoints_characterization_inference.py b/tests/unit/test_router/core/test_endpoints_characterization_inference.py index c975e45b1d3..ac0aa175874 100644 --- a/tests/unit/test_router/core/test_endpoints_characterization_inference.py +++ b/tests/unit/test_router/core/test_endpoints_characterization_inference.py @@ -14,6 +14,7 @@ from typing import Final import pytest import litellm +from litellm.litellm_core_utils.reasoning_effort_capability import REASONING_EFFORT_ADVERTISEMENT_ORDER from litellm.router_core import RouterCore from tests.unit.test_router.core.endpoints_helpers import EndpointHost from tests.unit.test_router.core.scripted_host import ScriptedHost, deployment, timeout @@ -487,6 +488,74 @@ async def test_classifier_call_keeps_reasoning_effort_on_a_deployment_with_suppo assert _sent(host)["reasoning_effort"] == "low" +LEVELED_FOUNDRY_MODEL: Final = "azure_ai/DeepSeek-V4-Flash-0731" + + +def _catalog_levels(model: str) -> tuple[str, ...]: + levels: Final = litellm.model_cost[model]["reasoning_effort_levels"] + assert isinstance(levels, list) and levels + return tuple(str(level) for level in levels) + + +def _unlisted_level(model: str) -> str: + return next(level for level in REASONING_EFFORT_ADVERTISEMENT_ORDER if level not in _catalog_levels(model)) + + +@pytest.mark.parametrize( + ("model", "model_info"), + [(LEVELED_FOUNDRY_MODEL, {}), ("azure_ai/litellm-qa-own-name", {"base_model": LEVELED_FOUNDRY_MODEL})], + ids=["catalog-model", "base-model"], +) +async def test_classifier_call_drops_a_reasoning_effort_level_the_deployment_refuses( + make_core: MakeCore, host: EndpointHost, model: str, model_info: dict[str, object] +): + core: Final = make_core(model_list=[deployment("g", "d1", model=model, model_info=model_info)]) + unlisted: Final = _unlisted_level(LEVELED_FOUNDRY_MODEL) + await core.acompletion( + model="g", + messages=MESSAGES, + reasoning_effort=unlisted, + litellm_metadata=dict(CLASSIFIER_METADATA), + proxy_server_request={"body": {"reasoning_effort": unlisted, "keep": 1}}, + ) + sent: Final = _sent(host) + assert "reasoning_effort" not in sent + assert sent["proxy_server_request"] == {"body": {"keep": 1}} + + +@pytest.mark.parametrize( + ("listed", "deployment_params"), + [(True, {}), (False, {"allowed_openai_params": ["reasoning_effort"]})], + ids=["catalog-level", "allowlisted"], +) +async def test_classifier_call_keeps_a_reasoning_effort_level_the_deployment_sends( + make_core: MakeCore, host: EndpointHost, listed: bool, deployment_params: dict[str, object] +): + level: Final = ( + _catalog_levels(LEVELED_FOUNDRY_MODEL)[-1] if listed else _unlisted_level(LEVELED_FOUNDRY_MODEL) + ) + core: Final = make_core(model_list=[deployment("g", "d1", model=LEVELED_FOUNDRY_MODEL, **deployment_params)]) + await core.acompletion( + model="g", messages=MESSAGES, reasoning_effort=level, litellm_metadata=dict(CLASSIFIER_METADATA) + ) + assert _sent(host)["reasoning_effort"] == level + + +async def test_classifier_call_keeps_a_level_the_deployment_declares_itself(make_core: MakeCore, host: EndpointHost): + unlisted: Final = _unlisted_level(LEVELED_FOUNDRY_MODEL) + core: Final = make_core( + model_list=[ + deployment( + "g", "d1", model=LEVELED_FOUNDRY_MODEL, model_info={"reasoning_effort_levels": [unlisted, "high"]} + ) + ] + ) + await core.acompletion( + model="g", messages=MESSAGES, reasoning_effort=unlisted, litellm_metadata=dict(CLASSIFIER_METADATA) + ) + assert _sent(host)["reasoning_effort"] == unlisted + + # --- silent experiments ---------------------------------------------------------------------