diff --git a/litellm/litellm_core_utils/get_model_cost_map.py b/litellm/litellm_core_utils/get_model_cost_map.py index 77f07fa60c5..89fb631eea8 100644 --- a/litellm/litellm_core_utils/get_model_cost_map.py +++ b/litellm/litellm_core_utils/get_model_cost_map.py @@ -113,11 +113,10 @@ class GetModelCostMap: return _loaded_snapshot(model_cost_inputs.provenance.revision).rows @staticmethod - def loaded_model_cost_map_entry(key: str) -> Mapping[str, object] | None: - """The loaded catalog's entry for ``key``, else for the key that differs from it only in case.""" + def loaded_model_cost_map_key(key: str) -> str | None: + """``key`` as the loaded catalog spells it, matched exactly first and then ignoring case.""" snapshot: Final = _loaded_snapshot(model_cost_inputs.provenance.revision) - exact: Final = snapshot.rows.get(key) - return exact if exact is not None else snapshot.rows.get(snapshot.keys_by_casefold.get(key.casefold(), key)) + return key if key in snapshot.rows else snapshot.keys_by_casefold.get(key.casefold()) @staticmethod def loaded_model_cost_map_fields() -> frozenset[str]: diff --git a/litellm/litellm_core_utils/reasoning_effort_capability.py b/litellm/litellm_core_utils/reasoning_effort_capability.py index c315a31b65d..8c72c388b1f 100644 --- a/litellm/litellm_core_utils/reasoning_effort_capability.py +++ b/litellm/litellm_core_utils/reasoning_effort_capability.py @@ -1,7 +1,7 @@ """Resolve which reasoning_effort values a deployment, and by intersection a model group, accepts. An entry that states its levels outright in reasoning_effort_levels wins whole, for a model whose set -the per-level flags cannot express (Kimi K3 takes low, high and max, and no flag can drop medium). +the per-level flags cannot express (DeepSeek V4 on Foundry takes none, high and max, and no flag can drop medium). Every other entry answers through its supports_*_reasoning_effort flags, read with the polarity the chat gate uses: medium and high are unconditional, minimal and low are opt-out (refused only on an explicit false), xhigh is opt-in, and none is opt-out except on the azure gpt-5 family, whose chat diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py index a6bbbb35c8c..e257c3ceb26 100644 --- a/litellm/llms/azure_ai/chat/transformation.py +++ b/litellm/llms/azure_ai/chat/transformation.py @@ -1,13 +1,14 @@ import copy import enum import re -from collections.abc import AsyncIterator, Collection, Iterator, Mapping +from collections.abc import AsyncIterator, Collection, Iterator, Mapping, Sequence from dataclasses import dataclass from typing import TYPE_CHECKING, Final, cast import httpx from httpx import Response from pydantic import TypeAdapter, ValidationError +from typing_extensions import assert_never import litellm from litellm._logging import WarningLogger, verbose_logger @@ -50,7 +51,7 @@ from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues, PromptCacheBreakpoint from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ModelResponse, ModelResponseStream, ProviderField -from litellm.utils import add_path_to_api_base, supports_tool_choice +from litellm.utils import add_path_to_api_base, catalog_with_runtime_registrations, supports_tool_choice if TYPE_CHECKING: from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer @@ -152,76 +153,13 @@ def _deployment_reads_breakpoints(model: str, litellm_params: Mapping[str, objec ) -_DEFAULT_REASONING_EFFORT: Final = "default" _REASONING_CONTENT_MODEL_PREFIXES: Final = (AZURE_FOUNDRY_FIREWORKS_MODEL_ID_PREFIX.casefold(), "kimi-") -_FOUNDRY_REASONING_EFFORTS: Final = (*REASONING_EFFORT_ADVERTISEMENT_ORDER, "auto") - - -@dataclass(frozen=True, slots=True) -class _RefusedReasoningEffort: - requested: object def _capability_models(model: str, base_model: str | None) -> tuple[str, ...]: return tuple(name.removeprefix("azure_ai/") for name in (model, base_model) if name) -def _declares_reasoning_effort(model: str) -> bool: - row: Final = GetModelCostMap.loaded_model_cost_map_entry(f"azure_ai/{model.removeprefix('azure_ai/')}") - return row is not None and bool(declared_reasoning_efforts(row)) - - -def _refused_reasoning_effort_error(refused: _RefusedReasoningEffort, model: str) -> UnsupportedParamsError: - return UnsupportedParamsError( - message=f"azure_ai takes reasoning_effort as one of {', '.join(_FOUNDRY_REASONING_EFFORTS)}, " - f"got {refused.requested!r}. To drop unsupported openai params from the call, set `litellm.drop_params = True`", - llm_provider="azure_ai", - model=model, - status_code=400, - ) - - -def _unsupported_reasoning_effort( - requested: object, model: str, drop_params: bool -) -> Mapping[str, str] | _RefusedReasoningEffort: - if not drop_params and litellm.drop_params is not True: - return _RefusedReasoningEffort(requested) - warn_dropped_reasoning( - "reasoning_effort", requested, model, "Azure AI Foundry takes only a level name.", drop_params_gated=True - ) - return {} - - -def _reasoning_effort_level( - requested: object, model: str, drop_params: bool -) -> Mapping[str, str] | _RefusedReasoningEffort: - fields: Final = as_str_mapping(requested) - level: Final = fields.get("effort") if fields is not None else requested - if level == _DEFAULT_REASONING_EFFORT: - return {} - if not isinstance(level, str) or level not in _FOUNDRY_REASONING_EFFORTS: - return _unsupported_reasoning_effort(requested, model, drop_params) - dropped: Final = {key: value for key, value in fields.items() if key != "effort"} if fields is not None else {} - if dropped: - warn_dropped_reasoning( - "reasoning_effort", - dropped, - model, - f"Azure AI Foundry takes only a level name, so only {level!r} is sent.", - drop_params_gated=False, - ) - return {"reasoning_effort": level} - - -def _declared_reasoning_effort_param( - non_default_params: Mapping[str, object], model: str, base_model: str | None, drop_params: bool -) -> Mapping[str, str] | _RefusedReasoningEffort: - declared: Final = any(_declares_reasoning_effort(name) for name in _capability_models(model, base_model)) - if "reasoning_effort" not in non_default_params or not declared: - return {} - return _reasoning_effort_level(non_default_params["reasoning_effort"], model, drop_params) - - def _message_fields_to_strip(model: str, base_model: str | None) -> tuple[str, ...]: names: Final = _capability_models(model, base_model) if not any(name.casefold().startswith(_REASONING_CONTENT_MODEL_PREFIXES) for name in names): @@ -257,6 +195,130 @@ class AzureAIGPT5Config(OpenAIGPT5Config): azureAIGPT5Config: Final = AzureAIGPT5Config() +_REASONING_EFFORT: Final = "reasoning_effort" +_LEVELS_LEFT_TO_THE_MODEL: Final = frozenset(("default", "auto")) + + +def _catalog_row(key: str) -> Mapping[str, object] | None: + catalog: Final = catalog_with_runtime_registrations() + canonical: Final = GetModelCostMap.loaded_model_cost_map_key(key) + catalog_row: Final = catalog.get(canonical) if canonical is not None else None + registered_row: Final = catalog.get(key) + if catalog_row is None or registered_row is None: + return registered_row if catalog_row is None else catalog_row + return {**catalog_row, **registered_row} + + +def _catalog_levels(model: str) -> tuple[str, ...] | None: + row: Final = _catalog_row(f"azure_ai/{model.removeprefix('azure_ai/')}") + return declared_reasoning_efforts(row) if row is not None else None + + +def _catalog_reasoning_efforts(model: str, base_model: str | None) -> tuple[str, ...] | None: + return next(filter(None, (_catalog_levels(name) for name in (base_model, model) if name)), None) + + +def _with_reasoning_effort( + supported_params: Sequence[str], model: str +) -> list[str]: # mutable-ok: get_supported_openai_params returns a list its callers extend + if _REASONING_EFFORT in supported_params or not _catalog_levels(model): + return list(supported_params) + return [*supported_params, _REASONING_EFFORT] + + +@dataclass(frozen=True, slots=True) +class _SendReasoningEffort: + level: str + dropped_fields: Mapping[str, object] + + +@dataclass(frozen=True, slots=True) +class _LeaveReasoningEffortToModel: + pass + + +@dataclass(frozen=True, slots=True) +class _UnsendableReasoningEffort: + reason: str + + +def _unsendable_reason(level: object, accepted_levels: tuple[str, ...]) -> str: + if isinstance(level, str) and level in REASONING_EFFORT_ADVERTISEMENT_ORDER: + return f"the model catalog lists only {list(accepted_levels)} in its reasoning_effort_levels." + return f"it is not one of the levels {', '.join(REASONING_EFFORT_ADVERTISEMENT_ORDER)}." + + +def _foundry_reasoning_effort( + requested: object, accepted_levels: tuple[str, ...] | None +) -> _SendReasoningEffort | _LeaveReasoningEffortToModel | _UnsendableReasoningEffort: + fields: Final = as_str_mapping(requested) + level: Final = fields.get("effort") if fields is not None else requested + if isinstance(level, str) and level in _LEVELS_LEFT_TO_THE_MODEL: + return _LeaveReasoningEffortToModel() + if isinstance(level, str) and (accepted_levels is None or level in accepted_levels): + return _SendReasoningEffort(level, {key: value for key, value in (fields or {}).items() if key != "effort"}) + return _UnsendableReasoningEffort( + _unsendable_reason(level, accepted_levels or ()) + if isinstance(level, str) + else "it names no level such as 'high'." + ) + + +def _catalog_reasoning_effort_outcome( + requested: object, model: str, *, base_model: str | None, level_check_waived: bool +) -> _SendReasoningEffort | _LeaveReasoningEffortToModel | _UnsendableReasoningEffort | None: + catalog_levels: Final = _catalog_reasoning_efforts(model, base_model) + if catalog_levels is None: + return None + return _foundry_reasoning_effort(requested, None if level_check_waived else catalog_levels) + + +def _reasoning_effort_params( + non_default_params: dict[str, object], # mutable-ok: a drop removes the effort from the caller's params + model: str, + *, + base_model: str | None, + level_check_waived: bool, + drop_params: bool, +) -> Mapping[str, str]: + if _REASONING_EFFORT not in non_default_params: + return {} + requested: Final = non_default_params[_REASONING_EFFORT] + outcome: Final = _catalog_reasoning_effort_outcome( + requested, model, base_model=base_model, level_check_waived=level_check_waived + ) + match outcome: + case None: + return {} + case _SendReasoningEffort(level=level, dropped_fields=dropped_fields): + if dropped_fields: + warn_dropped_reasoning( + _REASONING_EFFORT, + dropped_fields, + model, + f"Azure AI Foundry takes only a level name, so only {level!r} is sent.", + drop_params_gated=False, + ) + return {_REASONING_EFFORT: level} + case _LeaveReasoningEffortToModel(): + non_default_params.pop(_REASONING_EFFORT) + return {} + case _UnsendableReasoningEffort(reason=reason) if drop_params: + warn_dropped_reasoning(_REASONING_EFFORT, requested, model, reason, drop_params_gated=True) + non_default_params.pop(_REASONING_EFFORT) + return {} + case _UnsendableReasoningEffort(reason=reason): + raise UnsupportedParamsError( + message=f"azure_ai cannot send reasoning_effort={requested!r} for model={model} because {reason} " + "To drop it from the call, set `litellm.drop_params=True` or for proxy:\n\n" + "`litellm_settings:\n drop_params: true`\n", + llm_provider="azure_ai", + model=model, + status_code=400, + ) + case _: + assert_never(outcome) + class AzureAIStudioConfig(OpenAIConfig): def get_supported_openai_params(self, model: str) -> list[str]: @@ -266,7 +328,7 @@ class AzureAIStudioConfig(OpenAIConfig): supported_params = ( azureAIGPT5Config.get_supported_openai_params(model) if azureAIGPT5Config.is_model_gpt_5_model(model) - else super().get_supported_openai_params(model) + else _with_reasoning_effort(super().get_supported_openai_params(model), model) ) if not model_supports_tool_choice: filtered_supported_params: Final = [] @@ -279,12 +341,7 @@ class AzureAIStudioConfig(OpenAIConfig): if not self._supports_stop_reason(model): supported_params = [param for param in supported_params if param != "stop"] - declared_extra: Final = ( - ("reasoning_effort",) - if "reasoning_effort" not in supported_params and _declares_reasoning_effort(model) - else () - ) - return [*supported_params, *declared_extra] + return supported_params def map_openai_params( self, @@ -296,6 +353,7 @@ class AzureAIStudioConfig(OpenAIConfig): declared_reasoning_efforts: Collection[str] | None = None, reasoning_drop_logger: WarningLogger = verbose_logger, base_model: str | None = None, + allowed_openai_params: Collection[str] = (), ) -> dict[str, object]: # mutable-ok: OpenAIConfig.map_openai_params signature if azureAIGPT5Config.is_model_gpt_5_model(model): return azureAIGPT5Config.map_openai_params( @@ -306,14 +364,27 @@ class AzureAIStudioConfig(OpenAIConfig): declared_reasoning_efforts=declared_reasoning_efforts, reasoning_drop_logger=reasoning_drop_logger, ) - effort: Final = _declared_reasoning_effort_param(non_default_params, model, base_model, drop_params) - if isinstance(effort, _RefusedReasoningEffort): - raise _refused_reasoning_effort_error(effort, model) - return super().map_openai_params( - non_default_params=non_default_params, - optional_params={**optional_params, **effort}, - model=model, - drop_params=drop_params, + foundry_effort: Final = _reasoning_effort_params( + non_default_params, + model, + base_model=base_model, + level_check_waived=bool(declared_reasoning_efforts) or _REASONING_EFFORT in allowed_openai_params, + drop_params=drop_params or litellm.drop_params is True, + ) + return { + **super().map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model=model, + drop_params=drop_params, + ), + **foundry_effort, + } + + def refuses_reasoning_effort(self, model: str, reasoning_effort: object, *, base_model: str | None) -> bool: + return not azureAIGPT5Config.is_model_gpt_5_model(model) and isinstance( + _catalog_reasoning_effort_outcome(reasoning_effort, model, base_model=base_model, level_check_waived=False), + _UnsendableReasoningEffort, ) def _supports_stop_reason(self, model: str) -> bool: diff --git a/litellm/llms/base_llm/chat/transformation.py b/litellm/llms/base_llm/chat/transformation.py index 08f986de85a..8e2cc03543f 100644 --- a/litellm/llms/base_llm/chat/transformation.py +++ b/litellm/llms/base_llm/chat/transformation.py @@ -515,6 +515,9 @@ class BaseConfig(ABC): def accepts_reasoning_summary_in_reasoning_effort(self) -> bool: return False + def refuses_reasoning_effort(self, model: str, reasoning_effort: object, *, base_model: str | None) -> bool: + return False + def post_stream_processing(self, stream: object) -> object: """Hook for providers to post-process streaming responses. Default: pass-through.""" return stream diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 668cc68f453..4ed07e8c103 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -13275,8 +13275,12 @@ "mode": "chat", "output_cost_per_token": 1.5e-05, "reasoning_effort_levels": [ + "none", + "minimal", "low", + "medium", "high", + "xhigh", "max" ], "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -13736,6 +13740,11 @@ "max_output_tokens": 384000, "mode": "chat", "output_cost_per_token": 3.48e-06, + "reasoning_effort_levels": [ + "none", + "high", + "max" + ], "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_pdf_input": false, @@ -13757,6 +13766,11 @@ "max_output_tokens": 384000, "mode": "chat", "output_cost_per_token": 5.1e-07, + "reasoning_effort_levels": [ + "none", + "high", + "max" + ], "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_pdf_input": false, @@ -13778,6 +13792,11 @@ "max_output_tokens": 384000, "mode": "chat", "output_cost_per_token": 1.32e-06, + "reasoning_effort_levels": [ + "none", + "high", + "max" + ], "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", "supports_function_calling": true, "supports_pdf_input": false, diff --git a/litellm/router_rust/invoke/shaping.py b/litellm/router_rust/invoke/shaping.py index fec11c3d642..3162d8fa32b 100644 --- a/litellm/router_rust/invoke/shaping.py +++ b/litellm/router_rust/invoke/shaping.py @@ -31,6 +31,7 @@ from litellm.litellm_core_utils.completion_timeout import CompletionTimeout from litellm.litellm_core_utils.core_helpers import allowed_openai_param_names, without from litellm.litellm_core_utils.get_litellm_params import SUPPLEMENTAL_KWARGS_KEYS from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider, inferred_provider +from litellm.litellm_core_utils.reasoning_effort_capability import deployment_reasoning_effort_levels from litellm.litellm_core_utils.request_timeout_resolver import get_service_tier_timeout from litellm.litellm_core_utils.routing_common_utils import deployment_model_id_pin, metadata_defaults_under_request from litellm.litellm_core_utils.tool_fields import TOOL_FIELDS, drop_policy @@ -49,7 +50,8 @@ from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import ( from litellm.router_utils.stream_usage_injection import router_config_requests_stream_usage from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD from litellm.types.router import Deployment, GenericLiteLLMParams, LiteLLM_Params, validate_litellm_params_strictly -from litellm.types.utils import AUTOROUTER_CLASSIFIER_CALL_ORIGIN +from litellm.types.utils import AUTOROUTER_CLASSIFIER_CALL_ORIGIN, LlmProviders +from litellm.utils import ProviderConfigManager if TYPE_CHECKING: from litellm.redis.stores.router_usage import RouterUsage @@ -245,38 +247,65 @@ def _declared_param_allowlist(params: Mapping[str, object]) -> frozenset[str]: return frozenset(allowed_openai_param_names(params.get("allowed_openai_params"))) +def _resolved_upstream(params: Mapping[str, object], group: str, host: "PythonHost") -> tuple[str, str]: + model_label: Final = str(params.get("model") or group) + provider_param: Final = params.get("custom_llm_provider") + authenticating: Final = declared_authenticating_provider( + model_label, provider_param if isinstance(provider_param, str) else None + ) + if authenticating is not None: + return model_label.removeprefix(f"{authenticating}/"), authenticating + model, custom_llm_provider, _, _ = host.invoke("get_llm_provider")( + model=model_label, + custom_llm_provider=provider_param, + litellm_params=LiteLLM_Params.model_validate(params), + ) + return model, custom_llm_provider + + +def _deployment_base_model(deployment: Mapping[str, object], params: Mapping[str, object]) -> str | None: + base_model: Final = as_dict(deployment.get("model_info")).get("base_model") or params.get("base_model") + return base_model if isinstance(base_model, str) else None + + def _deployment_supports_param(deployment: Mapping[str, object], group: str, param: str, host: "PythonHost") -> bool: params: Final = as_dict(deployment.get("litellm_params")) if not params: return True - model_info: Final = as_dict(deployment.get("model_info")) - base_model: Final = model_info.get("base_model") or params.get("base_model") try: - model_label: Final = str(params.get("model") or group) - provider_param: Final = params.get("custom_llm_provider") - authenticating: Final = declared_authenticating_provider( - model_label, provider_param if isinstance(provider_param, str) else None - ) - if authenticating is not None: - model, custom_llm_provider = model_label.removeprefix(f"{authenticating}/"), authenticating - else: - model, custom_llm_provider, _, _ = host.invoke( # rebind-ok: rebound as the Python router rebinds it - "get_llm_provider" - )( - model=model_label, - custom_llm_provider=provider_param, - litellm_params=LiteLLM_Params.model_validate(params), - ) + model, custom_llm_provider = _resolved_upstream(params, group, host) supported: Final = host.invoke("get_supported_openai_params")( model=model, custom_llm_provider=custom_llm_provider, - base_model=base_model if isinstance(base_model, str) else None, + base_model=_deployment_base_model(deployment, params), ) except Exception: # noqa: BLE001 # best-effort filter: an unresolvable provider must not narrow the request return True return supported is None or param in supported +def _deployment_refuses_reasoning_effort( + deployment: Mapping[str, object], group: str, reasoning_effort: object, host: "PythonHost" +) -> bool: + params: Final = as_dict(deployment.get("litellm_params")) + if ( + not params + or "reasoning_effort" in _declared_param_allowlist(params) + or deployment_reasoning_effort_levels(deployment.get("model_info")) + ): + return False + try: + model, custom_llm_provider = _resolved_upstream(params, group, host) + chat_config: Final = ProviderConfigManager.get_provider_chat_config( + model=model, provider=LlmProviders(custom_llm_provider) + ) + except Exception: # noqa: BLE001 # best-effort filter: an unresolvable provider must not narrow the request + return False + return chat_config is not None and chat_config.refuses_reasoning_effort( + model, reasoning_effort, base_model=_deployment_base_model(deployment, params) + ) + + def deployment_accepts_param(deployment: Mapping[str, object], group: str, param: str, host: "PythonHost") -> bool: params: Final = as_dict(deployment.get("litellm_params")) if not params: @@ -298,9 +327,12 @@ def drop_unsupported_classifier_reasoning_effort( host: "PythonHost", ) -> None: """``_drop_unsupported_classifier_reasoning_effort``.""" - if kwargs.get("reasoning_effort") is None or not _is_classifier_internal_call(kwargs): + reasoning_effort: Final = kwargs.get("reasoning_effort") + if reasoning_effort is None or not _is_classifier_internal_call(kwargs): return - if deployment_accepts_param(deployment, model, "reasoning_effort", host): + if deployment_accepts_param(deployment, model, "reasoning_effort", host) and not ( + _deployment_refuses_reasoning_effort(deployment, model, reasoning_effort, host) + ): return kwargs.pop("reasoning_effort", None) proxy_server_request: Final = kwargs.get("proxy_server_request") diff --git a/litellm/utils.py b/litellm/utils.py index d963bc3304f..62e778b7766 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5597,6 +5597,7 @@ def _map_params_with_provider_config( model_info: Mapping[str, object] | None, reasoning_drop_logger: WarningLogger, base_model: str | None, + allowed_openai_params: Sequence[str] | None, ) -> dict[str, object]: # mutable-ok: get_optional_params keeps building on the mapped params if custom_llm_provider == "azure_ai" and isinstance(provider_config, litellm.AzureAIStudioConfig): return provider_config.map_openai_params( @@ -5607,6 +5608,7 @@ def _map_params_with_provider_config( declared_reasoning_efforts=_declared_reasoning_efforts(model_info), reasoning_drop_logger=reasoning_drop_logger, base_model=base_model, + allowed_openai_params=allowed_openai_param_names(allowed_openai_params), ) if isinstance(provider_config, (litellm.ChatGPTConfig, litellm.GithubCopilotConfig)): return provider_config.map_openai_params( @@ -6284,6 +6286,7 @@ def get_optional_params( model_info=model_info, reasoning_drop_logger=_litellm_reasoning_drop_logger, base_model=base_model, + allowed_openai_params=allowed_openai_params, ) else: # assume passing in params for openai-like api optional_params = litellm.OpenAILikeChatConfig().map_openai_params( diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 668cc68f453..4ed07e8c103 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -13275,8 +13275,12 @@ "mode": "chat", "output_cost_per_token": 1.5e-05, "reasoning_effort_levels": [ + "none", + "minimal", "low", + "medium", "high", + "xhigh", "max" ], "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", @@ -13736,6 +13740,11 @@ "max_output_tokens": 384000, "mode": "chat", "output_cost_per_token": 3.48e-06, + "reasoning_effort_levels": [ + "none", + "high", + "max" + ], "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_pdf_input": false, @@ -13757,6 +13766,11 @@ "max_output_tokens": 384000, "mode": "chat", "output_cost_per_token": 5.1e-07, + "reasoning_effort_levels": [ + "none", + "high", + "max" + ], "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supports_function_calling": true, "supports_pdf_input": false, @@ -13778,6 +13792,11 @@ "max_output_tokens": 384000, "mode": "chat", "output_cost_per_token": 1.32e-06, + "reasoning_effort_levels": [ + "none", + "high", + "max" + ], "source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure", "supports_function_calling": true, "supports_pdf_input": false, diff --git a/tests/unit/llms/anthropic/messages/test_reasoning_effort_fields.py b/tests/unit/llms/anthropic/messages/test_reasoning_effort_fields.py index 7a26f403408..53acf1f4bfa 100644 --- a/tests/unit/llms/anthropic/messages/test_reasoning_effort_fields.py +++ b/tests/unit/llms/anthropic/messages/test_reasoning_effort_fields.py @@ -223,18 +223,18 @@ class TestAdvertisedLevelsAreTheForwardedLevels: "which levels does this deployment take" through two different readers, so the proxy advertised kimi-k3 max while /v1/messages quietly forwarded high. Both now resolve through one owner.""" - KIMI_K3_SPELLINGS = ( + KIMI_K3_PASSTHROUGH_SPELLINGS = ( ("kimi-k3", "moonshot"), ("kimi-k3", "fireworks_ai"), ("kimi-k3-us", "fireworks_ai"), - ("FW-Kimi-K3", "azure_ai"), ) + KIMI_K3_SPELLINGS = (*KIMI_K3_PASSTHROUGH_SPELLINGS, ("FW-Kimi-K3", "azure_ai")) @pytest.mark.parametrize("model, provider", KIMI_K3_SPELLINGS) def test_a_declared_level_is_forwarded_rather_than_degraded(self, local_model_cost_map, model, provider): assert normalize_reasoning_effort_value("max", model, provider) == "max" - @pytest.mark.parametrize("model, provider", KIMI_K3_SPELLINGS) + @pytest.mark.parametrize("model, provider", KIMI_K3_PASSTHROUGH_SPELLINGS) def test_a_level_the_entry_does_not_declare_still_degrades(self, local_model_cost_map, model, provider): """kimi-k3 declares low, high and max, so xhigh and minimal are absent from its set and keep falling through the chain rather than being waved past by the presence of a declaration.""" @@ -263,11 +263,12 @@ class TestAdvertisedLevelsAreTheForwardedLevels: for effort in ("minimal", "xhigh", "max"): assert normalize_reasoning_effort_value(effort, model, provider) in supported - def test_the_wider_perplexity_entry_keeps_the_levels_it_declares(self, local_model_cost_map): - """The entry describing that reseller declares a six-level set, and every one of them is + @pytest.mark.parametrize("model, provider", [("perplexity/kimi-k3", "perplexity"), ("FW-Kimi-K3", "azure_ai")]) + def test_a_wider_entry_keeps_the_levels_it_declares(self, local_model_cost_map, model, provider): + """The entries describing that reseller and Foundry declare wider sets, and every level in them is forwarded, which is what the declared list exists to express.""" - assert normalize_reasoning_effort_value("xhigh", "perplexity/kimi-k3", "perplexity") == "xhigh" - assert normalize_reasoning_effort_value("minimal", "perplexity/kimi-k3", "perplexity") == "minimal" + assert normalize_reasoning_effort_value("xhigh", model, provider) == "xhigh" + assert normalize_reasoning_effort_value("minimal", model, provider) == "minimal" def test_the_minimal_chain_clears_a_deployment_that_refuses_low(self, local_model_cost_map): """gpt-5.5-pro accepts medium, high and xhigh only, so the nearest level to ``minimal`` it diff --git a/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py b/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py index d924b0609e0..229b47bdc40 100644 --- a/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py +++ b/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py @@ -1,7 +1,7 @@ import json import logging import traceback -from collections.abc import Callable +from collections.abc import Callable, Iterator, Mapping from types import MappingProxyType from typing import Final from unittest.mock import MagicMock, patch @@ -12,7 +12,7 @@ from httpx import Response import litellm from litellm import Router -from litellm.llms.custom_httpx.http_handler import HTTPHandler +from litellm.constants import DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET from litellm.litellm_core_utils.get_model_cost_map import ( GetModelCostMap, ModelCostMapReloaded, @@ -23,6 +23,7 @@ from litellm.llms.azure_ai.azure_model_router.transformation import ( AzureModelRouterConfig, ) from litellm.llms.azure_ai.chat.transformation import AzureAIStudioConfig +from litellm.llms.custom_httpx.http_handler import HTTPHandler @pytest.mark.asyncio @@ -340,6 +341,276 @@ def test_foundry_gpt_6_astra_refuses_reasoning_effort_none_before_the_call(_loca ) +_FOUNDRY_HOST: Final = "qa-reasoning.services.ai.azure.com" +_FOUNDRY_MESSAGES: Final = ({"role": "user", "content": "Say hi."},) + + +def _foundry_row(**capabilities: object) -> Mapping[str, object]: + return { + "litellm_provider": "azure_ai", + "mode": "chat", + "input_cost_per_token": 0, + "output_cost_per_token": 0, + **capabilities, + } + + +_DECLARED_LEVELS: Final = ("none", "high", "max") +_FOUNDRY_ROWS: Final = MappingProxyType( + { + "azure_ai/qa-reasoner": _foundry_row(supports_reasoning=True, reasoning_effort_levels=list(_DECLARED_LEVELS)), + "azure_ai/qa-unleveled": _foundry_row(supports_reasoning=True), + "azure_ai/qa-chat-only": _foundry_row(supports_reasoning=False), + "azure_ai/QA-Cased-Up": _foundry_row(supports_reasoning=True, reasoning_effort_levels=list(_DECLARED_LEVELS)), + "azure_ai/qa-cased-down": _foundry_row(supports_reasoning=True, reasoning_effort_levels=list(_DECLARED_LEVELS)), + "qa-bare-only": { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 0, + "output_cost_per_token": 0, + "supports_reasoning": True, + "reasoning_effort_levels": ["low", "high"], + }, + } +) + + +@pytest.fixture +def foundry_catalog(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + monkeypatch.setattr(litellm, "model_cost", litellm.model_cost) + catalog: Final = {**GetModelCostMap.load_local_model_cost_map(), **_FOUNDRY_ROWS} + assert isinstance(install_model_cost_map(json.dumps(catalog).encode()), ModelCostMapReloaded) + litellm.get_model_info.cache_clear() + yield + litellm.get_model_info.cache_clear() + + +def _mock_foundry_chat(respx_mock: respx.MockRouter, model: str) -> respx.Route: + return respx_mock.post(host=_FOUNDRY_HOST, path="/models/chat/completions").mock( + return_value=Response( + 200, + json={ + "id": "chatcmpl-foundry", + "object": "chat.completion", + "created": 1700000000, + "model": model, + "choices": [{"index": 0, "message": {"role": "assistant", "content": "Hi"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}, + }, + ) + ) + + +def _foundry_request_body(respx_mock: respx.MockRouter, model: str, **params: object) -> dict[str, object]: + route: Final = _mock_foundry_chat(respx_mock, model) + litellm.completion( + model=f"azure_ai/{model}", + api_base=f"https://{_FOUNDRY_HOST}", + api_key="azure-ai-test-key", + messages=list(_FOUNDRY_MESSAGES), + **params, + ) + return json.loads(route.calls.last.request.content) + + +@pytest.mark.usefixtures("foundry_catalog") +@pytest.mark.parametrize( + ("requested", "sent"), + [ + pytest.param("max", "max", id="level"), + pytest.param({"effort": "high", "summary": "auto"}, "high", id="effort-with-summary"), + ], +) +def test_azure_ai_sends_reasoning_effort_to_a_foundry_reasoning_model( + respx_mock: respx.MockRouter, requested: object, sent: str +) -> None: + body: Final = _foundry_request_body(respx_mock, "qa-reasoner", reasoning_effort=requested) + + assert body == {"model": "qa-reasoner", "messages": list(_FOUNDRY_MESSAGES), "reasoning_effort": sent} + + +@pytest.mark.usefixtures("foundry_catalog") +@pytest.mark.parametrize( + "model", ["qa-reasoner", "qa-unleveled"], ids=["catalog-declares-other-levels", "catalog-declares-none"] +) +def test_azure_ai_sends_a_level_the_deployment_declares_whatever_the_catalog_says( + respx_mock: respx.MockRouter, model: str +) -> None: + body: Final = _foundry_request_body( + respx_mock, model, reasoning_effort="low", model_info={"reasoning_effort_levels": ["low", "high"]} + ) + + assert body["reasoning_effort"] == "low" + + +@pytest.mark.usefixtures("foundry_catalog") +@pytest.mark.parametrize( + "requested", + [ + pytest.param("auto", id="auto"), + pytest.param("default", id="default"), + pytest.param({"effort": "high", "summary": "auto"}, id="effort-with-summary"), + ], +) +def test_azure_ai_sends_reasoning_effort_unchanged_for_a_model_the_catalog_lists_no_levels_for( + respx_mock: respx.MockRouter, requested: object +) -> None: + body: Final = _foundry_request_body(respx_mock, "o3-mini", reasoning_effort=requested) + + assert body["reasoning_effort"] == requested + + +@pytest.mark.usefixtures("foundry_catalog") +def test_azure_ai_refuses_reasoning_effort_for_a_foundry_model_without_reasoning(respx_mock: respx.MockRouter) -> None: + with pytest.raises(litellm.UnsupportedParamsError): + _foundry_request_body(respx_mock, "qa-chat-only", reasoning_effort="high") + + +@pytest.mark.usefixtures("foundry_catalog") +def test_azure_ai_sends_a_level_the_foundry_model_declares(respx_mock: respx.MockRouter) -> None: + body: Final = _foundry_request_body(respx_mock, "qa-reasoner", reasoning_effort="none", drop_params=True) + + assert body["reasoning_effort"] == "none" + + +@pytest.mark.usefixtures("foundry_catalog") +@pytest.mark.parametrize( + ("model", "requested"), + [ + pytest.param("qa-reasoner", "low", id="undeclared"), + pytest.param("qa-reasoner", {"summary": "auto"}, id="no-level"), + pytest.param("qa-unleveled", "high", id="catalog-declares-no-levels"), + ], +) +@pytest.mark.parametrize("drop_params_global", [False, True], ids=["per-request", "global"]) +def test_azure_ai_drops_a_reasoning_effort_the_foundry_model_cannot_take_under_drop_params( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + model: str, + requested: object, + drop_params_global: bool, +) -> None: + monkeypatch.setattr(litellm, "drop_params", drop_params_global) + drop_params: Final = {} if drop_params_global else {"drop_params": True} + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + body: Final = _foundry_request_body(respx_mock, model, reasoning_effort=requested, **drop_params) + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] + + assert "reasoning_effort" not in body + assert len(drop_warnings) == 1 + assert "reasoning_effort=" in drop_warnings[0] + + +@pytest.mark.usefixtures("foundry_catalog") +@pytest.mark.parametrize( + ("requested", "named"), + [ + pytest.param("low", "['none', 'high', 'max']", id="undeclared"), + pytest.param({"summary": "auto"}, "no level", id="no-level"), + ], +) +def test_azure_ai_refuses_a_reasoning_effort_the_foundry_model_cannot_take_without_drop_params( + respx_mock: respx.MockRouter, requested: object, named: str +) -> None: + with pytest.raises(litellm.UnsupportedParamsError) as refused: + _foundry_request_body(respx_mock, "qa-reasoner", reasoning_effort=requested) + + assert refused.value.status_code == 400 + assert named in str(refused.value) + + +@pytest.mark.usefixtures("foundry_catalog") +@pytest.mark.parametrize("drop_params", [{}, {"drop_params": True}], ids=["strict", "drop-params"]) +def test_azure_ai_sends_any_level_when_the_request_allows_reasoning_effort( + respx_mock: respx.MockRouter, drop_params: dict[str, object] +) -> None: + body: Final = _foundry_request_body( + respx_mock, "qa-reasoner", reasoning_effort="medium", allowed_openai_params=["reasoning_effort"], **drop_params + ) + + assert body["reasoning_effort"] == "medium" + + +@pytest.mark.usefixtures("foundry_catalog") +@pytest.mark.parametrize("base_model", ["azure_ai/qa-reasoner", "qa-reasoner"], ids=["prefixed", "bare"]) +@pytest.mark.parametrize(("requested", "sent"), [("max", "max"), ("low", None)], ids=["declared", "undeclared"]) +def test_azure_ai_checks_levels_against_the_base_model_row( + respx_mock: respx.MockRouter, base_model: str, requested: str, sent: str | None +) -> None: + body: Final = _foundry_request_body( + respx_mock, "qa-custom-deployment", reasoning_effort=requested, drop_params=True, base_model=base_model + ) + + assert body.get("reasoning_effort") == sent + + +@pytest.mark.usefixtures("foundry_catalog") +def test_azure_ai_checks_levels_against_the_catalog_not_a_sibling_deployments_model_info( + respx_mock: respx.MockRouter, +) -> None: + route: Final = _mock_foundry_chat(respx_mock, "qa-reasoner") + deployment: Final = {"model": "azure_ai/qa-reasoner", "api_base": f"https://{_FOUNDRY_HOST}", "api_key": "k"} + router: Final = Router( + model_list=[ + {"model_name": "narrow", "litellm_params": deployment, "model_info": {"reasoning_effort_levels": ["none"]}}, + {"model_name": "plain", "litellm_params": deployment}, + ] + ) + + router.completion(model="plain", messages=list(_FOUNDRY_MESSAGES), reasoning_effort="max") + + assert json.loads(route.calls.last.request.content)["reasoning_effort"] == "max" + + +@pytest.mark.usefixtures("foundry_catalog") +def test_azure_ai_reads_no_levels_from_a_row_that_does_not_describe_the_foundry_model( + respx_mock: respx.MockRouter, +) -> None: + with pytest.raises(litellm.UnsupportedParamsError, match="does not support"): + _foundry_request_body(respx_mock, "qa-bare-only", reasoning_effort="low") + + +@pytest.mark.usefixtures("foundry_catalog") +@pytest.mark.parametrize( + "requested_as", + [pytest.param("qa-cased-up", id="catalog-mixed-case"), pytest.param("QA-Cased-Down", id="request-mixed-case")], +) +def test_azure_ai_checks_declared_levels_whatever_the_model_name_casing( + respx_mock: respx.MockRouter, requested_as: str +) -> None: + with pytest.raises(litellm.UnsupportedParamsError): + _foundry_request_body(respx_mock, requested_as, reasoning_effort="low") + + +@pytest.mark.usefixtures("foundry_catalog") +@pytest.mark.parametrize("model", ["qa-registered", "qa-reasoner"], ids=["new-row", "over-catalog-row"]) +@pytest.mark.parametrize(("requested", "sent"), [("low", "low"), ("max", None)], ids=["declared", "undeclared"]) +def test_azure_ai_checks_levels_against_a_register_model_registration( + respx_mock: respx.MockRouter, model: str, requested: str, sent: str | None +) -> None: + litellm.register_model( + {f"azure_ai/{model}": _foundry_row(supports_reasoning=True, reasoning_effort_levels=["low", "high"])} + ) + + body: Final = _foundry_request_body(respx_mock, model, reasoning_effort=requested, drop_params=True) + + assert body.get("reasoning_effort") == sent + + +@pytest.mark.usefixtures("foundry_catalog") +@pytest.mark.parametrize(("requested", "sent"), [("max", "max"), ("low", None)], ids=["declared", "undeclared"]) +def test_azure_ai_keeps_catalog_levels_under_a_pricing_registration_spelled_in_another_case( + respx_mock: respx.MockRouter, requested: str, sent: str | None +) -> None: + litellm.register_model({"azure_ai/qa-cased-up": _foundry_row(input_cost_per_token=1e-6)}) + + body: Final = _foundry_request_body(respx_mock, "qa-cased-up", reasoning_effort=requested, drop_params=True) + + assert body.get("reasoning_effort") == sent + + FOUNDRY_MODELS_CHAT_URL: Final = "https://example-resource.services.ai.azure.com/models/chat/completions" FOUNDRY_CHAT_COMPLETION: Final = MappingProxyType( { @@ -376,9 +647,6 @@ def foundry_reasoner(monkeypatch: pytest.MonkeyPatch) -> str: ("model", "requested", "sent"), [ pytest.param("FW-Reasoner", "high", "high", id="declared-level"), - pytest.param("FW-Reasoner", "medium", "medium", id="level-the-row-does-not-list-is-sent-as-asked"), - pytest.param("FW-Reasoner", "none", "none", id="none"), - pytest.param("FW-Reasoner", "auto", "auto", id="auto"), pytest.param("FW-Reasoner", {"effort": "low", "summary": "detailed"}, "low", id="effort-with-summary"), pytest.param("fw-reasoner", "high", "high", id="model-named-in-another-case"), ], @@ -447,11 +715,19 @@ def test_a_foundry_reasoner_named_with_its_provider_prefix_advertises_reasoning_ assert "reasoning_effort" in AzureAIStudioConfig().get_supported_openai_params(f"azure_ai/{foundry_reasoner}") -@pytest.mark.parametrize("drop_params", [False, True]) +@pytest.mark.parametrize( + "request_options", + [ + pytest.param(MappingProxyType({}), id="checked"), + pytest.param(MappingProxyType({"drop_params": True}), id="drop-params"), + pytest.param(MappingProxyType({"allowed_openai_params": ["reasoning_effort"]}), id="check-waived"), + ], +) +@pytest.mark.parametrize("requested", ["default", "auto"]) @pytest.mark.usefixtures("foundry_reasoner") @pytest.mark.asyncio -async def test_foundry_reasoning_effort_default_leaves_the_level_to_the_model( - respx_mock: respx.MockRouter, drop_params: bool +async def test_foundry_reasoning_effort_default_or_auto_leaves_the_level_to_the_model( + respx_mock: respx.MockRouter, request_options: Mapping[str, object], requested: str ) -> None: route: Final = respx_mock.post(FOUNDRY_MODELS_CHAT_URL).mock( return_value=Response(200, json=dict(FOUNDRY_CHAT_COMPLETION)) @@ -462,8 +738,8 @@ async def test_foundry_reasoning_effort_default_leaves_the_level_to_the_model( api_key="azure-ai-test-key", api_base="https://example-resource.services.ai.azure.com", messages=[{"role": "user", "content": "What is 17*23?"}], - reasoning_effort="default", - drop_params=drop_params, + reasoning_effort=requested, + **request_options, ) assert "reasoning_effort" not in json.loads(route.calls.last.request.content) @@ -492,15 +768,17 @@ def test_foundry_reasoning_effort_object_sends_its_level_and_warns_about_the_res assert any("summary" in message and "'effort'" not in message for message in warnings) is warns -NOT_A_FOUNDRY_LEVEL: Final = ( +UNSENDABLE_FOUNDRY_EFFORT: Final = ( + pytest.param("medium", id="level-the-row-does-not-list"), + pytest.param("none", id="none-the-row-does-not-list"), pytest.param("High", id="level-in-another-case"), pytest.param("ultra", id="unknown-level"), pytest.param({"summary": "detailed"}, id="object-without-a-level"), ) -@pytest.mark.parametrize("requested", NOT_A_FOUNDRY_LEVEL) -def test_foundry_reasoning_effort_that_is_not_a_level_is_refused_without_drop_params( +@pytest.mark.parametrize("requested", UNSENDABLE_FOUNDRY_EFFORT) +def test_foundry_reasoning_effort_the_row_does_not_take_is_refused_without_drop_params( foundry_reasoner: str, requested: object ) -> None: with pytest.raises(litellm.UnsupportedParamsError): @@ -512,12 +790,12 @@ def test_foundry_reasoning_effort_that_is_not_a_level_is_refused_without_drop_pa ) -@pytest.mark.parametrize("requested", NOT_A_FOUNDRY_LEVEL) +@pytest.mark.parametrize("requested", UNSENDABLE_FOUNDRY_EFFORT) @pytest.mark.parametrize( ("request_drop_params", "global_drop_params"), (pytest.param(True, False, id="request-drop-params"), pytest.param(False, True, id="global-drop-params")), ) -def test_foundry_reasoning_effort_that_is_not_a_level_is_dropped_with_a_warning_under_drop_params( +def test_foundry_reasoning_effort_the_row_does_not_take_is_dropped_with_a_warning_under_drop_params( foundry_reasoner: str, caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch, @@ -555,7 +833,7 @@ async def test_anthropic_messages_thinking_reaches_a_foundry_reasoner_as_reasoni api_key="azure-ai-test-key", api_base="https://example-resource.services.ai.azure.com", max_tokens=1024, - thinking={"type": "enabled", "budget_tokens": 4000}, + thinking={"type": "enabled", "budget_tokens": DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET}, messages=[ {"role": "user", "content": "What is the weather in SF?"}, { @@ -577,7 +855,7 @@ async def test_anthropic_messages_thinking_reaches_a_foundry_reasoner_as_reasoni ) sent: Final = json.loads(route.calls.last.request.content) - assert isinstance(sent["reasoning_effort"], str) + assert sent["reasoning_effort"] == "high" assert sent["messages"][1]["reasoning_content"] == earlier_thinking assert not _find_key_anywhere(sent["messages"], "thinking_blocks") @@ -594,7 +872,7 @@ async def test_anthropic_messages_thinking_reaches_a_foundry_reasoner_as_reasoni "reasoning_effort_levels": ["low", "high"], } }, - id="levels-a-sibling-deployment-registered-on-the-shared-row", + id="levels-a-router-deployment-registered-on-the-shared-row", ), ], ) @@ -602,7 +880,7 @@ async def test_anthropic_messages_thinking_reaches_a_foundry_reasoner_as_reasoni async def test_foundry_model_without_declared_reasoning_efforts_still_refuses_reasoning_effort( respx_mock: respx.MockRouter, foundry_reasoner: str, registered_at_runtime: dict[str, dict[str, object]] ) -> None: - litellm.register_model(registered_at_runtime) + litellm.register_model(registered_at_runtime, persist_across_reloads=False) route: Final = respx_mock.post(FOUNDRY_MODELS_CHAT_URL).mock( return_value=Response(200, json=dict(FOUNDRY_CHAT_COMPLETION)) ) @@ -1704,7 +1982,7 @@ def test_azure_ai_services_handler(api_base, expected_url): def test_azure_ai_services_with_api_version(): - from litellm.llms.custom_httpx.http_handler import HTTPHandler, AsyncHTTPHandler + from litellm.llms.custom_httpx.http_handler import HTTPHandler client = HTTPHandler() diff --git a/tests/unit/router_utils/test_reasoning_effort_capability.py b/tests/unit/router_utils/test_reasoning_effort_capability.py index c0ff141ffab..88d595a921b 100644 --- a/tests/unit/router_utils/test_reasoning_effort_capability.py +++ b/tests/unit/router_utils/test_reasoning_effort_capability.py @@ -322,7 +322,6 @@ class TestDeclaredEffortList: KIMI_K3_PASSTHROUGH_KEYS = ( - "azure_ai/FW-Kimi-K3", "moonshot/kimi-k3", "together_ai/moonshotai/Kimi-K3", "fireworks_ai/kimi-k3", @@ -345,6 +344,21 @@ class TestKimiK3AdvertisesItsDocumentedLevels: assert resolve_supported_reasoning_efforts(entry, deployment_is_mapped=True) == ("low", "high", "max") + def test_the_foundry_entry_advertises_every_level_foundry_takes(self, local_model_cost_map): + """Foundry's Kimi K3 answered 200 for every level from none to max on a live deployment, + checked on 2026-11-14 for PR #51747, so its row declares all seven.""" + entry = dict(litellm.model_cost["azure_ai/FW-Kimi-K3"], key="azure_ai/FW-Kimi-K3") + + assert resolve_supported_reasoning_efforts(entry, deployment_is_mapped=True) == ( + "none", + "minimal", + "low", + "medium", + "high", + "xhigh", + "max", + ) + def test_the_perplexity_entry_advertises_the_wider_set_it_maps_down(self, local_model_cost_map): """Perplexity's Agent API takes a six-value enum and maps it down internally, so this deployment is legitimately wider than a passthrough. One blanket list could not say both.""" diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index a44e14f662d..ed6a03afe5c 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -10810,12 +10810,13 @@ def foundry_reasoner_row(monkeypatch: pytest.MonkeyPatch) -> str: return model -def test_a_model_map_declaration_alone_forwards_reasoning_effort_without_rounding_it( +def test_a_model_map_declaration_alone_checks_reasoning_effort_without_rounding_it( foundry_reasoner_row: str, respx_mock: respx.MockRouter ) -> None: - assert _chat_body_sent(respx_mock, foundry_reasoner_row, "medium", {"id": "reasoner-1"}, drop_params=True)[ - "reasoning_effort" - ] == "medium" + assert _chat_body_sent(respx_mock, foundry_reasoner_row, "low", {"id": "reasoner-1"})["reasoning_effort"] == "low" + assert "reasoning_effort" not in _chat_body_sent( + respx_mock, foundry_reasoner_row, "medium", {"id": "reasoner-1"}, drop_params=True + ) @pytest.mark.parametrize("model_info, expected", [(_KIMI_DEPLOYMENT, "high"), (None, "medium")]) diff --git a/tests/unit/test_router/core/test_endpoints_characterization_inference.py b/tests/unit/test_router/core/test_endpoints_characterization_inference.py index c975e45b1d3..ac0aa175874 100644 --- a/tests/unit/test_router/core/test_endpoints_characterization_inference.py +++ b/tests/unit/test_router/core/test_endpoints_characterization_inference.py @@ -14,6 +14,7 @@ from typing import Final import pytest import litellm +from litellm.litellm_core_utils.reasoning_effort_capability import REASONING_EFFORT_ADVERTISEMENT_ORDER from litellm.router_core import RouterCore from tests.unit.test_router.core.endpoints_helpers import EndpointHost from tests.unit.test_router.core.scripted_host import ScriptedHost, deployment, timeout @@ -487,6 +488,74 @@ async def test_classifier_call_keeps_reasoning_effort_on_a_deployment_with_suppo assert _sent(host)["reasoning_effort"] == "low" +LEVELED_FOUNDRY_MODEL: Final = "azure_ai/DeepSeek-V4-Flash-0731" + + +def _catalog_levels(model: str) -> tuple[str, ...]: + levels: Final = litellm.model_cost[model]["reasoning_effort_levels"] + assert isinstance(levels, list) and levels + return tuple(str(level) for level in levels) + + +def _unlisted_level(model: str) -> str: + return next(level for level in REASONING_EFFORT_ADVERTISEMENT_ORDER if level not in _catalog_levels(model)) + + +@pytest.mark.parametrize( + ("model", "model_info"), + [(LEVELED_FOUNDRY_MODEL, {}), ("azure_ai/litellm-qa-own-name", {"base_model": LEVELED_FOUNDRY_MODEL})], + ids=["catalog-model", "base-model"], +) +async def test_classifier_call_drops_a_reasoning_effort_level_the_deployment_refuses( + make_core: MakeCore, host: EndpointHost, model: str, model_info: dict[str, object] +): + core: Final = make_core(model_list=[deployment("g", "d1", model=model, model_info=model_info)]) + unlisted: Final = _unlisted_level(LEVELED_FOUNDRY_MODEL) + await core.acompletion( + model="g", + messages=MESSAGES, + reasoning_effort=unlisted, + litellm_metadata=dict(CLASSIFIER_METADATA), + proxy_server_request={"body": {"reasoning_effort": unlisted, "keep": 1}}, + ) + sent: Final = _sent(host) + assert "reasoning_effort" not in sent + assert sent["proxy_server_request"] == {"body": {"keep": 1}} + + +@pytest.mark.parametrize( + ("listed", "deployment_params"), + [(True, {}), (False, {"allowed_openai_params": ["reasoning_effort"]})], + ids=["catalog-level", "allowlisted"], +) +async def test_classifier_call_keeps_a_reasoning_effort_level_the_deployment_sends( + make_core: MakeCore, host: EndpointHost, listed: bool, deployment_params: dict[str, object] +): + level: Final = ( + _catalog_levels(LEVELED_FOUNDRY_MODEL)[-1] if listed else _unlisted_level(LEVELED_FOUNDRY_MODEL) + ) + core: Final = make_core(model_list=[deployment("g", "d1", model=LEVELED_FOUNDRY_MODEL, **deployment_params)]) + await core.acompletion( + model="g", messages=MESSAGES, reasoning_effort=level, litellm_metadata=dict(CLASSIFIER_METADATA) + ) + assert _sent(host)["reasoning_effort"] == level + + +async def test_classifier_call_keeps_a_level_the_deployment_declares_itself(make_core: MakeCore, host: EndpointHost): + unlisted: Final = _unlisted_level(LEVELED_FOUNDRY_MODEL) + core: Final = make_core( + model_list=[ + deployment( + "g", "d1", model=LEVELED_FOUNDRY_MODEL, model_info={"reasoning_effort_levels": [unlisted, "high"]} + ) + ] + ) + await core.acompletion( + model="g", messages=MESSAGES, reasoning_effort=unlisted, litellm_metadata=dict(CLASSIFIER_METADATA) + ) + assert _sent(host)["reasoning_effort"] == unlisted + + # --- silent experiments ---------------------------------------------------------------------