mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(azure_ai): send reasoning_effort to Foundry reasoning models (#51775)
* fix(azure_ai): send reasoning_effort to Foundry reasoning models Foundry models outside the GPT-5 family that the cost map marks as reasoning models, like DeepSeek V4, now accept reasoning_effort. A level the model row declares is sent as is. A level it does not declare is dropped under drop_params and gets a 400 otherwise. Declares the levels DeepSeek V4 Flash and Pro accept on Foundry * fix(azure_ai): send reasoning_effort only to Foundry rows that declare levels Gate reasoning_effort on the cost map's reasoning_effort_levels instead of supports_reasoning, so undeclared reasoning rows keep the drop_params behavior they had before. Look the levels up case-insensitively, keep a dropped effort out of the body when allowed_openai_params lists it, model the level check as values, and name the proxy drop_params setting in the 400 * fix(azure_ai): read Foundry reasoning levels from the loaded catalog row and honor allowed_openai_params Levels now come only from the model's own azure_ai catalog row, checked for the deployment's base_model first, so a sibling deployment's model_info or a bare openai row no longer decides them. A request that allows reasoning_effort sends any level, as before this PR * fix(azure_ai): honor register_model reasoning levels on Foundry rows The level check read only the loaded cost map, so a model registered with litellm.register_model lost its declared reasoning_effort_levels. Read the loaded map with register_model registrations on top, still without the router's per-deployment registrations, so sibling deployments stay separate * fix(azure_ai): leave default and auto to the model and keep rows without levels unchanged default is stripped even when allowed_openai_params skips the level check, since Foundry refuses it on DeepSeek V4 auto leaves the level to the model, the same way the Fireworks config treats it, so Kimi K3 auto keeps working A model whose catalog row lists no levels gets reasoning_effort exactly as it was sent Updated the Kimi K3 level tests to the seven-level Foundry row, and the catalog-only test to the level check * fix(router): drop a classifier reasoning_effort level the deployment refuses The complexity router's classifier kept reasoning_effort once azure_ai listed it, so a level Foundry's row refuses failed the classifier call and sent every request to the capable tier. The classifier now drops a level the deployment's provider config refuses, unless the deployment declares its own levels or allows reasoning_effort * fix(azure_ai): keep the catalog row's reasoning levels under a registration spelled in another case A price-only register_model call under azure_ai/fw-kimi-k3 hid the azure_ai/FW-Kimi-K3 row, so its levels were lost and reasoning_effort answered 400 or was dropped The level lookup now merges the catalog row under the registration, the way register_model itself merges The Foundry Kimi K3 level test cites the live check behind its seven levels --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
parent
dd7d77fea1
commit
b45fae77cd
13 changed files with 647 additions and 138 deletions
|
|
@ -113,11 +113,10 @@ class GetModelCostMap:
|
|||
return _loaded_snapshot(model_cost_inputs.provenance.revision).rows
|
||||
|
||||
@staticmethod
|
||||
def loaded_model_cost_map_entry(key: str) -> Mapping[str, object] | None:
|
||||
"""The loaded catalog's entry for ``key``, else for the key that differs from it only in case."""
|
||||
def loaded_model_cost_map_key(key: str) -> str | None:
|
||||
"""``key`` as the loaded catalog spells it, matched exactly first and then ignoring case."""
|
||||
snapshot: Final = _loaded_snapshot(model_cost_inputs.provenance.revision)
|
||||
exact: Final = snapshot.rows.get(key)
|
||||
return exact if exact is not None else snapshot.rows.get(snapshot.keys_by_casefold.get(key.casefold(), key))
|
||||
return key if key in snapshot.rows else snapshot.keys_by_casefold.get(key.casefold())
|
||||
|
||||
@staticmethod
|
||||
def loaded_model_cost_map_fields() -> frozenset[str]:
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
"""Resolve which reasoning_effort values a deployment, and by intersection a model group, accepts.
|
||||
|
||||
An entry that states its levels outright in reasoning_effort_levels wins whole, for a model whose set
|
||||
the per-level flags cannot express (Kimi K3 takes low, high and max, and no flag can drop medium).
|
||||
the per-level flags cannot express (DeepSeek V4 on Foundry takes none, high and max, and no flag can drop medium).
|
||||
Every other entry answers through its supports_*_reasoning_effort flags, read with the polarity the
|
||||
chat gate uses: medium and high are unconditional, minimal and low are opt-out (refused only on an
|
||||
explicit false), xhigh is opt-in, and none is opt-out except on the azure gpt-5 family, whose chat
|
||||
|
|
|
|||
|
|
@ -1,13 +1,14 @@
|
|||
import copy
|
||||
import enum
|
||||
import re
|
||||
from collections.abc import AsyncIterator, Collection, Iterator, Mapping
|
||||
from collections.abc import AsyncIterator, Collection, Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Final, cast
|
||||
|
||||
import httpx
|
||||
from httpx import Response
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
from typing_extensions import assert_never
|
||||
|
||||
import litellm
|
||||
from litellm._logging import WarningLogger, verbose_logger
|
||||
|
|
@ -50,7 +51,7 @@ from litellm.secret_managers.main import get_secret_str
|
|||
from litellm.types.llms.openai import AllMessageValues, PromptCacheBreakpoint
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import ModelResponse, ModelResponseStream, ProviderField
|
||||
from litellm.utils import add_path_to_api_base, supports_tool_choice
|
||||
from litellm.utils import add_path_to_api_base, catalog_with_runtime_registrations, supports_tool_choice
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
|
||||
|
|
@ -152,76 +153,13 @@ def _deployment_reads_breakpoints(model: str, litellm_params: Mapping[str, objec
|
|||
)
|
||||
|
||||
|
||||
_DEFAULT_REASONING_EFFORT: Final = "default"
|
||||
_REASONING_CONTENT_MODEL_PREFIXES: Final = (AZURE_FOUNDRY_FIREWORKS_MODEL_ID_PREFIX.casefold(), "kimi-")
|
||||
_FOUNDRY_REASONING_EFFORTS: Final = (*REASONING_EFFORT_ADVERTISEMENT_ORDER, "auto")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _RefusedReasoningEffort:
|
||||
requested: object
|
||||
|
||||
|
||||
def _capability_models(model: str, base_model: str | None) -> tuple[str, ...]:
|
||||
return tuple(name.removeprefix("azure_ai/") for name in (model, base_model) if name)
|
||||
|
||||
|
||||
def _declares_reasoning_effort(model: str) -> bool:
|
||||
row: Final = GetModelCostMap.loaded_model_cost_map_entry(f"azure_ai/{model.removeprefix('azure_ai/')}")
|
||||
return row is not None and bool(declared_reasoning_efforts(row))
|
||||
|
||||
|
||||
def _refused_reasoning_effort_error(refused: _RefusedReasoningEffort, model: str) -> UnsupportedParamsError:
|
||||
return UnsupportedParamsError(
|
||||
message=f"azure_ai takes reasoning_effort as one of {', '.join(_FOUNDRY_REASONING_EFFORTS)}, "
|
||||
f"got {refused.requested!r}. To drop unsupported openai params from the call, set `litellm.drop_params = True`",
|
||||
llm_provider="azure_ai",
|
||||
model=model,
|
||||
status_code=400,
|
||||
)
|
||||
|
||||
|
||||
def _unsupported_reasoning_effort(
|
||||
requested: object, model: str, drop_params: bool
|
||||
) -> Mapping[str, str] | _RefusedReasoningEffort:
|
||||
if not drop_params and litellm.drop_params is not True:
|
||||
return _RefusedReasoningEffort(requested)
|
||||
warn_dropped_reasoning(
|
||||
"reasoning_effort", requested, model, "Azure AI Foundry takes only a level name.", drop_params_gated=True
|
||||
)
|
||||
return {}
|
||||
|
||||
|
||||
def _reasoning_effort_level(
|
||||
requested: object, model: str, drop_params: bool
|
||||
) -> Mapping[str, str] | _RefusedReasoningEffort:
|
||||
fields: Final = as_str_mapping(requested)
|
||||
level: Final = fields.get("effort") if fields is not None else requested
|
||||
if level == _DEFAULT_REASONING_EFFORT:
|
||||
return {}
|
||||
if not isinstance(level, str) or level not in _FOUNDRY_REASONING_EFFORTS:
|
||||
return _unsupported_reasoning_effort(requested, model, drop_params)
|
||||
dropped: Final = {key: value for key, value in fields.items() if key != "effort"} if fields is not None else {}
|
||||
if dropped:
|
||||
warn_dropped_reasoning(
|
||||
"reasoning_effort",
|
||||
dropped,
|
||||
model,
|
||||
f"Azure AI Foundry takes only a level name, so only {level!r} is sent.",
|
||||
drop_params_gated=False,
|
||||
)
|
||||
return {"reasoning_effort": level}
|
||||
|
||||
|
||||
def _declared_reasoning_effort_param(
|
||||
non_default_params: Mapping[str, object], model: str, base_model: str | None, drop_params: bool
|
||||
) -> Mapping[str, str] | _RefusedReasoningEffort:
|
||||
declared: Final = any(_declares_reasoning_effort(name) for name in _capability_models(model, base_model))
|
||||
if "reasoning_effort" not in non_default_params or not declared:
|
||||
return {}
|
||||
return _reasoning_effort_level(non_default_params["reasoning_effort"], model, drop_params)
|
||||
|
||||
|
||||
def _message_fields_to_strip(model: str, base_model: str | None) -> tuple[str, ...]:
|
||||
names: Final = _capability_models(model, base_model)
|
||||
if not any(name.casefold().startswith(_REASONING_CONTENT_MODEL_PREFIXES) for name in names):
|
||||
|
|
@ -257,6 +195,130 @@ class AzureAIGPT5Config(OpenAIGPT5Config):
|
|||
|
||||
azureAIGPT5Config: Final = AzureAIGPT5Config()
|
||||
|
||||
_REASONING_EFFORT: Final = "reasoning_effort"
|
||||
_LEVELS_LEFT_TO_THE_MODEL: Final = frozenset(("default", "auto"))
|
||||
|
||||
|
||||
def _catalog_row(key: str) -> Mapping[str, object] | None:
|
||||
catalog: Final = catalog_with_runtime_registrations()
|
||||
canonical: Final = GetModelCostMap.loaded_model_cost_map_key(key)
|
||||
catalog_row: Final = catalog.get(canonical) if canonical is not None else None
|
||||
registered_row: Final = catalog.get(key)
|
||||
if catalog_row is None or registered_row is None:
|
||||
return registered_row if catalog_row is None else catalog_row
|
||||
return {**catalog_row, **registered_row}
|
||||
|
||||
|
||||
def _catalog_levels(model: str) -> tuple[str, ...] | None:
|
||||
row: Final = _catalog_row(f"azure_ai/{model.removeprefix('azure_ai/')}")
|
||||
return declared_reasoning_efforts(row) if row is not None else None
|
||||
|
||||
|
||||
def _catalog_reasoning_efforts(model: str, base_model: str | None) -> tuple[str, ...] | None:
|
||||
return next(filter(None, (_catalog_levels(name) for name in (base_model, model) if name)), None)
|
||||
|
||||
|
||||
def _with_reasoning_effort(
|
||||
supported_params: Sequence[str], model: str
|
||||
) -> list[str]: # mutable-ok: get_supported_openai_params returns a list its callers extend
|
||||
if _REASONING_EFFORT in supported_params or not _catalog_levels(model):
|
||||
return list(supported_params)
|
||||
return [*supported_params, _REASONING_EFFORT]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _SendReasoningEffort:
|
||||
level: str
|
||||
dropped_fields: Mapping[str, object]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _LeaveReasoningEffortToModel:
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _UnsendableReasoningEffort:
|
||||
reason: str
|
||||
|
||||
|
||||
def _unsendable_reason(level: object, accepted_levels: tuple[str, ...]) -> str:
|
||||
if isinstance(level, str) and level in REASONING_EFFORT_ADVERTISEMENT_ORDER:
|
||||
return f"the model catalog lists only {list(accepted_levels)} in its reasoning_effort_levels."
|
||||
return f"it is not one of the levels {', '.join(REASONING_EFFORT_ADVERTISEMENT_ORDER)}."
|
||||
|
||||
|
||||
def _foundry_reasoning_effort(
|
||||
requested: object, accepted_levels: tuple[str, ...] | None
|
||||
) -> _SendReasoningEffort | _LeaveReasoningEffortToModel | _UnsendableReasoningEffort:
|
||||
fields: Final = as_str_mapping(requested)
|
||||
level: Final = fields.get("effort") if fields is not None else requested
|
||||
if isinstance(level, str) and level in _LEVELS_LEFT_TO_THE_MODEL:
|
||||
return _LeaveReasoningEffortToModel()
|
||||
if isinstance(level, str) and (accepted_levels is None or level in accepted_levels):
|
||||
return _SendReasoningEffort(level, {key: value for key, value in (fields or {}).items() if key != "effort"})
|
||||
return _UnsendableReasoningEffort(
|
||||
_unsendable_reason(level, accepted_levels or ())
|
||||
if isinstance(level, str)
|
||||
else "it names no level such as 'high'."
|
||||
)
|
||||
|
||||
|
||||
def _catalog_reasoning_effort_outcome(
|
||||
requested: object, model: str, *, base_model: str | None, level_check_waived: bool
|
||||
) -> _SendReasoningEffort | _LeaveReasoningEffortToModel | _UnsendableReasoningEffort | None:
|
||||
catalog_levels: Final = _catalog_reasoning_efforts(model, base_model)
|
||||
if catalog_levels is None:
|
||||
return None
|
||||
return _foundry_reasoning_effort(requested, None if level_check_waived else catalog_levels)
|
||||
|
||||
|
||||
def _reasoning_effort_params(
|
||||
non_default_params: dict[str, object], # mutable-ok: a drop removes the effort from the caller's params
|
||||
model: str,
|
||||
*,
|
||||
base_model: str | None,
|
||||
level_check_waived: bool,
|
||||
drop_params: bool,
|
||||
) -> Mapping[str, str]:
|
||||
if _REASONING_EFFORT not in non_default_params:
|
||||
return {}
|
||||
requested: Final = non_default_params[_REASONING_EFFORT]
|
||||
outcome: Final = _catalog_reasoning_effort_outcome(
|
||||
requested, model, base_model=base_model, level_check_waived=level_check_waived
|
||||
)
|
||||
match outcome:
|
||||
case None:
|
||||
return {}
|
||||
case _SendReasoningEffort(level=level, dropped_fields=dropped_fields):
|
||||
if dropped_fields:
|
||||
warn_dropped_reasoning(
|
||||
_REASONING_EFFORT,
|
||||
dropped_fields,
|
||||
model,
|
||||
f"Azure AI Foundry takes only a level name, so only {level!r} is sent.",
|
||||
drop_params_gated=False,
|
||||
)
|
||||
return {_REASONING_EFFORT: level}
|
||||
case _LeaveReasoningEffortToModel():
|
||||
non_default_params.pop(_REASONING_EFFORT)
|
||||
return {}
|
||||
case _UnsendableReasoningEffort(reason=reason) if drop_params:
|
||||
warn_dropped_reasoning(_REASONING_EFFORT, requested, model, reason, drop_params_gated=True)
|
||||
non_default_params.pop(_REASONING_EFFORT)
|
||||
return {}
|
||||
case _UnsendableReasoningEffort(reason=reason):
|
||||
raise UnsupportedParamsError(
|
||||
message=f"azure_ai cannot send reasoning_effort={requested!r} for model={model} because {reason} "
|
||||
"To drop it from the call, set `litellm.drop_params=True` or for proxy:\n\n"
|
||||
"`litellm_settings:\n drop_params: true`\n",
|
||||
llm_provider="azure_ai",
|
||||
model=model,
|
||||
status_code=400,
|
||||
)
|
||||
case _:
|
||||
assert_never(outcome)
|
||||
|
||||
|
||||
class AzureAIStudioConfig(OpenAIConfig):
|
||||
def get_supported_openai_params(self, model: str) -> list[str]:
|
||||
|
|
@ -266,7 +328,7 @@ class AzureAIStudioConfig(OpenAIConfig):
|
|||
supported_params = (
|
||||
azureAIGPT5Config.get_supported_openai_params(model)
|
||||
if azureAIGPT5Config.is_model_gpt_5_model(model)
|
||||
else super().get_supported_openai_params(model)
|
||||
else _with_reasoning_effort(super().get_supported_openai_params(model), model)
|
||||
)
|
||||
if not model_supports_tool_choice:
|
||||
filtered_supported_params: Final = []
|
||||
|
|
@ -279,12 +341,7 @@ class AzureAIStudioConfig(OpenAIConfig):
|
|||
if not self._supports_stop_reason(model):
|
||||
supported_params = [param for param in supported_params if param != "stop"]
|
||||
|
||||
declared_extra: Final = (
|
||||
("reasoning_effort",)
|
||||
if "reasoning_effort" not in supported_params and _declares_reasoning_effort(model)
|
||||
else ()
|
||||
)
|
||||
return [*supported_params, *declared_extra]
|
||||
return supported_params
|
||||
|
||||
def map_openai_params(
|
||||
self,
|
||||
|
|
@ -296,6 +353,7 @@ class AzureAIStudioConfig(OpenAIConfig):
|
|||
declared_reasoning_efforts: Collection[str] | None = None,
|
||||
reasoning_drop_logger: WarningLogger = verbose_logger,
|
||||
base_model: str | None = None,
|
||||
allowed_openai_params: Collection[str] = (),
|
||||
) -> dict[str, object]: # mutable-ok: OpenAIConfig.map_openai_params signature
|
||||
if azureAIGPT5Config.is_model_gpt_5_model(model):
|
||||
return azureAIGPT5Config.map_openai_params(
|
||||
|
|
@ -306,14 +364,27 @@ class AzureAIStudioConfig(OpenAIConfig):
|
|||
declared_reasoning_efforts=declared_reasoning_efforts,
|
||||
reasoning_drop_logger=reasoning_drop_logger,
|
||||
)
|
||||
effort: Final = _declared_reasoning_effort_param(non_default_params, model, base_model, drop_params)
|
||||
if isinstance(effort, _RefusedReasoningEffort):
|
||||
raise _refused_reasoning_effort_error(effort, model)
|
||||
return super().map_openai_params(
|
||||
non_default_params=non_default_params,
|
||||
optional_params={**optional_params, **effort},
|
||||
model=model,
|
||||
drop_params=drop_params,
|
||||
foundry_effort: Final = _reasoning_effort_params(
|
||||
non_default_params,
|
||||
model,
|
||||
base_model=base_model,
|
||||
level_check_waived=bool(declared_reasoning_efforts) or _REASONING_EFFORT in allowed_openai_params,
|
||||
drop_params=drop_params or litellm.drop_params is True,
|
||||
)
|
||||
return {
|
||||
**super().map_openai_params(
|
||||
non_default_params=non_default_params,
|
||||
optional_params=optional_params,
|
||||
model=model,
|
||||
drop_params=drop_params,
|
||||
),
|
||||
**foundry_effort,
|
||||
}
|
||||
|
||||
def refuses_reasoning_effort(self, model: str, reasoning_effort: object, *, base_model: str | None) -> bool:
|
||||
return not azureAIGPT5Config.is_model_gpt_5_model(model) and isinstance(
|
||||
_catalog_reasoning_effort_outcome(reasoning_effort, model, base_model=base_model, level_check_waived=False),
|
||||
_UnsendableReasoningEffort,
|
||||
)
|
||||
|
||||
def _supports_stop_reason(self, model: str) -> bool:
|
||||
|
|
|
|||
|
|
@ -515,6 +515,9 @@ class BaseConfig(ABC):
|
|||
def accepts_reasoning_summary_in_reasoning_effort(self) -> bool:
|
||||
return False
|
||||
|
||||
def refuses_reasoning_effort(self, model: str, reasoning_effort: object, *, base_model: str | None) -> bool:
|
||||
return False
|
||||
|
||||
def post_stream_processing(self, stream: object) -> object:
|
||||
"""Hook for providers to post-process streaming responses. Default: pass-through."""
|
||||
return stream
|
||||
|
|
|
|||
|
|
@ -13275,8 +13275,12 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 1.5e-05,
|
||||
"reasoning_effort_levels": [
|
||||
"none",
|
||||
"minimal",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh",
|
||||
"max"
|
||||
],
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
|
|
@ -13736,6 +13740,11 @@
|
|||
"max_output_tokens": 384000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3.48e-06,
|
||||
"reasoning_effort_levels": [
|
||||
"none",
|
||||
"high",
|
||||
"max"
|
||||
],
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
|
|
@ -13757,6 +13766,11 @@
|
|||
"max_output_tokens": 384000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 5.1e-07,
|
||||
"reasoning_effort_levels": [
|
||||
"none",
|
||||
"high",
|
||||
"max"
|
||||
],
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
|
|
@ -13778,6 +13792,11 @@
|
|||
"max_output_tokens": 384000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.32e-06,
|
||||
"reasoning_effort_levels": [
|
||||
"none",
|
||||
"high",
|
||||
"max"
|
||||
],
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
|
|||
from litellm.litellm_core_utils.core_helpers import allowed_openai_param_names, without
|
||||
from litellm.litellm_core_utils.get_litellm_params import SUPPLEMENTAL_KWARGS_KEYS
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider, inferred_provider
|
||||
from litellm.litellm_core_utils.reasoning_effort_capability import deployment_reasoning_effort_levels
|
||||
from litellm.litellm_core_utils.request_timeout_resolver import get_service_tier_timeout
|
||||
from litellm.litellm_core_utils.routing_common_utils import deployment_model_id_pin, metadata_defaults_under_request
|
||||
from litellm.litellm_core_utils.tool_fields import TOOL_FIELDS, drop_policy
|
||||
|
|
@ -49,7 +50,8 @@ from litellm.router_utils.pre_call_checks.io_token_rate_limit_check import (
|
|||
from litellm.router_utils.stream_usage_injection import router_config_requests_stream_usage
|
||||
from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD
|
||||
from litellm.types.router import Deployment, GenericLiteLLMParams, LiteLLM_Params, validate_litellm_params_strictly
|
||||
from litellm.types.utils import AUTOROUTER_CLASSIFIER_CALL_ORIGIN
|
||||
from litellm.types.utils import AUTOROUTER_CLASSIFIER_CALL_ORIGIN, LlmProviders
|
||||
from litellm.utils import ProviderConfigManager
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.redis.stores.router_usage import RouterUsage
|
||||
|
|
@ -245,38 +247,65 @@ def _declared_param_allowlist(params: Mapping[str, object]) -> frozenset[str]:
|
|||
return frozenset(allowed_openai_param_names(params.get("allowed_openai_params")))
|
||||
|
||||
|
||||
def _resolved_upstream(params: Mapping[str, object], group: str, host: "PythonHost") -> tuple[str, str]:
|
||||
model_label: Final = str(params.get("model") or group)
|
||||
provider_param: Final = params.get("custom_llm_provider")
|
||||
authenticating: Final = declared_authenticating_provider(
|
||||
model_label, provider_param if isinstance(provider_param, str) else None
|
||||
)
|
||||
if authenticating is not None:
|
||||
return model_label.removeprefix(f"{authenticating}/"), authenticating
|
||||
model, custom_llm_provider, _, _ = host.invoke("get_llm_provider")(
|
||||
model=model_label,
|
||||
custom_llm_provider=provider_param,
|
||||
litellm_params=LiteLLM_Params.model_validate(params),
|
||||
)
|
||||
return model, custom_llm_provider
|
||||
|
||||
|
||||
def _deployment_base_model(deployment: Mapping[str, object], params: Mapping[str, object]) -> str | None:
|
||||
base_model: Final = as_dict(deployment.get("model_info")).get("base_model") or params.get("base_model")
|
||||
return base_model if isinstance(base_model, str) else None
|
||||
|
||||
|
||||
def _deployment_supports_param(deployment: Mapping[str, object], group: str, param: str, host: "PythonHost") -> bool:
|
||||
params: Final = as_dict(deployment.get("litellm_params"))
|
||||
if not params:
|
||||
return True
|
||||
model_info: Final = as_dict(deployment.get("model_info"))
|
||||
base_model: Final = model_info.get("base_model") or params.get("base_model")
|
||||
try:
|
||||
model_label: Final = str(params.get("model") or group)
|
||||
provider_param: Final = params.get("custom_llm_provider")
|
||||
authenticating: Final = declared_authenticating_provider(
|
||||
model_label, provider_param if isinstance(provider_param, str) else None
|
||||
)
|
||||
if authenticating is not None:
|
||||
model, custom_llm_provider = model_label.removeprefix(f"{authenticating}/"), authenticating
|
||||
else:
|
||||
model, custom_llm_provider, _, _ = host.invoke( # rebind-ok: rebound as the Python router rebinds it
|
||||
"get_llm_provider"
|
||||
)(
|
||||
model=model_label,
|
||||
custom_llm_provider=provider_param,
|
||||
litellm_params=LiteLLM_Params.model_validate(params),
|
||||
)
|
||||
model, custom_llm_provider = _resolved_upstream(params, group, host)
|
||||
supported: Final = host.invoke("get_supported_openai_params")(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
base_model=base_model if isinstance(base_model, str) else None,
|
||||
base_model=_deployment_base_model(deployment, params),
|
||||
)
|
||||
except Exception: # noqa: BLE001 # best-effort filter: an unresolvable provider must not narrow the request
|
||||
return True
|
||||
return supported is None or param in supported
|
||||
|
||||
|
||||
def _deployment_refuses_reasoning_effort(
|
||||
deployment: Mapping[str, object], group: str, reasoning_effort: object, host: "PythonHost"
|
||||
) -> bool:
|
||||
params: Final = as_dict(deployment.get("litellm_params"))
|
||||
if (
|
||||
not params
|
||||
or "reasoning_effort" in _declared_param_allowlist(params)
|
||||
or deployment_reasoning_effort_levels(deployment.get("model_info"))
|
||||
):
|
||||
return False
|
||||
try:
|
||||
model, custom_llm_provider = _resolved_upstream(params, group, host)
|
||||
chat_config: Final = ProviderConfigManager.get_provider_chat_config(
|
||||
model=model, provider=LlmProviders(custom_llm_provider)
|
||||
)
|
||||
except Exception: # noqa: BLE001 # best-effort filter: an unresolvable provider must not narrow the request
|
||||
return False
|
||||
return chat_config is not None and chat_config.refuses_reasoning_effort(
|
||||
model, reasoning_effort, base_model=_deployment_base_model(deployment, params)
|
||||
)
|
||||
|
||||
|
||||
def deployment_accepts_param(deployment: Mapping[str, object], group: str, param: str, host: "PythonHost") -> bool:
|
||||
params: Final = as_dict(deployment.get("litellm_params"))
|
||||
if not params:
|
||||
|
|
@ -298,9 +327,12 @@ def drop_unsupported_classifier_reasoning_effort(
|
|||
host: "PythonHost",
|
||||
) -> None:
|
||||
"""``_drop_unsupported_classifier_reasoning_effort``."""
|
||||
if kwargs.get("reasoning_effort") is None or not _is_classifier_internal_call(kwargs):
|
||||
reasoning_effort: Final = kwargs.get("reasoning_effort")
|
||||
if reasoning_effort is None or not _is_classifier_internal_call(kwargs):
|
||||
return
|
||||
if deployment_accepts_param(deployment, model, "reasoning_effort", host):
|
||||
if deployment_accepts_param(deployment, model, "reasoning_effort", host) and not (
|
||||
_deployment_refuses_reasoning_effort(deployment, model, reasoning_effort, host)
|
||||
):
|
||||
return
|
||||
kwargs.pop("reasoning_effort", None)
|
||||
proxy_server_request: Final = kwargs.get("proxy_server_request")
|
||||
|
|
|
|||
|
|
@ -5597,6 +5597,7 @@ def _map_params_with_provider_config(
|
|||
model_info: Mapping[str, object] | None,
|
||||
reasoning_drop_logger: WarningLogger,
|
||||
base_model: str | None,
|
||||
allowed_openai_params: Sequence[str] | None,
|
||||
) -> dict[str, object]: # mutable-ok: get_optional_params keeps building on the mapped params
|
||||
if custom_llm_provider == "azure_ai" and isinstance(provider_config, litellm.AzureAIStudioConfig):
|
||||
return provider_config.map_openai_params(
|
||||
|
|
@ -5607,6 +5608,7 @@ def _map_params_with_provider_config(
|
|||
declared_reasoning_efforts=_declared_reasoning_efforts(model_info),
|
||||
reasoning_drop_logger=reasoning_drop_logger,
|
||||
base_model=base_model,
|
||||
allowed_openai_params=allowed_openai_param_names(allowed_openai_params),
|
||||
)
|
||||
if isinstance(provider_config, (litellm.ChatGPTConfig, litellm.GithubCopilotConfig)):
|
||||
return provider_config.map_openai_params(
|
||||
|
|
@ -6284,6 +6286,7 @@ def get_optional_params(
|
|||
model_info=model_info,
|
||||
reasoning_drop_logger=_litellm_reasoning_drop_logger,
|
||||
base_model=base_model,
|
||||
allowed_openai_params=allowed_openai_params,
|
||||
)
|
||||
else: # assume passing in params for openai-like api
|
||||
optional_params = litellm.OpenAILikeChatConfig().map_openai_params(
|
||||
|
|
|
|||
|
|
@ -13275,8 +13275,12 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 1.5e-05,
|
||||
"reasoning_effort_levels": [
|
||||
"none",
|
||||
"minimal",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh",
|
||||
"max"
|
||||
],
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
|
|
@ -13736,6 +13740,11 @@
|
|||
"max_output_tokens": 384000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3.48e-06,
|
||||
"reasoning_effort_levels": [
|
||||
"none",
|
||||
"high",
|
||||
"max"
|
||||
],
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
|
|
@ -13757,6 +13766,11 @@
|
|||
"max_output_tokens": 384000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 5.1e-07,
|
||||
"reasoning_effort_levels": [
|
||||
"none",
|
||||
"high",
|
||||
"max"
|
||||
],
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
|
|
@ -13778,6 +13792,11 @@
|
|||
"max_output_tokens": 384000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.32e-06,
|
||||
"reasoning_effort_levels": [
|
||||
"none",
|
||||
"high",
|
||||
"max"
|
||||
],
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
|
|
|
|||
|
|
@ -223,18 +223,18 @@ class TestAdvertisedLevelsAreTheForwardedLevels:
|
|||
"which levels does this deployment take" through two different readers, so the proxy advertised
|
||||
kimi-k3 max while /v1/messages quietly forwarded high. Both now resolve through one owner."""
|
||||
|
||||
KIMI_K3_SPELLINGS = (
|
||||
KIMI_K3_PASSTHROUGH_SPELLINGS = (
|
||||
("kimi-k3", "moonshot"),
|
||||
("kimi-k3", "fireworks_ai"),
|
||||
("kimi-k3-us", "fireworks_ai"),
|
||||
("FW-Kimi-K3", "azure_ai"),
|
||||
)
|
||||
KIMI_K3_SPELLINGS = (*KIMI_K3_PASSTHROUGH_SPELLINGS, ("FW-Kimi-K3", "azure_ai"))
|
||||
|
||||
@pytest.mark.parametrize("model, provider", KIMI_K3_SPELLINGS)
|
||||
def test_a_declared_level_is_forwarded_rather_than_degraded(self, local_model_cost_map, model, provider):
|
||||
assert normalize_reasoning_effort_value("max", model, provider) == "max"
|
||||
|
||||
@pytest.mark.parametrize("model, provider", KIMI_K3_SPELLINGS)
|
||||
@pytest.mark.parametrize("model, provider", KIMI_K3_PASSTHROUGH_SPELLINGS)
|
||||
def test_a_level_the_entry_does_not_declare_still_degrades(self, local_model_cost_map, model, provider):
|
||||
"""kimi-k3 declares low, high and max, so xhigh and minimal are absent from its set and keep
|
||||
falling through the chain rather than being waved past by the presence of a declaration."""
|
||||
|
|
@ -263,11 +263,12 @@ class TestAdvertisedLevelsAreTheForwardedLevels:
|
|||
for effort in ("minimal", "xhigh", "max"):
|
||||
assert normalize_reasoning_effort_value(effort, model, provider) in supported
|
||||
|
||||
def test_the_wider_perplexity_entry_keeps_the_levels_it_declares(self, local_model_cost_map):
|
||||
"""The entry describing that reseller declares a six-level set, and every one of them is
|
||||
@pytest.mark.parametrize("model, provider", [("perplexity/kimi-k3", "perplexity"), ("FW-Kimi-K3", "azure_ai")])
|
||||
def test_a_wider_entry_keeps_the_levels_it_declares(self, local_model_cost_map, model, provider):
|
||||
"""The entries describing that reseller and Foundry declare wider sets, and every level in them is
|
||||
forwarded, which is what the declared list exists to express."""
|
||||
assert normalize_reasoning_effort_value("xhigh", "perplexity/kimi-k3", "perplexity") == "xhigh"
|
||||
assert normalize_reasoning_effort_value("minimal", "perplexity/kimi-k3", "perplexity") == "minimal"
|
||||
assert normalize_reasoning_effort_value("xhigh", model, provider) == "xhigh"
|
||||
assert normalize_reasoning_effort_value("minimal", model, provider) == "minimal"
|
||||
|
||||
def test_the_minimal_chain_clears_a_deployment_that_refuses_low(self, local_model_cost_map):
|
||||
"""gpt-5.5-pro accepts medium, high and xhigh only, so the nearest level to ``minimal`` it
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import json
|
||||
import logging
|
||||
import traceback
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Callable, Iterator, Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
|
@ -12,7 +12,7 @@ from httpx import Response
|
|||
|
||||
import litellm
|
||||
from litellm import Router
|
||||
from litellm.llms.custom_httpx.http_handler import HTTPHandler
|
||||
from litellm.constants import DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET
|
||||
from litellm.litellm_core_utils.get_model_cost_map import (
|
||||
GetModelCostMap,
|
||||
ModelCostMapReloaded,
|
||||
|
|
@ -23,6 +23,7 @@ from litellm.llms.azure_ai.azure_model_router.transformation import (
|
|||
AzureModelRouterConfig,
|
||||
)
|
||||
from litellm.llms.azure_ai.chat.transformation import AzureAIStudioConfig
|
||||
from litellm.llms.custom_httpx.http_handler import HTTPHandler
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -340,6 +341,276 @@ def test_foundry_gpt_6_astra_refuses_reasoning_effort_none_before_the_call(_loca
|
|||
)
|
||||
|
||||
|
||||
_FOUNDRY_HOST: Final = "qa-reasoning.services.ai.azure.com"
|
||||
_FOUNDRY_MESSAGES: Final = ({"role": "user", "content": "Say hi."},)
|
||||
|
||||
|
||||
def _foundry_row(**capabilities: object) -> Mapping[str, object]:
|
||||
return {
|
||||
"litellm_provider": "azure_ai",
|
||||
"mode": "chat",
|
||||
"input_cost_per_token": 0,
|
||||
"output_cost_per_token": 0,
|
||||
**capabilities,
|
||||
}
|
||||
|
||||
|
||||
_DECLARED_LEVELS: Final = ("none", "high", "max")
|
||||
_FOUNDRY_ROWS: Final = MappingProxyType(
|
||||
{
|
||||
"azure_ai/qa-reasoner": _foundry_row(supports_reasoning=True, reasoning_effort_levels=list(_DECLARED_LEVELS)),
|
||||
"azure_ai/qa-unleveled": _foundry_row(supports_reasoning=True),
|
||||
"azure_ai/qa-chat-only": _foundry_row(supports_reasoning=False),
|
||||
"azure_ai/QA-Cased-Up": _foundry_row(supports_reasoning=True, reasoning_effort_levels=list(_DECLARED_LEVELS)),
|
||||
"azure_ai/qa-cased-down": _foundry_row(supports_reasoning=True, reasoning_effort_levels=list(_DECLARED_LEVELS)),
|
||||
"qa-bare-only": {
|
||||
"litellm_provider": "openai",
|
||||
"mode": "chat",
|
||||
"input_cost_per_token": 0,
|
||||
"output_cost_per_token": 0,
|
||||
"supports_reasoning": True,
|
||||
"reasoning_effort_levels": ["low", "high"],
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def foundry_catalog(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
|
||||
monkeypatch.setattr(litellm, "model_cost", litellm.model_cost)
|
||||
catalog: Final = {**GetModelCostMap.load_local_model_cost_map(), **_FOUNDRY_ROWS}
|
||||
assert isinstance(install_model_cost_map(json.dumps(catalog).encode()), ModelCostMapReloaded)
|
||||
litellm.get_model_info.cache_clear()
|
||||
yield
|
||||
litellm.get_model_info.cache_clear()
|
||||
|
||||
|
||||
def _mock_foundry_chat(respx_mock: respx.MockRouter, model: str) -> respx.Route:
|
||||
return respx_mock.post(host=_FOUNDRY_HOST, path="/models/chat/completions").mock(
|
||||
return_value=Response(
|
||||
200,
|
||||
json={
|
||||
"id": "chatcmpl-foundry",
|
||||
"object": "chat.completion",
|
||||
"created": 1700000000,
|
||||
"model": model,
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "Hi"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4},
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _foundry_request_body(respx_mock: respx.MockRouter, model: str, **params: object) -> dict[str, object]:
|
||||
route: Final = _mock_foundry_chat(respx_mock, model)
|
||||
litellm.completion(
|
||||
model=f"azure_ai/{model}",
|
||||
api_base=f"https://{_FOUNDRY_HOST}",
|
||||
api_key="azure-ai-test-key",
|
||||
messages=list(_FOUNDRY_MESSAGES),
|
||||
**params,
|
||||
)
|
||||
return json.loads(route.calls.last.request.content)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("foundry_catalog")
|
||||
@pytest.mark.parametrize(
|
||||
("requested", "sent"),
|
||||
[
|
||||
pytest.param("max", "max", id="level"),
|
||||
pytest.param({"effort": "high", "summary": "auto"}, "high", id="effort-with-summary"),
|
||||
],
|
||||
)
|
||||
def test_azure_ai_sends_reasoning_effort_to_a_foundry_reasoning_model(
|
||||
respx_mock: respx.MockRouter, requested: object, sent: str
|
||||
) -> None:
|
||||
body: Final = _foundry_request_body(respx_mock, "qa-reasoner", reasoning_effort=requested)
|
||||
|
||||
assert body == {"model": "qa-reasoner", "messages": list(_FOUNDRY_MESSAGES), "reasoning_effort": sent}
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("foundry_catalog")
|
||||
@pytest.mark.parametrize(
|
||||
"model", ["qa-reasoner", "qa-unleveled"], ids=["catalog-declares-other-levels", "catalog-declares-none"]
|
||||
)
|
||||
def test_azure_ai_sends_a_level_the_deployment_declares_whatever_the_catalog_says(
|
||||
respx_mock: respx.MockRouter, model: str
|
||||
) -> None:
|
||||
body: Final = _foundry_request_body(
|
||||
respx_mock, model, reasoning_effort="low", model_info={"reasoning_effort_levels": ["low", "high"]}
|
||||
)
|
||||
|
||||
assert body["reasoning_effort"] == "low"
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("foundry_catalog")
|
||||
@pytest.mark.parametrize(
|
||||
"requested",
|
||||
[
|
||||
pytest.param("auto", id="auto"),
|
||||
pytest.param("default", id="default"),
|
||||
pytest.param({"effort": "high", "summary": "auto"}, id="effort-with-summary"),
|
||||
],
|
||||
)
|
||||
def test_azure_ai_sends_reasoning_effort_unchanged_for_a_model_the_catalog_lists_no_levels_for(
|
||||
respx_mock: respx.MockRouter, requested: object
|
||||
) -> None:
|
||||
body: Final = _foundry_request_body(respx_mock, "o3-mini", reasoning_effort=requested)
|
||||
|
||||
assert body["reasoning_effort"] == requested
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("foundry_catalog")
|
||||
def test_azure_ai_refuses_reasoning_effort_for_a_foundry_model_without_reasoning(respx_mock: respx.MockRouter) -> None:
|
||||
with pytest.raises(litellm.UnsupportedParamsError):
|
||||
_foundry_request_body(respx_mock, "qa-chat-only", reasoning_effort="high")
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("foundry_catalog")
|
||||
def test_azure_ai_sends_a_level_the_foundry_model_declares(respx_mock: respx.MockRouter) -> None:
|
||||
body: Final = _foundry_request_body(respx_mock, "qa-reasoner", reasoning_effort="none", drop_params=True)
|
||||
|
||||
assert body["reasoning_effort"] == "none"
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("foundry_catalog")
|
||||
@pytest.mark.parametrize(
|
||||
("model", "requested"),
|
||||
[
|
||||
pytest.param("qa-reasoner", "low", id="undeclared"),
|
||||
pytest.param("qa-reasoner", {"summary": "auto"}, id="no-level"),
|
||||
pytest.param("qa-unleveled", "high", id="catalog-declares-no-levels"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("drop_params_global", [False, True], ids=["per-request", "global"])
|
||||
def test_azure_ai_drops_a_reasoning_effort_the_foundry_model_cannot_take_under_drop_params(
|
||||
respx_mock: respx.MockRouter,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
model: str,
|
||||
requested: object,
|
||||
drop_params_global: bool,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", drop_params_global)
|
||||
drop_params: Final = {} if drop_params_global else {"drop_params": True}
|
||||
|
||||
with caplog.at_level(logging.WARNING, logger="LiteLLM"):
|
||||
body: Final = _foundry_request_body(respx_mock, model, reasoning_effort=requested, **drop_params)
|
||||
drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()]
|
||||
|
||||
assert "reasoning_effort" not in body
|
||||
assert len(drop_warnings) == 1
|
||||
assert "reasoning_effort=" in drop_warnings[0]
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("foundry_catalog")
|
||||
@pytest.mark.parametrize(
|
||||
("requested", "named"),
|
||||
[
|
||||
pytest.param("low", "['none', 'high', 'max']", id="undeclared"),
|
||||
pytest.param({"summary": "auto"}, "no level", id="no-level"),
|
||||
],
|
||||
)
|
||||
def test_azure_ai_refuses_a_reasoning_effort_the_foundry_model_cannot_take_without_drop_params(
|
||||
respx_mock: respx.MockRouter, requested: object, named: str
|
||||
) -> None:
|
||||
with pytest.raises(litellm.UnsupportedParamsError) as refused:
|
||||
_foundry_request_body(respx_mock, "qa-reasoner", reasoning_effort=requested)
|
||||
|
||||
assert refused.value.status_code == 400
|
||||
assert named in str(refused.value)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("foundry_catalog")
|
||||
@pytest.mark.parametrize("drop_params", [{}, {"drop_params": True}], ids=["strict", "drop-params"])
|
||||
def test_azure_ai_sends_any_level_when_the_request_allows_reasoning_effort(
|
||||
respx_mock: respx.MockRouter, drop_params: dict[str, object]
|
||||
) -> None:
|
||||
body: Final = _foundry_request_body(
|
||||
respx_mock, "qa-reasoner", reasoning_effort="medium", allowed_openai_params=["reasoning_effort"], **drop_params
|
||||
)
|
||||
|
||||
assert body["reasoning_effort"] == "medium"
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("foundry_catalog")
|
||||
@pytest.mark.parametrize("base_model", ["azure_ai/qa-reasoner", "qa-reasoner"], ids=["prefixed", "bare"])
|
||||
@pytest.mark.parametrize(("requested", "sent"), [("max", "max"), ("low", None)], ids=["declared", "undeclared"])
|
||||
def test_azure_ai_checks_levels_against_the_base_model_row(
|
||||
respx_mock: respx.MockRouter, base_model: str, requested: str, sent: str | None
|
||||
) -> None:
|
||||
body: Final = _foundry_request_body(
|
||||
respx_mock, "qa-custom-deployment", reasoning_effort=requested, drop_params=True, base_model=base_model
|
||||
)
|
||||
|
||||
assert body.get("reasoning_effort") == sent
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("foundry_catalog")
|
||||
def test_azure_ai_checks_levels_against_the_catalog_not_a_sibling_deployments_model_info(
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
route: Final = _mock_foundry_chat(respx_mock, "qa-reasoner")
|
||||
deployment: Final = {"model": "azure_ai/qa-reasoner", "api_base": f"https://{_FOUNDRY_HOST}", "api_key": "k"}
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{"model_name": "narrow", "litellm_params": deployment, "model_info": {"reasoning_effort_levels": ["none"]}},
|
||||
{"model_name": "plain", "litellm_params": deployment},
|
||||
]
|
||||
)
|
||||
|
||||
router.completion(model="plain", messages=list(_FOUNDRY_MESSAGES), reasoning_effort="max")
|
||||
|
||||
assert json.loads(route.calls.last.request.content)["reasoning_effort"] == "max"
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("foundry_catalog")
|
||||
def test_azure_ai_reads_no_levels_from_a_row_that_does_not_describe_the_foundry_model(
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
with pytest.raises(litellm.UnsupportedParamsError, match="does not support"):
|
||||
_foundry_request_body(respx_mock, "qa-bare-only", reasoning_effort="low")
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("foundry_catalog")
|
||||
@pytest.mark.parametrize(
|
||||
"requested_as",
|
||||
[pytest.param("qa-cased-up", id="catalog-mixed-case"), pytest.param("QA-Cased-Down", id="request-mixed-case")],
|
||||
)
|
||||
def test_azure_ai_checks_declared_levels_whatever_the_model_name_casing(
|
||||
respx_mock: respx.MockRouter, requested_as: str
|
||||
) -> None:
|
||||
with pytest.raises(litellm.UnsupportedParamsError):
|
||||
_foundry_request_body(respx_mock, requested_as, reasoning_effort="low")
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("foundry_catalog")
|
||||
@pytest.mark.parametrize("model", ["qa-registered", "qa-reasoner"], ids=["new-row", "over-catalog-row"])
|
||||
@pytest.mark.parametrize(("requested", "sent"), [("low", "low"), ("max", None)], ids=["declared", "undeclared"])
|
||||
def test_azure_ai_checks_levels_against_a_register_model_registration(
|
||||
respx_mock: respx.MockRouter, model: str, requested: str, sent: str | None
|
||||
) -> None:
|
||||
litellm.register_model(
|
||||
{f"azure_ai/{model}": _foundry_row(supports_reasoning=True, reasoning_effort_levels=["low", "high"])}
|
||||
)
|
||||
|
||||
body: Final = _foundry_request_body(respx_mock, model, reasoning_effort=requested, drop_params=True)
|
||||
|
||||
assert body.get("reasoning_effort") == sent
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("foundry_catalog")
|
||||
@pytest.mark.parametrize(("requested", "sent"), [("max", "max"), ("low", None)], ids=["declared", "undeclared"])
|
||||
def test_azure_ai_keeps_catalog_levels_under_a_pricing_registration_spelled_in_another_case(
|
||||
respx_mock: respx.MockRouter, requested: str, sent: str | None
|
||||
) -> None:
|
||||
litellm.register_model({"azure_ai/qa-cased-up": _foundry_row(input_cost_per_token=1e-6)})
|
||||
|
||||
body: Final = _foundry_request_body(respx_mock, "qa-cased-up", reasoning_effort=requested, drop_params=True)
|
||||
|
||||
assert body.get("reasoning_effort") == sent
|
||||
|
||||
|
||||
FOUNDRY_MODELS_CHAT_URL: Final = "https://example-resource.services.ai.azure.com/models/chat/completions"
|
||||
FOUNDRY_CHAT_COMPLETION: Final = MappingProxyType(
|
||||
{
|
||||
|
|
@ -376,9 +647,6 @@ def foundry_reasoner(monkeypatch: pytest.MonkeyPatch) -> str:
|
|||
("model", "requested", "sent"),
|
||||
[
|
||||
pytest.param("FW-Reasoner", "high", "high", id="declared-level"),
|
||||
pytest.param("FW-Reasoner", "medium", "medium", id="level-the-row-does-not-list-is-sent-as-asked"),
|
||||
pytest.param("FW-Reasoner", "none", "none", id="none"),
|
||||
pytest.param("FW-Reasoner", "auto", "auto", id="auto"),
|
||||
pytest.param("FW-Reasoner", {"effort": "low", "summary": "detailed"}, "low", id="effort-with-summary"),
|
||||
pytest.param("fw-reasoner", "high", "high", id="model-named-in-another-case"),
|
||||
],
|
||||
|
|
@ -447,11 +715,19 @@ def test_a_foundry_reasoner_named_with_its_provider_prefix_advertises_reasoning_
|
|||
assert "reasoning_effort" in AzureAIStudioConfig().get_supported_openai_params(f"azure_ai/{foundry_reasoner}")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("drop_params", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"request_options",
|
||||
[
|
||||
pytest.param(MappingProxyType({}), id="checked"),
|
||||
pytest.param(MappingProxyType({"drop_params": True}), id="drop-params"),
|
||||
pytest.param(MappingProxyType({"allowed_openai_params": ["reasoning_effort"]}), id="check-waived"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("requested", ["default", "auto"])
|
||||
@pytest.mark.usefixtures("foundry_reasoner")
|
||||
@pytest.mark.asyncio
|
||||
async def test_foundry_reasoning_effort_default_leaves_the_level_to_the_model(
|
||||
respx_mock: respx.MockRouter, drop_params: bool
|
||||
async def test_foundry_reasoning_effort_default_or_auto_leaves_the_level_to_the_model(
|
||||
respx_mock: respx.MockRouter, request_options: Mapping[str, object], requested: str
|
||||
) -> None:
|
||||
route: Final = respx_mock.post(FOUNDRY_MODELS_CHAT_URL).mock(
|
||||
return_value=Response(200, json=dict(FOUNDRY_CHAT_COMPLETION))
|
||||
|
|
@ -462,8 +738,8 @@ async def test_foundry_reasoning_effort_default_leaves_the_level_to_the_model(
|
|||
api_key="azure-ai-test-key",
|
||||
api_base="https://example-resource.services.ai.azure.com",
|
||||
messages=[{"role": "user", "content": "What is 17*23?"}],
|
||||
reasoning_effort="default",
|
||||
drop_params=drop_params,
|
||||
reasoning_effort=requested,
|
||||
**request_options,
|
||||
)
|
||||
|
||||
assert "reasoning_effort" not in json.loads(route.calls.last.request.content)
|
||||
|
|
@ -492,15 +768,17 @@ def test_foundry_reasoning_effort_object_sends_its_level_and_warns_about_the_res
|
|||
assert any("summary" in message and "'effort'" not in message for message in warnings) is warns
|
||||
|
||||
|
||||
NOT_A_FOUNDRY_LEVEL: Final = (
|
||||
UNSENDABLE_FOUNDRY_EFFORT: Final = (
|
||||
pytest.param("medium", id="level-the-row-does-not-list"),
|
||||
pytest.param("none", id="none-the-row-does-not-list"),
|
||||
pytest.param("High", id="level-in-another-case"),
|
||||
pytest.param("ultra", id="unknown-level"),
|
||||
pytest.param({"summary": "detailed"}, id="object-without-a-level"),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("requested", NOT_A_FOUNDRY_LEVEL)
|
||||
def test_foundry_reasoning_effort_that_is_not_a_level_is_refused_without_drop_params(
|
||||
@pytest.mark.parametrize("requested", UNSENDABLE_FOUNDRY_EFFORT)
|
||||
def test_foundry_reasoning_effort_the_row_does_not_take_is_refused_without_drop_params(
|
||||
foundry_reasoner: str, requested: object
|
||||
) -> None:
|
||||
with pytest.raises(litellm.UnsupportedParamsError):
|
||||
|
|
@ -512,12 +790,12 @@ def test_foundry_reasoning_effort_that_is_not_a_level_is_refused_without_drop_pa
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("requested", NOT_A_FOUNDRY_LEVEL)
|
||||
@pytest.mark.parametrize("requested", UNSENDABLE_FOUNDRY_EFFORT)
|
||||
@pytest.mark.parametrize(
|
||||
("request_drop_params", "global_drop_params"),
|
||||
(pytest.param(True, False, id="request-drop-params"), pytest.param(False, True, id="global-drop-params")),
|
||||
)
|
||||
def test_foundry_reasoning_effort_that_is_not_a_level_is_dropped_with_a_warning_under_drop_params(
|
||||
def test_foundry_reasoning_effort_the_row_does_not_take_is_dropped_with_a_warning_under_drop_params(
|
||||
foundry_reasoner: str,
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
|
|
@ -555,7 +833,7 @@ async def test_anthropic_messages_thinking_reaches_a_foundry_reasoner_as_reasoni
|
|||
api_key="azure-ai-test-key",
|
||||
api_base="https://example-resource.services.ai.azure.com",
|
||||
max_tokens=1024,
|
||||
thinking={"type": "enabled", "budget_tokens": 4000},
|
||||
thinking={"type": "enabled", "budget_tokens": DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET},
|
||||
messages=[
|
||||
{"role": "user", "content": "What is the weather in SF?"},
|
||||
{
|
||||
|
|
@ -577,7 +855,7 @@ async def test_anthropic_messages_thinking_reaches_a_foundry_reasoner_as_reasoni
|
|||
)
|
||||
|
||||
sent: Final = json.loads(route.calls.last.request.content)
|
||||
assert isinstance(sent["reasoning_effort"], str)
|
||||
assert sent["reasoning_effort"] == "high"
|
||||
assert sent["messages"][1]["reasoning_content"] == earlier_thinking
|
||||
assert not _find_key_anywhere(sent["messages"], "thinking_blocks")
|
||||
|
||||
|
|
@ -594,7 +872,7 @@ async def test_anthropic_messages_thinking_reaches_a_foundry_reasoner_as_reasoni
|
|||
"reasoning_effort_levels": ["low", "high"],
|
||||
}
|
||||
},
|
||||
id="levels-a-sibling-deployment-registered-on-the-shared-row",
|
||||
id="levels-a-router-deployment-registered-on-the-shared-row",
|
||||
),
|
||||
],
|
||||
)
|
||||
|
|
@ -602,7 +880,7 @@ async def test_anthropic_messages_thinking_reaches_a_foundry_reasoner_as_reasoni
|
|||
async def test_foundry_model_without_declared_reasoning_efforts_still_refuses_reasoning_effort(
|
||||
respx_mock: respx.MockRouter, foundry_reasoner: str, registered_at_runtime: dict[str, dict[str, object]]
|
||||
) -> None:
|
||||
litellm.register_model(registered_at_runtime)
|
||||
litellm.register_model(registered_at_runtime, persist_across_reloads=False)
|
||||
route: Final = respx_mock.post(FOUNDRY_MODELS_CHAT_URL).mock(
|
||||
return_value=Response(200, json=dict(FOUNDRY_CHAT_COMPLETION))
|
||||
)
|
||||
|
|
@ -1704,7 +1982,7 @@ def test_azure_ai_services_handler(api_base, expected_url):
|
|||
|
||||
|
||||
def test_azure_ai_services_with_api_version():
|
||||
from litellm.llms.custom_httpx.http_handler import HTTPHandler, AsyncHTTPHandler
|
||||
from litellm.llms.custom_httpx.http_handler import HTTPHandler
|
||||
|
||||
client = HTTPHandler()
|
||||
|
||||
|
|
|
|||
|
|
@ -322,7 +322,6 @@ class TestDeclaredEffortList:
|
|||
|
||||
|
||||
KIMI_K3_PASSTHROUGH_KEYS = (
|
||||
"azure_ai/FW-Kimi-K3",
|
||||
"moonshot/kimi-k3",
|
||||
"together_ai/moonshotai/Kimi-K3",
|
||||
"fireworks_ai/kimi-k3",
|
||||
|
|
@ -345,6 +344,21 @@ class TestKimiK3AdvertisesItsDocumentedLevels:
|
|||
|
||||
assert resolve_supported_reasoning_efforts(entry, deployment_is_mapped=True) == ("low", "high", "max")
|
||||
|
||||
def test_the_foundry_entry_advertises_every_level_foundry_takes(self, local_model_cost_map):
|
||||
"""Foundry's Kimi K3 answered 200 for every level from none to max on a live deployment,
|
||||
checked on 2026-11-14 for PR #51747, so its row declares all seven."""
|
||||
entry = dict(litellm.model_cost["azure_ai/FW-Kimi-K3"], key="azure_ai/FW-Kimi-K3")
|
||||
|
||||
assert resolve_supported_reasoning_efforts(entry, deployment_is_mapped=True) == (
|
||||
"none",
|
||||
"minimal",
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"xhigh",
|
||||
"max",
|
||||
)
|
||||
|
||||
def test_the_perplexity_entry_advertises_the_wider_set_it_maps_down(self, local_model_cost_map):
|
||||
"""Perplexity's Agent API takes a six-value enum and maps it down internally, so this
|
||||
deployment is legitimately wider than a passthrough. One blanket list could not say both."""
|
||||
|
|
|
|||
|
|
@ -10810,12 +10810,13 @@ def foundry_reasoner_row(monkeypatch: pytest.MonkeyPatch) -> str:
|
|||
return model
|
||||
|
||||
|
||||
def test_a_model_map_declaration_alone_forwards_reasoning_effort_without_rounding_it(
|
||||
def test_a_model_map_declaration_alone_checks_reasoning_effort_without_rounding_it(
|
||||
foundry_reasoner_row: str, respx_mock: respx.MockRouter
|
||||
) -> None:
|
||||
assert _chat_body_sent(respx_mock, foundry_reasoner_row, "medium", {"id": "reasoner-1"}, drop_params=True)[
|
||||
"reasoning_effort"
|
||||
] == "medium"
|
||||
assert _chat_body_sent(respx_mock, foundry_reasoner_row, "low", {"id": "reasoner-1"})["reasoning_effort"] == "low"
|
||||
assert "reasoning_effort" not in _chat_body_sent(
|
||||
respx_mock, foundry_reasoner_row, "medium", {"id": "reasoner-1"}, drop_params=True
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_info, expected", [(_KIMI_DEPLOYMENT, "high"), (None, "medium")])
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ from typing import Final
|
|||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.reasoning_effort_capability import REASONING_EFFORT_ADVERTISEMENT_ORDER
|
||||
from litellm.router_core import RouterCore
|
||||
from tests.unit.test_router.core.endpoints_helpers import EndpointHost
|
||||
from tests.unit.test_router.core.scripted_host import ScriptedHost, deployment, timeout
|
||||
|
|
@ -487,6 +488,74 @@ async def test_classifier_call_keeps_reasoning_effort_on_a_deployment_with_suppo
|
|||
assert _sent(host)["reasoning_effort"] == "low"
|
||||
|
||||
|
||||
LEVELED_FOUNDRY_MODEL: Final = "azure_ai/DeepSeek-V4-Flash-0731"
|
||||
|
||||
|
||||
def _catalog_levels(model: str) -> tuple[str, ...]:
|
||||
levels: Final = litellm.model_cost[model]["reasoning_effort_levels"]
|
||||
assert isinstance(levels, list) and levels
|
||||
return tuple(str(level) for level in levels)
|
||||
|
||||
|
||||
def _unlisted_level(model: str) -> str:
|
||||
return next(level for level in REASONING_EFFORT_ADVERTISEMENT_ORDER if level not in _catalog_levels(model))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "model_info"),
|
||||
[(LEVELED_FOUNDRY_MODEL, {}), ("azure_ai/litellm-qa-own-name", {"base_model": LEVELED_FOUNDRY_MODEL})],
|
||||
ids=["catalog-model", "base-model"],
|
||||
)
|
||||
async def test_classifier_call_drops_a_reasoning_effort_level_the_deployment_refuses(
|
||||
make_core: MakeCore, host: EndpointHost, model: str, model_info: dict[str, object]
|
||||
):
|
||||
core: Final = make_core(model_list=[deployment("g", "d1", model=model, model_info=model_info)])
|
||||
unlisted: Final = _unlisted_level(LEVELED_FOUNDRY_MODEL)
|
||||
await core.acompletion(
|
||||
model="g",
|
||||
messages=MESSAGES,
|
||||
reasoning_effort=unlisted,
|
||||
litellm_metadata=dict(CLASSIFIER_METADATA),
|
||||
proxy_server_request={"body": {"reasoning_effort": unlisted, "keep": 1}},
|
||||
)
|
||||
sent: Final = _sent(host)
|
||||
assert "reasoning_effort" not in sent
|
||||
assert sent["proxy_server_request"] == {"body": {"keep": 1}}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("listed", "deployment_params"),
|
||||
[(True, {}), (False, {"allowed_openai_params": ["reasoning_effort"]})],
|
||||
ids=["catalog-level", "allowlisted"],
|
||||
)
|
||||
async def test_classifier_call_keeps_a_reasoning_effort_level_the_deployment_sends(
|
||||
make_core: MakeCore, host: EndpointHost, listed: bool, deployment_params: dict[str, object]
|
||||
):
|
||||
level: Final = (
|
||||
_catalog_levels(LEVELED_FOUNDRY_MODEL)[-1] if listed else _unlisted_level(LEVELED_FOUNDRY_MODEL)
|
||||
)
|
||||
core: Final = make_core(model_list=[deployment("g", "d1", model=LEVELED_FOUNDRY_MODEL, **deployment_params)])
|
||||
await core.acompletion(
|
||||
model="g", messages=MESSAGES, reasoning_effort=level, litellm_metadata=dict(CLASSIFIER_METADATA)
|
||||
)
|
||||
assert _sent(host)["reasoning_effort"] == level
|
||||
|
||||
|
||||
async def test_classifier_call_keeps_a_level_the_deployment_declares_itself(make_core: MakeCore, host: EndpointHost):
|
||||
unlisted: Final = _unlisted_level(LEVELED_FOUNDRY_MODEL)
|
||||
core: Final = make_core(
|
||||
model_list=[
|
||||
deployment(
|
||||
"g", "d1", model=LEVELED_FOUNDRY_MODEL, model_info={"reasoning_effort_levels": [unlisted, "high"]}
|
||||
)
|
||||
]
|
||||
)
|
||||
await core.acompletion(
|
||||
model="g", messages=MESSAGES, reasoning_effort=unlisted, litellm_metadata=dict(CLASSIFIER_METADATA)
|
||||
)
|
||||
assert _sent(host)["reasoning_effort"] == unlisted
|
||||
|
||||
|
||||
# --- silent experiments ---------------------------------------------------------------------
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue