From 1805db8c8abe2c923873910ef384c8e25b31795f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 08:18:19 -0700 Subject: [PATCH 1/6] fix(logging): warn when drop_params drops a requested reasoning level on the remaining paths --- .../reasoning_effort_utils.py | 54 +++++++++++++- litellm/llms/anthropic/common_utils.py | 33 ++++++--- .../messages/adapters/transformation.py | 7 +- .../llms/anthropic/messages/native/handler.py | 8 ++- .../responses_adapters/transformation.py | 4 +- litellm/llms/azure_ai/chat/transformation.py | 20 +++++- .../chat/chat_completions/transformation.py | 16 +++++ .../bedrock/chat/converse_transformation.py | 8 ++- litellm/llms/fal_ai/chat/transformation.py | 13 +++- .../llms/hosted_vllm/chat/transformation.py | 2 +- litellm/llms/oci/chat/transformation.py | 16 +++++ litellm/llms/openai/common_utils.py | 22 +++--- litellm/llms/openai/openai.py | 8 ++- .../reasoning_effort_capability.py | 8 +-- ...hropic_messages_adapters_transformation.py | 16 +++-- ..._handler_reasoning_effort_normalization.py | 26 ++++--- .../native/test_anthropic_messages_handler.py | 10 ++- .../native/test_anthropic_messages_router.py | 17 +++-- .../test_responses_adapters_transformation.py | 33 +++++---- .../messages/test_reasoning_effort_fields.py | 5 +- .../anthropic/test_anthropic_common_utils.py | 18 +++-- .../chat/test_azure_ai_transformation.py | 33 +++++++++ ...bedrock_chat_completions_transformation.py | 42 +++++++---- .../chat/test_converse_transformation.py | 19 +++++ .../chat/test_fal_ai_chat_transformation.py | 33 +++++++-- .../oci/chat/test_oci_chat_transformation.py | 41 +++++++---- .../llms/openai/test_openai_common_utils.py | 70 +++++++++++++++++++ tests/unit/test_main.py | 36 ++++++++-- 28 files changed, 502 insertions(+), 116 deletions(-) diff --git a/litellm/litellm_core_utils/reasoning_effort_utils.py b/litellm/litellm_core_utils/reasoning_effort_utils.py index acd20aeb37d..8a017f01e7a 100644 --- a/litellm/litellm_core_utils/reasoning_effort_utils.py +++ b/litellm/litellm_core_utils/reasoning_effort_utils.py @@ -1,6 +1,9 @@ +from collections.abc import Iterable, Mapping +from dataclasses import dataclass from typing import Final, Literal import litellm +from litellm._logging import verbose_logger from litellm.constants import ( DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET, @@ -11,6 +14,34 @@ from litellm.litellm_core_utils.core_helpers import as_str_mapping OpenAIStyleReasoningEffort = Literal["minimal", "low", "medium", "high"] +_DROP_WARNING: Final = "Dropping %s=%r for model=%r because %s" +_DROP_PARAMS_DROP_WARNING: Final = "Dropping %s=%r for model=%r (drop_params=True) because %s" +_NON_NUMERIC_BUDGET_REASON: Final = "it is not a number. Send budget_tokens as an integer to set a reasoning level." +_REASONING_PARAMS: Final = frozenset(("reasoning_effort", "reasoning", "thinking")) + + +def warn_dropped_reasoning(param: str, value: object, model: object, reason: str, *, drop_params_gated: bool) -> None: + verbose_logger.warning( + _DROP_PARAMS_DROP_WARNING if drop_params_gated else _DROP_WARNING, param, value, model, reason + ) + + +def warn_dropped_reasoning_params( + dropped: Iterable[str], request: Mapping[str, object], model: object, reason: str, *, drop_params_gated: bool +) -> None: + for param in sorted(_REASONING_PARAMS.intersection(dropped, request)): + warn_dropped_reasoning(param, request[param], model, reason, drop_params_gated=drop_params_gated) + + +@dataclass(frozen=True, slots=True) +class DroppedThinkingBudget: + budget_tokens: object + + def warn(self, model: str) -> None: + warn_dropped_reasoning( + "thinking.budget_tokens", self.budget_tokens, model, _NON_NUMERIC_BUDGET_REASON, drop_params_gated=True + ) + def reasoning_effort_from_thinking_budget( budget_tokens: float, @@ -29,7 +60,9 @@ def reasoning_effort_from_thinking_budget( return "minimal" -def reasoning_effort_from_thinking(thinking: object, *, drop_params: bool) -> OpenAIStyleReasoningEffort | None: +def _effort_or_dropped_budget( + thinking: object, *, drop_params: bool +) -> OpenAIStyleReasoningEffort | DroppedThinkingBudget | None: fields: Final = as_str_mapping(thinking) if fields is None or fields.get("type") != "enabled": return None @@ -37,10 +70,27 @@ def reasoning_effort_from_thinking(thinking: object, *, drop_params: bool) -> Op if isinstance(budget, (int, float)) and not isinstance(budget, bool): return reasoning_effort_from_thinking_budget(budget) if drop_params or litellm.drop_params is True: - return None + return DroppedThinkingBudget(budget) raise non_numeric_thinking_budget_error(budget) +def reasoning_effort_from_thinking( + thinking: object, *, model: str, drop_params: bool +) -> OpenAIStyleReasoningEffort | None: + effort: Final = _effort_or_dropped_budget(thinking, drop_params=drop_params) + if not isinstance(effort, DroppedThinkingBudget): + return effort + effort.warn(model) + return None + + +def reasoning_effort_from_thinking_silently( + thinking: object, *, drop_params: bool +) -> OpenAIStyleReasoningEffort | None: + effort: Final = _effort_or_dropped_budget(thinking, drop_params=drop_params) + return None if isinstance(effort, DroppedThinkingBudget) else effort + + def non_numeric_thinking_budget_error(budget: object) -> UnsupportedParamsError: return UnsupportedParamsError( message=( diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 1f5e4e7de39..f8999edcc31 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -34,7 +34,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of, ) -from litellm.litellm_core_utils.reasoning_effort_utils import non_numeric_thinking_budget_error +from litellm.litellm_core_utils.reasoning_effort_utils import DroppedThinkingBudget, non_numeric_thinking_budget_error from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.anthropic.wif import ( aget_anthropic_wif_token, @@ -512,11 +512,11 @@ def _numeric_budget_tokens(budget_tokens: object) -> int | None: return None -def _legacy_budget_tokens(budget_tokens: object, *, drop_params: bool) -> int | None: +def _legacy_budget_tokens(budget_tokens: object, *, drop_params: bool) -> int | DroppedThinkingBudget: if (numeric := _numeric_budget_tokens(budget_tokens)) is not None: return numeric if drop_params or litellm.drop_params is True: - return None + return DroppedThinkingBudget(budget_tokens) raise non_numeric_thinking_budget_error(budget_tokens) @@ -985,6 +985,22 @@ class AnthropicModelInfo(BaseLLMModelInfo): custom_llm_provider: str, drop_params: bool, ) -> None: + dropped: Final = AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model_silently( + model=model, + optional_params=optional_params, + custom_llm_provider=custom_llm_provider, + drop_params=drop_params, + ) + if dropped is not None: + dropped.warn(model) + + @staticmethod + def translate_legacy_thinking_for_adaptive_model_silently( + model: str, + optional_params: MutableMapping[str, object], # mutable-ok: in-place out-param like the sibling helpers + custom_llm_provider: str, + drop_params: bool, + ) -> DroppedThinkingBudget | None: """Translate legacy ``thinking.type=enabled`` to adaptive for the adaptive-thinking models that reject it (4.7+ and the 5 families). Models flagged ``supports_legacy_thinking`` (the 4.6 family) accept the @@ -993,12 +1009,12 @@ class AnthropicModelInfo(BaseLLMModelInfo): ``output_config.effort`` is never overridden. """ if not AnthropicModelInfo._is_adaptive_thinking_model(model, custom_llm_provider): - return + return None if AnthropicModelInfo._supports_legacy_thinking(model, custom_llm_provider): - return + return None thinking: Final = validated_object_mapping(optional_params.get("thinking")) if thinking is None or thinking.get("type") != "enabled": - return + return None budget_tokens: Final = _legacy_budget_tokens(thinking.get("budget_tokens", 0), drop_params=drop_params) display: Final = thinking.get("display") @@ -1007,8 +1023,8 @@ class AnthropicModelInfo(BaseLLMModelInfo): thinking, rebuilt_keys=frozenset(("type", "budget_tokens", "display")), drop_params=drop_params ) optional_params["thinking"] = {**carried_keys, "type": "adaptive", **supported_display} - if budget_tokens is None: - return + if isinstance(budget_tokens, DroppedThinkingBudget): + return budget_tokens effort: Final = AnthropicModelInfo._legacy_budget_to_effort( model=model, budget_tokens=budget_tokens, custom_llm_provider=custom_llm_provider ) @@ -1017,6 +1033,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): "effort": effort, **(existing_output_config if isinstance(existing_output_config, dict) else MappingProxyType({})), } + return None @staticmethod def _legacy_budget_to_effort(model: str, budget_tokens: int, custom_llm_provider: str) -> str: diff --git a/litellm/llms/anthropic/messages/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py index 1ed552d46e3..0c346873ed2 100644 --- a/litellm/llms/anthropic/messages/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -656,6 +656,7 @@ class LiteLLMAnthropicMessagesAdapter: def translate_anthropic_thinking_to_reasoning_effort( thinking: AnthropicThinkingParam, *, + model: str, drop_params: bool = True, ) -> str | None: """ @@ -675,7 +676,7 @@ class LiteLLMAnthropicMessagesAdapter: if thinking_type == "disabled": return "none" elif thinking_type == "enabled": - return reasoning_effort_from_thinking(thinking, drop_params=drop_params) + return reasoning_effort_from_thinking(thinking, model=model, drop_params=drop_params) elif thinking_type == "adaptive": # Adaptive thinking: effort is controlled by output_config.effort, # not budget_tokens. Return a default; caller should override with @@ -742,7 +743,7 @@ class LiteLLMAnthropicMessagesAdapter: return {"thinking": thinking} else: reasoning_effort: Final = LiteLLMAnthropicMessagesAdapter.translate_anthropic_thinking_to_reasoning_effort( - thinking + thinking, model=model ) if reasoning_effort: return { @@ -1180,7 +1181,7 @@ class LiteLLMAnthropicMessagesAdapter: return reasoning_effort: Final = declared_effort or self.translate_anthropic_thinking_to_reasoning_effort( - cast(AnthropicThinkingParam, thinking), drop_params=drop_params + cast(AnthropicThinkingParam, thinking), model=model, drop_params=drop_params ) if not reasoning_effort: return diff --git a/litellm/llms/anthropic/messages/native/handler.py b/litellm/llms/anthropic/messages/native/handler.py index 6f97a019a96..1da7c2efdda 100644 --- a/litellm/llms/anthropic/messages/native/handler.py +++ b/litellm/llms/anthropic/messages/native/handler.py @@ -20,6 +20,7 @@ from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.exception_mapping_utils import exception_type from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.anthropic.common_utils import ( + AnthropicModelInfo, strip_provider_specific_fields_from_anthropic_messages, tools_marked_for_a_tool_config_point, ) @@ -36,7 +37,6 @@ from litellm.router_utils.reasoning_effort_capability import ( keeps_effort, model_group_from_metadata, resolve_deployment_reasoning_effort, - with_adaptive_thinking_as_effort, ) from litellm.types.llms.anthropic import AllAnthropicToolsValues from litellm.types.llms.anthropic_messages.anthropic_request import AnthropicMetadata @@ -185,9 +185,11 @@ def _with_deployment_output_effort( ).value def translate(source: Mapping[str, object]) -> Mapping[str, object]: - return with_adaptive_thinking_as_effort( - source, model=model, custom_llm_provider=custom_llm_provider, drop_params=drop_params + translated: Final = dict(source) + AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model( + model=model, optional_params=translated, custom_llm_provider=custom_llm_provider, drop_params=drop_params ) + return translated requested_output: Final = as_str_mapping(params.get("output_config")) caller_output: Final = None if requested_output is None else resolve(requested_output) diff --git a/litellm/llms/anthropic/messages/responses_adapters/transformation.py b/litellm/llms/anthropic/messages/responses_adapters/transformation.py index 1dd07c1739b..285fc179aa8 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/messages/responses_adapters/transformation.py @@ -542,6 +542,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: output_config: dict[str, object] | None = None, auto_summary_level: str = "detailed", *, + model: str, drop_params: bool = True, ) -> dict[str, object] | None: """ @@ -563,7 +564,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: if isinstance(output_config, dict) and output_config.get("effort"): effort = output_config["effort"] elif thinking_type == "enabled": - budget_effort: Final = reasoning_effort_from_thinking(thinking, drop_params=drop_params) + budget_effort: Final = reasoning_effort_from_thinking(thinking, model=model, drop_params=drop_params) if budget_effort is None: return None effort = budget_effort @@ -675,6 +676,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: thinking, output_config=cast(dict[str, object] | None, output_config), auto_summary_level=auto_summary_level, + model=model, drop_params=drop_params, ) if reasoning: diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py index 6aed7eb3c54..72f20c3148b 100644 --- a/litellm/llms/azure_ai/chat/transformation.py +++ b/litellm/llms/azure_ai/chat/transformation.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Final, cast import httpx from httpx import Response +from pydantic import TypeAdapter import litellm from litellm._logging import verbose_logger @@ -15,6 +16,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( drop_tool_reference_parts_from_tool_messages, filter_value_from_dict, ) +from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning_params from litellm.llms.azure.common_utils import BaseAzureLLM from litellm.llms.azure_ai.common_utils import ( api_key_header_for_base, @@ -35,6 +37,9 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer +_REQUEST_BODY: Final = TypeAdapter(dict[str, object]) + + class AzureFoundryErrorStrings(str, enum.Enum): SET_EXTRA_PARAMETERS_TO_PASS_THROUGH = "Set extra-parameters to 'pass-through'" @@ -393,7 +398,12 @@ class AzureAIStudioConfig(OpenAIConfig): request_data = self._drop_extra_params_from_request_data(request_data, error_text) if "Extra inputs are not permitted" in error_text and self._error_has_tool_level_extra_fields(error_text): request_data = self._drop_tool_level_extra_fields(request_data, error_text) - data: Final = drop_params_from_unprocessable_entity_error(e=e, data=request_data) + data: Final = drop_params_from_unprocessable_entity_error( + e=e, + data=request_data, + model=_REQUEST_BODY.validate_python(request_data).get("model"), + drop_params_gated=False, + ) return data def _drop_tool_level_extra_fields(self, request_data: dict, error_text: str) -> dict: @@ -409,6 +419,14 @@ class AzureAIStudioConfig(OpenAIConfig): def _drop_extra_params_from_request_data(self, request_data: dict, error_text: str) -> dict: params_to_drop: Final = self._extract_params_to_drop_from_error_text(error_text) if params_to_drop: + request: Final = _REQUEST_BODY.validate_python(request_data) + warn_dropped_reasoning_params( + params_to_drop, + request, + request.get("model"), + "Azure AI rejected it as an extra parameter its deployment does not pass through.", + drop_params_gated=False, + ) for param in params_to_drop: if param in request_data: request_data.pop(param, None) diff --git a/litellm/llms/bedrock/chat/chat_completions/transformation.py b/litellm/llms/bedrock/chat/chat_completions/transformation.py index 123e4a9e6b4..0762bdbb1df 100644 --- a/litellm/llms/bedrock/chat/chat_completions/transformation.py +++ b/litellm/llms/bedrock/chat/chat_completions/transformation.py @@ -33,6 +33,7 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import ( inline_remote_image_urls, inline_remote_media, ) +from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning from litellm.litellm_core_utils.url_utils import ensure_api_route, join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, bedrock_bearer_token @@ -121,6 +122,13 @@ def without_refused_reasoning_effort(model: str, params: Mapping[str, object]) - effort: Final = params.get("reasoning_effort") if not isinstance(effort, str) or effort not in chat_completions_reasoning_efforts_refused_for(model): return params + warn_dropped_reasoning( + "reasoning_effort", + effort, + model, + "AWS's Chat Completions endpoint rejects that level for this model family, so AWS applies its default effort.", + drop_params_gated=False, + ) return _without_params(params, frozenset(("reasoning_effort",))) @@ -445,6 +453,14 @@ class AmazonBedrockRuntimeChatCompletionsConfig(OpenAILikeChatConfig): ), status_code=400, ) + if malformed_effort: + warn_dropped_reasoning( + "reasoning_effort", + raw_params["reasoning_effort"], + model, + "Bedrock's Chat Completions endpoint takes it only as a string, so AWS applies its default effort.", + drop_params_gated=True, + ) rejected_stop: Final = frozenset(("stop",)) if bedrock_rejects_stop_sequences(model) else frozenset[str]() return dict( without_refused_reasoning_effort( diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index f54c352fd6e..4a8323ecc23 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -48,6 +48,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of, ) +from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning from litellm.llms.anthropic.chat.transformation import ( DROP_UNSUPPORTED_ADAPTIVE_THINKING_WARNING, DROP_UNSUPPORTED_OUTPUT_CONFIG_WARNING, @@ -1174,9 +1175,12 @@ class AmazonConverseConfig(BaseConfig): drop_params=drop_params, ) elif param == "reasoning_effort" and isinstance(value, str) and drop_reasoning_effort_param: - verbose_logger.debug( - "Dropping unsupported `reasoning_effort` param for Bedrock model=%s; it always reasons and rejects it.", + warn_dropped_reasoning( + param, + value, model, + "it always reasons and Bedrock rejects reasoning_effort for it.", + drop_params_gated=False, ) elif ( param == "reasoning_effort" diff --git a/litellm/llms/fal_ai/chat/transformation.py b/litellm/llms/fal_ai/chat/transformation.py index 115b1f840de..848ae498cdd 100644 --- a/litellm/llms/fal_ai/chat/transformation.py +++ b/litellm/llms/fal_ai/chat/transformation.py @@ -12,7 +12,9 @@ from typing import TYPE_CHECKING, Final import httpx from pydantic import ConfigDict, TypeAdapter +import litellm from litellm.litellm_core_utils.core_helpers import map_finish_reason +from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.secret_managers.main import get_secret_str from litellm.types.llms.base import LiteLLMBaseModel @@ -29,6 +31,10 @@ PROVIDER_PREFIX: Final[str] = "fal_ai/" PASSTHROUGH_PARAMS: Final[frozenset[str]] = frozenset(("reasoning", "temperature", "top_p")) REASONING_DISABLED_EFFORTS: Final[frozenset[str]] = frozenset(("none", "minimal")) REASONING_ENABLED_EFFORTS: Final[frozenset[str]] = frozenset(("low", "medium", "high")) +_UNSUPPORTED_EFFORT_REASON: Final = ( + f"fal only turns reasoning off for {sorted(REASONING_DISABLED_EFFORTS)} " + f"and on for {sorted(REASONING_ENABLED_EFFORTS)}." +) class _FalUsage(LiteLLMBaseModel): @@ -120,9 +126,10 @@ class FalAIChatConfig(BaseConfig): return False if isinstance(value, str) and value in REASONING_ENABLED_EFFORTS: return True - if drop_params: - return None - raise FalAIError(status_code=400, message=f"Unsupported reasoning_effort {value!r} for {model}") + if not (drop_params or litellm.drop_params is True): + raise FalAIError(status_code=400, message=f"Unsupported reasoning_effort {value!r} for {model}") + warn_dropped_reasoning("reasoning_effort", value, model, _UNSUPPORTED_EFFORT_REASON, drop_params_gated=True) + return None def _translate_param(self, param: str, value: object, model: str, drop_params: bool) -> tuple[str, object] | None: if param in ("temperature", "top_p"): diff --git a/litellm/llms/hosted_vllm/chat/transformation.py b/litellm/llms/hosted_vllm/chat/transformation.py index 63e6cb2b5da..9f9503f93b6 100644 --- a/litellm/llms/hosted_vllm/chat/transformation.py +++ b/litellm/llms/hosted_vllm/chat/transformation.py @@ -99,7 +99,7 @@ class HostedVLLMChatConfig(OpenAIGPTConfig): thinking_effort: Final = ( None if "reasoning_effort" in non_default_params - else reasoning_effort_from_thinking(thinking, drop_params=drop_params) + else reasoning_effort_from_thinking(thinking, model=model, drop_params=drop_params) ) if thinking_effort is not None: non_default_params["reasoning_effort"] = thinking_effort diff --git a/litellm/llms/oci/chat/transformation.py b/litellm/llms/oci/chat/transformation.py index e8d37f857e6..2af845c6f60 100644 --- a/litellm/llms/oci/chat/transformation.py +++ b/litellm/llms/oci/chat/transformation.py @@ -20,6 +20,7 @@ import litellm from litellm._logging import verbose_logger from litellm.constants import DEFAULT_OCI_CHAT_MAX_TOKENS from litellm.litellm_core_utils.logging_utils import track_llm_api_timing +from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException @@ -427,6 +428,13 @@ class OCIChatConfig(BaseConfig): alias = param_map.get(key) if key == "reasoning_effort" and alias and not _model_supports_reasoning_effort(model): if drop_params or litellm.drop_params: + warn_dropped_reasoning( + key, + value, + model, + "the model map lists no reasoning_effort levels for it, so OCI would reject reasoningEffort.", + drop_params_gated=True, + ) continue raise OCIError( status_code=400, @@ -450,6 +458,14 @@ class OCIChatConfig(BaseConfig): if key == "n" and (value is None or value == 1): continue if drop_params or litellm.drop_params: + if key == "reasoning_effort": + warn_dropped_reasoning( + key, + value, + model, + "OCI's Cohere chat API has no reasoning effort field.", + drop_params_gated=True, + ) continue raise OCIError( status_code=400, diff --git a/litellm/llms/openai/common_utils.py b/litellm/llms/openai/common_utils.py index 12ead582975..f2eb40980a1 100644 --- a/litellm/llms/openai/common_utils.py +++ b/litellm/llms/openai/common_utils.py @@ -31,6 +31,7 @@ if TYPE_CHECKING: import litellm from litellm.litellm_core_utils.completion_timeout import CompletionTimeout from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object +from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning_params from litellm.litellm_core_utils.token_counter import token_counter from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.custom_httpx.http_handler import ( @@ -143,17 +144,11 @@ _DECODED_JSON: Final = TypeAdapter(object) def drop_params_from_unprocessable_entity_error( e: openai.UnprocessableEntityError | httpx.HTTPStatusError, data: Mapping[str, object], + *, + model: object, + drop_params_gated: bool, ) -> dict[str, object]: - """ - Helper function to read OpenAI UnprocessableEntityError and drop the params that raised an error from the error message. - - Args: - e (UnprocessableEntityError): The UnprocessableEntityError exception - data (Dict[str, Any]): The original data dictionary containing all parameters - - Returns: - Dict[str, Any]: A new dictionary with invalid parameters removed - """ + """Return a copy of ``data`` without the params a 422 response names as invalid.""" invalid_params: Final[list[str]] = [] if isinstance(e, httpx.HTTPStatusError): error_json: Final = decoded_json_object(_DECODED_JSON.validate_python(e.response.json())) @@ -181,6 +176,13 @@ def drop_params_from_unprocessable_entity_error( invalid_params.append(error_dict["loc"][1]) new_data: Final = {k: v for k, v in data.items() if k not in invalid_params} + warn_dropped_reasoning_params( + invalid_params, + data, + model, + "the provider answered 422 naming it an invalid param.", + drop_params_gated=drop_params_gated, + ) return new_data diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index 9d4df354a30..25a0d9beffe 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -910,7 +910,9 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): except openai.UnprocessableEntityError as e: ## check if body contains unprocessable params - related issue https://github.com/BerriAI/litellm/issues/4800 if litellm.drop_params is True or drop_params is True: - inference_params = drop_params_from_unprocessable_entity_error(e, inference_params) + inference_params = drop_params_from_unprocessable_entity_error( + e, inference_params, model=model, drop_params_gated=True + ) else: raise e # e.message @@ -1066,7 +1068,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): except openai.UnprocessableEntityError as e: ## check if body contains unprocessable params - related issue https://github.com/BerriAI/litellm/issues/4800 if litellm.drop_params is True or drop_params is True: - data = drop_params_from_unprocessable_entity_error(e, data) + data = drop_params_from_unprocessable_entity_error(e, data, model=model, drop_params_gated=True) else: raise e # e.message @@ -1220,7 +1222,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): except openai.UnprocessableEntityError as e: ## check if body contains unprocessable params - related issue https://github.com/BerriAI/litellm/issues/4800 if litellm.drop_params is True or drop_params is True: - data = drop_params_from_unprocessable_entity_error(e, data) + data = drop_params_from_unprocessable_entity_error(e, data, model=model, drop_params_gated=True) else: raise e except ( diff --git a/litellm/router_utils/reasoning_effort_capability.py b/litellm/router_utils/reasoning_effort_capability.py index 16b8a22a40f..997d9e02b91 100644 --- a/litellm/router_utils/reasoning_effort_capability.py +++ b/litellm/router_utils/reasoning_effort_capability.py @@ -35,7 +35,7 @@ import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.get_supported_openai_params import get_supported_openai_params -from litellm.litellm_core_utils.reasoning_effort_utils import reasoning_effort_from_thinking +from litellm.litellm_core_utils.reasoning_effort_utils import reasoning_effort_from_thinking_silently from litellm.llms.anthropic.common_utils import AnthropicModelInfo, validated_object_mapping from litellm.types.llms.anthropic import AnthropicThinkingParam from litellm.types.llms.openai import REASONING_EFFORT @@ -170,7 +170,7 @@ def with_adaptive_thinking_as_effort( params: Mapping[str, object], *, model: str, custom_llm_provider: str, drop_params: bool ) -> Mapping[str, object]: translated: Final = dict(params) - AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model( + AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model_silently( model=model, optional_params=translated, custom_llm_provider=custom_llm_provider, drop_params=drop_params ) return translated @@ -178,7 +178,7 @@ def with_adaptive_thinking_as_effort( def thinking_budget_effort(thinking: object, *, model: str, custom_llm_provider: str, drop_params: bool) -> str | None: if custom_llm_provider in _PROVIDERS_SENDING_THINKING_AS_EFFORT: - return reasoning_effort_from_thinking(thinking, drop_params=drop_params) + return reasoning_effort_from_thinking_silently(thinking, drop_params=drop_params) adaptive_config: Final = as_str_mapping( with_adaptive_thinking_as_effort( {"thinking": thinking}, model=model, custom_llm_provider=custom_llm_provider, drop_params=drop_params @@ -189,7 +189,7 @@ def thinking_budget_effort(thinking: object, *, model: str, custom_llm_provider: return adaptive_effort if isinstance(adaptive_effort, str) else None if "thinking" in (get_supported_openai_params(model=model, custom_llm_provider=custom_llm_provider) or ()): return None - return reasoning_effort_from_thinking(thinking, drop_params=drop_params) + return reasoning_effort_from_thinking_silently(thinking, drop_params=drop_params) def provider_reads_thinking(thinking: object, *, model: str, custom_llm_provider: str, drop_params: bool) -> bool: diff --git a/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py b/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py index 4e4b89b764b..417eb19bc12 100644 --- a/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py @@ -1,6 +1,7 @@ import base64 import functools import json +import logging from collections.abc import Mapping from types import MappingProxyType from typing import Any, Final, cast @@ -2871,14 +2872,21 @@ def test_translate_completion_input_params_rejects_a_non_numeric_budget_without_ @pytest.mark.parametrize("budget", ["2048", True]) def test_translate_completion_input_params_drops_a_non_numeric_budget_under_drop_params( - budget: object, monkeypatch: pytest.MonkeyPatch + budget: object, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture ): monkeypatch.setattr(litellm, "drop_params", False) - translated: Final = AnthropicAdapter().translate_completion_input_params( - _gemini_request_with_budget(budget, drop_params=True) - ) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + translated: Final = AnthropicAdapter().translate_completion_input_params( + _gemini_request_with_budget(budget, drop_params=True) + ) + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] assert translated is not None assert not {"reasoning_effort", "thinking"} & translated.keys() + assert len(drop_warnings) == 1 + assert all( + fragment in drop_warnings[0] + for fragment in (f"thinking.budget_tokens={budget!r}", "gemini-2.5-flash", "(drop_params=True)") + ) def test_translate_openai_content_to_anthropic_reasoning_content_without_thinking_blocks(): diff --git a/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py b/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py index e504513dd36..27ed1f5ebd6 100644 --- a/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py @@ -8,6 +8,7 @@ leaves the adapter, in the shape the target expects. import functools import json +import logging from collections.abc import Mapping from typing import Final @@ -510,26 +511,33 @@ class TestADeclaredDeploymentRoundsTheMessagesTierOntoItsLevels: @pytest.mark.asyncio @pytest.mark.parametrize("budget", ["2048", True]) @pytest.mark.parametrize("litellm_params, model_info, upstream_url, upstream_body", _MALFORMED_BUDGET_TARGETS) - async def test_drop_params_drops_a_non_numeric_budget( + async def test_drop_params_drops_a_non_numeric_budget_with_one_warning( self, local_model_cost_map: None, respx_mock: respx.MockRouter, + caplog: pytest.LogCaptureFixture, litellm_params: Mapping[str, object], model_info: Mapping[str, object], upstream_url: str, upstream_body: Mapping[str, object], budget: object, ) -> None: - sent: Final = await _body_sent_upstream( - respx_mock, - {**litellm_params, "drop_params": True}, - model_info, - upstream_url, - upstream_body, - thinking={"type": "enabled", "budget_tokens": budget}, - ) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + sent: Final = await _body_sent_upstream( + respx_mock, + {**litellm_params, "drop_params": True}, + model_info, + upstream_url, + upstream_body, + thinking={"type": "enabled", "budget_tokens": budget}, + ) assert not {"reasoning_effort", "reasoning", "thinking"} & sent.keys() + dropped: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] + assert len(dropped) == 1 + assert f"thinking.budget_tokens={budget!r}" in dropped[0] + assert "(drop_params=True)" in dropped[0] + assert str(litellm_params["model"]).split("/", 1)[1] in dropped[0] @pytest.mark.asyncio @pytest.mark.parametrize( diff --git a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py index c11f58faec4..e5e5b91bb7c 100644 --- a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_handler.py @@ -158,7 +158,7 @@ async def test_openai_model_does_not_forward_stream_options_to_responses_api(): await litellm.anthropic.messages.acreate( max_tokens=100, messages=[{"role": "user", "content": "Hello, how are you?"}], - model="openai/gpt-5.5", + model="openai/gpt-6.1-sol", api_key="test-api-key", stream_options={"include_usage": True}, ) @@ -668,7 +668,9 @@ class TestThinkingSummaryPreservation: ) thinking = {"type": "enabled", "budget_tokens": 5000, "summary": "concise"} - result = LiteLLMAnthropicToResponsesAPIAdapter.translate_thinking_to_reasoning(thinking) + result = LiteLLMAnthropicToResponsesAPIAdapter.translate_thinking_to_reasoning( + thinking, model="openai/gpt-6.1-sol" + ) assert result == {"effort": "high", "summary": "concise"} def test_responses_adapter_no_summary_by_default(self): @@ -682,7 +684,9 @@ class TestThinkingSummaryPreservation: try: litellm.reasoning_auto_summary = False thinking = {"type": "enabled", "budget_tokens": 5000} - result = LiteLLMAnthropicToResponsesAPIAdapter.translate_thinking_to_reasoning(thinking) + result = LiteLLMAnthropicToResponsesAPIAdapter.translate_thinking_to_reasoning( + thinking, model="openai/gpt-6.1-sol" + ) assert result == {"effort": "high"} assert result is not None and "summary" not in result finally: diff --git a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_router.py b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_router.py index 19e03d8ce46..56e650d3351 100644 --- a/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_router.py +++ b/tests/unit/llms/anthropic/messages/native/test_anthropic_messages_router.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio import base64 import json +import logging import struct import uuid from collections.abc import AsyncIterable, Mapping @@ -760,24 +761,32 @@ async def test_native_messages_send_no_effort_for_a_default_effort_beside_a_thin @pytest.mark.asyncio @pytest.mark.parametrize("model_info", [None, _DECLARING_LOW_AND_HIGH]) @pytest.mark.parametrize("drop_globally", [False, True]) -async def test_native_messages_drop_params_drops_a_non_numeric_thinking_budget_on_adaptive_claude( +async def test_native_messages_drop_params_drops_a_non_numeric_thinking_budget_on_adaptive_claude_with_one_warning( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, model_info: Mapping[str, object] | None, drop_globally: bool, ) -> None: monkeypatch.setattr(litellm, "drop_params", drop_globally) request_drop: Final = {} if drop_globally else {"drop_params": True} - dropped: Final = await _native_opus_body_sent( - respx_mock, model_info, thinking={"type": "enabled", "budget_tokens": "lots"}, **request_drop - ) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + dropped: Final = await _native_opus_body_sent( + respx_mock, model_info, thinking={"type": "enabled", "budget_tokens": "lots"}, **request_drop + ) + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] adaptive_alone: Final = await _native_opus_body_sent( respx_mock, model_info, thinking={"type": "adaptive"}, **request_drop ) assert "output_config" not in adaptive_alone assert dropped == adaptive_alone + assert len(drop_warnings) == 1 + assert all( + fragment in drop_warnings[0] + for fragment in ("thinking.budget_tokens='lots'", "claude-opus-5-5", "(drop_params=True)") + ) _MIXED_GROUP: Final = "gpt-5.6-mixed" diff --git a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py index f6fcb9e6f1e..ece0cf5d423 100644 --- a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py @@ -44,6 +44,7 @@ def _make_request(**overrides) -> AnthropicMessagesRequest: _ADAPTER = LiteLLMAnthropicToResponsesAPIAdapter() +_MODEL: Final = "openai/gpt-6.1-sol" # --------------------------------------------------------------------------- @@ -957,13 +958,13 @@ class TestTranslateThinkingToReasoning: """Anthropic thinking param -> Responses API reasoning param.""" def test_budget_high_effort(self): - result = _ADAPTER.translate_thinking_to_reasoning({"type": "enabled", "budget_tokens": 10000}) + result = _ADAPTER.translate_thinking_to_reasoning({"type": "enabled", "budget_tokens": 10000}, model=_MODEL) # Default (reasoning_auto_summary=False): only effort, no summary assert result == {"effort": "high"} assert result is not None and "summary" not in result def test_budget_above_threshold_high_effort(self): - result = _ADAPTER.translate_thinking_to_reasoning({"type": "enabled", "budget_tokens": 50000}) + result = _ADAPTER.translate_thinking_to_reasoning({"type": "enabled", "budget_tokens": 50000}, model=_MODEL) assert result is not None assert result["effort"] == "high" assert "summary" not in result @@ -973,7 +974,8 @@ class TestTranslateThinkingToReasoning: { "type": "enabled", "budget_tokens": DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, - } + }, + model=_MODEL, ) assert result == {"effort": "medium"} assert result is not None and "summary" not in result @@ -983,13 +985,14 @@ class TestTranslateThinkingToReasoning: { "type": "enabled", "budget_tokens": DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET, - } + }, + model=_MODEL, ) assert result == {"effort": "low"} assert result is not None and "summary" not in result def test_budget_minimal_effort(self): - result = _ADAPTER.translate_thinking_to_reasoning({"type": "enabled", "budget_tokens": 500}) + result = _ADAPTER.translate_thinking_to_reasoning({"type": "enabled", "budget_tokens": 500}, model=_MODEL) assert result == {"effort": "minimal"} assert result is not None and "summary" not in result @@ -998,7 +1001,8 @@ class TestTranslateThinkingToReasoning: { "type": "enabled", "budget_tokens": DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, - } + }, + model=_MODEL, ) assert result_high is not None assert result_high["effort"] == "high" @@ -1006,7 +1010,8 @@ class TestTranslateThinkingToReasoning: { "type": "enabled", "budget_tokens": DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, - } + }, + model=_MODEL, ) assert result_medium is not None assert result_medium["effort"] == "medium" @@ -1015,23 +1020,24 @@ class TestTranslateThinkingToReasoning: { "type": "enabled", "budget_tokens": DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET, - } + }, + model=_MODEL, ) assert result_low is not None assert result_low["effort"] == "low" assert "summary" not in result_low def test_disabled_type_returns_none(self): - result = _ADAPTER.translate_thinking_to_reasoning({"type": "disabled"}) + result = _ADAPTER.translate_thinking_to_reasoning({"type": "disabled"}, model=_MODEL) assert result is None def test_non_dict_returns_none(self): - result = _ADAPTER.translate_thinking_to_reasoning("enabled") # type: ignore + result = _ADAPTER.translate_thinking_to_reasoning("enabled", model=_MODEL) # type: ignore assert result is None def test_missing_budget_defaults_to_minimal(self): """Missing budget_tokens defaults to 0, below the low threshold -> minimal.""" - result = _ADAPTER.translate_thinking_to_reasoning({"type": "enabled"}) + result = _ADAPTER.translate_thinking_to_reasoning({"type": "enabled"}, model=_MODEL) assert result == {"effort": "minimal"} assert result is not None and "summary" not in result @@ -1042,7 +1048,7 @@ class TestTranslateThinkingToReasoning: original = litellm.reasoning_auto_summary try: litellm.reasoning_auto_summary = True - result = _ADAPTER.translate_thinking_to_reasoning({"type": "enabled", "budget_tokens": 10000}) + result = _ADAPTER.translate_thinking_to_reasoning({"type": "enabled", "budget_tokens": 10000}, model=_MODEL) assert result == {"effort": "high", "summary": "detailed"} finally: litellm.reasoning_auto_summary = original @@ -1059,7 +1065,8 @@ class TestTranslateThinkingToReasoning: { "type": "enabled", "budget_tokens": DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, - } + }, + model=_MODEL, ) assert result == {"effort": "medium", "summary": "detailed"} finally: diff --git a/tests/unit/llms/anthropic/messages/test_reasoning_effort_fields.py b/tests/unit/llms/anthropic/messages/test_reasoning_effort_fields.py index 3432c9503de..7128c04a056 100644 --- a/tests/unit/llms/anthropic/messages/test_reasoning_effort_fields.py +++ b/tests/unit/llms/anthropic/messages/test_reasoning_effort_fields.py @@ -162,7 +162,8 @@ class TestAdapterAdaptiveThinking: adapter = LiteLLMAnthropicMessagesAdapter() result = adapter.translate_anthropic_thinking_to_reasoning_effort( - {"type": "adaptive"} + {"type": "adaptive"}, + model="openai/gpt-6.1-sol", ) assert result == "medium" @@ -198,6 +199,7 @@ class TestAdapterAdaptiveThinking: result = LiteLLMAnthropicToResponsesAPIAdapter.translate_thinking_to_reasoning( thinking={"type": "adaptive"}, output_config={"effort": "xhigh"}, + model="openai/gpt-6.1-sol", ) assert result is not None assert result["effort"] == "xhigh" @@ -210,6 +212,7 @@ class TestAdapterAdaptiveThinking: result = LiteLLMAnthropicToResponsesAPIAdapter.translate_thinking_to_reasoning( thinking={"type": "adaptive"}, + model="openai/gpt-6.1-sol", ) assert result is not None assert result["effort"] == "medium" diff --git a/tests/unit/llms/anthropic/test_anthropic_common_utils.py b/tests/unit/llms/anthropic/test_anthropic_common_utils.py index eb5c1598c17..06430011443 100644 --- a/tests/unit/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/unit/llms/anthropic/test_anthropic_common_utils.py @@ -11,6 +11,7 @@ Verifies that: """ import json +import logging import os import sys import threading @@ -4512,13 +4513,22 @@ def test_shared_legacy_thinking_translation_rejects_a_non_numeric_budget(budget_ @pytest.mark.usefixtures("local_model_cost_map") @pytest.mark.parametrize("budget_tokens", _NON_NUMERIC_BUDGETS) -def test_shared_legacy_thinking_translation_drops_a_non_numeric_budget_under_drop_params(budget_tokens: object) -> None: +def test_shared_legacy_thinking_translation_drops_a_non_numeric_budget_under_drop_params( + budget_tokens: object, caplog: pytest.LogCaptureFixture +) -> None: from litellm.llms.anthropic.common_utils import AnthropicModelInfo optional_params: Final = {"thinking": {"type": "enabled", "budget_tokens": budget_tokens}} - AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model( - model="claude-opus-5", optional_params=optional_params, custom_llm_provider="azure_ai", drop_params=True - ) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model( + model="claude-opus-5", optional_params=optional_params, custom_llm_provider="azure_ai", drop_params=True + ) + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] assert optional_params == {"thinking": {"type": "adaptive"}} + assert len(drop_warnings) == 1 + assert all( + fragment in drop_warnings[0] + for fragment in (f"thinking.budget_tokens={budget_tokens!r}", "claude-opus-5", "(drop_params=True)") + ) diff --git a/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py b/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py index ccac6ffc309..86844265b21 100644 --- a/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py +++ b/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py @@ -1,4 +1,5 @@ import json +import logging import traceback from typing import Final from unittest.mock import MagicMock, patch @@ -755,6 +756,38 @@ def test_drop_tool_level_extra_fields_strips_copilot_mcp_server_name(): assert result["tools"][1]["function"]["name"] == "read_file" +def test_extra_parameters_error_drops_reasoning_effort_with_one_warning(caplog: pytest.LogCaptureFixture) -> None: + import httpx + + error_text: Final = json.dumps( + { + "error": { + "message": "Extra parameters ['reasoning_effort'] are not allowed when extra-parameters is not set " + "or set to be 'error'. Set extra-parameters to 'pass-through' to pass to the model." + } + } + ) + error: Final = httpx.HTTPStatusError( + message="422", + request=httpx.Request("POST", "https://example.services.ai.azure.com/models/chat/completions"), + response=httpx.Response(422, text=error_text), + ) + request_data: Final = { + "model": "Kimi-K2.6", + "messages": [{"role": "user", "content": "Say hi."}], + "reasoning_effort": "high", + } + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + retried: Final = AzureAIStudioConfig().transform_request_on_unprocessable_entity_error(error, request_data) + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] + + assert retried == {"model": "Kimi-K2.6", "messages": [{"role": "user", "content": "Say hi."}]} + assert len(drop_warnings) == 1 + assert all(fragment in drop_warnings[0] for fragment in ("reasoning_effort='high'", "Kimi-K2.6")) + assert "drop_params" not in drop_warnings[0] + + def _find_key_anywhere(obj, key: str) -> bool: if isinstance(obj, dict): if key in obj: diff --git a/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py b/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py index 3faae2e3289..63212911fb4 100644 --- a/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py +++ b/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py @@ -1,6 +1,7 @@ """Bedrock Runtime Chat Completions: the default for GPT 5.6 and newer, ``bedrock/chat_completions/`` for the rest.""" import json +import logging import re import httpx @@ -709,15 +710,21 @@ def test_with_max_completion_tokens_leaves_other_params_alone(): "model", ["us.xai.grok-4.6", "bedrock/us-gov-west-1/us.xai.grok-4.6"], ) -def test_map_openai_params_drops_reasoning_effort_none_for_grok(model): +def test_map_openai_params_drops_reasoning_effort_none_for_grok_with_one_warning(model, caplog): cfg = AmazonBedrockRuntimeChatCompletionsConfig() - mapped = cfg.map_openai_params( - non_default_params={"reasoning_effort": "none", "max_tokens": 64}, - optional_params={}, - model=model, - drop_params=False, - ) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + mapped = cfg.map_openai_params( + non_default_params={"reasoning_effort": "none", "max_tokens": 64}, + optional_params={}, + model=model, + drop_params=False, + ) + drop_warnings = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] assert "reasoning_effort" not in mapped + assert len(drop_warnings) == 1 + assert "reasoning_effort='none'" in drop_warnings[0] + assert repr(model) in drop_warnings[0] + assert "drop_params" not in drop_warnings[0] def test_map_openai_params_keeps_reasoning_effort_low_for_grok(): @@ -750,17 +757,24 @@ def test_map_openai_params_refuses_a_non_string_reasoning_effort_without_drop_pa @pytest.mark.parametrize("reasoning_effort", [["low"], {"effort": "low"}, 5], ids=["list", "object", "int"]) @pytest.mark.parametrize("drop_params_via", ["request", "litellm.drop_params"]) def test_map_openai_params_drops_a_non_string_reasoning_effort_under_drop_params( - monkeypatch, model, reasoning_effort, drop_params_via + monkeypatch, caplog, model, reasoning_effort, drop_params_via ): monkeypatch.setattr(litellm, "drop_params", drop_params_via == "litellm.drop_params") - mapped = AmazonBedrockRuntimeChatCompletionsConfig().map_openai_params( - non_default_params={"reasoning_effort": reasoning_effort, "max_tokens": 64}, - optional_params={}, - model=model, - drop_params=drop_params_via == "request", - ) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + mapped = AmazonBedrockRuntimeChatCompletionsConfig().map_openai_params( + non_default_params={"reasoning_effort": reasoning_effort, "max_tokens": 64}, + optional_params={}, + model=model, + drop_params=drop_params_via == "request", + ) + drop_warnings = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] assert "reasoning_effort" not in mapped assert mapped["max_completion_tokens"] == 64 + assert len(drop_warnings) == 1 + assert all( + fragment in drop_warnings[0] + for fragment in (f"reasoning_effort={reasoning_effort!r}", repr(model), "(drop_params=True)") + ) def test_map_openai_params_keeps_reasoning_effort_none_for_gpt56(): diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 154797bab5a..45fd6d22d1e 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -1,6 +1,7 @@ import base64 import copy import json +import logging import os import re from collections.abc import Iterator, Mapping @@ -1731,6 +1732,24 @@ def test_bedrock_deepseek_r1_reasoning_params_not_forwarded_by_map(param): assert request.get("additionalModelRequestFields") is None +def test_bedrock_deepseek_r1_reasoning_effort_drop_warns_once(caplog: pytest.LogCaptureFixture) -> None: + model: Final = "bedrock/converse/us.deepseek.r1-v1:0" + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + optional_params: Final = AmazonConverseConfig().map_openai_params( + non_default_params={"reasoning_effort": "high", "max_tokens": 100}, + optional_params={}, + model=model, + drop_params=False, + ) + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] + assert "reasoning_effort" not in optional_params + assert optional_params["maxTokens"] == 100 + assert len(drop_warnings) == 1 + assert "reasoning_effort='high'" in drop_warnings[0] + assert repr(model) in drop_warnings[0] + assert "drop_params" not in drop_warnings[0] + + def test_bedrock_deepseek_v3_reasoning_effort_forwarded_raw(): """DeepSeek V3 takes reasoning_effort verbatim in additionalModelRequestFields, never converted into the Anthropic `thinking` block that Claude models get.""" diff --git a/tests/unit/llms/fal_ai/chat/test_fal_ai_chat_transformation.py b/tests/unit/llms/fal_ai/chat/test_fal_ai_chat_transformation.py index b7a79f03f8d..f4a7455ff27 100644 --- a/tests/unit/llms/fal_ai/chat/test_fal_ai_chat_transformation.py +++ b/tests/unit/llms/fal_ai/chat/test_fal_ai_chat_transformation.py @@ -1,3 +1,6 @@ +import logging +from typing import Final + import httpx import pytest @@ -242,11 +245,33 @@ def test_map_openai_params_maps_reasoning_effort(effort, expected): assert mapped["reasoning"] is expected -def test_map_openai_params_drops_unknown_reasoning_effort_when_dropping(): - mapped = FalAIChatConfig().map_openai_params( - non_default_params={"reasoning_effort": "extreme"}, optional_params={}, model=MODEL, drop_params=True - ) +@pytest.mark.parametrize("drop_globally", [False, True]) +def test_map_openai_params_drops_unknown_reasoning_effort_with_one_warning( + drop_globally: bool, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + monkeypatch.setattr(litellm, "drop_params", drop_globally) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + mapped: Final = FalAIChatConfig().map_openai_params( + non_default_params={"reasoning_effort": "xhigh"}, + optional_params={}, + model=MODEL, + drop_params=not drop_globally, + ) + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] assert "reasoning" not in mapped + assert len(drop_warnings) == 1 + assert all(fragment in drop_warnings[0] for fragment in ("reasoning_effort='xhigh'", MODEL, "(drop_params=True)")) + + +def test_map_openai_params_rejects_unknown_reasoning_effort_without_drop_params( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + with pytest.raises(FalAIError) as exc_info: + FalAIChatConfig().map_openai_params( + non_default_params={"reasoning_effort": "xhigh"}, optional_params={}, model=MODEL, drop_params=False + ) + assert exc_info.value.status_code == 400 @pytest.mark.parametrize("effort", [{"level": "low"}, ["low"], 1]) diff --git a/tests/unit/llms/oci/chat/test_oci_chat_transformation.py b/tests/unit/llms/oci/chat/test_oci_chat_transformation.py index 8c96bd91049..54981cce3cd 100644 --- a/tests/unit/llms/oci/chat/test_oci_chat_transformation.py +++ b/tests/unit/llms/oci/chat/test_oci_chat_transformation.py @@ -1,4 +1,5 @@ import datetime +import logging import httpx import pytest import json @@ -1102,19 +1103,25 @@ class TestOCIReasoningEffort: ) assert chat_request.get("reasoningEffort") == "HIGH" - def test_reasoning_effort_unsupported_on_cohere_dropped(self): - """drop_params=True → silently drop reasoning_effort for Cohere.""" + def test_reasoning_effort_unsupported_on_cohere_dropped_with_one_warning(self, caplog): from litellm.llms.oci.chat.transformation import OCIChatConfig config = OCIChatConfig() - result = config.map_openai_params( - non_default_params={"reasoning_effort": "low"}, - optional_params={}, - model="cohere.command-latest", - drop_params=True, - ) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + result = config.map_openai_params( + non_default_params={"reasoning_effort": "low"}, + optional_params={}, + model="cohere.command-latest", + drop_params=True, + ) + drop_warnings = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] assert "reasoning_effort" not in result assert "reasoningEffort" not in result + assert len(drop_warnings) == 1 + assert all( + fragment in drop_warnings[0] + for fragment in ("reasoning_effort='low'", "cohere.command-latest", "(drop_params=True)") + ) def test_reasoning_effort_unsupported_on_cohere_raises(self): """drop_params=False → raise rather than ship a payload Cohere will reject.""" @@ -2109,16 +2116,20 @@ class TestReasoningEffortGating: assert "reasoning_effort" not in OCIChatConfig().get_supported_openai_params("meta.llama-3.3-70b-instruct") @pytest.mark.parametrize("model", SERVED_GROK_4_20_MODELS) - def test_reasoning_effort_dropped_under_drop_params(self, local_model_cost_map, model): - result = OCIChatConfig().map_openai_params( - non_default_params={"reasoning_effort": "low", "temperature": 0.1}, - optional_params={}, - model=model, - drop_params=True, - ) + def test_reasoning_effort_dropped_under_drop_params_with_one_warning(self, local_model_cost_map, model, caplog): + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + result = OCIChatConfig().map_openai_params( + non_default_params={"reasoning_effort": "low", "temperature": 0.1}, + optional_params={}, + model=model, + drop_params=True, + ) + drop_warnings = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] assert "reasoningEffort" not in result assert "reasoning_effort" not in result assert result["temperature"] == 0.1 + assert len(drop_warnings) == 1 + assert all(fragment in drop_warnings[0] for fragment in ("reasoning_effort='low'", model, "(drop_params=True)")) def test_reasoning_effort_raises_without_drop_params_for_gated_grok(self, local_model_cost_map, monkeypatch): monkeypatch.setattr(litellm, "drop_params", False) diff --git a/tests/unit/llms/openai/test_openai_common_utils.py b/tests/unit/llms/openai/test_openai_common_utils.py index 831df30c98a..b033ce9fc4c 100644 --- a/tests/unit/llms/openai/test_openai_common_utils.py +++ b/tests/unit/llms/openai/test_openai_common_utils.py @@ -1,3 +1,6 @@ +import asyncio +import json +import logging from typing import Final from unittest.mock import MagicMock, patch @@ -306,6 +309,73 @@ def _completion_kwargs(provider: str, client, **overrides) -> dict: } +_REASONING_EFFORT_422: Final = { + "error": { + "message": json.dumps( + {"detail": [{"loc": ["body", "reasoning_effort"], "msg": "Extra inputs are not permitted"}]} + ) + } +} +_CHAT_COMPLETION: Final = { + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 1, + "model": "gpt-6.1-sol", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, +} +_CHAT_COMPLETION_STREAM: Final = ( + 'data: {"id":"chatcmpl-1","object":"chat.completion.chunk","created":1,"model":"gpt-6.1-sol",' + '"choices":[{"index":0,"delta":{"role":"assistant","content":"ok"},"finish_reason":"stop"}]}\n\n' + "data: [DONE]\n\n" +) + + +def _answer_422_while_reasoning_effort_is_sent(request: httpx.Request) -> httpx.Response: + body: Final = json.loads(request.content) + if "reasoning_effort" in body: + return httpx.Response(422, json=_REASONING_EFFORT_422) + if body.get("stream"): + return httpx.Response(200, text=_CHAT_COMPLETION_STREAM, headers={"content-type": "text/event-stream"}) + return httpx.Response(200, json=_CHAT_COMPLETION) + + +async def _complete_through_the_422_retry(call_shape: str) -> str: + transport: Final = httpx.MockTransport(_answer_422_while_reasoning_effort_is_sent) + kwargs: Final = { + "model": "gpt-6.1-sol", + "messages": [{"role": "user", "content": "hi"}], + "reasoning_effort": "high", + } + if call_shape == "sync": + sync_client: Final = openai.OpenAI(api_key="sk-not-a-real-key", http_client=httpx.Client(transport=transport)) + return litellm.completion(**kwargs, client=sync_client).choices[0].message.content + async_client: Final = openai.AsyncOpenAI( + api_key="sk-not-a-real-key", http_client=httpx.AsyncClient(transport=transport) + ) + if call_shape == "async": + return (await litellm.acompletion(**kwargs, client=async_client)).choices[0].message.content + stream: Final = await litellm.acompletion(**kwargs, client=async_client, stream=True) + return "".join([chunk.choices[0].delta.content or "" async for chunk in stream]) + + +@pytest.mark.parametrize("call_shape", ["sync", "async", "async_stream"]) +def test_a_422_naming_reasoning_effort_drops_it_with_one_warning( + call_shape: str, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + monkeypatch.setattr(litellm, "drop_params", True) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + content: Final = asyncio.run(_complete_through_the_422_retry(call_shape)) + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] + + assert content == "ok" + assert len(drop_warnings) == 1 + assert all( + fragment in drop_warnings[0] + for fragment in ("reasoning_effort='high'", "gpt-6.1-sol", "(drop_params=True)", "422") + ) + + @pytest.mark.parametrize("provider", ["openai", "azure"]) def test_sync_output_limit_400_maps_to_length_truncated_response(provider): response = litellm.completion( diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index f12129af1fc..cac176de4e5 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -9342,9 +9342,10 @@ def test_a_non_numeric_thinking_budget_on_adaptive_claude_is_rejected_with_a_400 @pytest.mark.parametrize("drop_globally", [False, True]) @pytest.mark.parametrize("model_info", [None, _OPUS_DECLARING_LOW_AND_HIGH]) @pytest.mark.parametrize("body_sent", [_anthropic_chat_body_sent, _bedrock_converse_request_fields_sent]) -def test_drop_params_drops_a_non_numeric_thinking_budget_on_adaptive_claude( +def test_drop_params_drops_a_non_numeric_thinking_budget_on_adaptive_claude_with_one_warning( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, body_sent: Callable[..., Mapping[str, object]], model_info: Mapping[str, object] | None, drop_globally: bool, @@ -9352,13 +9353,40 @@ def test_drop_params_drops_a_non_numeric_thinking_budget_on_adaptive_claude( monkeypatch.setattr(litellm, "drop_params", drop_globally) request_drop: Final = {} if drop_globally else {"drop_params": True} - dropped: Final = body_sent( - respx_mock, model_info, thinking={"type": "enabled", "budget_tokens": "lots"}, **request_drop - ) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + dropped: Final = body_sent( + respx_mock, model_info, thinking={"type": "enabled", "budget_tokens": "lots"}, **request_drop + ) + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] adaptive_alone: Final = body_sent(respx_mock, model_info, thinking={"type": "adaptive"}, **request_drop) assert "output_config" not in adaptive_alone assert dropped == adaptive_alone + assert len(drop_warnings) == 1 + assert all( + fragment in drop_warnings[0] + for fragment in ("thinking.budget_tokens='lots'", "claude-opus-5-5", "(drop_params=True)") + ) + + +@pytest.mark.parametrize("model_info", [None, _KIMI_DEPLOYMENT]) +def test_drop_params_drops_a_non_numeric_thinking_budget_on_hosted_vllm_with_one_warning( + respx_mock: respx.MockRouter, caplog: pytest.LogCaptureFixture, model_info: Mapping[str, object] | None +) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + sent: Final = _chat_body_sent( + respx_mock, + "hosted_vllm/my-kimi", + None, + model_info, + thinking={"type": "enabled", "budget_tokens": "lots"}, + drop_params=True, + ) + + assert not {"reasoning_effort", "thinking"} & sent.keys() + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] + assert len(drop_warnings) == 1 + assert all(fragment in drop_warnings[0] for fragment in ("thinking.budget_tokens='lots'", "my-kimi")) @pytest.mark.parametrize("model", ["azure_ai/my-kimi", "hosted_vllm/my-kimi"]) From 17511cc039a5009fbee00fbf09e27c045a8057c7 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 13:38:39 -0700 Subject: [PATCH 2/6] fix(logging): keep guardrail and token-count reads of a request from logging drop warnings Only the real Anthropic-messages adapter request reports a dropped thinking budget. Guardrail, shadow-eval and token-count translations of the same request now drop it silently, so one request logs one warning. The 422 retry also no longer hashes each loc entry, so a list inside loc cannot raise TypeError. --- .../reasoning_effort_utils.py | 11 ++++--- .../messages/adapters/transformation.py | 15 +++++++-- .../test_anthropic_guardrail_handler.py | 16 ++++++++++ .../llms/openai/test_openai_common_utils.py | 31 +++++++++++++++++++ 4 files changed, 66 insertions(+), 7 deletions(-) diff --git a/litellm/litellm_core_utils/reasoning_effort_utils.py b/litellm/litellm_core_utils/reasoning_effort_utils.py index 8a017f01e7a..334ddea205c 100644 --- a/litellm/litellm_core_utils/reasoning_effort_utils.py +++ b/litellm/litellm_core_utils/reasoning_effort_utils.py @@ -1,4 +1,4 @@ -from collections.abc import Iterable, Mapping +from collections.abc import Collection, Mapping from dataclasses import dataclass from typing import Final, Literal @@ -17,7 +17,7 @@ OpenAIStyleReasoningEffort = Literal["minimal", "low", "medium", "high"] _DROP_WARNING: Final = "Dropping %s=%r for model=%r because %s" _DROP_PARAMS_DROP_WARNING: Final = "Dropping %s=%r for model=%r (drop_params=True) because %s" _NON_NUMERIC_BUDGET_REASON: Final = "it is not a number. Send budget_tokens as an integer to set a reasoning level." -_REASONING_PARAMS: Final = frozenset(("reasoning_effort", "reasoning", "thinking")) +_REASONING_PARAMS: Final = ("reasoning", "reasoning_effort", "thinking") def warn_dropped_reasoning(param: str, value: object, model: object, reason: str, *, drop_params_gated: bool) -> None: @@ -27,10 +27,11 @@ def warn_dropped_reasoning(param: str, value: object, model: object, reason: str def warn_dropped_reasoning_params( - dropped: Iterable[str], request: Mapping[str, object], model: object, reason: str, *, drop_params_gated: bool + dropped: Collection[object], request: Mapping[str, object], model: object, reason: str, *, drop_params_gated: bool ) -> None: - for param in sorted(_REASONING_PARAMS.intersection(dropped, request)): - warn_dropped_reasoning(param, request[param], model, reason, drop_params_gated=drop_params_gated) + for param in _REASONING_PARAMS: + if param in dropped and param in request: + warn_dropped_reasoning(param, request[param], model, reason, drop_params_gated=drop_params_gated) @dataclass(frozen=True, slots=True) diff --git a/litellm/llms/anthropic/messages/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py index 0c346873ed2..721f12451d0 100644 --- a/litellm/llms/anthropic/messages/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -132,6 +132,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( ) from litellm.litellm_core_utils.reasoning_effort_utils import ( reasoning_effort_from_thinking, + reasoning_effort_from_thinking_silently, ) from litellm.llms.anthropic.common_utils import ( eager_input_streaming_flag, @@ -287,6 +288,7 @@ class AnthropicAdapter: anthropic_message_request=request_body, custom_llm_provider=custom_llm_provider, drop_params=drop_params, + report_reasoning_drops=True, ) return translated_body, tool_name_mapping @@ -658,6 +660,7 @@ class LiteLLMAnthropicMessagesAdapter: *, model: str, drop_params: bool = True, + report_reasoning_drops: bool = False, ) -> str | None: """ Translate Anthropic's thinking parameter to OpenAI's reasoning_effort. @@ -676,7 +679,9 @@ class LiteLLMAnthropicMessagesAdapter: if thinking_type == "disabled": return "none" elif thinking_type == "enabled": - return reasoning_effort_from_thinking(thinking, model=model, drop_params=drop_params) + if report_reasoning_drops: + return reasoning_effort_from_thinking(thinking, model=model, drop_params=drop_params) + return reasoning_effort_from_thinking_silently(thinking, drop_params=drop_params) elif thinking_type == "adaptive": # Adaptive thinking: effort is controlled by output_config.effort, # not budget_tokens. Return a default; caller should override with @@ -1124,6 +1129,7 @@ class LiteLLMAnthropicMessagesAdapter: *, custom_llm_provider: str | None = None, drop_params: bool = True, + report_reasoning_drops: bool = False, ) -> None: """Translate Anthropic thinking to either thinking or reasoning_effort. @@ -1181,7 +1187,10 @@ class LiteLLMAnthropicMessagesAdapter: return reasoning_effort: Final = declared_effort or self.translate_anthropic_thinking_to_reasoning_effort( - cast(AnthropicThinkingParam, thinking), model=model, drop_params=drop_params + cast(AnthropicThinkingParam, thinking), + model=model, + drop_params=drop_params, + report_reasoning_drops=report_reasoning_drops, ) if not reasoning_effort: return @@ -1240,6 +1249,7 @@ class LiteLLMAnthropicMessagesAdapter: custom_llm_provider: str | None = None, preserve_midturn_system: bool = False, drop_params: bool = True, + report_reasoning_drops: bool = False, ) -> tuple[ChatCompletionRequest, dict[str, str]]: """ This is used by the beta Anthropic Adapter, for translating anthropic `/v1/messages` requests to the openai format. @@ -1293,6 +1303,7 @@ class LiteLLMAnthropicMessagesAdapter: new_kwargs=new_kwargs, custom_llm_provider=custom_llm_provider, drop_params=drop_params, + report_reasoning_drops=report_reasoning_drops, ) ## CONVERT STOP_SEQUENCES self._translate_stop_sequences_to_openai( diff --git a/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py index 9e88b59adc9..58f7f5f32ce 100644 --- a/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py +++ b/tests/unit/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py @@ -6,6 +6,7 @@ with guardrail transformations, specifically testing edge cases with empty choic """ import json +import logging from typing import Any, Literal, Optional, Final from unittest.mock import MagicMock, patch @@ -3043,3 +3044,18 @@ class TestAnthropicMessagesHandlerPostCallHookResponse: native = {"type": "message", "role": "assistant", "content": [{"type": "text", "text": "hi"}]} assert AnthropicMessagesHandler().post_call_hook_response(native) is native + + +def test_reading_a_dropped_budget_for_a_guardrail_logs_no_drop_warning(caplog: pytest.LogCaptureFixture) -> None: + data: Final = { + "model": "gemini-2.5-flash", + "max_tokens": 64, + "thinking": {"type": "enabled", "budget_tokens": "high"}, + "messages": [{"role": "user", "content": "hi"}], + } + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + messages: Final = AnthropicMessagesHandler().get_structured_messages(data) + + assert messages == [{"role": "user", "content": "hi"}] + assert not [record for record in caplog.records if "Dropping" in record.getMessage()] diff --git a/tests/unit/llms/openai/test_openai_common_utils.py b/tests/unit/llms/openai/test_openai_common_utils.py index b033ce9fc4c..378d205613c 100644 --- a/tests/unit/llms/openai/test_openai_common_utils.py +++ b/tests/unit/llms/openai/test_openai_common_utils.py @@ -376,6 +376,37 @@ def test_a_422_naming_reasoning_effort_drops_it_with_one_warning( ) +def test_a_422_with_an_unhashable_loc_still_drops_reasoning_effort_with_one_warning( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + detail: Final = [ + {"loc": ["body", ["messages", 0]], "msg": "Invalid"}, + {"loc": ["body", "reasoning_effort"], "msg": "Extra inputs are not permitted"}, + ] + + def answer(request: httpx.Request) -> httpx.Response: + if "reasoning_effort" in json.loads(request.content): + return httpx.Response(422, json={"error": {"message": json.dumps({"detail": detail})}}) + return httpx.Response(200, json=_CHAT_COMPLETION) + + monkeypatch.setattr(litellm, "drop_params", True) + client: Final = openai.OpenAI( + api_key="sk-not-a-real-key", http_client=httpx.Client(transport=httpx.MockTransport(answer)) + ) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + response: Final = litellm.completion( + model="gpt-6.1-sol", + messages=[{"role": "user", "content": "hi"}], + reasoning_effort="high", + client=client, + ) + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] + + assert response.choices[0].message.content == "ok" + assert len(drop_warnings) == 1 + assert "reasoning_effort='high'" in drop_warnings[0] + + @pytest.mark.parametrize("provider", ["openai", "azure"]) def test_sync_output_limit_400_maps_to_length_truncated_response(provider): response = litellm.completion( From 1f4baa6d9155d8e7602eecc8536f3a8d45be21f3 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 17:08:39 -0700 Subject: [PATCH 3/6] fix(bedrock): keep the DeepSeek R1 Converse reasoning_effort drop at debug level Only allowed_openai_params reaches that branch, and it puts reasoning_effort back after the provider mapping, so a warning there claimed a drop that never happened --- .../bedrock/chat/converse_transformation.py | 8 ++------ .../chat/test_converse_transformation.py | 19 ------------------- 2 files changed, 2 insertions(+), 25 deletions(-) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 4a8323ecc23..f54c352fd6e 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -48,7 +48,6 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of, ) -from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning from litellm.llms.anthropic.chat.transformation import ( DROP_UNSUPPORTED_ADAPTIVE_THINKING_WARNING, DROP_UNSUPPORTED_OUTPUT_CONFIG_WARNING, @@ -1175,12 +1174,9 @@ class AmazonConverseConfig(BaseConfig): drop_params=drop_params, ) elif param == "reasoning_effort" and isinstance(value, str) and drop_reasoning_effort_param: - warn_dropped_reasoning( - param, - value, + verbose_logger.debug( + "Dropping unsupported `reasoning_effort` param for Bedrock model=%s; it always reasons and rejects it.", model, - "it always reasons and Bedrock rejects reasoning_effort for it.", - drop_params_gated=False, ) elif ( param == "reasoning_effort" diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 45fd6d22d1e..154797bab5a 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -1,7 +1,6 @@ import base64 import copy import json -import logging import os import re from collections.abc import Iterator, Mapping @@ -1732,24 +1731,6 @@ def test_bedrock_deepseek_r1_reasoning_params_not_forwarded_by_map(param): assert request.get("additionalModelRequestFields") is None -def test_bedrock_deepseek_r1_reasoning_effort_drop_warns_once(caplog: pytest.LogCaptureFixture) -> None: - model: Final = "bedrock/converse/us.deepseek.r1-v1:0" - with caplog.at_level(logging.WARNING, logger="LiteLLM"): - optional_params: Final = AmazonConverseConfig().map_openai_params( - non_default_params={"reasoning_effort": "high", "max_tokens": 100}, - optional_params={}, - model=model, - drop_params=False, - ) - drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] - assert "reasoning_effort" not in optional_params - assert optional_params["maxTokens"] == 100 - assert len(drop_warnings) == 1 - assert "reasoning_effort='high'" in drop_warnings[0] - assert repr(model) in drop_warnings[0] - assert "drop_params" not in drop_warnings[0] - - def test_bedrock_deepseek_v3_reasoning_effort_forwarded_raw(): """DeepSeek V3 takes reasoning_effort verbatim in additionalModelRequestFields, never converted into the Anthropic `thinking` block that Claude models get.""" From 1172cb6ab7c5eeb2e4af648b77da0271b3cd8e5a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 00:30:00 +0000 Subject: [PATCH 4/6] fix(logging): warn on Gemini non-integer budget drops and keep the auto-router baseline silent Gemini 2.x now logs the drop of a non-integer thinking budget. The auto-router savings baseline builds a body it never sends, so it no longer warns. Logged values are capped with reprlib, Azure AI marks only its drop_params retry as gated, and Bedrock Chat Completions names the model without its route prefix. --- .../reasoning_effort_utils.py | 18 ++++++--- .../messages/native/transformation.py | 5 ++- .../llms/anthropic/prompt_cache_prediction.py | 8 +++- litellm/llms/azure_ai/chat/transformation.py | 14 +++++-- .../chat/chat_completions/transformation.py | 5 ++- litellm/llms/openai/common_utils.py | 2 +- .../vertex_and_google_ai_studio_gemini.py | 14 ++++++- .../test_anthropic_prompt_cache_prediction.py | 22 +++++++++++ .../chat/test_azure_ai_transformation.py | 28 ++++++++++++++ ...bedrock_chat_completions_transformation.py | 14 +++++-- .../llms/openai/test_openai_common_utils.py | 27 ++++++++++++- ...test_vertex_and_google_ai_studio_gemini.py | 38 +++++++++++++++++++ 12 files changed, 174 insertions(+), 21 deletions(-) diff --git a/litellm/litellm_core_utils/reasoning_effort_utils.py b/litellm/litellm_core_utils/reasoning_effort_utils.py index 334ddea205c..8dc80ae7b81 100644 --- a/litellm/litellm_core_utils/reasoning_effort_utils.py +++ b/litellm/litellm_core_utils/reasoning_effort_utils.py @@ -1,3 +1,4 @@ +import reprlib from collections.abc import Collection, Mapping from dataclasses import dataclass from typing import Final, Literal @@ -14,20 +15,27 @@ from litellm.litellm_core_utils.core_helpers import as_str_mapping OpenAIStyleReasoningEffort = Literal["minimal", "low", "medium", "high"] -_DROP_WARNING: Final = "Dropping %s=%r for model=%r because %s" -_DROP_PARAMS_DROP_WARNING: Final = "Dropping %s=%r for model=%r (drop_params=True) because %s" +_DROP_WARNING: Final = "Dropping %s=%s for model=%r because %s" +_DROP_PARAMS_DROP_WARNING: Final = "Dropping %s=%s for model=%r (drop_params=True) because %s" _NON_NUMERIC_BUDGET_REASON: Final = "it is not a number. Send budget_tokens as an integer to set a reasoning level." _REASONING_PARAMS: Final = ("reasoning", "reasoning_effort", "thinking") -def warn_dropped_reasoning(param: str, value: object, model: object, reason: str, *, drop_params_gated: bool) -> None: +def warn_dropped_reasoning( + param: str, value: object, model: str | None, reason: str, *, drop_params_gated: bool +) -> None: verbose_logger.warning( - _DROP_PARAMS_DROP_WARNING if drop_params_gated else _DROP_WARNING, param, value, model, reason + _DROP_PARAMS_DROP_WARNING if drop_params_gated else _DROP_WARNING, param, reprlib.repr(value), model, reason ) def warn_dropped_reasoning_params( - dropped: Collection[object], request: Mapping[str, object], model: object, reason: str, *, drop_params_gated: bool + dropped: Collection[object], + request: Mapping[str, object], + model: str | None, + reason: str, + *, + drop_params_gated: bool, ) -> None: for param in _REASONING_PARAMS: if param in dropped and param in request: diff --git a/litellm/llms/anthropic/messages/native/transformation.py b/litellm/llms/anthropic/messages/native/transformation.py index cc53858f842..a995ba100c4 100644 --- a/litellm/llms/anthropic/messages/native/transformation.py +++ b/litellm/llms/anthropic/messages/native/transformation.py @@ -83,6 +83,7 @@ def _ensure_stop_sequence(response: AnthropicMessagesResponse) -> AnthropicMessa class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): _workload_identity_eligible: ClassVar[bool] = True + _report_reasoning_drops: ClassVar[bool] = True @property def custom_llm_provider(self) -> str | None: @@ -653,12 +654,14 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): custom_llm_provider=self._resolved_provider, ) - AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model( + dropped_budget: Final = AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model_silently( model=model, optional_params=anthropic_messages_optional_request_params, custom_llm_provider=self._resolved_provider, drop_params=getattr(litellm_params, "drop_params", None) is True, ) + if dropped_budget is not None and self._report_reasoning_drops: + dropped_budget.warn(model) self._translate_adaptive_effort_for_non_adaptive_model( model=model, diff --git a/litellm/llms/anthropic/prompt_cache_prediction.py b/litellm/llms/anthropic/prompt_cache_prediction.py index 7a4c707860d..6f69464277a 100644 --- a/litellm/llms/anthropic/prompt_cache_prediction.py +++ b/litellm/llms/anthropic/prompt_cache_prediction.py @@ -9,7 +9,7 @@ from dataclasses import dataclass, field from functools import reduce from itertools import accumulate, groupby from types import MappingProxyType -from typing import Annotated, Final, Literal, Protocol, TypeAlias +from typing import Annotated, ClassVar, Final, Literal, Protocol, TypeAlias import httpx from pydantic import ConfigDict, Field, JsonValue, StrictInt, TypeAdapter, ValidationError @@ -81,6 +81,10 @@ _DEPLOYMENT_OPTIONS: Final = frozenset( ) +class _BaselineAnthropicMessagesConfig(AnthropicMessagesConfig): + _report_reasoning_drops: ClassVar[bool] = False + + class _StrictModel(LiteLLMBaseModel): model_config = ConfigDict(extra="forbid", frozen=True, strict=True) @@ -661,7 +665,7 @@ def prepare_native_baseline_body(request: Mapping[str, object], model: str) -> M TypeAdapter(tuple[str, ...]).validate_python(owned.get("additional_drop_params") or ()), dict(options), ) - body: Final = AnthropicMessagesConfig().transform_anthropic_messages_request( + body: Final = _BaselineAnthropicMessagesConfig().transform_anthropic_messages_request( model=resolved_model, messages=strip_provider_specific_fields_from_anthropic_messages(messages), anthropic_messages_optional_request_params=filtered, diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py index 72f20c3148b..adf44b4f8a2 100644 --- a/litellm/llms/azure_ai/chat/transformation.py +++ b/litellm/llms/azure_ai/chat/transformation.py @@ -1,7 +1,7 @@ import copy import enum import re -from collections.abc import AsyncIterator, Collection, Iterator +from collections.abc import AsyncIterator, Collection, Iterator, Mapping from typing import TYPE_CHECKING, Final, cast import httpx @@ -40,6 +40,11 @@ if TYPE_CHECKING: _REQUEST_BODY: Final = TypeAdapter(dict[str, object]) +def _request_model(request: Mapping[str, object]) -> str | None: + model: Final = request.get("model") + return model if isinstance(model, str) else None + + class AzureFoundryErrorStrings(str, enum.Enum): SET_EXTRA_PARAMETERS_TO_PASS_THROUGH = "Set extra-parameters to 'pass-through'" @@ -401,8 +406,9 @@ class AzureAIStudioConfig(OpenAIConfig): data: Final = drop_params_from_unprocessable_entity_error( e=e, data=request_data, - model=_REQUEST_BODY.validate_python(request_data).get("model"), - drop_params_gated=False, + model=_request_model(_REQUEST_BODY.validate_python(request_data)), + drop_params_gated="Extra inputs are not permitted" in error_text + and not self._error_has_tool_level_extra_fields(error_text), ) return data @@ -423,7 +429,7 @@ class AzureAIStudioConfig(OpenAIConfig): warn_dropped_reasoning_params( params_to_drop, request, - request.get("model"), + _request_model(request), "Azure AI rejected it as an extra parameter its deployment does not pass through.", drop_params_gated=False, ) diff --git a/litellm/llms/bedrock/chat/chat_completions/transformation.py b/litellm/llms/bedrock/chat/chat_completions/transformation.py index 0762bdbb1df..dd2277a6c35 100644 --- a/litellm/llms/bedrock/chat/chat_completions/transformation.py +++ b/litellm/llms/bedrock/chat/chat_completions/transformation.py @@ -43,6 +43,7 @@ from litellm.llms.bedrock.common_utils import ( bedrock_rejects_stop_sequences, bedrock_runtime_chat_completions_serves_reasoning_inline, split_bedrock_region_path, + strip_bedrock_routing_prefix, ) from litellm.llms.openai.chat.gpt_transformation import OpenAIChatCompletionStreamingHandler from litellm.llms.openai_like.chat.transformation import OpenAILikeChatConfig @@ -125,7 +126,7 @@ def without_refused_reasoning_effort(model: str, params: Mapping[str, object]) - warn_dropped_reasoning( "reasoning_effort", effort, - model, + strip_bedrock_routing_prefix(model), "AWS's Chat Completions endpoint rejects that level for this model family, so AWS applies its default effort.", drop_params_gated=False, ) @@ -457,7 +458,7 @@ class AmazonBedrockRuntimeChatCompletionsConfig(OpenAILikeChatConfig): warn_dropped_reasoning( "reasoning_effort", raw_params["reasoning_effort"], - model, + strip_bedrock_routing_prefix(model), "Bedrock's Chat Completions endpoint takes it only as a string, so AWS applies its default effort.", drop_params_gated=True, ) diff --git a/litellm/llms/openai/common_utils.py b/litellm/llms/openai/common_utils.py index f2eb40980a1..36b314480e4 100644 --- a/litellm/llms/openai/common_utils.py +++ b/litellm/llms/openai/common_utils.py @@ -145,7 +145,7 @@ def drop_params_from_unprocessable_entity_error( e: openai.UnprocessableEntityError | httpx.HTTPStatusError, data: Mapping[str, object], *, - model: object, + model: str | None, drop_params_gated: bool, ) -> dict[str, object]: """Return a copy of ``data`` without the params a 422 response names as invalid.""" diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 9acbc9c459c..fa52e1d4e0f 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -34,6 +34,7 @@ from litellm.litellm_core_utils.json_fragment_accumulator import JSONFragmentAcc from litellm.litellm_core_utils.prompt_templates.factory import ( encode_tool_call_id_with_signature, ) +from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, @@ -200,6 +201,9 @@ def _unsupported_reasoning_effort(reasoning_effort: str) -> UnsupportedParamsErr _SAMPLING_DEPRECATION_UNMAPPED_MODELS: Final = "models outside litellm's model map" +_NON_INTEGER_BUDGET_REASON: Final = ( + "Gemini takes budget_tokens only as an integer, so it uses its default thinking budget." +) _sampling_deprecation_lock: Final = threading.Lock() @@ -1477,8 +1481,16 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): # For older Gemini models, use thinkingBudget if thinking_enabled and not VertexGeminiConfig._is_thinking_budget_zero(thinking_budget): params["includeThoughts"] = sends_thoughts - if thinking_budget is not None and isinstance(thinking_budget, int): + if isinstance(thinking_budget, int): params["thinkingBudget"] = thinking_budget + elif thinking_budget is not None: + warn_dropped_reasoning( + "thinking.budget_tokens", + thinking_budget, + model, + _NON_INTEGER_BUDGET_REASON, + drop_params_gated=False, + ) return params diff --git a/tests/unit/llms/anthropic/test_anthropic_prompt_cache_prediction.py b/tests/unit/llms/anthropic/test_anthropic_prompt_cache_prediction.py index 7a6b6534131..e91df2b36c3 100644 --- a/tests/unit/llms/anthropic/test_anthropic_prompt_cache_prediction.py +++ b/tests/unit/llms/anthropic/test_anthropic_prompt_cache_prediction.py @@ -1,4 +1,5 @@ import json +import logging from collections.abc import Mapping from datetime import datetime from types import SimpleNamespace @@ -26,6 +27,7 @@ from litellm.llms.anthropic.prompt_cache_prediction import ( parse_cache_plan, parse_observed_cache, parse_prompt, + prepare_native_baseline_body, resolve_baseline_prediction_target, resolve_prediction_target, supported_prediction_headers, @@ -452,3 +454,23 @@ async def test_native_count_preserves_settings_and_requires_every_prefix( assert tuple(marker.prefix_tokens for marker in result.breakpoints) == (5000, 5800) assert tuple(route.call_count for route in routes) == (1, 1, 1) assert unexpected.call_count == 0 + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_native_baseline_body_drops_a_non_numeric_budget_without_logging_a_drop( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + monkeypatch.setattr(litellm, "drop_params", True) + request: Final = { + "model": "claude-opus-5", + "max_tokens": 64, + "messages": [{"role": "user", "content": "hi"}], + "thinking": {"type": "enabled", "budget_tokens": "high"}, + } + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + body: Final = prepare_native_baseline_body(request, "anthropic/claude-opus-5") + + assert body is not None + assert body["thinking"] == {"type": "adaptive"} + assert [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] == [] diff --git a/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py b/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py index 86844265b21..7cedb7cd269 100644 --- a/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py +++ b/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py @@ -788,6 +788,34 @@ def test_extra_parameters_error_drops_reasoning_effort_with_one_warning(caplog: assert "drop_params" not in drop_warnings[0] +def test_extra_inputs_error_names_drop_params_in_the_reasoning_drop_warning(caplog: pytest.LogCaptureFixture) -> None: + import httpx + + detail: Final = [ + {"type": "extra_forbidden", "loc": ["body", "reasoning_effort"], "msg": "Extra inputs are not permitted"} + ] + error: Final = httpx.HTTPStatusError( + message="422", + request=httpx.Request("POST", "https://example.services.ai.azure.com/models/chat/completions"), + response=httpx.Response(422, text=json.dumps({"error": {"message": json.dumps({"detail": detail})}})), + ) + request_data: Final = { + "model": "Mistral-Large-3", + "messages": [{"role": "user", "content": "Say hi."}], + "reasoning_effort": "high", + } + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + retried: Final = AzureAIStudioConfig().transform_request_on_unprocessable_entity_error(error, request_data) + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] + + assert retried == {"model": "Mistral-Large-3", "messages": [{"role": "user", "content": "Say hi."}]} + assert len(drop_warnings) == 1 + assert all( + fragment in drop_warnings[0] for fragment in ("reasoning_effort='high'", "Mistral-Large-3", "(drop_params=True)") + ) + + def _find_key_anywhere(obj, key: str) -> bool: if isinstance(obj, dict): if key in obj: diff --git a/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py b/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py index 63212911fb4..f4dcff1969e 100644 --- a/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py +++ b/tests/unit/llms/bedrock/chat/chat_completions/test_bedrock_chat_completions_transformation.py @@ -707,10 +707,16 @@ def test_with_max_completion_tokens_leaves_other_params_alone(): @pytest.mark.parametrize( - "model", - ["us.xai.grok-4.6", "bedrock/us-gov-west-1/us.xai.grok-4.6"], + ("model", "logged_model"), + [ + ("us.xai.grok-4.6", "us.xai.grok-4.6"), + ("chat_completions/us.xai.grok-4.6", "us.xai.grok-4.6"), + ("bedrock/us-gov-west-1/us.xai.grok-4.6", "us-gov-west-1/us.xai.grok-4.6"), + ], ) -def test_map_openai_params_drops_reasoning_effort_none_for_grok_with_one_warning(model, caplog): +def test_map_openai_params_drops_reasoning_effort_none_for_grok_with_one_warning( + model: str, logged_model: str, caplog: pytest.LogCaptureFixture +) -> None: cfg = AmazonBedrockRuntimeChatCompletionsConfig() with caplog.at_level(logging.WARNING, logger="LiteLLM"): mapped = cfg.map_openai_params( @@ -723,7 +729,7 @@ def test_map_openai_params_drops_reasoning_effort_none_for_grok_with_one_warning assert "reasoning_effort" not in mapped assert len(drop_warnings) == 1 assert "reasoning_effort='none'" in drop_warnings[0] - assert repr(model) in drop_warnings[0] + assert f"model={logged_model!r}" in drop_warnings[0] assert "drop_params" not in drop_warnings[0] diff --git a/tests/unit/llms/openai/test_openai_common_utils.py b/tests/unit/llms/openai/test_openai_common_utils.py index 378d205613c..af7d1938455 100644 --- a/tests/unit/llms/openai/test_openai_common_utils.py +++ b/tests/unit/llms/openai/test_openai_common_utils.py @@ -11,7 +11,11 @@ import respx import litellm from litellm.litellm_core_utils.token_counter import token_counter -from litellm.llms.openai.common_utils import BaseOpenAILLM, is_openai_backed_api_base +from litellm.llms.openai.common_utils import ( + BaseOpenAILLM, + drop_params_from_unprocessable_entity_error, + is_openai_backed_api_base, +) # Test parameters for different API functions API_FUNCTION_PARAMS = [ @@ -647,3 +651,24 @@ async def test_azure_gateway_and_image_variations_use_the_callers_async_session( ) assert response.data[0].url == "https://example.com/image.png" assert not session.is_closed + + + +def test_reasoning_drop_warning_caps_a_large_caller_value(caplog: pytest.LogCaptureFixture) -> None: + error: Final = httpx.HTTPStatusError( + message="422", + request=httpx.Request("POST", "https://example.invalid/v1/chat/completions"), + response=httpx.Response(422, json=_REASONING_EFFORT_422), + ) + data: Final[dict[str, object]] = {"model": "gpt-6.1-sol", "messages": [], "reasoning_effort": "x" * 5000} + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + retried: Final = drop_params_from_unprocessable_entity_error( + error, data, model="gpt-6.1-sol", drop_params_gated=True + ) + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] + + assert "reasoning_effort" not in retried + assert len(drop_warnings) == 1 + assert len(drop_warnings[0]) < 300 + assert all(fragment in drop_warnings[0] for fragment in ("reasoning_effort='xxx", "'gpt-6.1-sol'", "422")) diff --git a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 065d894e658..3d8c4649a00 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -1692,6 +1692,44 @@ def test_vertex_ai_map_thinking_param_without_budget_tokens_for_gemini_3(): assert result["thinkingConfig"] == {"includeThoughts": True} +def _budget_drop_warnings(caplog: pytest.LogCaptureFixture) -> tuple[str, ...]: + return tuple( + r.getMessage() + for r in caplog.records + if r.levelno == logging.WARNING and "thinking.budget_tokens" in r.getMessage() + ) + + +@pytest.mark.parametrize("budget", ("high", 1024.5, {"tokens": 1024})) +def test_gemini_non_integer_thinking_budget_drop_warns_once(budget: object, caplog: pytest.LogCaptureFixture) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + result: Final = VertexGeminiConfig().map_openai_params( + non_default_params={"thinking": {"type": "enabled", "budget_tokens": budget}}, + optional_params={}, + model="gemini-2.5-flash", + drop_params=False, + ) + + warnings: Final = _budget_drop_warnings(caplog) + assert "thinkingBudget" not in result["thinkingConfig"] + assert len(warnings) == 1 + assert f"thinking.budget_tokens={budget!r}" in warnings[0] + assert "gemini-2.5-flash" in warnings[0] + + +def test_gemini_integer_thinking_budget_is_sent_without_warning(caplog: pytest.LogCaptureFixture) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + result: Final = VertexGeminiConfig().map_openai_params( + non_default_params={"thinking": {"type": "enabled", "budget_tokens": 2048}}, + optional_params={}, + model="gemini-2.5-flash", + drop_params=False, + ) + + assert result["thinkingConfig"]["thinkingBudget"] == 2048 + assert _budget_drop_warnings(caplog) == () + + def test_vertex_ai_map_tools(): v = VertexGeminiConfig() optional_params = {} From 5b85f78300ec2bfaf931e0dbaf5b04729cef8f93 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 10:30:00 +0000 Subject: [PATCH 5/6] fix(logging): warn on the Gemini 3, Magistral, Ollama, Fireworks and DeepSeek ARN reasoning drops Gemini 3 and later now log a dropped thinking budget, Magistral logs the level it swaps for a reasoning prompt, Ollama logs a level it maps to think off, Fireworks logs chat_template_kwargs effort keys dropped for a model the map does not list as reasoning, and DeepSeek on Bedrock logs a drop when the model is addressed by an ARN that advertises the param. --- .../reasoning_effort_utils.py | 3 ++ .../bedrock/chat/converse_transformation.py | 17 ++++--- .../llms/fireworks_ai/chat/transformation.py | 14 ++++-- .../fireworks_ai/completion/transformation.py | 7 ++- litellm/llms/mistral/chat/transformation.py | 5 ++ litellm/llms/ollama/chat/transformation.py | 10 +++- .../vertex_and_google_ai_studio_gemini.py | 12 +++++ .../chat/test_converse_transformation.py | 48 +++++++++++++++++++ .../test_fireworks_ai_chat_transformation.py | 35 ++++++++++++++ ...works_ai_text_completion_transformation.py | 37 ++++++++++++++ .../test_mistral_chat_transformation.py | 41 ++++++++++++++++ .../ollama/test_ollama_chat_transformation.py | 35 ++++++++++++++ ...test_vertex_and_google_ai_studio_gemini.py | 34 +++++++++++++ 13 files changed, 282 insertions(+), 16 deletions(-) diff --git a/litellm/litellm_core_utils/reasoning_effort_utils.py b/litellm/litellm_core_utils/reasoning_effort_utils.py index 8dc80ae7b81..4d4bfdf9bb9 100644 --- a/litellm/litellm_core_utils/reasoning_effort_utils.py +++ b/litellm/litellm_core_utils/reasoning_effort_utils.py @@ -3,6 +3,8 @@ from collections.abc import Collection, Mapping from dataclasses import dataclass from typing import Final, Literal +from pydantic import TypeAdapter + import litellm from litellm._logging import verbose_logger from litellm.constants import ( @@ -19,6 +21,7 @@ _DROP_WARNING: Final = "Dropping %s=%s for model=%r because %s" _DROP_PARAMS_DROP_WARNING: Final = "Dropping %s=%s for model=%r (drop_params=True) because %s" _NON_NUMERIC_BUDGET_REASON: Final = "it is not a number. Send budget_tokens as an integer to set a reasoning level." _REASONING_PARAMS: Final = ("reasoning", "reasoning_effort", "thinking") +OPAQUE_VALUE: Final = TypeAdapter(object) def warn_dropped_reasoning( diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index f54c352fd6e..03071bfa905 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -48,6 +48,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of, ) +from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_dropped_reasoning from litellm.llms.anthropic.chat.transformation import ( DROP_UNSUPPORTED_ADAPTIVE_THINKING_WARNING, DROP_UNSUPPORTED_OUTPUT_CONFIG_WARNING, @@ -183,6 +184,8 @@ def _tools_the_model_accepts( _OUTPUT_CONFIG_BLOCK: Final = TypeAdapter(OutputConfigBlock) +_DEEPSEEK_THINKING_REASON: Final = "DeepSeek on Bedrock reasons natively and takes no thinking field." +_DEEPSEEK_R1_EFFORT_REASON: Final = "DeepSeek R1 on Bedrock always reasons and rejects reasoning_effort." def _has_native_text_format(output_config: object) -> bool: @@ -635,6 +638,10 @@ class AmazonConverseConfig(BaseConfig): or supports_reasoning(model=base_model, custom_llm_provider=self.custom_llm_provider) ) + def _warn_native_reasoning_drop(self, param: str, value: object, model: str, reason: str) -> None: + if param in self.get_supported_openai_params(model): + warn_dropped_reasoning(param, value, model, reason, drop_params_gated=False) + def _model_rejects_reasoning_effort_param(self, model: str, base_model: str) -> bool: """Whether the model returns a 400 for every ``reasoning_effort`` shape on Converse. @@ -1138,9 +1145,8 @@ class AmazonConverseConfig(BaseConfig): "tool_choice": {"type": "auto", "disable_parallel_tool_use": not value} } if param == "thinking" and drop_thinking_param: - verbose_logger.debug( - "Dropping unsupported `thinking` param for Bedrock model=%s; it reasons natively.", - model, + self._warn_native_reasoning_drop( + param, OPAQUE_VALUE.validate_python(value), model, _DEEPSEEK_THINKING_REASON ) elif param == "thinking" and not self._is_openai_gpt_reasoning_model(model): if ( @@ -1174,10 +1180,7 @@ class AmazonConverseConfig(BaseConfig): drop_params=drop_params, ) elif param == "reasoning_effort" and isinstance(value, str) and drop_reasoning_effort_param: - verbose_logger.debug( - "Dropping unsupported `reasoning_effort` param for Bedrock model=%s; it always reasons and rejects it.", - model, - ) + self._warn_native_reasoning_drop(param, value, model, _DEEPSEEK_R1_EFFORT_REASON) elif ( param == "reasoning_effort" and isinstance(value, str) diff --git a/litellm/llms/fireworks_ai/chat/transformation.py b/litellm/llms/fireworks_ai/chat/transformation.py index fa5794d71b3..92c6c8439b3 100644 --- a/litellm/llms/fireworks_ai/chat/transformation.py +++ b/litellm/llms/fireworks_ai/chat/transformation.py @@ -17,6 +17,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( strip_litellm_response_only_fields, unpack_legacy_defs, ) +from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_dropped_reasoning from litellm.litellm_core_utils.url_utils import join_api_base from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ( @@ -97,6 +98,7 @@ def _json_schema_response_format(schema: object, name: str) -> Mapping[str, obje EFFORT_KWARG_KEYS: Final = frozenset({"enable_thinking", "thinking", "reasoning_budget", "low_effort"}) +_NOT_A_REASONING_MODEL_REASON: Final = "litellm's model map does not list it as a reasoning model." def _bool_from_kwargs(kwargs: Mapping[str, object], keys: tuple[str, ...]) -> bool | None: @@ -107,6 +109,13 @@ def _bool_from_kwargs(kwargs: Mapping[str, object], keys: tuple[str, ...]) -> bo return None +def warn_dropped_effort_kwargs(chat_template_kwargs: object, effort: object, model: str) -> None: + if effort != "none": + warn_dropped_reasoning( + "chat_template_kwargs", chat_template_kwargs, model, _NOT_A_REASONING_MODEL_REASON, drop_params_gated=False + ) + + def effort_from_chat_template_kwargs(kwargs: Mapping[str, object]) -> object: enable_thinking: Final = _bool_from_kwargs(kwargs, ("enable_thinking", "thinking")) if enable_thinking is False: @@ -438,10 +447,7 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): if effort is None: return () if not supports_reasoning(model=model, custom_llm_provider="fireworks_ai"): - verbose_logger.debug( - "fireworks_ai model %r does not support reasoning; dropping chat_template_kwargs effort keys.", - model, - ) + warn_dropped_effort_kwargs(OPAQUE_VALUE.validate_python(chat_template_kwargs), effort, model) return () return (("reasoning_effort", effort),) diff --git a/litellm/llms/fireworks_ai/completion/transformation.py b/litellm/llms/fireworks_ai/completion/transformation.py index 1f5d4f306a9..611e60cedd4 100644 --- a/litellm/llms/fireworks_ai/completion/transformation.py +++ b/litellm/llms/fireworks_ai/completion/transformation.py @@ -2,6 +2,7 @@ from collections.abc import Mapping from typing import Final from litellm._logging import verbose_logger +from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE from litellm.types.llms.openai import AllMessageValues, OpenAITextCompletionUserMessage from litellm.utils import supports_reasoning @@ -12,6 +13,7 @@ from ..chat.transformation import ( NIM_VLLM_STRIP_PARAMS, FireworksAIConfig, effort_from_chat_template_kwargs, + warn_dropped_effort_kwargs, ) from ..common_utils import FireworksAIMixin, resolve_fireworks_resource_name @@ -129,10 +131,7 @@ class FireworksAITextCompletionConfig(FireworksAIMixin, BaseTextCompletionConfig ) return result if not supports_reasoning(model=model, custom_llm_provider="fireworks_ai"): - verbose_logger.debug( - "fireworks_ai model %r does not support reasoning; dropping chat_template_kwargs effort keys.", - model, - ) + warn_dropped_effort_kwargs(OPAQUE_VALUE.validate_python(chat_template_kwargs), effort, model) return result return {**result, "reasoning_effort": effort} diff --git a/litellm/llms/mistral/chat/transformation.py b/litellm/llms/mistral/chat/transformation.py index 2fb9402ed03..673acc73007 100644 --- a/litellm/llms/mistral/chat/transformation.py +++ b/litellm/llms/mistral/chat/transformation.py @@ -20,6 +20,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( strip_litellm_internal_message_fields, strip_none_values_from_message, ) +from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_dropped_reasoning from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.openai.chat.gpt_transformation import ( OpenAIChatCompletionStreamingHandler, @@ -57,6 +58,7 @@ def _accepted_reasoning_effort(model: str, requested: str, custom_llm_provider: _NO_TOOL_CALL_TURN: Final[tuple[int, frozenset[str]]] = (-1, frozenset()) +_REASONING_PROMPT_REASON: Final = "litellm sends Magistral a reasoning system prompt instead of the requested level." def _is_json_array(value: object) -> TypeIs[Sequence[object]]: # guard-ok: a decoded JSON array is a list @@ -216,6 +218,9 @@ class MistralConfig(OpenAIGPTConfig): if param in ("reasoning_effort", "thinking") and "magistral" in model.lower(): # Flag that we need to add reasoning system prompt optional_params["_add_reasoning_prompt"] = True + warn_dropped_reasoning( + param, OPAQUE_VALUE.validate_python(value), model, _REASONING_PROMPT_REASON, drop_params_gated=False + ) if param == "parallel_tool_calls": optional_params["parallel_tool_calls"] = value return optional_params diff --git a/litellm/llms/ollama/chat/transformation.py b/litellm/llms/ollama/chat/transformation.py index f9e7071db41..5b179466c54 100644 --- a/litellm/llms/ollama/chat/transformation.py +++ b/litellm/llms/ollama/chat/transformation.py @@ -14,6 +14,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( extract_images_from_message, extract_reasoning_content, ) +from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException @@ -39,6 +40,9 @@ if TYPE_CHECKING: else: LiteLLMLoggingObj = Any +_THINK_ON_EFFORTS: Final = frozenset({"low", "medium", "high"}) +_THINK_OFF_REASON: Final = "Ollama takes think only as on or off, and litellm maps this level to off." + class OllamaChatConfig(BaseConfig): """ @@ -175,7 +179,11 @@ class OllamaChatConfig(BaseConfig): if model.startswith("gpt-oss"): optional_params["think"] = value else: - optional_params["think"] = value in {"low", "medium", "high"} + optional_params["think"] = value in _THINK_ON_EFFORTS + if isinstance(value, str) and value not in _THINK_ON_EFFORTS and value != "none": + warn_dropped_reasoning( + "reasoning_effort", value, model, _THINK_OFF_REASON, drop_params_gated=False + ) ### FUNCTION CALLING LOGIC ### # Ollama 0.4+ supports native tool calling - pass tools directly # and let Ollama handle model capability detection diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index fa52e1d4e0f..812f162aff6 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -204,6 +204,10 @@ _SAMPLING_DEPRECATION_UNMAPPED_MODELS: Final = "models outside litellm's model m _NON_INTEGER_BUDGET_REASON: Final = ( "Gemini takes budget_tokens only as an integer, so it uses its default thinking budget." ) +_LEVEL_ONLY_BUDGET_REASON: Final = ( + "Gemini 3 and later set reasoning by thinking level, not budget, so it uses its default level. " + "Send reasoning_effort to pick a level." +) _sampling_deprecation_lock: Final = threading.Lock() @@ -1469,6 +1473,14 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): params["includeThoughts"] = False else: params["includeThoughts"] = sends_thoughts + if thinking_budget is not None: + warn_dropped_reasoning( + "thinking.budget_tokens", + thinking_budget, + model, + _LEVEL_ONLY_BUDGET_REASON, + drop_params_gated=False, + ) # Follow provider defaults unless explicitly opted into legacy behavior. if litellm.enable_gemini_default_thinking_level_low is True: params["thinkingLevel"] = ( diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 154797bab5a..19796cc10af 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -1,6 +1,7 @@ import base64 import copy import json +import logging import os import re from collections.abc import Iterator, Mapping @@ -12,6 +13,7 @@ import pytest import respx import litellm +from litellm._logging import verbose_logger from litellm import ModelResponse, Router, completion from litellm.litellm_core_utils.prompt_templates.factory import ( _bedrock_converse_messages_pt, @@ -10789,3 +10791,49 @@ def test_a_deployment_with_a_batch_control_plane_endpoint_keeps_it_out_of_chat_r assert response.choices[0].message.content == "ok" assert "aws_bedrock_control_plane_endpoint" not in route.calls[0].request.content.decode() + + +def _warnings_naming(caplog: pytest.LogCaptureFixture, param: str) -> tuple[str, ...]: + return tuple( + r.getMessage() for r in caplog.records if r.levelno == logging.WARNING and f"Dropping {param}=" in r.getMessage() + ) + + +_DEEPSEEK_R1_PROFILE_ARN: Final = "arn:aws:bedrock:us-east-1:123456789012:inference-profile/us.deepseek.r1-v1:0" + + +@pytest.mark.parametrize( + "param, value", (("thinking", {"type": "enabled", "budget_tokens": 1024}), ("reasoning_effort", "high")) +) +def test_bedrock_deepseek_arn_reasoning_drop_warns_once( + param: str, value: object, caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + optional_params: Final = AmazonConverseConfig().map_openai_params( + non_default_params={param: value, "max_tokens": 100}, + optional_params={}, + model=_DEEPSEEK_R1_PROFILE_ARN, + drop_params=False, + ) + + warnings: Final = _warnings_naming(caplog, param) + assert param not in optional_params + assert len(warnings) == 1 + assert _DEEPSEEK_R1_PROFILE_ARN in warnings[0] + + +@pytest.mark.parametrize( + "param, value", (("thinking", {"type": "enabled", "budget_tokens": 1024}), ("reasoning_effort", "high")) +) +def test_bedrock_deepseek_r1_drop_of_an_unadvertised_param_does_not_warn( + param: str, value: object, caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + AmazonConverseConfig().map_openai_params( + non_default_params={param: value, "max_tokens": 100}, + optional_params={}, + model="us.deepseek.r1-v1:0", + drop_params=False, + ) + + assert _warnings_naming(caplog, param) == () diff --git a/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py b/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py index 04b791cf289..d4a6329842e 100644 --- a/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py +++ b/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py @@ -1,4 +1,5 @@ import json +import logging from typing import Final from unittest.mock import MagicMock, patch @@ -8,6 +9,7 @@ import respx from pydantic import ValidationError import litellm +from litellm._logging import verbose_logger from litellm.constants import SESSION_ID_GENERATED_METADATA_KEY from litellm.litellm_core_utils.get_supported_openai_params import get_supported_openai_params from litellm.llms.custom_httpx.http_handler import HTTPHandler @@ -2358,3 +2360,36 @@ def test_global_disable_flag_with_transform_messages_helper( ) assert json.loads(route.calls.last.request.content)["messages"] == [{"role": "user", "content": content}] + + +def _warnings_naming(caplog: pytest.LogCaptureFixture, param: str) -> tuple[str, ...]: + return tuple( + r.getMessage() for r in caplog.records if r.levelno == logging.WARNING and f"Dropping {param}=" in r.getMessage() + ) + + +def test_chat_template_kwargs_effort_dropped_for_non_reasoning_model_warns_once( + caplog: pytest.LogCaptureFixture, +) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + FireworksAIConfig().map_extra_body_params( + {"extra_body": {"chat_template_kwargs": {"reasoning_budget": 512}}}, + _NON_REASONING_MODEL, + ) + + warnings: Final = _warnings_naming(caplog, "chat_template_kwargs") + assert len(warnings) == 1 + assert "'reasoning_budget': 512" in warnings[0] + assert "llama-v3-70b-instruct" in warnings[0] + + +def test_chat_template_kwargs_thinking_off_for_non_reasoning_model_does_not_warn( + caplog: pytest.LogCaptureFixture, +) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + FireworksAIConfig().map_extra_body_params( + {"extra_body": {"chat_template_kwargs": {"enable_thinking": False}}}, + _NON_REASONING_MODEL, + ) + + assert _warnings_naming(caplog, "chat_template_kwargs") == () diff --git a/tests/unit/llms/fireworks_ai/completion/test_fireworks_ai_text_completion_transformation.py b/tests/unit/llms/fireworks_ai/completion/test_fireworks_ai_text_completion_transformation.py index e5a77aa8d41..d35442d4095 100644 --- a/tests/unit/llms/fireworks_ai/completion/test_fireworks_ai_text_completion_transformation.py +++ b/tests/unit/llms/fireworks_ai/completion/test_fireworks_ai_text_completion_transformation.py @@ -1,7 +1,11 @@ +import logging +from typing import Final + import pytest import litellm +from litellm._logging import verbose_logger from litellm.llms.fireworks_ai.completion.transformation import ( @@ -204,3 +208,36 @@ def test_transform_text_completion_request_keeps_sdk_rejected_keys_in_extra_body assert "chat_template_kwargs" not in data["extra_body"] assert "best_of" not in data["extra_body"] assert "response_format" not in data + + +def _warnings_naming(caplog: pytest.LogCaptureFixture, param: str) -> tuple[str, ...]: + return tuple( + r.getMessage() for r in caplog.records if r.levelno == logging.WARNING and f"Dropping {param}=" in r.getMessage() + ) + + +def test_chat_template_kwargs_effort_dropped_for_non_reasoning_model_warns_once( + caplog: pytest.LogCaptureFixture, +) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + FireworksAITextCompletionConfig().map_extra_body_params( + {"extra_body": {"chat_template_kwargs": {"reasoning_budget": 512}}}, + _NON_REASONING_MODEL, + ) + + warnings: Final = _warnings_naming(caplog, "chat_template_kwargs") + assert len(warnings) == 1 + assert "'reasoning_budget': 512" in warnings[0] + assert "llama-v3-70b-instruct" in warnings[0] + + +def test_chat_template_kwargs_thinking_off_for_non_reasoning_model_does_not_warn( + caplog: pytest.LogCaptureFixture, +) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + FireworksAITextCompletionConfig().map_extra_body_params( + {"extra_body": {"chat_template_kwargs": {"enable_thinking": False}}}, + _NON_REASONING_MODEL, + ) + + assert _warnings_naming(caplog, "chat_template_kwargs") == () diff --git a/tests/unit/llms/mistral/test_mistral_chat_transformation.py b/tests/unit/llms/mistral/test_mistral_chat_transformation.py index b76e1707805..b16220b841b 100644 --- a/tests/unit/llms/mistral/test_mistral_chat_transformation.py +++ b/tests/unit/llms/mistral/test_mistral_chat_transformation.py @@ -1,8 +1,10 @@ +import logging from typing import Final, List, cast from unittest.mock import MagicMock, patch import pytest +from litellm._logging import verbose_logger from litellm.litellm_core_utils.prompt_templates.common_utils import TOOL_RESULT_IMAGE_BOUNDARY from litellm.llms.mistral.chat.transformation import ( MistralChatResponseIterator, @@ -1665,3 +1667,42 @@ def test_mistral_transform_request_drops_an_empty_tool_message_whose_call_id_is_ ) assert [m.get("role") for m in request["messages"]] == ["user", "assistant"] + + +def _warnings_naming(caplog: pytest.LogCaptureFixture, param: str) -> tuple[str, ...]: + return tuple( + r.getMessage() for r in caplog.records if r.levelno == logging.WARNING and f"Dropping {param}=" in r.getMessage() + ) + + +@pytest.mark.parametrize( + "param, value", (("reasoning_effort", "none"), ("thinking", {"type": "enabled", "budget_tokens": 1024})) +) +def test_magistral_reasoning_request_replaced_by_prompt_warns_once( + param: str, value: object, caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + result: Final = MistralConfig().map_openai_params( + non_default_params={param: value}, + optional_params={}, + model="magistral-medium-latest", + drop_params=False, + ) + + warnings: Final = _warnings_naming(caplog, param) + assert result["_add_reasoning_prompt"] is True + assert len(warnings) == 1 + assert "magistral-medium-latest" in warnings[0] + + +def test_non_magistral_reasoning_effort_is_sent_without_warning(caplog: pytest.LogCaptureFixture) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + result: Final = MistralConfig().map_openai_params( + non_default_params={"reasoning_effort": "high"}, + optional_params={}, + model="mistral-medium-latest", + drop_params=False, + ) + + assert result["reasoning_effort"] == "high" + assert _warnings_naming(caplog, "reasoning_effort") == () diff --git a/tests/unit/llms/ollama/test_ollama_chat_transformation.py b/tests/unit/llms/ollama/test_ollama_chat_transformation.py index df9f5fd9d14..af1b85cf325 100644 --- a/tests/unit/llms/ollama/test_ollama_chat_transformation.py +++ b/tests/unit/llms/ollama/test_ollama_chat_transformation.py @@ -1,5 +1,6 @@ import asyncio import inspect +import logging import os import sys from typing import Final, cast @@ -23,6 +24,7 @@ import json from unittest.mock import ANY, MagicMock, patch import litellm +from litellm._logging import verbose_logger from litellm.types.utils import Choices, EmbeddingResponse, Message, ModelResponse, ModelResponseStream from unittest import mock @@ -1114,3 +1116,36 @@ def test_get_complete_url_ignores_a_trailing_slash_on_api_base(api_base: str): ) == "http://gateway.example.com/ollama/api/chat" ) + + +def _warnings_naming(caplog: pytest.LogCaptureFixture, param: str) -> tuple[str, ...]: + return tuple( + r.getMessage() for r in caplog.records if r.levelno == logging.WARNING and f"Dropping {param}=" in r.getMessage() + ) + + +@pytest.mark.parametrize("effort", ("minimal", "xhigh")) +def test_reasoning_effort_mapped_to_think_off_warns_once(effort: str, caplog: pytest.LogCaptureFixture) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + result: Final = OllamaChatConfig().map_openai_params( + non_default_params={"reasoning_effort": effort}, optional_params={}, model="qwen3", drop_params=False + ) + + warnings: Final = _warnings_naming(caplog, "reasoning_effort") + assert result["think"] is False + assert len(warnings) == 1 + assert f"reasoning_effort={effort!r}" in warnings[0] + assert "qwen3" in warnings[0] + + +@pytest.mark.parametrize("effort, think", (("none", False), ("low", True), ("high", True))) +def test_reasoning_effort_with_a_matching_think_value_does_not_warn( + effort: str, think: bool, caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + result: Final = OllamaChatConfig().map_openai_params( + non_default_params={"reasoning_effort": effort}, optional_params={}, model="qwen3", drop_params=False + ) + + assert result["think"] is think + assert _warnings_naming(caplog, "reasoning_effort") == () diff --git a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 3d8c4649a00..79129f60744 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -1730,6 +1730,40 @@ def test_gemini_integer_thinking_budget_is_sent_without_warning(caplog: pytest.L assert _budget_drop_warnings(caplog) == () +def test_gemini_3_thinking_budget_drop_warns_once(caplog: pytest.LogCaptureFixture) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + result: Final = VertexGeminiConfig().map_openai_params( + non_default_params={"thinking": {"type": "enabled", "budget_tokens": 2048}}, + optional_params={}, + model="gemini-3-pro-preview", + drop_params=False, + ) + + warnings: Final = _budget_drop_warnings(caplog) + assert "thinkingBudget" not in result["thinkingConfig"] + assert len(warnings) == 1 + assert "thinking.budget_tokens=2048" in warnings[0] + assert "gemini-3-pro-preview" in warnings[0] + + +@pytest.mark.parametrize( + "thinking", + ({"type": "enabled"}, {"type": "enabled", "budget_tokens": 0}, {"type": "disabled", "budget_tokens": 2048}), +) +def test_gemini_3_thinking_without_a_dropped_budget_does_not_warn( + thinking: dict[str, object], caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + VertexGeminiConfig().map_openai_params( + non_default_params={"thinking": thinking}, + optional_params={}, + model="gemini-3-pro-preview", + drop_params=False, + ) + + assert _budget_drop_warnings(caplog) == () + + def test_vertex_ai_map_tools(): v = VertexGeminiConfig() optional_params = {} From 949c6365dbb3248e90169ea681b5f392f7566395 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 15:00:00 +0000 Subject: [PATCH 6/6] fix(logging): warn on the remaining Ollama, DeepSeek and Tencent reasoning drops The ollama/ completion route now shares the chat route's think mapping, so a level it turns off logs the same warning. DeepSeek warns when it drops thinking.budget_tokens, and Tencent warns when an adaptive-only model gets a non-integer budget. Magistral no longer warns on low, medium, high or enabled thinking, the values its reasoning prompt stands in for. The Gemini 3 budget warning no longer claims the model uses its default level. --- litellm/llms/deepseek/chat/transformation.py | 13 ++++++ litellm/llms/mistral/chat/transformation.py | 16 ++++++-- litellm/llms/ollama/chat/transformation.py | 16 ++------ litellm/llms/ollama/common_utils.py | 12 ++++++ .../llms/ollama/completion/transformation.py | 8 ++-- litellm/llms/tencent/chat/transformation.py | 7 ++++ .../vertex_and_google_ai_studio_gemini.py | 2 +- .../chat/test_deepseek_chat_transformation.py | 41 +++++++++++++++++++ .../test_mistral_chat_transformation.py | 27 +++++++++++- .../test_ollama_completion_transformation.py | 39 ++++++++++++++++++ .../chat/test_tencent_chat_transformation.py | 26 ++++++++++++ 11 files changed, 184 insertions(+), 23 deletions(-) diff --git a/litellm/llms/deepseek/chat/transformation.py b/litellm/llms/deepseek/chat/transformation.py index ecaa0a2ef16..5b7c2a2c410 100644 --- a/litellm/llms/deepseek/chat/transformation.py +++ b/litellm/llms/deepseek/chat/transformation.py @@ -6,10 +6,12 @@ from collections.abc import Coroutine, Mapping, Sequence from typing import Final, Literal, cast, overload import litellm +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.prompt_templates.common_utils import ( convert_content_list_to_str, extract_search_results_text, ) +from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_dropped_reasoning from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues @@ -18,6 +20,16 @@ from litellm.utils import supports_reasoning, supports_vision from ...openai.chat.gpt_transformation import OpenAIGPTConfig SUPPORTED_REASONING_EFFORTS: Final = ("minimal", "low", "medium", "high", "xhigh", "max", "ultra") +_NO_BUDGET_REASON: Final = "DeepSeek takes thinking only as enabled or disabled, with no budget." + + +def _warn_on_dropped_budget(thinking: object, model: str) -> None: + fields: Final = as_str_mapping(thinking) + if fields is None or fields.get("type") != "enabled" or "budget_tokens" not in fields: + return + warn_dropped_reasoning( + "thinking.budget_tokens", fields["budget_tokens"], model, _NO_BUDGET_REASON, drop_params_gated=False + ) class DeepSeekChatConfig(OpenAIGPTConfig): @@ -53,6 +65,7 @@ class DeepSeekChatConfig(OpenAIGPTConfig): # Then re-add only if valid for DeepSeek thinking_value: Final = optional_params.pop("thinking", None) reasoning_effort: Final = optional_params.pop("reasoning_effort", None) + _warn_on_dropped_budget(OPAQUE_VALUE.validate_python(thinking_value), model) # Handle thinking parameter - accept both enabled and disabled, ignore budget_tokens if isinstance(thinking_value, dict) and thinking_value.get("type") in ("enabled", "disabled"): diff --git a/litellm/llms/mistral/chat/transformation.py b/litellm/llms/mistral/chat/transformation.py index 673acc73007..24f04eb76a1 100644 --- a/litellm/llms/mistral/chat/transformation.py +++ b/litellm/llms/mistral/chat/transformation.py @@ -14,6 +14,7 @@ import httpx from typing_extensions import TypeIs # noqa: TID251 # TypeIs reaches typing only on 3.13 from litellm._logging import verbose_logger +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.prompt_templates.common_utils import ( convert_content_list_to_str, @@ -59,6 +60,17 @@ def _accepted_reasoning_effort(model: str, requested: str, custom_llm_provider: _NO_TOOL_CALL_TURN: Final[tuple[int, frozenset[str]]] = (-1, frozenset()) _REASONING_PROMPT_REASON: Final = "litellm sends Magistral a reasoning system prompt instead of the requested level." +_DOCUMENTED_MAGISTRAL_EFFORTS: Final = frozenset({"low", "medium", "high"}) +_REASONING_ON_THINKING_TYPES: Final = frozenset({"enabled", "adaptive"}) + + +def _warn_unless_reasoning_prompt_honors(param: str, value: object, model: str) -> None: + if param == "reasoning_effort" and isinstance(value, str) and value in _DOCUMENTED_MAGISTRAL_EFFORTS: + return + thinking: Final = as_str_mapping(value) + if param == "thinking" and thinking is not None and thinking.get("type") in _REASONING_ON_THINKING_TYPES: + return + warn_dropped_reasoning(param, value, model, _REASONING_PROMPT_REASON, drop_params_gated=False) def _is_json_array(value: object) -> TypeIs[Sequence[object]]: # guard-ok: a decoded JSON array is a list @@ -218,9 +230,7 @@ class MistralConfig(OpenAIGPTConfig): if param in ("reasoning_effort", "thinking") and "magistral" in model.lower(): # Flag that we need to add reasoning system prompt optional_params["_add_reasoning_prompt"] = True - warn_dropped_reasoning( - param, OPAQUE_VALUE.validate_python(value), model, _REASONING_PROMPT_REASON, drop_params_gated=False - ) + _warn_unless_reasoning_prompt_honors(param, OPAQUE_VALUE.validate_python(value), model) if param == "parallel_tool_calls": optional_params["parallel_tool_calls"] = value return optional_params diff --git a/litellm/llms/ollama/chat/transformation.py b/litellm/llms/ollama/chat/transformation.py index 5b179466c54..755d647da1d 100644 --- a/litellm/llms/ollama/chat/transformation.py +++ b/litellm/llms/ollama/chat/transformation.py @@ -14,7 +14,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( extract_images_from_message, extract_reasoning_content, ) -from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning +from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException @@ -30,7 +30,7 @@ from litellm.types.llms.openai import ( ) from litellm.types.utils import ModelResponse, ModelResponseStream -from ..common_utils import OllamaError +from ..common_utils import OllamaError, ollama_think if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -40,9 +40,6 @@ if TYPE_CHECKING: else: LiteLLMLoggingObj = Any -_THINK_ON_EFFORTS: Final = frozenset({"low", "medium", "high"}) -_THINK_OFF_REASON: Final = "Ollama takes think only as on or off, and litellm maps this level to off." - class OllamaChatConfig(BaseConfig): """ @@ -176,14 +173,7 @@ class OllamaChatConfig(BaseConfig): if value.get("json_schema") and value["json_schema"].get("schema"): optional_params["format"] = value["json_schema"]["schema"] if param == "reasoning_effort" and value is not None: - if model.startswith("gpt-oss"): - optional_params["think"] = value - else: - optional_params["think"] = value in _THINK_ON_EFFORTS - if isinstance(value, str) and value not in _THINK_ON_EFFORTS and value != "none": - warn_dropped_reasoning( - "reasoning_effort", value, model, _THINK_OFF_REASON, drop_params_gated=False - ) + optional_params["think"] = ollama_think(model, OPAQUE_VALUE.validate_python(value)) ### FUNCTION CALLING LOGIC ### # Ollama 0.4+ supports native tool calling - pass tools directly # and let Ollama handle model capability detection diff --git a/litellm/llms/ollama/common_utils.py b/litellm/llms/ollama/common_utils.py index f313c62f026..fcf6a396a13 100644 --- a/litellm/llms/ollama/common_utils.py +++ b/litellm/llms/ollama/common_utils.py @@ -8,11 +8,14 @@ from pydantic import BaseModel, TypeAdapter from litellm import verbose_logger from litellm._uuid import uuid +from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.types.utils import ModelResponse, ModelResponseStream _PROVIDER_FIELDS: Final = TypeAdapter(dict[str, object]) +_THINK_ON_EFFORTS: Final = frozenset({"low", "medium", "high"}) +_THINK_OFF_REASON: Final = "Ollama takes think only as on or off, and litellm maps this level to off." _DECODED_JSON: Final = TypeAdapter(object) @@ -296,3 +299,12 @@ class OllamaModelInfo(BaseLLMModelInfo): Return the base model name for Ollama (no-op). """ return model + + +def ollama_think(model: str, reasoning_effort: object) -> object: + if model.startswith("gpt-oss"): + return reasoning_effort + think: Final = reasoning_effort in _THINK_ON_EFFORTS + if isinstance(reasoning_effort, str) and not think and reasoning_effort != "none": + warn_dropped_reasoning("reasoning_effort", reasoning_effort, model, _THINK_OFF_REASON, drop_params_gated=False) + return think diff --git a/litellm/llms/ollama/completion/transformation.py b/litellm/llms/ollama/completion/transformation.py index 239b4e32645..6dbcb050e11 100644 --- a/litellm/llms/ollama/completion/transformation.py +++ b/litellm/llms/ollama/completion/transformation.py @@ -23,6 +23,7 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import ( async_inline_remote_media, inline_remote_image_urls, ) +from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException @@ -43,7 +44,7 @@ from litellm.types.utils import ( generate_id, ) -from ..common_utils import OllamaError, OllamaModelInfo, convert_image +from ..common_utils import OllamaError, OllamaModelInfo, convert_image, ollama_think if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -249,10 +250,7 @@ class OllamaConfig(BaseConfig): elif param == "stop": optional_params["stop"] = value elif param == "reasoning_effort" and value is not None: - if model.startswith("gpt-oss"): - optional_params["think"] = value - else: - optional_params["think"] = value in {"low", "medium", "high"} + optional_params["think"] = ollama_think(model, OPAQUE_VALUE.validate_python(value)) elif param == "response_format" and isinstance(value, dict): if value["type"] == "json_object": optional_params["format"] = "json" diff --git a/litellm/llms/tencent/chat/transformation.py b/litellm/llms/tencent/chat/transformation.py index 1f0bcd218d6..23a6ccd52c6 100644 --- a/litellm/llms/tencent/chat/transformation.py +++ b/litellm/llms/tencent/chat/transformation.py @@ -9,12 +9,15 @@ from typing import Final, TypedDict from typing_extensions import ReadOnly import litellm +from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning from litellm.litellm_core_utils.url_utils import ensure_api_route, join_api_base from litellm.secret_managers.main import get_secret_str from litellm.utils import supports_reasoning from ...openai.chat.gpt_transformation import OpenAIGPTConfig +_ADAPTIVE_BUDGET_REASON: Final = "it is not an integer, so litellm sends adaptive thinking with no budget." + class ThinkingPayload(TypedDict, total=False): """Tencent TokenHub `thinking` object. @@ -102,6 +105,10 @@ class TencentChatConfig(OpenAIGPTConfig): if isinstance(budget, int): coerced_with_budget: Final[ThinkingPayload] = {"type": "adaptive", "budget_tokens": budget} return coerced_with_budget + if budget is not None: + warn_dropped_reasoning( + "thinking.budget_tokens", budget, model, _ADAPTIVE_BUDGET_REASON, drop_params_gated=False + ) coerced: Final[ThinkingPayload] = {"type": "adaptive"} return coerced diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 9ae0d8a9651..1ef5267df14 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -206,7 +206,7 @@ _NON_INTEGER_BUDGET_REASON: Final = ( "Gemini takes budget_tokens only as an integer, so it uses its default thinking budget." ) _LEVEL_ONLY_BUDGET_REASON: Final = ( - "Gemini 3 and later set reasoning by thinking level, not budget, so it uses its default level. " + "Gemini 3 and later set reasoning by thinking level, not budget, so litellm does not send the budget. " "Send reasoning_effort to pick a level." ) _sampling_deprecation_lock: Final = threading.Lock() diff --git a/tests/unit/llms/deepseek/chat/test_deepseek_chat_transformation.py b/tests/unit/llms/deepseek/chat/test_deepseek_chat_transformation.py index 3fbb12881da..56af47aeebe 100644 --- a/tests/unit/llms/deepseek/chat/test_deepseek_chat_transformation.py +++ b/tests/unit/llms/deepseek/chat/test_deepseek_chat_transformation.py @@ -1,4 +1,5 @@ import json +import logging from collections.abc import Iterator from typing import Final @@ -6,6 +7,7 @@ import httpx import litellm import pytest import respx +from litellm._logging import verbose_logger from litellm.caching.llm_caching_handler import LLMClientCache from litellm.llms.deepseek.chat.transformation import DeepSeekChatConfig @@ -951,3 +953,42 @@ def test_get_complete_url_ignores_a_trailing_slash_on_api_base(api_base: str): ) == "https://gateway.example.com/deepseek/chat/completions" ) + + +def _budget_warnings(caplog: pytest.LogCaptureFixture) -> tuple[str, ...]: + return tuple( + r.getMessage() + for r in caplog.records + if r.levelno == logging.WARNING and "Dropping thinking.budget_tokens=" in r.getMessage() + ) + + +def test_enabled_thinking_budget_dropped_for_deepseek_warns_once(caplog: pytest.LogCaptureFixture) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + result: Final = DeepSeekChatConfig().map_openai_params( + non_default_params={"thinking": {"type": "enabled", "budget_tokens": 4000}}, + optional_params={}, + model="deepseek-reasoner", + drop_params=False, + ) + + warnings: Final = _budget_warnings(caplog) + assert result["thinking"] == {"type": "enabled"} + assert len(warnings) == 1 + assert "budget_tokens=4000" in warnings[0] + assert "deepseek-reasoner" in warnings[0] + + +@pytest.mark.parametrize( + "thinking", ({"type": "enabled"}, {"type": "disabled", "budget_tokens": 4000}), ids=("no-budget", "disabled") +) +def test_deepseek_thinking_without_a_requested_budget_does_not_warn( + thinking: dict[str, object], caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + result: Final = DeepSeekChatConfig().map_openai_params( + non_default_params={"thinking": thinking}, optional_params={}, model="deepseek-reasoner", drop_params=False + ) + + assert result["thinking"] == {"type": thinking["type"]} + assert _budget_warnings(caplog) == () diff --git a/tests/unit/llms/mistral/test_mistral_chat_transformation.py b/tests/unit/llms/mistral/test_mistral_chat_transformation.py index b16220b841b..24fb7762f99 100644 --- a/tests/unit/llms/mistral/test_mistral_chat_transformation.py +++ b/tests/unit/llms/mistral/test_mistral_chat_transformation.py @@ -1676,7 +1676,8 @@ def _warnings_naming(caplog: pytest.LogCaptureFixture, param: str) -> tuple[str, @pytest.mark.parametrize( - "param, value", (("reasoning_effort", "none"), ("thinking", {"type": "enabled", "budget_tokens": 1024})) + "param, value", + (("reasoning_effort", "none"), ("reasoning_effort", "xhigh"), ("thinking", {"type": "disabled"})), ) def test_magistral_reasoning_request_replaced_by_prompt_warns_once( param: str, value: object, caplog: pytest.LogCaptureFixture @@ -1695,6 +1696,30 @@ def test_magistral_reasoning_request_replaced_by_prompt_warns_once( assert "magistral-medium-latest" in warnings[0] +@pytest.mark.parametrize( + "param, value", + ( + ("reasoning_effort", "low"), + ("reasoning_effort", "medium"), + ("reasoning_effort", "high"), + ("thinking", {"type": "enabled", "budget_tokens": 1024}), + ), +) +def test_magistral_reasoning_request_the_prompt_honors_does_not_warn( + param: str, value: object, caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + result: Final = MistralConfig().map_openai_params( + non_default_params={param: value}, + optional_params={}, + model="magistral-medium-latest", + drop_params=False, + ) + + assert result["_add_reasoning_prompt"] is True + assert _warnings_naming(caplog, param) == () + + def test_non_magistral_reasoning_effort_is_sent_without_warning(caplog: pytest.LogCaptureFixture) -> None: with caplog.at_level(logging.WARNING, logger=verbose_logger.name): result: Final = MistralConfig().map_openai_params( diff --git a/tests/unit/llms/ollama/test_ollama_completion_transformation.py b/tests/unit/llms/ollama/test_ollama_completion_transformation.py index 248e745ebff..edea39bb10c 100644 --- a/tests/unit/llms/ollama/test_ollama_completion_transformation.py +++ b/tests/unit/llms/ollama/test_ollama_completion_transformation.py @@ -1,6 +1,7 @@ import base64 import io import json +import logging import sys from typing import Final from unittest.mock import MagicMock, patch @@ -10,6 +11,7 @@ import pytest import respx import litellm +from litellm._logging import verbose_logger from litellm._uuid import uuid from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.ollama.completion.transformation import ( @@ -990,3 +992,40 @@ def test_get_complete_url_ignores_a_trailing_slash_on_api_base(api_base: str): ) == "http://gateway.example.com/ollama/api/generate" ) + +def _reasoning_effort_warnings(caplog: pytest.LogCaptureFixture) -> tuple[str, ...]: + return tuple( + r.getMessage() + for r in caplog.records + if r.levelno == logging.WARNING and "Dropping reasoning_effort=" in r.getMessage() + ) + + +@pytest.mark.parametrize("effort", ("minimal", "xhigh")) +def test_reasoning_effort_mapped_to_think_off_warns_once(effort: str, caplog: pytest.LogCaptureFixture) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + result: Final = OllamaConfig().map_openai_params( + non_default_params={"reasoning_effort": effort}, optional_params={}, model="qwen3:0.6b", drop_params=False + ) + + warnings: Final = _reasoning_effort_warnings(caplog) + assert result["think"] is False + assert len(warnings) == 1 + assert f"reasoning_effort={effort!r}" in warnings[0] + assert "qwen3:0.6b" in warnings[0] + + +@pytest.mark.parametrize( + "model, effort, think", + (("qwen3:0.6b", "none", False), ("qwen3:0.6b", "high", True), ("gpt-oss:20b", "xhigh", "xhigh")), +) +def test_reasoning_effort_ollama_honors_does_not_warn( + model: str, effort: str, think: object, caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + result: Final = OllamaConfig().map_openai_params( + non_default_params={"reasoning_effort": effort}, optional_params={}, model=model, drop_params=False + ) + + assert result["think"] == think + assert _reasoning_effort_warnings(caplog) == () diff --git a/tests/unit/llms/tencent/chat/test_tencent_chat_transformation.py b/tests/unit/llms/tencent/chat/test_tencent_chat_transformation.py index 4d6d252ae6e..1bf646fc2ad 100644 --- a/tests/unit/llms/tencent/chat/test_tencent_chat_transformation.py +++ b/tests/unit/llms/tencent/chat/test_tencent_chat_transformation.py @@ -1,5 +1,10 @@ +import logging +from typing import Final from unittest.mock import patch +import pytest + +from litellm._logging import verbose_logger from litellm.llms.tencent.chat.transformation import TencentChatConfig @@ -208,6 +213,27 @@ class TestAdaptiveThinkingCoercion: assert result["extra_body"]["thinking"] == {"type": "adaptive", "budget_tokens": 4096} + @pytest.mark.parametrize("budget, warned", (("high", 1), (4096, 0)), ids=("non-integer", "integer")) + def test_enabled_thinking_on_adaptive_only_model_warns_only_when_the_budget_is_dropped( + self, local_model_cost_map, budget: object, warned: int, caplog: pytest.LogCaptureFixture + ): + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + result: Final = TencentChatConfig().map_openai_params( + non_default_params={"thinking": {"type": "enabled", "budget_tokens": budget}}, + optional_params={}, + model="tencent/minimax-m3", + drop_params=False, + ) + + warnings: Final = tuple( + r.getMessage() + for r in caplog.records + if r.levelno == logging.WARNING and f"Dropping thinking.budget_tokens={budget!r}" in r.getMessage() + ) + assert result["extra_body"]["thinking"]["type"] == "adaptive" + assert result["extra_body"]["thinking"].get("budget_tokens") == (None if warned else budget) + assert len(warnings) == warned + def test_disabled_thinking_kept_for_adaptive_only_model(self, local_model_cost_map): config = TencentChatConfig() result = config.map_openai_params(