From 8d4f593b6a7f38894e6c69830b944e226c1fe8b4 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 19:48:49 -0700 Subject: [PATCH 1/4] fix(openai): warn when the GPT-5 reasoning effort gate drops a level under drop_params A GPT-5 request whose reasoning_effort the model map does not enable for that model lost the effort with no log line under drop_params. An Azure deployment name missing from the cost map lost reasoning_effort none this way and ran at the model's default effort. The gate now logs a warning naming the model, the level, the map entry it checked, and the model_info.base_model and model_info.reasoning_effort_levels fixes. --- .../llms/openai/chat/gpt_5_transformation.py | 13 ++++++ .../llms/openai/test_gpt5_transformation.py | 42 +++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/litellm/llms/openai/chat/gpt_5_transformation.py b/litellm/llms/openai/chat/gpt_5_transformation.py index 0311f6c6590..73a712f3b98 100644 --- a/litellm/llms/openai/chat/gpt_5_transformation.py +++ b/litellm/llms/openai/chat/gpt_5_transformation.py @@ -5,6 +5,7 @@ from collections.abc import Collection from typing import ClassVar, Final import litellm +from litellm._logging import verbose_logger from litellm.utils import ( declared_value_factory, is_explicitly_disabled_factory, @@ -15,6 +16,12 @@ from .gpt_transformation import OpenAIGPTConfig _GPT_SERIES_VERSION: Final = re.compile(r"^gpt-(\d+)(?:\.(\d+))?(?=[.-]|$)") +DROP_REFUSED_REASONING_EFFORT_WARNING: Final = ( + "Dropping reasoning_effort=%r for model=%r (drop_params=True) because the model map does not mark that level " + "supported for %r. If the deployment supports it, set model_info.base_model to its underlying model or add " + "the level to model_info.reasoning_effort_levels." +) + def _catalogue_declares_default_effort() -> bool: """Whether the loaded cost map carries default_reasoning_effort for ANY entry. @@ -342,6 +349,12 @@ class OpenAIGPT5Config(OpenAIGPTConfig): and self.refuses_reasoning_effort_level(model, requested_effort) ): if litellm.drop_params or drop_params: + verbose_logger.warning( + DROP_REFUSED_REASONING_EFFORT_WARNING, + requested_effort, + model, + self._model_map_lookup_name(model), + ) non_default_params.pop("reasoning_effort", None) optional_params.pop("reasoning_effort", None) else: diff --git a/tests/unit/llms/openai/test_gpt5_transformation.py b/tests/unit/llms/openai/test_gpt5_transformation.py index 7b10411ba20..6fc0d4d9a9b 100644 --- a/tests/unit/llms/openai/test_gpt5_transformation.py +++ b/tests/unit/llms/openai/test_gpt5_transformation.py @@ -1,3 +1,4 @@ +import logging import re from collections.abc import Mapping from typing import Final @@ -1589,3 +1590,44 @@ def test_gpt_6_astra_takes_the_reasoning_series_request_shape(): assert "max_tokens" not in params assert params["reasoning_effort"] == "max" assert params["verbosity"] == "low" + + +@pytest.mark.parametrize( + ("custom_llm_provider", "level"), + [("azure", "none"), ("azure", "xhigh"), ("openai", "xhigh")], +) +def test_a_reasoning_effort_the_level_gate_drops_logs_a_warning_naming_the_fix( + caplog: pytest.LogCaptureFixture, custom_llm_provider: str, level: str +) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + params = litellm.get_optional_params( + model="gpt-5.6-terra-ptu", + custom_llm_provider=custom_llm_provider, + reasoning_effort=level, + drop_params=True, + ) + + assert "reasoning_effort" not in params + drop_warnings = [ + record.getMessage() + for record in caplog.records + if record.levelno == logging.WARNING and "reasoning_effort" in record.getMessage() + ] + assert len(drop_warnings) == 1 + assert all( + fragment in drop_warnings[0] + for fragment in (level, "gpt-5.6-terra-ptu", "model_info.base_model", "model_info.reasoning_effort_levels") + ) + + +def test_a_reasoning_effort_the_level_gate_forwards_logs_no_drop_warning(caplog: pytest.LogCaptureFixture) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + params = litellm.get_optional_params( + model="gpt-5.6-terra-ptu", + custom_llm_provider="azure", + reasoning_effort="medium", + drop_params=True, + ) + + assert params["reasoning_effort"] == "medium" + assert not [record for record in caplog.records if "reasoning_effort" in record.getMessage()] From 54cf2e68bc10745db15ec06cca3dcae811c677ac Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 20:45:23 -0700 Subject: [PATCH 2/4] fix(openai): warn when the Responses route or a deployment's declared levels drop a reasoning effort The /v1/responses gate dropped a row-disabled GPT-5 effort, and reasoning for an OpenAI model without supports_reasoning, with no log line, and the router logged a declared-level drop only at INFO. Both Responses drops now log a WARNING and the router logs its drop at WARNING. The warning now leads with model_info.reasoning_effort_levels, since base_model only helps a deployment named differently from its underlying model. --- .../llms/openai/chat/gpt_5_transformation.py | 7 +-- .../llms/openai/responses/transformation.py | 10 +++- .../reasoning_effort_capability.py | 3 +- .../test_openai_responses_transformation.py | 50 +++++++++++++++++++ .../llms/openai/test_gpt5_transformation.py | 6 +-- .../test_reasoning_effort_capability.py | 11 ++++ 6 files changed, 79 insertions(+), 8 deletions(-) diff --git a/litellm/llms/openai/chat/gpt_5_transformation.py b/litellm/llms/openai/chat/gpt_5_transformation.py index 73a712f3b98..d9c28cb6b15 100644 --- a/litellm/llms/openai/chat/gpt_5_transformation.py +++ b/litellm/llms/openai/chat/gpt_5_transformation.py @@ -17,9 +17,9 @@ from .gpt_transformation import OpenAIGPTConfig _GPT_SERIES_VERSION: Final = re.compile(r"^gpt-(\d+)(?:\.(\d+))?(?=[.-]|$)") DROP_REFUSED_REASONING_EFFORT_WARNING: Final = ( - "Dropping reasoning_effort=%r for model=%r (drop_params=True) because the model map does not mark that level " - "supported for %r. If the deployment supports it, set model_info.base_model to its underlying model or add " - "the level to model_info.reasoning_effort_levels." + "Dropping %s=%r for model=%r (drop_params=True) because the model map does not mark that level supported for " + "%r. If the deployment supports it, add the level to model_info.reasoning_effort_levels, or set " + "model_info.base_model when the deployment name is not its underlying model." ) @@ -351,6 +351,7 @@ class OpenAIGPT5Config(OpenAIGPTConfig): if litellm.drop_params or drop_params: verbose_logger.warning( DROP_REFUSED_REASONING_EFFORT_WARNING, + "reasoning_effort", requested_effort, model, self._model_map_lookup_name(model), diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 8f29a71d63b..7a72bb107fa 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -26,6 +26,7 @@ from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.llms.openai.chat.gpt_5_transformation import ( + DROP_REFUSED_REASONING_EFFORT_WARNING, is_gpt_reasoning_series_name, unsupported_reasoning_effort_message, ) @@ -50,6 +51,9 @@ from ..common_utils import OpenAIError from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config OPENAI_RESPONSES_API_MIN_MAX_OUTPUT_TOKENS: Final = 16 +DROP_UNSUPPORTED_REASONING_WARNING: Final = ( + "Dropping reasoning=%r for model=%r (drop_params=True) because its model cost map entry lacks supports_reasoning." +) _RAW_RESPONSE_JSON: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True)) if TYPE_CHECKING: @@ -401,7 +405,8 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): and not self._supports_reasoning_param(model=model) ): if drop_params or litellm.drop_params: - params.pop("reasoning", None) + dropped_reasoning: Final = params.pop("reasoning", None) + verbose_logger.warning(DROP_UNSUPPORTED_REASONING_WARNING, dropped_reasoning, model) else: raise litellm.UnsupportedParamsError( message=( @@ -427,6 +432,9 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): message=unsupported_reasoning_effort_message("reasoning.effort", requested_effort, model), status_code=400, ) + verbose_logger.warning( + DROP_REFUSED_REASONING_EFFORT_WARNING, "reasoning.effort", requested_effort, model, lookup_name + ) remaining: Final = reasoning.copy() remaining.pop("effort", None) if remaining: diff --git a/litellm/router_utils/reasoning_effort_capability.py b/litellm/router_utils/reasoning_effort_capability.py index 4357722970f..4d979046b66 100644 --- a/litellm/router_utils/reasoning_effort_capability.py +++ b/litellm/router_utils/reasoning_effort_capability.py @@ -426,7 +426,8 @@ def _log_remap( logged_deployment_id: Final = _single_line(deployment_id) logged_model_group: Final = _single_line(model_group) logged_deployment_model: Final = _single_line(deployment_model) - verbose_logger.info( + log: Final = verbose_logger.info if sent_level is not None else verbose_logger.warning + log( "reasoning_effort %s sent as %s (rounding %s) to deployment %s, model group %s, model %s", logged_requested_level, logged_sent_level, diff --git a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py index 020f491dcfd..65ccd49dd21 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py @@ -1,4 +1,5 @@ import json +import logging from collections.abc import Callable, Mapping from types import SimpleNamespace from typing import Final, cast @@ -2387,6 +2388,55 @@ class TestResponsesSurfaceSharesTheEffortRule: ) assert mapped["reasoning"] == {"summary": "auto"} + def test_dropping_a_refused_effort_logs_one_warning_naming_the_level_and_the_fix( + self, local_model_cost_map, caplog: pytest.LogCaptureFixture + ) -> None: + refused: Final = next( + level + for level in ("none", "minimal", "low") + if litellm.model_cost["gpt-6.1-sol"].get(f"supports_{level}_reasoning_effort") is False + ) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + mapped: Final = OpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params={"reasoning": {"effort": refused}}, + model="gpt-6.1-sol", + drop_params=True, + ) + + assert "reasoning" not in mapped + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] + assert len(drop_warnings) == 1 + assert all( + fragment in drop_warnings[0] + for fragment in (repr(refused), "gpt-6.1-sol", "model_info.reasoning_effort_levels") + ) + + def test_dropping_reasoning_for_a_model_without_supports_reasoning_logs_one_warning( + self, local_model_cost_map, caplog: pytest.LogCaptureFixture + ) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + mapped: Final = OpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params={"reasoning": {"effort": "medium"}}, + model="gpt-4o", + drop_params=True, + ) + + assert "reasoning" not in mapped + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] + assert len(drop_warnings) == 1 + assert all(fragment in drop_warnings[0] for fragment in ("'medium'", "gpt-4o", "supports_reasoning")) + + def test_a_forwarded_effort_logs_no_drop_warning(self, local_model_cost_map, caplog: pytest.LogCaptureFixture) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + mapped: Final = OpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params={"reasoning": {"effort": "medium"}}, + model="gpt-6.1-sol", + drop_params=True, + ) + + assert mapped["reasoning"] == {"effort": "medium"} + assert not [record for record in caplog.records if "Dropping" in record.getMessage()] + @pytest.mark.parametrize("level", ["none", "minimal", "low", "medium", "high", "xhigh", "max"]) def test_gpt5_3_codex_forwards_exactly_the_effort_levels_its_map_row_allows(self, local_model_cost_map, level): """The gate turns a stale false into a local refusal, so the Responses-only codex row has to diff --git a/tests/unit/llms/openai/test_gpt5_transformation.py b/tests/unit/llms/openai/test_gpt5_transformation.py index 6fc0d4d9a9b..76721ded9f1 100644 --- a/tests/unit/llms/openai/test_gpt5_transformation.py +++ b/tests/unit/llms/openai/test_gpt5_transformation.py @@ -1600,7 +1600,7 @@ def test_a_reasoning_effort_the_level_gate_drops_logs_a_warning_naming_the_fix( caplog: pytest.LogCaptureFixture, custom_llm_provider: str, level: str ) -> None: with caplog.at_level(logging.WARNING, logger="LiteLLM"): - params = litellm.get_optional_params( + params: Final = litellm.get_optional_params( model="gpt-5.6-terra-ptu", custom_llm_provider=custom_llm_provider, reasoning_effort=level, @@ -1608,7 +1608,7 @@ def test_a_reasoning_effort_the_level_gate_drops_logs_a_warning_naming_the_fix( ) assert "reasoning_effort" not in params - drop_warnings = [ + drop_warnings: Final = [ record.getMessage() for record in caplog.records if record.levelno == logging.WARNING and "reasoning_effort" in record.getMessage() @@ -1622,7 +1622,7 @@ def test_a_reasoning_effort_the_level_gate_drops_logs_a_warning_naming_the_fix( def test_a_reasoning_effort_the_level_gate_forwards_logs_no_drop_warning(caplog: pytest.LogCaptureFixture) -> None: with caplog.at_level(logging.WARNING, logger="LiteLLM"): - params = litellm.get_optional_params( + params: Final = litellm.get_optional_params( model="gpt-5.6-terra-ptu", custom_llm_provider="azure", reasoning_effort="medium", diff --git a/tests/unit/router_utils/test_reasoning_effort_capability.py b/tests/unit/router_utils/test_reasoning_effort_capability.py index d634e239ea9..9c91d8acea8 100644 --- a/tests/unit/router_utils/test_reasoning_effort_capability.py +++ b/tests/unit/router_utils/test_reasoning_effort_capability.py @@ -669,6 +669,17 @@ class TestResolveDeploymentReasoningEffort: "reasoning_effort_rounding": "up", } + def test_a_dropped_level_logs_a_warning_where_a_remap_logs_info(self, caplog: pytest.LogCaptureFixture) -> None: + verbose_logger.addHandler(caplog.handler) + verbose_logger.propagate = False + verbose_logger.setLevel(logging.INFO) + + _sent("none", {"reasoning_effort_levels": ["low", "high"]}, drop_params=True) + _sent("medium") + + assert [record.levelno for record in caplog.records] == [logging.WARNING, logging.INFO] + assert "reasoning_effort none sent as None" in caplog.records[0].getMessage() + class TestAzureGpt6SolAndLunaAdvertiseTheOpenAiLevels: @pytest.mark.parametrize( From add014b92ce8de82acecec7f7adc95714f0c7aa8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 21 Oct 2026 01:54:55 -0700 Subject: [PATCH 3/4] fix: warn only when a reasoning level is actually dropped, and on generic reasoning_effort drops --- .../llms/openai/chat/gpt_5_transformation.py | 2 +- .../reasoning_effort_capability.py | 33 +++++++++++++------ litellm/utils.py | 22 ++++++++----- .../test_openai_responses_transformation.py | 8 +++-- .../test_reasoning_effort_capability.py | 23 +++++++++---- tests/unit/test_utils_get_optional_params.py | 17 ++++++++++ 6 files changed, 77 insertions(+), 28 deletions(-) diff --git a/litellm/llms/openai/chat/gpt_5_transformation.py b/litellm/llms/openai/chat/gpt_5_transformation.py index d9c28cb6b15..55f8c1c0846 100644 --- a/litellm/llms/openai/chat/gpt_5_transformation.py +++ b/litellm/llms/openai/chat/gpt_5_transformation.py @@ -18,7 +18,7 @@ _GPT_SERIES_VERSION: Final = re.compile(r"^gpt-(\d+)(?:\.(\d+))?(?=[.-]|$)") DROP_REFUSED_REASONING_EFFORT_WARNING: Final = ( "Dropping %s=%r for model=%r (drop_params=True) because the model map does not mark that level supported for " - "%r. If the deployment supports it, add the level to model_info.reasoning_effort_levels, or set " + "%r. If the deployment supports it, list every level it supports in model_info.reasoning_effort_levels, or set " "model_info.base_model when the deployment name is not its underlying model." ) diff --git a/litellm/router_utils/reasoning_effort_capability.py b/litellm/router_utils/reasoning_effort_capability.py index 4d979046b66..12453adc6bf 100644 --- a/litellm/router_utils/reasoning_effort_capability.py +++ b/litellm/router_utils/reasoning_effort_capability.py @@ -426,8 +426,28 @@ def _log_remap( logged_deployment_id: Final = _single_line(deployment_id) logged_model_group: Final = _single_line(model_group) logged_deployment_model: Final = _single_line(deployment_model) - log: Final = verbose_logger.info if sent_level is not None else verbose_logger.warning - log( + extra: Final = MappingProxyType( + { + "reasoning_effort_deployment_id": logged_deployment_id, + "reasoning_effort_model_group": logged_model_group, + "reasoning_effort_deployment_model": logged_deployment_model, + "reasoning_effort_requested": logged_requested_level, + "reasoning_effort_sent": logged_sent_level, + "reasoning_effort_rounding": rounding, + } + ) + if sent_level is None and requested_level != _DEFAULT_EFFORT: + verbose_logger.warning( + "Dropping reasoning_effort %s (drop_params=True) for deployment %s, model group %s, model %s because its " + "model_info.reasoning_effort_levels does not list that level. List every level the deployment supports.", + logged_requested_level, + logged_deployment_id, + logged_model_group, + logged_deployment_model, + extra=extra, + ) + return + verbose_logger.info( "reasoning_effort %s sent as %s (rounding %s) to deployment %s, model group %s, model %s", logged_requested_level, logged_sent_level, @@ -435,14 +455,7 @@ def _log_remap( logged_deployment_id, logged_model_group, logged_deployment_model, - extra={ - "reasoning_effort_deployment_id": logged_deployment_id, - "reasoning_effort_model_group": logged_model_group, - "reasoning_effort_deployment_model": logged_deployment_model, - "reasoning_effort_requested": logged_requested_level, - "reasoning_effort_sent": logged_sent_level, - "reasoning_effort_rounding": rounding, - }, + extra=extra, ) diff --git a/litellm/utils.py b/litellm/utils.py index 379eddb3155..cb95146fd21 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5036,19 +5036,25 @@ def get_optional_params( verbose_logger.info("\nLiteLLM completion() model= %s; provider = %s", model, custom_llm_provider) verbose_logger.debug("\nLiteLLM: Params passed to completion() %s", passed_params) verbose_logger.debug("\nLiteLLM: Non-Default params passed to completion() %s", non_default_params) - unsupported_params: Final = {} - for k in non_default_params: - if k not in supported_params: - if k in PROVIDER_UNVALIDATED_PARAMS: - continue - if k == "n" and n == 1: # langchain sends n=1 as a default value - continue # skip this param - unsupported_params[k] = non_default_params[k] + unsupported_params: Final[Mapping[str, object]] = { + k: non_default_params[k] + for k in non_default_params + if k not in supported_params and k not in PROVIDER_UNVALIDATED_PARAMS and not (k == "n" and n == 1) + } if unsupported_params: if litellm.drop_params is True or (drop_params is not None and drop_params is True): for k in unsupported_params: non_default_params.pop(k, None) + if "reasoning_effort" in unsupported_params: + verbose_logger.warning( + "Dropping reasoning_effort=%r for model=%r (drop_params=True) because %s does not support " + "reasoning_effort for it. If the deployment is a reasoning model, set model_info.base_model " + "to its underlying model.", + unsupported_params["reasoning_effort"], + model, + custom_llm_provider, + ) else: raise UnsupportedParamsError( status_code=500, diff --git a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py index 65ccd49dd21..59788a3a299 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py @@ -2389,7 +2389,7 @@ class TestResponsesSurfaceSharesTheEffortRule: assert mapped["reasoning"] == {"summary": "auto"} def test_dropping_a_refused_effort_logs_one_warning_naming_the_level_and_the_fix( - self, local_model_cost_map, caplog: pytest.LogCaptureFixture + self, local_model_cost_map: None, caplog: pytest.LogCaptureFixture ) -> None: refused: Final = next( level @@ -2412,7 +2412,7 @@ class TestResponsesSurfaceSharesTheEffortRule: ) def test_dropping_reasoning_for_a_model_without_supports_reasoning_logs_one_warning( - self, local_model_cost_map, caplog: pytest.LogCaptureFixture + self, local_model_cost_map: None, caplog: pytest.LogCaptureFixture ) -> None: with caplog.at_level(logging.WARNING, logger="LiteLLM"): mapped: Final = OpenAIResponsesAPIConfig().map_openai_params( @@ -2426,7 +2426,9 @@ class TestResponsesSurfaceSharesTheEffortRule: assert len(drop_warnings) == 1 assert all(fragment in drop_warnings[0] for fragment in ("'medium'", "gpt-4o", "supports_reasoning")) - def test_a_forwarded_effort_logs_no_drop_warning(self, local_model_cost_map, caplog: pytest.LogCaptureFixture) -> None: + def test_a_forwarded_effort_logs_no_drop_warning( + self, local_model_cost_map: None, caplog: pytest.LogCaptureFixture + ) -> None: with caplog.at_level(logging.WARNING, logger="LiteLLM"): mapped: Final = OpenAIResponsesAPIConfig().map_openai_params( response_api_optional_params={"reasoning": {"effort": "medium"}}, diff --git a/tests/unit/router_utils/test_reasoning_effort_capability.py b/tests/unit/router_utils/test_reasoning_effort_capability.py index 9c91d8acea8..f3b001bf229 100644 --- a/tests/unit/router_utils/test_reasoning_effort_capability.py +++ b/tests/unit/router_utils/test_reasoning_effort_capability.py @@ -626,8 +626,9 @@ class TestResolveDeploymentReasoningEffort: _sent("turbo\nforged line", drop_params=True) assert [record.getMessage() for record in caplog.records] == [ - "reasoning_effort unrecognized sent as None (rounding up) to deployment deployment-1, " - "model group kimi-k3, model hosted_vllm/my-kimi" + "Dropping reasoning_effort unrecognized (drop_params=True) for deployment deployment-1, model group " + "kimi-k3, model hosted_vllm/my-kimi because its model_info.reasoning_effort_levels does not list that " + "level. List every level the deployment supports." ] def test_line_breaks_in_logged_names_cannot_forge_a_log_line(self, caplog: pytest.LogCaptureFixture) -> None: @@ -669,16 +670,26 @@ class TestResolveDeploymentReasoningEffort: "reasoning_effort_rounding": "up", } - def test_a_dropped_level_logs_a_warning_where_a_remap_logs_info(self, caplog: pytest.LogCaptureFixture) -> None: + def test_only_a_dropped_level_logs_a_warning(self, caplog: pytest.LogCaptureFixture) -> None: verbose_logger.addHandler(caplog.handler) verbose_logger.propagate = False verbose_logger.setLevel(logging.INFO) + declared: Final = {"reasoning_effort_levels": ["low", "high"]} - _sent("none", {"reasoning_effort_levels": ["low", "high"]}, drop_params=True) + _sent("none", declared, drop_params=True) _sent("medium") + _sent("default", declared, drop_params=True) + _sent("default", declared) - assert [record.levelno for record in caplog.records] == [logging.WARNING, logging.INFO] - assert "reasoning_effort none sent as None" in caplog.records[0].getMessage() + assert [record.levelno for record in caplog.records] == [ + logging.WARNING, + logging.INFO, + logging.INFO, + logging.INFO, + ] + dropped: Final = caplog.records[0].getMessage() + assert "Dropping reasoning_effort none" in dropped + assert "model_info.reasoning_effort_levels" in dropped class TestAzureGpt6SolAndLunaAdvertiseTheOpenAiLevels: diff --git a/tests/unit/test_utils_get_optional_params.py b/tests/unit/test_utils_get_optional_params.py index 13a64577efb..e763860f141 100644 --- a/tests/unit/test_utils_get_optional_params.py +++ b/tests/unit/test_utils_get_optional_params.py @@ -1,4 +1,5 @@ import json +import logging from typing import Final from unittest.mock import MagicMock, patch @@ -2097,3 +2098,19 @@ def test_supports_system_message_merges_system_prompt_into_user_message() -> Non ] assert map_system_message_pt(messages=messages) == [{"role": "user", "content": "Listen here! Hello there!"}] + + +def test_dropping_an_unsupported_reasoning_effort_logs_one_warning( + local_model_cost_map: None, caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + optional_params: Final = get_optional_params( + model="terra-ptu", custom_llm_provider="azure", reasoning_effort="none", drop_params=True + ) + + assert "reasoning_effort" not in optional_params + drop_warnings: Final = [ + record.getMessage() for record in caplog.records if "Dropping reasoning_effort" in record.getMessage() + ] + assert len(drop_warnings) == 1 + assert all(fragment in drop_warnings[0] for fragment in ("'none'", "terra-ptu", "model_info.base_model")) From 2f910080b6f4f3b335582123a4d3f0a032e15439 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 21 Oct 2026 05:25:23 -0700 Subject: [PATCH 4/4] fix: warn when thinking or a Bedrock Converse reasoning level is dropped, and name the router drop a reasoning level --- .../bedrock/chat/converse_transformation.py | 5 ++-- .../reasoning_effort_capability.py | 2 +- litellm/utils.py | 28 +++++++++++++------ .../chat/test_converse_transformation.py | 20 ++++++++----- .../test_bedrock_openai_responses.py | 20 +++++++++---- .../test_reasoning_effort_capability.py | 4 +-- tests/unit/test_utils_get_optional_params.py | 16 +++++++++++ 7 files changed, 68 insertions(+), 27 deletions(-) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index cd3727401ef..bf1efaa57ee 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -1182,8 +1182,9 @@ class AmazonConverseConfig(BaseConfig): ), status_code=400, ) - verbose_logger.debug( - "Dropping unsupported `reasoning_effort=%s` for Bedrock model=%s.", + verbose_logger.warning( + "Dropping reasoning_effort=%r for Bedrock model=%r (drop_params=True) because the model map " + "marks that level unsupported.", value, model, ) diff --git a/litellm/router_utils/reasoning_effort_capability.py b/litellm/router_utils/reasoning_effort_capability.py index 12453adc6bf..bc77497191f 100644 --- a/litellm/router_utils/reasoning_effort_capability.py +++ b/litellm/router_utils/reasoning_effort_capability.py @@ -438,7 +438,7 @@ def _log_remap( ) if sent_level is None and requested_level != _DEFAULT_EFFORT: verbose_logger.warning( - "Dropping reasoning_effort %s (drop_params=True) for deployment %s, model group %s, model %s because its " + "Dropping reasoning level %s (drop_params=True) for deployment %s, model group %s, model %s because its " "model_info.reasoning_effort_levels does not list that level. List every level the deployment supports.", logged_requested_level, logged_deployment_id, diff --git a/litellm/utils.py b/litellm/utils.py index cb95146fd21..aa63ede8172 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4949,6 +4949,14 @@ def _declared_reasoning_efforts(model_info: Mapping[str, object] | None) -> tupl return deployment_reasoning_effort_levels(model_info) or () +_DROPPED_REASONING_PARAM_FIXES: Final = MappingProxyType( + { + "reasoning_effort": "If the deployment is a reasoning model, set model_info.base_model to its underlying model.", + "thinking": "If the deployment is a reasoning model, send reasoning_effort instead.", + } +) + + def get_optional_params( # use the openai defaults # https://platform.openai.com/docs/api-reference/chat/create @@ -5046,15 +5054,17 @@ def get_optional_params( if litellm.drop_params is True or (drop_params is not None and drop_params is True): for k in unsupported_params: non_default_params.pop(k, None) - if "reasoning_effort" in unsupported_params: - verbose_logger.warning( - "Dropping reasoning_effort=%r for model=%r (drop_params=True) because %s does not support " - "reasoning_effort for it. If the deployment is a reasoning model, set model_info.base_model " - "to its underlying model.", - unsupported_params["reasoning_effort"], - model, - custom_llm_provider, - ) + for reasoning_param, fix in _DROPPED_REASONING_PARAM_FIXES.items(): + if reasoning_param in unsupported_params: + verbose_logger.warning( + "Dropping %s=%r for model=%r (drop_params=True) because %s does not support %s for it. %s", + reasoning_param, + unsupported_params[reasoning_param], + model, + custom_llm_provider, + reasoning_param, + fix, + ) else: raise UnsupportedParamsError( status_code=500, diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 154797bab5a..f6344c0840d 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -1,6 +1,7 @@ import base64 import copy import json +import logging import os import re from collections.abc import Iterator, Mapping @@ -966,7 +967,9 @@ def test_reasoning_effort_maps_to_reasoning_effort_for_openai_gpt5_converse(mode "us.openai.gpt-6-astra", ], ) -def test_openai_gpt5_converse_rejects_effort_level_disabled_in_model_map(model, local_model_cost_map): +def test_openai_gpt5_converse_rejects_effort_level_disabled_in_model_map( + model: str, local_model_cost_map: None, caplog: pytest.LogCaptureFixture +) -> None: config = AmazonConverseConfig() assert litellm.utils.is_explicitly_disabled_factory( model=model, custom_llm_provider="bedrock_converse", key="supports_minimal_reasoning_effort" @@ -980,15 +983,18 @@ def test_openai_gpt5_converse_rejects_effort_level_disabled_in_model_map(model, drop_params=False, ) - optional_params = config.map_openai_params( - non_default_params={"reasoning_effort": "minimal"}, - optional_params={}, - model=model, - drop_params=True, - ) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + optional_params = config.map_openai_params( + non_default_params={"reasoning_effort": "minimal"}, + optional_params={}, + model=model, + drop_params=True, + ) _, additional_request_params, _, _ = config._prepare_request_params(optional_params, model) assert "reasoning" not in additional_request_params assert "thinking" not in additional_request_params + dropped = [r.getMessage() for r in caplog.records if "Dropping reasoning_effort='minimal'" in r.getMessage()] + assert len(dropped) == 1 and model in dropped[0] @pytest.mark.parametrize( diff --git a/tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py b/tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py index 6da131f38cc..c0ea47b8008 100644 --- a/tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py +++ b/tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py @@ -337,13 +337,21 @@ class TestDisabledReasoningEffort: ) @pytest.mark.parametrize("model", ["us.openai.gpt-5.6-luna", MODEL]) - def test_effort_level_disabled_in_model_map_is_dropped_with_drop_params(self, model, local_model_cost_map): - params = _cfg().map_openai_params( - response_api_optional_params={"reasoning": {"effort": "minimal", "summary": "auto"}, "max_output_tokens": 64}, - model=model, - drop_params=True, - ) + def test_effort_level_disabled_in_model_map_is_dropped_with_a_warning( + self, model: str, local_model_cost_map: None, caplog: pytest.LogCaptureFixture + ) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + params = _cfg().map_openai_params( + response_api_optional_params={ + "reasoning": {"effort": "minimal", "summary": "auto"}, + "max_output_tokens": 64, + }, + model=model, + drop_params=True, + ) assert params == {"reasoning": {"summary": "auto"}, "max_output_tokens": 64} + dropped = [r.getMessage() for r in caplog.records if "Dropping reasoning.effort='minimal'" in r.getMessage()] + assert len(dropped) == 1 and model in dropped[0] def test_effort_only_reasoning_is_removed_when_dropped(self, local_model_cost_map): params = _cfg().map_openai_params( diff --git a/tests/unit/router_utils/test_reasoning_effort_capability.py b/tests/unit/router_utils/test_reasoning_effort_capability.py index f3b001bf229..d27d67a29a0 100644 --- a/tests/unit/router_utils/test_reasoning_effort_capability.py +++ b/tests/unit/router_utils/test_reasoning_effort_capability.py @@ -626,7 +626,7 @@ class TestResolveDeploymentReasoningEffort: _sent("turbo\nforged line", drop_params=True) assert [record.getMessage() for record in caplog.records] == [ - "Dropping reasoning_effort unrecognized (drop_params=True) for deployment deployment-1, model group " + "Dropping reasoning level unrecognized (drop_params=True) for deployment deployment-1, model group " "kimi-k3, model hosted_vllm/my-kimi because its model_info.reasoning_effort_levels does not list that " "level. List every level the deployment supports." ] @@ -688,7 +688,7 @@ class TestResolveDeploymentReasoningEffort: logging.INFO, ] dropped: Final = caplog.records[0].getMessage() - assert "Dropping reasoning_effort none" in dropped + assert "Dropping reasoning level none" in dropped assert "model_info.reasoning_effort_levels" in dropped diff --git a/tests/unit/test_utils_get_optional_params.py b/tests/unit/test_utils_get_optional_params.py index e763860f141..5cf59136b86 100644 --- a/tests/unit/test_utils_get_optional_params.py +++ b/tests/unit/test_utils_get_optional_params.py @@ -2114,3 +2114,19 @@ def test_dropping_an_unsupported_reasoning_effort_logs_one_warning( ] assert len(drop_warnings) == 1 assert all(fragment in drop_warnings[0] for fragment in ("'none'", "terra-ptu", "model_info.base_model")) + + +def test_dropping_an_unsupported_thinking_logs_one_warning( + local_model_cost_map: None, caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + optional_params: Final = get_optional_params( + model="terra-ptu", custom_llm_provider="azure", thinking={"type": "disabled"}, drop_params=True + ) + + assert "thinking" not in optional_params + drop_warnings: Final = [ + record.getMessage() for record in caplog.records if "Dropping thinking" in record.getMessage() + ] + assert len(drop_warnings) == 1 + assert all(fragment in drop_warnings[0] for fragment in ("'disabled'", "terra-ptu", "send reasoning_effort"))