From 8d4f593b6a7f38894e6c69830b944e226c1fe8b4 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 19:48:49 -0700 Subject: [PATCH 1/8] fix(openai): warn when the GPT-5 reasoning effort gate drops a level under drop_params A GPT-5 request whose reasoning_effort the model map does not enable for that model lost the effort with no log line under drop_params. An Azure deployment name missing from the cost map lost reasoning_effort none this way and ran at the model's default effort. The gate now logs a warning naming the model, the level, the map entry it checked, and the model_info.base_model and model_info.reasoning_effort_levels fixes. --- .../llms/openai/chat/gpt_5_transformation.py | 13 ++++++ .../llms/openai/test_gpt5_transformation.py | 42 +++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/litellm/llms/openai/chat/gpt_5_transformation.py b/litellm/llms/openai/chat/gpt_5_transformation.py index 0311f6c6590..73a712f3b98 100644 --- a/litellm/llms/openai/chat/gpt_5_transformation.py +++ b/litellm/llms/openai/chat/gpt_5_transformation.py @@ -5,6 +5,7 @@ from collections.abc import Collection from typing import ClassVar, Final import litellm +from litellm._logging import verbose_logger from litellm.utils import ( declared_value_factory, is_explicitly_disabled_factory, @@ -15,6 +16,12 @@ from .gpt_transformation import OpenAIGPTConfig _GPT_SERIES_VERSION: Final = re.compile(r"^gpt-(\d+)(?:\.(\d+))?(?=[.-]|$)") +DROP_REFUSED_REASONING_EFFORT_WARNING: Final = ( + "Dropping reasoning_effort=%r for model=%r (drop_params=True) because the model map does not mark that level " + "supported for %r. If the deployment supports it, set model_info.base_model to its underlying model or add " + "the level to model_info.reasoning_effort_levels." +) + def _catalogue_declares_default_effort() -> bool: """Whether the loaded cost map carries default_reasoning_effort for ANY entry. @@ -342,6 +349,12 @@ class OpenAIGPT5Config(OpenAIGPTConfig): and self.refuses_reasoning_effort_level(model, requested_effort) ): if litellm.drop_params or drop_params: + verbose_logger.warning( + DROP_REFUSED_REASONING_EFFORT_WARNING, + requested_effort, + model, + self._model_map_lookup_name(model), + ) non_default_params.pop("reasoning_effort", None) optional_params.pop("reasoning_effort", None) else: diff --git a/tests/unit/llms/openai/test_gpt5_transformation.py b/tests/unit/llms/openai/test_gpt5_transformation.py index 7b10411ba20..6fc0d4d9a9b 100644 --- a/tests/unit/llms/openai/test_gpt5_transformation.py +++ b/tests/unit/llms/openai/test_gpt5_transformation.py @@ -1,3 +1,4 @@ +import logging import re from collections.abc import Mapping from typing import Final @@ -1589,3 +1590,44 @@ def test_gpt_6_astra_takes_the_reasoning_series_request_shape(): assert "max_tokens" not in params assert params["reasoning_effort"] == "max" assert params["verbosity"] == "low" + + +@pytest.mark.parametrize( + ("custom_llm_provider", "level"), + [("azure", "none"), ("azure", "xhigh"), ("openai", "xhigh")], +) +def test_a_reasoning_effort_the_level_gate_drops_logs_a_warning_naming_the_fix( + caplog: pytest.LogCaptureFixture, custom_llm_provider: str, level: str +) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + params = litellm.get_optional_params( + model="gpt-5.6-terra-ptu", + custom_llm_provider=custom_llm_provider, + reasoning_effort=level, + drop_params=True, + ) + + assert "reasoning_effort" not in params + drop_warnings = [ + record.getMessage() + for record in caplog.records + if record.levelno == logging.WARNING and "reasoning_effort" in record.getMessage() + ] + assert len(drop_warnings) == 1 + assert all( + fragment in drop_warnings[0] + for fragment in (level, "gpt-5.6-terra-ptu", "model_info.base_model", "model_info.reasoning_effort_levels") + ) + + +def test_a_reasoning_effort_the_level_gate_forwards_logs_no_drop_warning(caplog: pytest.LogCaptureFixture) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + params = litellm.get_optional_params( + model="gpt-5.6-terra-ptu", + custom_llm_provider="azure", + reasoning_effort="medium", + drop_params=True, + ) + + assert params["reasoning_effort"] == "medium" + assert not [record for record in caplog.records if "reasoning_effort" in record.getMessage()] From 54cf2e68bc10745db15ec06cca3dcae811c677ac Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 20:45:23 -0700 Subject: [PATCH 2/8] fix(openai): warn when the Responses route or a deployment's declared levels drop a reasoning effort The /v1/responses gate dropped a row-disabled GPT-5 effort, and reasoning for an OpenAI model without supports_reasoning, with no log line, and the router logged a declared-level drop only at INFO. Both Responses drops now log a WARNING and the router logs its drop at WARNING. The warning now leads with model_info.reasoning_effort_levels, since base_model only helps a deployment named differently from its underlying model. --- .../llms/openai/chat/gpt_5_transformation.py | 7 +-- .../llms/openai/responses/transformation.py | 10 +++- .../reasoning_effort_capability.py | 3 +- .../test_openai_responses_transformation.py | 50 +++++++++++++++++++ .../llms/openai/test_gpt5_transformation.py | 6 +-- .../test_reasoning_effort_capability.py | 11 ++++ 6 files changed, 79 insertions(+), 8 deletions(-) diff --git a/litellm/llms/openai/chat/gpt_5_transformation.py b/litellm/llms/openai/chat/gpt_5_transformation.py index 73a712f3b98..d9c28cb6b15 100644 --- a/litellm/llms/openai/chat/gpt_5_transformation.py +++ b/litellm/llms/openai/chat/gpt_5_transformation.py @@ -17,9 +17,9 @@ from .gpt_transformation import OpenAIGPTConfig _GPT_SERIES_VERSION: Final = re.compile(r"^gpt-(\d+)(?:\.(\d+))?(?=[.-]|$)") DROP_REFUSED_REASONING_EFFORT_WARNING: Final = ( - "Dropping reasoning_effort=%r for model=%r (drop_params=True) because the model map does not mark that level " - "supported for %r. If the deployment supports it, set model_info.base_model to its underlying model or add " - "the level to model_info.reasoning_effort_levels." + "Dropping %s=%r for model=%r (drop_params=True) because the model map does not mark that level supported for " + "%r. If the deployment supports it, add the level to model_info.reasoning_effort_levels, or set " + "model_info.base_model when the deployment name is not its underlying model." ) @@ -351,6 +351,7 @@ class OpenAIGPT5Config(OpenAIGPTConfig): if litellm.drop_params or drop_params: verbose_logger.warning( DROP_REFUSED_REASONING_EFFORT_WARNING, + "reasoning_effort", requested_effort, model, self._model_map_lookup_name(model), diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 8f29a71d63b..7a72bb107fa 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -26,6 +26,7 @@ from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.llms.openai.chat.gpt_5_transformation import ( + DROP_REFUSED_REASONING_EFFORT_WARNING, is_gpt_reasoning_series_name, unsupported_reasoning_effort_message, ) @@ -50,6 +51,9 @@ from ..common_utils import OpenAIError from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config OPENAI_RESPONSES_API_MIN_MAX_OUTPUT_TOKENS: Final = 16 +DROP_UNSUPPORTED_REASONING_WARNING: Final = ( + "Dropping reasoning=%r for model=%r (drop_params=True) because its model cost map entry lacks supports_reasoning." +) _RAW_RESPONSE_JSON: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True)) if TYPE_CHECKING: @@ -401,7 +405,8 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): and not self._supports_reasoning_param(model=model) ): if drop_params or litellm.drop_params: - params.pop("reasoning", None) + dropped_reasoning: Final = params.pop("reasoning", None) + verbose_logger.warning(DROP_UNSUPPORTED_REASONING_WARNING, dropped_reasoning, model) else: raise litellm.UnsupportedParamsError( message=( @@ -427,6 +432,9 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): message=unsupported_reasoning_effort_message("reasoning.effort", requested_effort, model), status_code=400, ) + verbose_logger.warning( + DROP_REFUSED_REASONING_EFFORT_WARNING, "reasoning.effort", requested_effort, model, lookup_name + ) remaining: Final = reasoning.copy() remaining.pop("effort", None) if remaining: diff --git a/litellm/router_utils/reasoning_effort_capability.py b/litellm/router_utils/reasoning_effort_capability.py index 4357722970f..4d979046b66 100644 --- a/litellm/router_utils/reasoning_effort_capability.py +++ b/litellm/router_utils/reasoning_effort_capability.py @@ -426,7 +426,8 @@ def _log_remap( logged_deployment_id: Final = _single_line(deployment_id) logged_model_group: Final = _single_line(model_group) logged_deployment_model: Final = _single_line(deployment_model) - verbose_logger.info( + log: Final = verbose_logger.info if sent_level is not None else verbose_logger.warning + log( "reasoning_effort %s sent as %s (rounding %s) to deployment %s, model group %s, model %s", logged_requested_level, logged_sent_level, diff --git a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py index 020f491dcfd..65ccd49dd21 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py @@ -1,4 +1,5 @@ import json +import logging from collections.abc import Callable, Mapping from types import SimpleNamespace from typing import Final, cast @@ -2387,6 +2388,55 @@ class TestResponsesSurfaceSharesTheEffortRule: ) assert mapped["reasoning"] == {"summary": "auto"} + def test_dropping_a_refused_effort_logs_one_warning_naming_the_level_and_the_fix( + self, local_model_cost_map, caplog: pytest.LogCaptureFixture + ) -> None: + refused: Final = next( + level + for level in ("none", "minimal", "low") + if litellm.model_cost["gpt-6.1-sol"].get(f"supports_{level}_reasoning_effort") is False + ) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + mapped: Final = OpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params={"reasoning": {"effort": refused}}, + model="gpt-6.1-sol", + drop_params=True, + ) + + assert "reasoning" not in mapped + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] + assert len(drop_warnings) == 1 + assert all( + fragment in drop_warnings[0] + for fragment in (repr(refused), "gpt-6.1-sol", "model_info.reasoning_effort_levels") + ) + + def test_dropping_reasoning_for_a_model_without_supports_reasoning_logs_one_warning( + self, local_model_cost_map, caplog: pytest.LogCaptureFixture + ) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + mapped: Final = OpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params={"reasoning": {"effort": "medium"}}, + model="gpt-4o", + drop_params=True, + ) + + assert "reasoning" not in mapped + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] + assert len(drop_warnings) == 1 + assert all(fragment in drop_warnings[0] for fragment in ("'medium'", "gpt-4o", "supports_reasoning")) + + def test_a_forwarded_effort_logs_no_drop_warning(self, local_model_cost_map, caplog: pytest.LogCaptureFixture) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + mapped: Final = OpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params={"reasoning": {"effort": "medium"}}, + model="gpt-6.1-sol", + drop_params=True, + ) + + assert mapped["reasoning"] == {"effort": "medium"} + assert not [record for record in caplog.records if "Dropping" in record.getMessage()] + @pytest.mark.parametrize("level", ["none", "minimal", "low", "medium", "high", "xhigh", "max"]) def test_gpt5_3_codex_forwards_exactly_the_effort_levels_its_map_row_allows(self, local_model_cost_map, level): """The gate turns a stale false into a local refusal, so the Responses-only codex row has to diff --git a/tests/unit/llms/openai/test_gpt5_transformation.py b/tests/unit/llms/openai/test_gpt5_transformation.py index 6fc0d4d9a9b..76721ded9f1 100644 --- a/tests/unit/llms/openai/test_gpt5_transformation.py +++ b/tests/unit/llms/openai/test_gpt5_transformation.py @@ -1600,7 +1600,7 @@ def test_a_reasoning_effort_the_level_gate_drops_logs_a_warning_naming_the_fix( caplog: pytest.LogCaptureFixture, custom_llm_provider: str, level: str ) -> None: with caplog.at_level(logging.WARNING, logger="LiteLLM"): - params = litellm.get_optional_params( + params: Final = litellm.get_optional_params( model="gpt-5.6-terra-ptu", custom_llm_provider=custom_llm_provider, reasoning_effort=level, @@ -1608,7 +1608,7 @@ def test_a_reasoning_effort_the_level_gate_drops_logs_a_warning_naming_the_fix( ) assert "reasoning_effort" not in params - drop_warnings = [ + drop_warnings: Final = [ record.getMessage() for record in caplog.records if record.levelno == logging.WARNING and "reasoning_effort" in record.getMessage() @@ -1622,7 +1622,7 @@ def test_a_reasoning_effort_the_level_gate_drops_logs_a_warning_naming_the_fix( def test_a_reasoning_effort_the_level_gate_forwards_logs_no_drop_warning(caplog: pytest.LogCaptureFixture) -> None: with caplog.at_level(logging.WARNING, logger="LiteLLM"): - params = litellm.get_optional_params( + params: Final = litellm.get_optional_params( model="gpt-5.6-terra-ptu", custom_llm_provider="azure", reasoning_effort="medium", diff --git a/tests/unit/router_utils/test_reasoning_effort_capability.py b/tests/unit/router_utils/test_reasoning_effort_capability.py index d634e239ea9..9c91d8acea8 100644 --- a/tests/unit/router_utils/test_reasoning_effort_capability.py +++ b/tests/unit/router_utils/test_reasoning_effort_capability.py @@ -669,6 +669,17 @@ class TestResolveDeploymentReasoningEffort: "reasoning_effort_rounding": "up", } + def test_a_dropped_level_logs_a_warning_where_a_remap_logs_info(self, caplog: pytest.LogCaptureFixture) -> None: + verbose_logger.addHandler(caplog.handler) + verbose_logger.propagate = False + verbose_logger.setLevel(logging.INFO) + + _sent("none", {"reasoning_effort_levels": ["low", "high"]}, drop_params=True) + _sent("medium") + + assert [record.levelno for record in caplog.records] == [logging.WARNING, logging.INFO] + assert "reasoning_effort none sent as None" in caplog.records[0].getMessage() + class TestAzureGpt6SolAndLunaAdvertiseTheOpenAiLevels: @pytest.mark.parametrize( From add014b92ce8de82acecec7f7adc95714f0c7aa8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 21 Oct 2026 01:54:55 -0700 Subject: [PATCH 3/8] fix: warn only when a reasoning level is actually dropped, and on generic reasoning_effort drops --- .../llms/openai/chat/gpt_5_transformation.py | 2 +- .../reasoning_effort_capability.py | 33 +++++++++++++------ litellm/utils.py | 22 ++++++++----- .../test_openai_responses_transformation.py | 8 +++-- .../test_reasoning_effort_capability.py | 23 +++++++++---- tests/unit/test_utils_get_optional_params.py | 17 ++++++++++ 6 files changed, 77 insertions(+), 28 deletions(-) diff --git a/litellm/llms/openai/chat/gpt_5_transformation.py b/litellm/llms/openai/chat/gpt_5_transformation.py index d9c28cb6b15..55f8c1c0846 100644 --- a/litellm/llms/openai/chat/gpt_5_transformation.py +++ b/litellm/llms/openai/chat/gpt_5_transformation.py @@ -18,7 +18,7 @@ _GPT_SERIES_VERSION: Final = re.compile(r"^gpt-(\d+)(?:\.(\d+))?(?=[.-]|$)") DROP_REFUSED_REASONING_EFFORT_WARNING: Final = ( "Dropping %s=%r for model=%r (drop_params=True) because the model map does not mark that level supported for " - "%r. If the deployment supports it, add the level to model_info.reasoning_effort_levels, or set " + "%r. If the deployment supports it, list every level it supports in model_info.reasoning_effort_levels, or set " "model_info.base_model when the deployment name is not its underlying model." ) diff --git a/litellm/router_utils/reasoning_effort_capability.py b/litellm/router_utils/reasoning_effort_capability.py index 4d979046b66..12453adc6bf 100644 --- a/litellm/router_utils/reasoning_effort_capability.py +++ b/litellm/router_utils/reasoning_effort_capability.py @@ -426,8 +426,28 @@ def _log_remap( logged_deployment_id: Final = _single_line(deployment_id) logged_model_group: Final = _single_line(model_group) logged_deployment_model: Final = _single_line(deployment_model) - log: Final = verbose_logger.info if sent_level is not None else verbose_logger.warning - log( + extra: Final = MappingProxyType( + { + "reasoning_effort_deployment_id": logged_deployment_id, + "reasoning_effort_model_group": logged_model_group, + "reasoning_effort_deployment_model": logged_deployment_model, + "reasoning_effort_requested": logged_requested_level, + "reasoning_effort_sent": logged_sent_level, + "reasoning_effort_rounding": rounding, + } + ) + if sent_level is None and requested_level != _DEFAULT_EFFORT: + verbose_logger.warning( + "Dropping reasoning_effort %s (drop_params=True) for deployment %s, model group %s, model %s because its " + "model_info.reasoning_effort_levels does not list that level. List every level the deployment supports.", + logged_requested_level, + logged_deployment_id, + logged_model_group, + logged_deployment_model, + extra=extra, + ) + return + verbose_logger.info( "reasoning_effort %s sent as %s (rounding %s) to deployment %s, model group %s, model %s", logged_requested_level, logged_sent_level, @@ -435,14 +455,7 @@ def _log_remap( logged_deployment_id, logged_model_group, logged_deployment_model, - extra={ - "reasoning_effort_deployment_id": logged_deployment_id, - "reasoning_effort_model_group": logged_model_group, - "reasoning_effort_deployment_model": logged_deployment_model, - "reasoning_effort_requested": logged_requested_level, - "reasoning_effort_sent": logged_sent_level, - "reasoning_effort_rounding": rounding, - }, + extra=extra, ) diff --git a/litellm/utils.py b/litellm/utils.py index 379eddb3155..cb95146fd21 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5036,19 +5036,25 @@ def get_optional_params( verbose_logger.info("\nLiteLLM completion() model= %s; provider = %s", model, custom_llm_provider) verbose_logger.debug("\nLiteLLM: Params passed to completion() %s", passed_params) verbose_logger.debug("\nLiteLLM: Non-Default params passed to completion() %s", non_default_params) - unsupported_params: Final = {} - for k in non_default_params: - if k not in supported_params: - if k in PROVIDER_UNVALIDATED_PARAMS: - continue - if k == "n" and n == 1: # langchain sends n=1 as a default value - continue # skip this param - unsupported_params[k] = non_default_params[k] + unsupported_params: Final[Mapping[str, object]] = { + k: non_default_params[k] + for k in non_default_params + if k not in supported_params and k not in PROVIDER_UNVALIDATED_PARAMS and not (k == "n" and n == 1) + } if unsupported_params: if litellm.drop_params is True or (drop_params is not None and drop_params is True): for k in unsupported_params: non_default_params.pop(k, None) + if "reasoning_effort" in unsupported_params: + verbose_logger.warning( + "Dropping reasoning_effort=%r for model=%r (drop_params=True) because %s does not support " + "reasoning_effort for it. If the deployment is a reasoning model, set model_info.base_model " + "to its underlying model.", + unsupported_params["reasoning_effort"], + model, + custom_llm_provider, + ) else: raise UnsupportedParamsError( status_code=500, diff --git a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py index 65ccd49dd21..59788a3a299 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py @@ -2389,7 +2389,7 @@ class TestResponsesSurfaceSharesTheEffortRule: assert mapped["reasoning"] == {"summary": "auto"} def test_dropping_a_refused_effort_logs_one_warning_naming_the_level_and_the_fix( - self, local_model_cost_map, caplog: pytest.LogCaptureFixture + self, local_model_cost_map: None, caplog: pytest.LogCaptureFixture ) -> None: refused: Final = next( level @@ -2412,7 +2412,7 @@ class TestResponsesSurfaceSharesTheEffortRule: ) def test_dropping_reasoning_for_a_model_without_supports_reasoning_logs_one_warning( - self, local_model_cost_map, caplog: pytest.LogCaptureFixture + self, local_model_cost_map: None, caplog: pytest.LogCaptureFixture ) -> None: with caplog.at_level(logging.WARNING, logger="LiteLLM"): mapped: Final = OpenAIResponsesAPIConfig().map_openai_params( @@ -2426,7 +2426,9 @@ class TestResponsesSurfaceSharesTheEffortRule: assert len(drop_warnings) == 1 assert all(fragment in drop_warnings[0] for fragment in ("'medium'", "gpt-4o", "supports_reasoning")) - def test_a_forwarded_effort_logs_no_drop_warning(self, local_model_cost_map, caplog: pytest.LogCaptureFixture) -> None: + def test_a_forwarded_effort_logs_no_drop_warning( + self, local_model_cost_map: None, caplog: pytest.LogCaptureFixture + ) -> None: with caplog.at_level(logging.WARNING, logger="LiteLLM"): mapped: Final = OpenAIResponsesAPIConfig().map_openai_params( response_api_optional_params={"reasoning": {"effort": "medium"}}, diff --git a/tests/unit/router_utils/test_reasoning_effort_capability.py b/tests/unit/router_utils/test_reasoning_effort_capability.py index 9c91d8acea8..f3b001bf229 100644 --- a/tests/unit/router_utils/test_reasoning_effort_capability.py +++ b/tests/unit/router_utils/test_reasoning_effort_capability.py @@ -626,8 +626,9 @@ class TestResolveDeploymentReasoningEffort: _sent("turbo\nforged line", drop_params=True) assert [record.getMessage() for record in caplog.records] == [ - "reasoning_effort unrecognized sent as None (rounding up) to deployment deployment-1, " - "model group kimi-k3, model hosted_vllm/my-kimi" + "Dropping reasoning_effort unrecognized (drop_params=True) for deployment deployment-1, model group " + "kimi-k3, model hosted_vllm/my-kimi because its model_info.reasoning_effort_levels does not list that " + "level. List every level the deployment supports." ] def test_line_breaks_in_logged_names_cannot_forge_a_log_line(self, caplog: pytest.LogCaptureFixture) -> None: @@ -669,16 +670,26 @@ class TestResolveDeploymentReasoningEffort: "reasoning_effort_rounding": "up", } - def test_a_dropped_level_logs_a_warning_where_a_remap_logs_info(self, caplog: pytest.LogCaptureFixture) -> None: + def test_only_a_dropped_level_logs_a_warning(self, caplog: pytest.LogCaptureFixture) -> None: verbose_logger.addHandler(caplog.handler) verbose_logger.propagate = False verbose_logger.setLevel(logging.INFO) + declared: Final = {"reasoning_effort_levels": ["low", "high"]} - _sent("none", {"reasoning_effort_levels": ["low", "high"]}, drop_params=True) + _sent("none", declared, drop_params=True) _sent("medium") + _sent("default", declared, drop_params=True) + _sent("default", declared) - assert [record.levelno for record in caplog.records] == [logging.WARNING, logging.INFO] - assert "reasoning_effort none sent as None" in caplog.records[0].getMessage() + assert [record.levelno for record in caplog.records] == [ + logging.WARNING, + logging.INFO, + logging.INFO, + logging.INFO, + ] + dropped: Final = caplog.records[0].getMessage() + assert "Dropping reasoning_effort none" in dropped + assert "model_info.reasoning_effort_levels" in dropped class TestAzureGpt6SolAndLunaAdvertiseTheOpenAiLevels: diff --git a/tests/unit/test_utils_get_optional_params.py b/tests/unit/test_utils_get_optional_params.py index 13a64577efb..e763860f141 100644 --- a/tests/unit/test_utils_get_optional_params.py +++ b/tests/unit/test_utils_get_optional_params.py @@ -1,4 +1,5 @@ import json +import logging from typing import Final from unittest.mock import MagicMock, patch @@ -2097,3 +2098,19 @@ def test_supports_system_message_merges_system_prompt_into_user_message() -> Non ] assert map_system_message_pt(messages=messages) == [{"role": "user", "content": "Listen here! Hello there!"}] + + +def test_dropping_an_unsupported_reasoning_effort_logs_one_warning( + local_model_cost_map: None, caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + optional_params: Final = get_optional_params( + model="terra-ptu", custom_llm_provider="azure", reasoning_effort="none", drop_params=True + ) + + assert "reasoning_effort" not in optional_params + drop_warnings: Final = [ + record.getMessage() for record in caplog.records if "Dropping reasoning_effort" in record.getMessage() + ] + assert len(drop_warnings) == 1 + assert all(fragment in drop_warnings[0] for fragment in ("'none'", "terra-ptu", "model_info.base_model")) From 2f910080b6f4f3b335582123a4d3f0a032e15439 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 21 Oct 2026 05:25:23 -0700 Subject: [PATCH 4/8] fix: warn when thinking or a Bedrock Converse reasoning level is dropped, and name the router drop a reasoning level --- .../bedrock/chat/converse_transformation.py | 5 ++-- .../reasoning_effort_capability.py | 2 +- litellm/utils.py | 28 +++++++++++++------ .../chat/test_converse_transformation.py | 20 ++++++++----- .../test_bedrock_openai_responses.py | 20 +++++++++---- .../test_reasoning_effort_capability.py | 4 +-- tests/unit/test_utils_get_optional_params.py | 16 +++++++++++ 7 files changed, 68 insertions(+), 27 deletions(-) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index cd3727401ef..bf1efaa57ee 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -1182,8 +1182,9 @@ class AmazonConverseConfig(BaseConfig): ), status_code=400, ) - verbose_logger.debug( - "Dropping unsupported `reasoning_effort=%s` for Bedrock model=%s.", + verbose_logger.warning( + "Dropping reasoning_effort=%r for Bedrock model=%r (drop_params=True) because the model map " + "marks that level unsupported.", value, model, ) diff --git a/litellm/router_utils/reasoning_effort_capability.py b/litellm/router_utils/reasoning_effort_capability.py index 12453adc6bf..bc77497191f 100644 --- a/litellm/router_utils/reasoning_effort_capability.py +++ b/litellm/router_utils/reasoning_effort_capability.py @@ -438,7 +438,7 @@ def _log_remap( ) if sent_level is None and requested_level != _DEFAULT_EFFORT: verbose_logger.warning( - "Dropping reasoning_effort %s (drop_params=True) for deployment %s, model group %s, model %s because its " + "Dropping reasoning level %s (drop_params=True) for deployment %s, model group %s, model %s because its " "model_info.reasoning_effort_levels does not list that level. List every level the deployment supports.", logged_requested_level, logged_deployment_id, diff --git a/litellm/utils.py b/litellm/utils.py index cb95146fd21..aa63ede8172 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4949,6 +4949,14 @@ def _declared_reasoning_efforts(model_info: Mapping[str, object] | None) -> tupl return deployment_reasoning_effort_levels(model_info) or () +_DROPPED_REASONING_PARAM_FIXES: Final = MappingProxyType( + { + "reasoning_effort": "If the deployment is a reasoning model, set model_info.base_model to its underlying model.", + "thinking": "If the deployment is a reasoning model, send reasoning_effort instead.", + } +) + + def get_optional_params( # use the openai defaults # https://platform.openai.com/docs/api-reference/chat/create @@ -5046,15 +5054,17 @@ def get_optional_params( if litellm.drop_params is True or (drop_params is not None and drop_params is True): for k in unsupported_params: non_default_params.pop(k, None) - if "reasoning_effort" in unsupported_params: - verbose_logger.warning( - "Dropping reasoning_effort=%r for model=%r (drop_params=True) because %s does not support " - "reasoning_effort for it. If the deployment is a reasoning model, set model_info.base_model " - "to its underlying model.", - unsupported_params["reasoning_effort"], - model, - custom_llm_provider, - ) + for reasoning_param, fix in _DROPPED_REASONING_PARAM_FIXES.items(): + if reasoning_param in unsupported_params: + verbose_logger.warning( + "Dropping %s=%r for model=%r (drop_params=True) because %s does not support %s for it. %s", + reasoning_param, + unsupported_params[reasoning_param], + model, + custom_llm_provider, + reasoning_param, + fix, + ) else: raise UnsupportedParamsError( status_code=500, diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 154797bab5a..f6344c0840d 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -1,6 +1,7 @@ import base64 import copy import json +import logging import os import re from collections.abc import Iterator, Mapping @@ -966,7 +967,9 @@ def test_reasoning_effort_maps_to_reasoning_effort_for_openai_gpt5_converse(mode "us.openai.gpt-6-astra", ], ) -def test_openai_gpt5_converse_rejects_effort_level_disabled_in_model_map(model, local_model_cost_map): +def test_openai_gpt5_converse_rejects_effort_level_disabled_in_model_map( + model: str, local_model_cost_map: None, caplog: pytest.LogCaptureFixture +) -> None: config = AmazonConverseConfig() assert litellm.utils.is_explicitly_disabled_factory( model=model, custom_llm_provider="bedrock_converse", key="supports_minimal_reasoning_effort" @@ -980,15 +983,18 @@ def test_openai_gpt5_converse_rejects_effort_level_disabled_in_model_map(model, drop_params=False, ) - optional_params = config.map_openai_params( - non_default_params={"reasoning_effort": "minimal"}, - optional_params={}, - model=model, - drop_params=True, - ) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + optional_params = config.map_openai_params( + non_default_params={"reasoning_effort": "minimal"}, + optional_params={}, + model=model, + drop_params=True, + ) _, additional_request_params, _, _ = config._prepare_request_params(optional_params, model) assert "reasoning" not in additional_request_params assert "thinking" not in additional_request_params + dropped = [r.getMessage() for r in caplog.records if "Dropping reasoning_effort='minimal'" in r.getMessage()] + assert len(dropped) == 1 and model in dropped[0] @pytest.mark.parametrize( diff --git a/tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py b/tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py index 6da131f38cc..c0ea47b8008 100644 --- a/tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py +++ b/tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py @@ -337,13 +337,21 @@ class TestDisabledReasoningEffort: ) @pytest.mark.parametrize("model", ["us.openai.gpt-5.6-luna", MODEL]) - def test_effort_level_disabled_in_model_map_is_dropped_with_drop_params(self, model, local_model_cost_map): - params = _cfg().map_openai_params( - response_api_optional_params={"reasoning": {"effort": "minimal", "summary": "auto"}, "max_output_tokens": 64}, - model=model, - drop_params=True, - ) + def test_effort_level_disabled_in_model_map_is_dropped_with_a_warning( + self, model: str, local_model_cost_map: None, caplog: pytest.LogCaptureFixture + ) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + params = _cfg().map_openai_params( + response_api_optional_params={ + "reasoning": {"effort": "minimal", "summary": "auto"}, + "max_output_tokens": 64, + }, + model=model, + drop_params=True, + ) assert params == {"reasoning": {"summary": "auto"}, "max_output_tokens": 64} + dropped = [r.getMessage() for r in caplog.records if "Dropping reasoning.effort='minimal'" in r.getMessage()] + assert len(dropped) == 1 and model in dropped[0] def test_effort_only_reasoning_is_removed_when_dropped(self, local_model_cost_map): params = _cfg().map_openai_params( diff --git a/tests/unit/router_utils/test_reasoning_effort_capability.py b/tests/unit/router_utils/test_reasoning_effort_capability.py index f3b001bf229..d27d67a29a0 100644 --- a/tests/unit/router_utils/test_reasoning_effort_capability.py +++ b/tests/unit/router_utils/test_reasoning_effort_capability.py @@ -626,7 +626,7 @@ class TestResolveDeploymentReasoningEffort: _sent("turbo\nforged line", drop_params=True) assert [record.getMessage() for record in caplog.records] == [ - "Dropping reasoning_effort unrecognized (drop_params=True) for deployment deployment-1, model group " + "Dropping reasoning level unrecognized (drop_params=True) for deployment deployment-1, model group " "kimi-k3, model hosted_vllm/my-kimi because its model_info.reasoning_effort_levels does not list that " "level. List every level the deployment supports." ] @@ -688,7 +688,7 @@ class TestResolveDeploymentReasoningEffort: logging.INFO, ] dropped: Final = caplog.records[0].getMessage() - assert "Dropping reasoning_effort none" in dropped + assert "Dropping reasoning level none" in dropped assert "model_info.reasoning_effort_levels" in dropped diff --git a/tests/unit/test_utils_get_optional_params.py b/tests/unit/test_utils_get_optional_params.py index e763860f141..5cf59136b86 100644 --- a/tests/unit/test_utils_get_optional_params.py +++ b/tests/unit/test_utils_get_optional_params.py @@ -2114,3 +2114,19 @@ def test_dropping_an_unsupported_reasoning_effort_logs_one_warning( ] assert len(drop_warnings) == 1 assert all(fragment in drop_warnings[0] for fragment in ("'none'", "terra-ptu", "model_info.base_model")) + + +def test_dropping_an_unsupported_thinking_logs_one_warning( + local_model_cost_map: None, caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + optional_params: Final = get_optional_params( + model="terra-ptu", custom_llm_provider="azure", thinking={"type": "disabled"}, drop_params=True + ) + + assert "thinking" not in optional_params + drop_warnings: Final = [ + record.getMessage() for record in caplog.records if "Dropping thinking" in record.getMessage() + ] + assert len(drop_warnings) == 1 + assert all(fragment in drop_warnings[0] for fragment in ("'disabled'", "terra-ptu", "send reasoning_effort")) From d32087cd8e7a0b472543fd17c7f18daa10e8d8a8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 11:02:20 -0700 Subject: [PATCH 5/8] fix: skip the drop warning when a non-reasoning model is asked for no reasoning, and honor declared levels on Bedrock Responses --- .../llms/bedrock/responses/transformation.py | 18 +++---- .../responses/transformation.py | 19 +++++--- litellm/utils.py | 47 +++++++++++++------ .../chat/test_converse_transformation.py | 4 +- .../test_bedrock_openai_responses.py | 19 +++++++- ...bedrock_mantle_responses_transformation.py | 12 +++++ tests/unit/test_utils_get_optional_params.py | 39 ++++++++++++++- 7 files changed, 123 insertions(+), 35 deletions(-) diff --git a/litellm/llms/bedrock/responses/transformation.py b/litellm/llms/bedrock/responses/transformation.py index 7cdd7b08f99..bf936399e83 100644 --- a/litellm/llms/bedrock/responses/transformation.py +++ b/litellm/llms/bedrock/responses/transformation.py @@ -34,7 +34,7 @@ inference profile, so the model is named ``us.openai.gpt-5.6-sol`` or """ import asyncio -from collections.abc import Awaitable, Callable, Mapping +from collections.abc import Awaitable, Callable, Collection, Mapping from types import MappingProxyType from typing import Final @@ -157,13 +157,13 @@ def inline_remote_image_urls( def _without_disabled_reasoning_effort( - params: Mapping[str, object], model: str, drop_params: bool + params: Mapping[str, object], model: str, drop_params: bool, declared_reasoning_efforts: Collection[str] ) -> dict[str, object]: # mutable-ok: becomes the map_openai_params return value reasoning: Final = params.get("reasoning") effort: Final = reasoning.get("effort") if isinstance(reasoning, Mapping) else None if not isinstance(reasoning, Mapping) or not isinstance(effort, str): return dict(params) - if not bedrock_reasoning_effort_disabled(model=model, effort=effort): + if effort in declared_reasoning_efforts or not bedrock_reasoning_effort_disabled(model=model, effort=effort): return dict(params) if not (drop_params or litellm.drop_params): raise litellm.UnsupportedParamsError( @@ -281,14 +281,16 @@ class BedrockOpenAIResponsesConfig(BaseAWSLLM, OpenAIResponsesAPIConfig): fake_stream=fake_stream, ) - def map_openai_params( + def map_openai_params_for_deployment( self, response_api_optional_params: ResponsesAPIOptionalRequestParams, model: str, drop_params: bool, - ) -> dict: # mutable-ok: signature fixed by the override contract - mapped: Final = super().map_openai_params( - response_api_optional_params=response_api_optional_params, model=model, drop_params=drop_params + *, + declared_reasoning_efforts: Collection[str], + ) -> Mapping[str, object]: + mapped: Final = super().map_openai_params_for_deployment( + response_api_optional_params, model, drop_params, declared_reasoning_efforts=declared_reasoning_efforts ) unsupported: Final = tuple(sorted(BEDROCK_RUNTIME_UNSUPPORTED_RESPONSE_PARAMS & mapped.keys())) if unsupported: @@ -297,7 +299,7 @@ class BedrockOpenAIResponsesConfig(BaseAWSLLM, OpenAIResponsesAPIConfig): unsupported, ) supported: Final[dict[str, object]] = {key: value for key, value in mapped.items() if key not in unsupported} - params: Final = _without_disabled_reasoning_effort(supported, model, drop_params) + params: Final = _without_disabled_reasoning_effort(supported, model, drop_params, declared_reasoning_efforts) tools: Final = params.get("tools") if not isinstance(tools, list): return params diff --git a/litellm/llms/bedrock_mantle/responses/transformation.py b/litellm/llms/bedrock_mantle/responses/transformation.py index 3532cfbd963..31cbbcad39a 100644 --- a/litellm/llms/bedrock_mantle/responses/transformation.py +++ b/litellm/llms/bedrock_mantle/responses/transformation.py @@ -15,7 +15,7 @@ role / access key / profile / web identity), signed via the shared BaseAWSLLM._sign_request after the request body is finalized. """ -from collections.abc import Mapping, Sequence +from collections.abc import Collection, Mapping, Sequence from typing import Final, cast # noqa: TID251 # map_openai_params returns the filtered params as a bare dict import httpx @@ -229,18 +229,23 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI def _model_map_lookup_name(model: str) -> str: return model.split("/")[-1].removeprefix("openai.") - def map_openai_params( + def map_openai_params_for_deployment( self, response_api_optional_params: ResponsesAPIOptionalRequestParams, model: str, drop_params: bool, - ) -> dict: + *, + declared_reasoning_efforts: Collection[str], + ) -> Mapping[str, object]: params: Final = self._handle_unsupported_reasoning_summary( self._handle_unsupported_service_tier( - super().map_openai_params( - response_api_optional_params=response_api_optional_params, - model=model, - drop_params=drop_params, + dict( + super().map_openai_params_for_deployment( + response_api_optional_params, + model, + drop_params, + declared_reasoning_efforts=declared_reasoning_efforts, + ) ), drop_params=drop_params, ), diff --git a/litellm/utils.py b/litellm/utils.py index 7498bbe89f6..6dd43a45582 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4974,12 +4974,24 @@ def _declared_reasoning_efforts(model_info: Mapping[str, object] | None) -> tupl return deployment_reasoning_effort_levels(model_info) or () -_DROPPED_REASONING_PARAM_FIXES: Final = MappingProxyType( - { - "reasoning_effort": "If the deployment is a reasoning model, set model_info.base_model to its underlying model.", - "thinking": "If the deployment is a reasoning model, send reasoning_effort instead.", - } -) +_DROPPED_REASONING_PARAMS: Final = ("reasoning_effort", "thinking") + + +def _turns_reasoning_off(value: object) -> bool: + match value: + case "none" | {"effort": "none"} | {"type": "disabled"}: + return True + case _: + return False + + +def _is_mapped_non_reasoning_model(model: str, custom_llm_provider: str | None) -> bool: + info: Final = lookup_model_info_for_registration(model, custom_llm_provider) + return not isinstance(info, ModelInfoLookupMiss) and info.get("supports_reasoning") is not True + + +def _warns_on_dropped_reasoning(value: object, model: str, custom_llm_provider: str | None) -> bool: + return not (_turns_reasoning_off(value) and _is_mapped_non_reasoning_model(model, custom_llm_provider)) def get_optional_params( @@ -5069,26 +5081,31 @@ def get_optional_params( verbose_logger.info("\nLiteLLM completion() model= %s; provider = %s", model, custom_llm_provider) verbose_logger.debug("\nLiteLLM: Params passed to completion() %s", passed_params) verbose_logger.debug("\nLiteLLM: Non-Default params passed to completion() %s", non_default_params) - unsupported_params: Final[Mapping[str, object]] = { - k: non_default_params[k] - for k in non_default_params - if k not in supported_params and k not in PROVIDER_UNVALIDATED_PARAMS and not (k == "n" and n == 1) - } + unsupported_params: Final[MappingProxyType[str, object]] = MappingProxyType( + { + k: non_default_params[k] + for k in non_default_params + if k not in supported_params and k not in PROVIDER_UNVALIDATED_PARAMS and not (k == "n" and n == 1) + } + ) if unsupported_params: if litellm.drop_params is True or (drop_params is not None and drop_params is True): for k in unsupported_params: non_default_params.pop(k, None) - for reasoning_param, fix in _DROPPED_REASONING_PARAM_FIXES.items(): - if reasoning_param in unsupported_params: + for reasoning_param in _DROPPED_REASONING_PARAMS: + if reasoning_param in unsupported_params and _warns_on_dropped_reasoning( + unsupported_params[reasoning_param], base_model or model, custom_llm_provider + ): verbose_logger.warning( - "Dropping %s=%r for model=%r (drop_params=True) because %s does not support %s for it. %s", + "Dropping %s=%r for model=%r (drop_params=True) because %s does not support %s for it. " + "If the deployment is a reasoning model, set model_info.base_model to its underlying " + "model.", reasoning_param, unsupported_params[reasoning_param], model, custom_llm_provider, reasoning_param, - fix, ) else: raise UnsupportedParamsError( diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index f6344c0840d..798bebd4881 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -984,7 +984,7 @@ def test_openai_gpt5_converse_rejects_effort_level_disabled_in_model_map( ) with caplog.at_level(logging.WARNING, logger="LiteLLM"): - optional_params = config.map_openai_params( + optional_params: Final = config.map_openai_params( non_default_params={"reasoning_effort": "minimal"}, optional_params={}, model=model, @@ -993,7 +993,7 @@ def test_openai_gpt5_converse_rejects_effort_level_disabled_in_model_map( _, additional_request_params, _, _ = config._prepare_request_params(optional_params, model) assert "reasoning" not in additional_request_params assert "thinking" not in additional_request_params - dropped = [r.getMessage() for r in caplog.records if "Dropping reasoning_effort='minimal'" in r.getMessage()] + dropped: Final = [r.getMessage() for r in caplog.records if "Dropping reasoning_effort='minimal'" in r.getMessage()] assert len(dropped) == 1 and model in dropped[0] diff --git a/tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py b/tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py index c0ea47b8008..ffc24c308cd 100644 --- a/tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py +++ b/tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py @@ -7,6 +7,7 @@ falls back to the Chat Completions bridge and rides Converse. import json import logging from importlib.resources import files +from typing import Final from unittest.mock import patch import pytest @@ -15,6 +16,7 @@ import litellm from litellm.llms.bedrock.common_utils import bedrock_supports_openai_responses from litellm.llms.bedrock.responses.transformation import BedrockOpenAIResponsesConfig from litellm.responses.file_search.emulated_handler import should_use_emulated_file_search +from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders from litellm.utils import ProviderConfigManager @@ -341,7 +343,7 @@ class TestDisabledReasoningEffort: self, model: str, local_model_cost_map: None, caplog: pytest.LogCaptureFixture ) -> None: with caplog.at_level(logging.WARNING, logger="LiteLLM"): - params = _cfg().map_openai_params( + params: Final = _cfg().map_openai_params( response_api_optional_params={ "reasoning": {"effort": "minimal", "summary": "auto"}, "max_output_tokens": 64, @@ -350,7 +352,9 @@ class TestDisabledReasoningEffort: drop_params=True, ) assert params == {"reasoning": {"summary": "auto"}, "max_output_tokens": 64} - dropped = [r.getMessage() for r in caplog.records if "Dropping reasoning.effort='minimal'" in r.getMessage()] + dropped: Final = [ + r.getMessage() for r in caplog.records if "Dropping reasoning.effort='minimal'" in r.getMessage() + ] assert len(dropped) == 1 and model in dropped[0] def test_effort_only_reasoning_is_removed_when_dropped(self, local_model_cost_map): @@ -365,6 +369,17 @@ class TestDisabledReasoningEffort: ) assert params == {"reasoning": {"effort": "low"}} + @pytest.mark.parametrize("model", ["us.openai.gpt-5.6-luna", MODEL]) + def test_effort_level_the_deployment_declares_is_forwarded(self, model: str, local_model_cost_map: None) -> None: + params: Final = ResponsesAPIRequestUtils.get_optional_params_responses_api( + model=model, + responses_api_provider_config=_cfg(), + response_api_optional_params={"reasoning": {"effort": "minimal"}}, + drop_params=False, + declared_reasoning_efforts=("minimal", "high"), + ) + assert params["reasoning"] == {"effort": "minimal"} + def _never_fetch(url: str) -> str: raise AssertionError(f"unexpected sync fetch of {url}") diff --git a/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py b/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py index a2289167ad4..3657a0965b2 100644 --- a/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py +++ b/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py @@ -9,6 +9,7 @@ gate, the URL construction for both paths, and the shared Bearer auth. import copy import logging +from typing import Final import pytest from botocore.exceptions import ( @@ -21,6 +22,7 @@ import litellm from litellm.llms.bedrock_mantle.responses.transformation import ( BedrockMantleResponsesAPIConfig, ) +from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders @@ -399,6 +401,16 @@ class TestBedrockMantleSamplingParams: assert ("top_p" in mapped) is survives assert ("temperature" in mapped) is survives + def test_effort_level_the_deployment_declares_is_forwarded(self, local_cost_map) -> None: + params: Final = ResponsesAPIRequestUtils.get_optional_params_responses_api( + model="openai.gpt-5.6-luna", + responses_api_provider_config=BedrockMantleResponsesAPIConfig(), + response_api_optional_params={"reasoning": {"effort": "minimal"}}, + drop_params=False, + declared_reasoning_efforts=("minimal", "high"), + ) + assert params["reasoning"] == {"effort": "minimal"} + def test_top_p_without_drop_params_raises_only_while_reasoning_is_active(self, local_cost_map): with pytest.raises(litellm.UnsupportedParamsError): BedrockMantleResponsesAPIConfig().map_openai_params( diff --git a/tests/unit/test_utils_get_optional_params.py b/tests/unit/test_utils_get_optional_params.py index 5cf59136b86..8b0610d0d50 100644 --- a/tests/unit/test_utils_get_optional_params.py +++ b/tests/unit/test_utils_get_optional_params.py @@ -2129,4 +2129,41 @@ def test_dropping_an_unsupported_thinking_logs_one_warning( record.getMessage() for record in caplog.records if "Dropping thinking" in record.getMessage() ] assert len(drop_warnings) == 1 - assert all(fragment in drop_warnings[0] for fragment in ("'disabled'", "terra-ptu", "send reasoning_effort")) + assert all(fragment in drop_warnings[0] for fragment in ("'disabled'", "terra-ptu", "model_info.base_model")) + + +@pytest.mark.parametrize( + ("reasoning_kwargs", "warns"), + [ + ({"reasoning_effort": "none"}, False), + ({"thinking": {"type": "disabled"}}, False), + ({"reasoning_effort": "high"}, True), + ({"thinking": {"type": "enabled", "budget_tokens": 1024}}, True), + ], +) +def test_reasoning_drop_warns_unless_a_mapped_non_reasoning_model_was_asked_for_none( + local_model_cost_map: None, + caplog: pytest.LogCaptureFixture, + reasoning_kwargs: dict[str, object], + warns: bool, +) -> None: + litellm.register_model( + { + "azure/plain-chat-test-deployment": { + "litellm_provider": "azure", + "mode": "chat", + "max_tokens": 1024, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "supports_reasoning": False, + } + } + ) + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + optional_params: Final = get_optional_params( + model="plain-chat-test-deployment", custom_llm_provider="azure", drop_params=True, **reasoning_kwargs + ) + + assert not set(reasoning_kwargs) & set(optional_params) + assert any("Dropping" in record.getMessage() for record in caplog.records) is warns From 9eaba58c7308c1317f48d9282c04aa628ce5520e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 12:54:24 -0700 Subject: [PATCH 6/8] fix: stay silent when reasoning is turned off on /v1/responses and warn on the Bedrock Responses fallback drop --- litellm/llms/bedrock/responses/transformation.py | 3 ++- litellm/llms/openai/responses/transformation.py | 12 +++++++----- .../responses/test_bedrock_openai_responses.py | 15 ++++++++++++++- .../test_openai_responses_transformation.py | 13 +++++++++++++ 4 files changed, 36 insertions(+), 7 deletions(-) diff --git a/litellm/llms/bedrock/responses/transformation.py b/litellm/llms/bedrock/responses/transformation.py index bf936399e83..94e59b6e303 100644 --- a/litellm/llms/bedrock/responses/transformation.py +++ b/litellm/llms/bedrock/responses/transformation.py @@ -56,6 +56,7 @@ from litellm.llms.bedrock.common_utils import ( bedrock_reasoning_effort_disabled, bedrock_supports_openai_responses, ) +from litellm.llms.openai.chat.gpt_reasoning_transformation import DROP_REFUSED_REASONING_EFFORT_WARNING from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ResponseInputParam, ResponsesAPIOptionalRequestParams @@ -173,7 +174,7 @@ def _without_disabled_reasoning_effort( ), status_code=400, ) - verbose_logger.debug("Dropping unsupported `reasoning.effort=%s` for Bedrock model=%s.", effort, model) + verbose_logger.warning(DROP_REFUSED_REASONING_EFFORT_WARNING, "reasoning.effort", effort, model, model) rest: Final = {key: value for key, value in reasoning.items() if key != "effort"} without_reasoning: Final = {key: value for key, value in params.items() if key != "reasoning"} return {**without_reasoning, "reasoning": rest} if rest else without_reasoning diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 4eda47bc8f2..4c6760c73b9 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -330,10 +330,11 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): @staticmethod def _requests_reasoning_effort(reasoning: object) -> bool: - effort: Final = ( - reasoning.get("effort") if isinstance(reasoning, Mapping) else getattr(reasoning, "effort", None) - ) - return effort is not None + return OpenAIResponsesAPIConfig._requested_effort(reasoning) is not None + + @staticmethod + def _requested_effort(reasoning: object) -> object: + return reasoning.get("effort") if isinstance(reasoning, Mapping) else getattr(reasoning, "effort", None) @staticmethod def _enforce_min_max_output_tokens(max_output_tokens: "int | None") -> "int | None": @@ -406,7 +407,8 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): ): if drop_params or litellm.drop_params: dropped_reasoning: Final = params.pop("reasoning", None) - verbose_logger.warning(DROP_UNSUPPORTED_REASONING_WARNING, dropped_reasoning, model) + if self._requested_effort(dropped_reasoning) != "none": + verbose_logger.warning(DROP_UNSUPPORTED_REASONING_WARNING, dropped_reasoning, model) else: raise litellm.UnsupportedParamsError( message=( diff --git a/tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py b/tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py index ffc24c308cd..e8efc9970e5 100644 --- a/tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py +++ b/tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py @@ -14,7 +14,7 @@ import pytest import litellm from litellm.llms.bedrock.common_utils import bedrock_supports_openai_responses -from litellm.llms.bedrock.responses.transformation import BedrockOpenAIResponsesConfig +from litellm.llms.bedrock.responses.transformation import BedrockOpenAIResponsesConfig, _without_disabled_reasoning_effort from litellm.responses.file_search.emulated_handler import should_use_emulated_file_search from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.router import GenericLiteLLMParams @@ -357,6 +357,19 @@ class TestDisabledReasoningEffort: ] assert len(dropped) == 1 and model in dropped[0] + def test_bedrock_disabled_effort_drop_warns_on_its_own( + self, local_model_cost_map: None, caplog: pytest.LogCaptureFixture + ) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + params: Final = _without_disabled_reasoning_effort( + {"reasoning": {"effort": "minimal"}, "max_output_tokens": 64}, MODEL, True, () + ) + assert params == {"max_output_tokens": 64} + dropped: Final = [ + r.getMessage() for r in caplog.records if "Dropping reasoning.effort='minimal'" in r.getMessage() + ] + assert len(dropped) == 1 and MODEL in dropped[0] + def test_effort_only_reasoning_is_removed_when_dropped(self, local_model_cost_map): params = _cfg().map_openai_params( response_api_optional_params={"reasoning": {"effort": "minimal"}}, model=MODEL, drop_params=True diff --git a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py index 59788a3a299..e37b8717bf3 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py @@ -2426,6 +2426,19 @@ class TestResponsesSurfaceSharesTheEffortRule: assert len(drop_warnings) == 1 assert all(fragment in drop_warnings[0] for fragment in ("'medium'", "gpt-4o", "supports_reasoning")) + def test_turning_reasoning_off_for_a_model_without_supports_reasoning_drops_it_silently( + self, local_model_cost_map: None, caplog: pytest.LogCaptureFixture + ) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + mapped: Final = OpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params={"reasoning": {"effort": "none", "summary": "auto"}}, + model="gpt-4o", + drop_params=True, + ) + + assert "reasoning" not in mapped + assert not [record for record in caplog.records if "Dropping" in record.getMessage()] + def test_a_forwarded_effort_logs_no_drop_warning( self, local_model_cost_map: None, caplog: pytest.LogCaptureFixture ) -> None: From f5f5715ea72b917a3f42a3b93c3cb25b1f7abc1c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 15:14:30 -0700 Subject: [PATCH 7/8] fix(reasoning): warn when any reasoning-off request is dropped and give each route only the advice it honors A request to turn reasoning off on a model name the catalog lacks now warns on every route, since it may be a reasoning model. Chat suggests model_info.reasoning_effort_levels and, for a name with no catalog entry, model_info.base_model. OpenAI Responses suggests declared levels only where the config honors them, and Bedrock suggests neither. The Bedrock and Mantle Responses forwarding of declared levels is reverted until it can be proven live. --- .../bedrock/chat/converse_transformation.py | 8 +-- litellm/llms/bedrock/common_utils.py | 6 ++ .../llms/bedrock/responses/transformation.py | 22 ++++---- .../responses/transformation.py | 19 +++---- .../chat/gpt_reasoning_transformation.py | 29 ++++++++-- .../llms/openai/responses/transformation.py | 21 ++++++- litellm/responses/utils.py | 2 +- litellm/utils.py | 32 ++++++++--- ...test_azure_gpt_reasoning_transformation.py | 23 ++++++++ .../chat/test_converse_transformation.py | 1 + .../test_bedrock_openai_responses.py | 15 +---- ...bedrock_mantle_responses_transformation.py | 35 ++++++++---- .../test_openai_responses_transformation.py | 26 +++++++++ tests/unit/test_utils_get_optional_params.py | 55 +++++++++++++++++++ 14 files changed, 223 insertions(+), 71 deletions(-) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index b83d45aa5e5..05440d70017 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -117,6 +117,7 @@ from litellm.utils import ( ) from ..common_utils import ( + DROP_DISABLED_REASONING_EFFORT_WARNING, BedrockError, BedrockModelInfo, bedrock_cache_point_ttl_supported, @@ -1192,12 +1193,7 @@ class AmazonConverseConfig(BaseConfig): ), status_code=400, ) - verbose_logger.warning( - "Dropping reasoning_effort=%r for Bedrock model=%r (drop_params=True) because the model map " - "marks that level unsupported.", - value, - model, - ) + verbose_logger.warning(DROP_DISABLED_REASONING_EFFORT_WARNING, "reasoning_effort", value, model) elif param == "reasoning_effort" and isinstance(value, str): self._handle_reasoning_effort_parameter( model=model, reasoning_effort=value, optional_params=optional_params diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 28888f1a23a..df700da6aab 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -1260,6 +1260,12 @@ def _mantle_api_base_from_env() -> str | None: return next((base[: -len(suffix)] for suffix in _MANTLE_OPENAI_BASE_SUFFIXES if base.endswith(suffix)), base) +DROP_DISABLED_REASONING_EFFORT_WARNING: Final = ( + "Dropping %s=%r for Bedrock model=%r (drop_params=True) because the model map marks that level unsupported. " + "Send a level the model supports." +) + + def bedrock_reasoning_effort_disabled(model: str, effort: str) -> bool: from litellm.utils import is_explicitly_disabled_factory diff --git a/litellm/llms/bedrock/responses/transformation.py b/litellm/llms/bedrock/responses/transformation.py index 94e59b6e303..ff44d973d97 100644 --- a/litellm/llms/bedrock/responses/transformation.py +++ b/litellm/llms/bedrock/responses/transformation.py @@ -34,7 +34,7 @@ inference profile, so the model is named ``us.openai.gpt-5.6-sol`` or """ import asyncio -from collections.abc import Awaitable, Callable, Collection, Mapping +from collections.abc import Awaitable, Callable, Mapping from types import MappingProxyType from typing import Final @@ -52,11 +52,11 @@ from litellm.llms.base_llm.responses.codex_compat import drop_unsupported_tools, from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.bedrock.common_utils import ( BEDROCK_CHAT_COMPLETIONS_ROUTE_PREFIX, + DROP_DISABLED_REASONING_EFFORT_WARNING, BedrockError, bedrock_reasoning_effort_disabled, bedrock_supports_openai_responses, ) -from litellm.llms.openai.chat.gpt_reasoning_transformation import DROP_REFUSED_REASONING_EFFORT_WARNING from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ResponseInputParam, ResponsesAPIOptionalRequestParams @@ -158,13 +158,13 @@ def inline_remote_image_urls( def _without_disabled_reasoning_effort( - params: Mapping[str, object], model: str, drop_params: bool, declared_reasoning_efforts: Collection[str] + params: Mapping[str, object], model: str, drop_params: bool ) -> dict[str, object]: # mutable-ok: becomes the map_openai_params return value reasoning: Final = params.get("reasoning") effort: Final = reasoning.get("effort") if isinstance(reasoning, Mapping) else None if not isinstance(reasoning, Mapping) or not isinstance(effort, str): return dict(params) - if effort in declared_reasoning_efforts or not bedrock_reasoning_effort_disabled(model=model, effort=effort): + if not bedrock_reasoning_effort_disabled(model=model, effort=effort): return dict(params) if not (drop_params or litellm.drop_params): raise litellm.UnsupportedParamsError( @@ -174,7 +174,7 @@ def _without_disabled_reasoning_effort( ), status_code=400, ) - verbose_logger.warning(DROP_REFUSED_REASONING_EFFORT_WARNING, "reasoning.effort", effort, model, model) + verbose_logger.warning(DROP_DISABLED_REASONING_EFFORT_WARNING, "reasoning.effort", effort, model) rest: Final = {key: value for key, value in reasoning.items() if key != "effort"} without_reasoning: Final = {key: value for key, value in params.items() if key != "reasoning"} return {**without_reasoning, "reasoning": rest} if rest else without_reasoning @@ -282,16 +282,14 @@ class BedrockOpenAIResponsesConfig(BaseAWSLLM, OpenAIResponsesAPIConfig): fake_stream=fake_stream, ) - def map_openai_params_for_deployment( + def map_openai_params( self, response_api_optional_params: ResponsesAPIOptionalRequestParams, model: str, drop_params: bool, - *, - declared_reasoning_efforts: Collection[str], - ) -> Mapping[str, object]: - mapped: Final = super().map_openai_params_for_deployment( - response_api_optional_params, model, drop_params, declared_reasoning_efforts=declared_reasoning_efforts + ) -> dict: # mutable-ok: signature fixed by the override contract + mapped: Final = super().map_openai_params( + response_api_optional_params=response_api_optional_params, model=model, drop_params=drop_params ) unsupported: Final = tuple(sorted(BEDROCK_RUNTIME_UNSUPPORTED_RESPONSE_PARAMS & mapped.keys())) if unsupported: @@ -300,7 +298,7 @@ class BedrockOpenAIResponsesConfig(BaseAWSLLM, OpenAIResponsesAPIConfig): unsupported, ) supported: Final[dict[str, object]] = {key: value for key, value in mapped.items() if key not in unsupported} - params: Final = _without_disabled_reasoning_effort(supported, model, drop_params, declared_reasoning_efforts) + params: Final = _without_disabled_reasoning_effort(supported, model, drop_params) tools: Final = params.get("tools") if not isinstance(tools, list): return params diff --git a/litellm/llms/bedrock_mantle/responses/transformation.py b/litellm/llms/bedrock_mantle/responses/transformation.py index 31cbbcad39a..3532cfbd963 100644 --- a/litellm/llms/bedrock_mantle/responses/transformation.py +++ b/litellm/llms/bedrock_mantle/responses/transformation.py @@ -15,7 +15,7 @@ role / access key / profile / web identity), signed via the shared BaseAWSLLM._sign_request after the request body is finalized. """ -from collections.abc import Collection, Mapping, Sequence +from collections.abc import Mapping, Sequence from typing import Final, cast # noqa: TID251 # map_openai_params returns the filtered params as a bare dict import httpx @@ -229,23 +229,18 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI def _model_map_lookup_name(model: str) -> str: return model.split("/")[-1].removeprefix("openai.") - def map_openai_params_for_deployment( + def map_openai_params( self, response_api_optional_params: ResponsesAPIOptionalRequestParams, model: str, drop_params: bool, - *, - declared_reasoning_efforts: Collection[str], - ) -> Mapping[str, object]: + ) -> dict: params: Final = self._handle_unsupported_reasoning_summary( self._handle_unsupported_service_tier( - dict( - super().map_openai_params_for_deployment( - response_api_optional_params, - model, - drop_params, - declared_reasoning_efforts=declared_reasoning_efforts, - ) + super().map_openai_params( + response_api_optional_params=response_api_optional_params, + model=model, + drop_params=drop_params, ), drop_params=drop_params, ), diff --git a/litellm/llms/openai/chat/gpt_reasoning_transformation.py b/litellm/llms/openai/chat/gpt_reasoning_transformation.py index 55f8c1c0846..151db5d6887 100644 --- a/litellm/llms/openai/chat/gpt_reasoning_transformation.py +++ b/litellm/llms/openai/chat/gpt_reasoning_transformation.py @@ -8,6 +8,7 @@ import litellm from litellm._logging import verbose_logger from litellm.utils import ( declared_value_factory, + has_catalog_entry, is_explicitly_disabled_factory, supports_factory, ) @@ -16,11 +17,24 @@ from .gpt_transformation import OpenAIGPTConfig _GPT_SERIES_VERSION: Final = re.compile(r"^gpt-(\d+)(?:\.(\d+))?(?=[.-]|$)") -DROP_REFUSED_REASONING_EFFORT_WARNING: Final = ( - "Dropping %s=%r for model=%r (drop_params=True) because the model map does not mark that level supported for " - "%r. If the deployment supports it, list every level it supports in model_info.reasoning_effort_levels, or set " - "model_info.base_model when the deployment name is not its underlying model." -) + +def refused_reasoning_effort_warning(*, suggest_declared_levels: bool, suggest_base_model: bool) -> str: + return "".join( + ( + "Dropping %s=%r for model=%r (drop_params=True) because the model map does not mark that level " + "supported for %r.", + ( + " If the deployment supports it, list every level it supports in model_info.reasoning_effort_levels." + if suggest_declared_levels + else "" + ), + ( + " If the deployment name is not its underlying model, set model_info.base_model to that model." + if suggest_base_model + else "" + ), + ) + ) def _catalogue_declares_default_effort() -> bool: @@ -350,7 +364,10 @@ class OpenAIGPT5Config(OpenAIGPTConfig): ): if litellm.drop_params or drop_params: verbose_logger.warning( - DROP_REFUSED_REASONING_EFFORT_WARNING, + refused_reasoning_effort_warning( + suggest_declared_levels=True, + suggest_base_model=not has_catalog_entry(self._model_map_lookup_name(model), None), + ), "reasoning_effort", requested_effort, model, diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 4c6760c73b9..21f19e8175c 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -26,8 +26,8 @@ from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.llms.openai.chat.gpt_reasoning_transformation import ( - DROP_REFUSED_REASONING_EFFORT_WARNING, is_gpt_reasoning_series_name, + refused_reasoning_effort_warning, unsupported_reasoning_effort_message, ) from litellm.responses.litellm_completion_transformation.custom_tools import TOOL_CALL_ITEM_ID_PREFIX_BY_TYPE @@ -328,6 +328,12 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): return declared return info["key"] in _bundled_openai_reasoning_models() + @staticmethod + def _is_known_non_reasoning_model(model: str) -> bool: + from litellm.utils import is_known_non_reasoning_model + + return is_known_non_reasoning_model(model.split("/")[-1], LlmProviders.OPENAI.value) + @staticmethod def _requests_reasoning_effort(reasoning: object) -> bool: return OpenAIResponsesAPIConfig._requested_effort(reasoning) is not None @@ -380,6 +386,9 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): ) ) + def honors_declared_reasoning_efforts(self) -> bool: + return type(self).map_openai_params is OpenAIResponsesAPIConfig.map_openai_params + def map_openai_params_for_deployment( self, response_api_optional_params: ResponsesAPIOptionalRequestParams, @@ -407,7 +416,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): ): if drop_params or litellm.drop_params: dropped_reasoning: Final = params.pop("reasoning", None) - if self._requested_effort(dropped_reasoning) != "none": + if self._requested_effort(dropped_reasoning) != "none" or not self._is_known_non_reasoning_model(model): verbose_logger.warning(DROP_UNSUPPORTED_REASONING_WARNING, dropped_reasoning, model) else: raise litellm.UnsupportedParamsError( @@ -435,7 +444,13 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): status_code=400, ) verbose_logger.warning( - DROP_REFUSED_REASONING_EFFORT_WARNING, "reasoning.effort", requested_effort, model, lookup_name + refused_reasoning_effort_warning( + suggest_declared_levels=self.honors_declared_reasoning_efforts(), suggest_base_model=False + ), + "reasoning.effort", + requested_effort, + model, + lookup_name, ) remaining: Final = reasoning.copy() remaining.pop("effort", None) diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index a6b042a2234..25f18a760d0 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -134,7 +134,7 @@ def _provider_mapped_params( if ( declared_reasoning_efforts and isinstance(config, OpenAIResponsesAPIConfig) - and type(config).map_openai_params is OpenAIResponsesAPIConfig.map_openai_params + and config.honors_declared_reasoning_efforts() ): return config.map_openai_params_for_deployment( params, model, drop_params, declared_reasoning_efforts=declared_reasoning_efforts diff --git a/litellm/utils.py b/litellm/utils.py index 6dd43a45582..eb40d0c80d9 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -91,7 +91,7 @@ from litellm.litellm_core_utils.fallback_generalizations import ( match_capability_generalizations, match_fill_missing_generalizations, ) -from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry +from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry, GetModelCostMap from litellm.litellm_core_utils.model_param_helper import peek_reasoning_summary_aliases from litellm.litellm_core_utils.offline_mode import is_offline_mode from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload @@ -4985,13 +4985,33 @@ def _turns_reasoning_off(value: object) -> bool: return False -def _is_mapped_non_reasoning_model(model: str, custom_llm_provider: str | None) -> bool: +def is_known_non_reasoning_model(model: str, custom_llm_provider: str | None) -> bool: info: Final = lookup_model_info_for_registration(model, custom_llm_provider) - return not isinstance(info, ModelInfoLookupMiss) and info.get("supports_reasoning") is not True + if isinstance(info, ModelInfoLookupMiss) or info.get("supports_reasoning") is True: + return False + return info.get("supports_reasoning") is False or info["key"] in GetModelCostMap.loaded_model_cost_map() + + +def has_catalog_entry(model: str, custom_llm_provider: str | None) -> bool: + info: Final = lookup_model_info_for_registration(model, custom_llm_provider) + return not isinstance(info, ModelInfoLookupMiss) and info["key"] in GetModelCostMap.loaded_model_cost_map() + + +def _dropped_reasoning_warning(model: str, custom_llm_provider: str | None) -> str: + return "".join( + ( + "Dropping %s=%r for model=%r (drop_params=True) because %s does not support %s for it.", + ( + "" + if has_catalog_entry(model, custom_llm_provider) + else " If the deployment is a reasoning model, set model_info.base_model to its underlying model." + ), + ) + ) def _warns_on_dropped_reasoning(value: object, model: str, custom_llm_provider: str | None) -> bool: - return not (_turns_reasoning_off(value) and _is_mapped_non_reasoning_model(model, custom_llm_provider)) + return not (_turns_reasoning_off(value) and is_known_non_reasoning_model(model, custom_llm_provider)) def get_optional_params( @@ -5098,9 +5118,7 @@ def get_optional_params( unsupported_params[reasoning_param], base_model or model, custom_llm_provider ): verbose_logger.warning( - "Dropping %s=%r for model=%r (drop_params=True) because %s does not support %s for it. " - "If the deployment is a reasoning model, set model_info.base_model to its underlying " - "model.", + _dropped_reasoning_warning(base_model or model, custom_llm_provider), reasoning_param, unsupported_params[reasoning_param], model, diff --git a/tests/unit/llms/azure/chat/test_azure_gpt_reasoning_transformation.py b/tests/unit/llms/azure/chat/test_azure_gpt_reasoning_transformation.py index a82dd98e31d..1d5d0c808de 100644 --- a/tests/unit/llms/azure/chat/test_azure_gpt_reasoning_transformation.py +++ b/tests/unit/llms/azure/chat/test_azure_gpt_reasoning_transformation.py @@ -1,3 +1,4 @@ +import logging from collections.abc import Mapping from typing import Final @@ -279,6 +280,28 @@ def test_azure_gpt5_reasoning_effort_none_dropped(config: AzureOpenAIGPT5Config) assert "reasoning_effort" not in params or params.get("reasoning_effort") != "none" +@pytest.mark.parametrize(("model", "names_base_model"), [("azure/litellm-qa-sol-alias", True), ("azure/gpt-5", False)]) +def test_azure_dropped_effort_warning_suggests_base_model_only_for_a_name_the_map_lacks( + config: AzureOpenAIGPT5Config, caplog: pytest.LogCaptureFixture, model: str, names_base_model: bool +) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + params: Final = config.map_openai_params( + non_default_params={"reasoning_effort": "none"}, + optional_params={}, + model=model, + drop_params=True, + api_version="2025-04-01-preview", + ) + + assert "reasoning_effort" not in params + drop_warnings: Final = [ + record.getMessage() for record in caplog.records if "Dropping reasoning_effort='none'" in record.getMessage() + ] + assert len(drop_warnings) == 1 + assert "model_info.reasoning_effort_levels" in drop_warnings[0] + assert ("model_info.base_model" in drop_warnings[0]) is names_base_model + + # Logprobs support tests for Azure GPT-5.2 def test_azure_gpt5_2_supports_logprobs(config: AzureOpenAIGPT5Config): """Test that Azure GPT-5.2 models support logprobs parameters. diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 798bebd4881..2d75ce1f22f 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -995,6 +995,7 @@ def test_openai_gpt5_converse_rejects_effort_level_disabled_in_model_map( assert "thinking" not in additional_request_params dropped: Final = [r.getMessage() for r in caplog.records if "Dropping reasoning_effort='minimal'" in r.getMessage()] assert len(dropped) == 1 and model in dropped[0] + assert "model_info" not in dropped[0] @pytest.mark.parametrize( diff --git a/tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py b/tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py index e8efc9970e5..c598abc86e5 100644 --- a/tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py +++ b/tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py @@ -16,7 +16,6 @@ import litellm from litellm.llms.bedrock.common_utils import bedrock_supports_openai_responses from litellm.llms.bedrock.responses.transformation import BedrockOpenAIResponsesConfig, _without_disabled_reasoning_effort from litellm.responses.file_search.emulated_handler import should_use_emulated_file_search -from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders from litellm.utils import ProviderConfigManager @@ -356,13 +355,14 @@ class TestDisabledReasoningEffort: r.getMessage() for r in caplog.records if "Dropping reasoning.effort='minimal'" in r.getMessage() ] assert len(dropped) == 1 and model in dropped[0] + assert "model_info" not in dropped[0] def test_bedrock_disabled_effort_drop_warns_on_its_own( self, local_model_cost_map: None, caplog: pytest.LogCaptureFixture ) -> None: with caplog.at_level(logging.WARNING, logger="LiteLLM"): params: Final = _without_disabled_reasoning_effort( - {"reasoning": {"effort": "minimal"}, "max_output_tokens": 64}, MODEL, True, () + {"reasoning": {"effort": "minimal"}, "max_output_tokens": 64}, MODEL, True ) assert params == {"max_output_tokens": 64} dropped: Final = [ @@ -382,17 +382,6 @@ class TestDisabledReasoningEffort: ) assert params == {"reasoning": {"effort": "low"}} - @pytest.mark.parametrize("model", ["us.openai.gpt-5.6-luna", MODEL]) - def test_effort_level_the_deployment_declares_is_forwarded(self, model: str, local_model_cost_map: None) -> None: - params: Final = ResponsesAPIRequestUtils.get_optional_params_responses_api( - model=model, - responses_api_provider_config=_cfg(), - response_api_optional_params={"reasoning": {"effort": "minimal"}}, - drop_params=False, - declared_reasoning_efforts=("minimal", "high"), - ) - assert params["reasoning"] == {"effort": "minimal"} - def _never_fetch(url: str) -> str: raise AssertionError(f"unexpected sync fetch of {url}") diff --git a/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py b/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py index 3657a0965b2..03b57d0c318 100644 --- a/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py +++ b/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py @@ -22,7 +22,6 @@ import litellm from litellm.llms.bedrock_mantle.responses.transformation import ( BedrockMantleResponsesAPIConfig, ) -from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders @@ -401,16 +400,6 @@ class TestBedrockMantleSamplingParams: assert ("top_p" in mapped) is survives assert ("temperature" in mapped) is survives - def test_effort_level_the_deployment_declares_is_forwarded(self, local_cost_map) -> None: - params: Final = ResponsesAPIRequestUtils.get_optional_params_responses_api( - model="openai.gpt-5.6-luna", - responses_api_provider_config=BedrockMantleResponsesAPIConfig(), - response_api_optional_params={"reasoning": {"effort": "minimal"}}, - drop_params=False, - declared_reasoning_efforts=("minimal", "high"), - ) - assert params["reasoning"] == {"effort": "minimal"} - def test_top_p_without_drop_params_raises_only_while_reasoning_is_active(self, local_cost_map): with pytest.raises(litellm.UnsupportedParamsError): BedrockMantleResponsesAPIConfig().map_openai_params( @@ -584,6 +573,30 @@ class TestBedrockMantleServiceTier: assert "priority" in str(mock_warning.call_args) +class TestBedrockMantleRefusedEffort: + def test_dropping_a_refused_effort_does_not_suggest_declaring_levels_mantle_ignores( + self, local_cost_map: None, caplog: pytest.LogCaptureFixture + ) -> None: + refused: Final = next( + level + for level in ("none", "minimal", "low") + if litellm.model_cost["gpt-6.1-sol"].get(f"supports_{level}_reasoning_effort") is False + ) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + params: Final = BedrockMantleResponsesAPIConfig().map_openai_params( + response_api_optional_params={"reasoning": {"effort": refused}}, + model="openai.gpt-6.1-sol", + drop_params=True, + ) + + assert "reasoning" not in params + drop_warnings: Final = [ + record.getMessage() for record in caplog.records if f"Dropping reasoning.effort={refused!r}" in record.getMessage() + ] + assert len(drop_warnings) == 1 + assert "model_info" not in drop_warnings[0] + + class TestBedrockMantleReasoningSummary: @pytest.mark.parametrize("summary", ["concise", "detailed"]) def test_unsupported_reasoning_summary_dropped_when_drop_params_true(self, summary): diff --git a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py index e37b8717bf3..d48617536b4 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py @@ -2410,6 +2410,7 @@ class TestResponsesSurfaceSharesTheEffortRule: fragment in drop_warnings[0] for fragment in (repr(refused), "gpt-6.1-sol", "model_info.reasoning_effort_levels") ) + assert "model_info.base_model" not in drop_warnings[0] def test_dropping_reasoning_for_a_model_without_supports_reasoning_logs_one_warning( self, local_model_cost_map: None, caplog: pytest.LogCaptureFixture @@ -2439,6 +2440,31 @@ class TestResponsesSurfaceSharesTheEffortRule: assert "reasoning" not in mapped assert not [record for record in caplog.records if "Dropping" in record.getMessage()] + def test_turning_reasoning_off_warns_when_only_custom_pricing_describes_the_model( + self, local_model_cost_map: None, caplog: pytest.LogCaptureFixture + ) -> None: + litellm.register_model( + { + "pricing-only-test-model": { + "litellm_provider": "openai", + "mode": "responses", + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.000002, + } + } + ) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + mapped: Final = OpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params={"reasoning": {"effort": "none"}}, + model="pricing-only-test-model", + drop_params=True, + ) + + assert "reasoning" not in mapped + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] + assert len(drop_warnings) == 1 + assert all(fragment in drop_warnings[0] for fragment in ("'none'", "pricing-only-test-model")) + def test_a_forwarded_effort_logs_no_drop_warning( self, local_model_cost_map: None, caplog: pytest.LogCaptureFixture ) -> None: diff --git a/tests/unit/test_utils_get_optional_params.py b/tests/unit/test_utils_get_optional_params.py index 8b0610d0d50..74e0e33a5ff 100644 --- a/tests/unit/test_utils_get_optional_params.py +++ b/tests/unit/test_utils_get_optional_params.py @@ -2167,3 +2167,58 @@ def test_reasoning_drop_warns_unless_a_mapped_non_reasoning_model_was_asked_for_ assert not set(reasoning_kwargs) & set(optional_params) assert any("Dropping" in record.getMessage() for record in caplog.records) is warns + + +@pytest.mark.parametrize("reasoning_kwargs", [{"reasoning_effort": "none"}, {"thinking": {"type": "disabled"}}]) +def test_turning_reasoning_off_warns_when_only_custom_pricing_describes_the_model( + local_model_cost_map: None, caplog: pytest.LogCaptureFixture, reasoning_kwargs: dict[str, object] +) -> None: + litellm.register_model( + { + "azure/pricing-only-test-deployment": { + "litellm_provider": "azure", + "mode": "chat", + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.000002, + } + } + ) + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + optional_params: Final = get_optional_params( + model="pricing-only-test-deployment", custom_llm_provider="azure", drop_params=True, **reasoning_kwargs + ) + + assert not set(reasoning_kwargs) & set(optional_params) + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] + assert len(drop_warnings) == 1 + assert all(fragment in drop_warnings[0] for fragment in ("pricing-only-test-deployment", "model_info.base_model")) + + +def test_dropping_reasoning_for_a_catalog_reasoning_model_does_not_suggest_base_model( + local_model_cost_map: None, caplog: pytest.LogCaptureFixture +) -> None: + assert litellm.model_cost["azure/gpt-5.6-sol"].get("supports_reasoning") is True + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + optional_params: Final = get_optional_params( + model="gpt-5.6-sol", custom_llm_provider="azure", thinking={"type": "disabled"}, drop_params=True + ) + + assert "thinking" not in optional_params + drop_warnings: Final = [ + record.getMessage() for record in caplog.records if "Dropping thinking" in record.getMessage() + ] + assert len(drop_warnings) == 1 + assert "model_info.base_model" not in drop_warnings[0] + + +def test_turning_reasoning_off_for_a_catalog_non_reasoning_model_drops_it_silently( + local_model_cost_map: None, caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + optional_params: Final = get_optional_params( + model="gpt-4o", custom_llm_provider="azure", reasoning_effort="none", drop_params=True + ) + + assert "reasoning_effort" not in optional_params + assert not [record for record in caplog.records if "Dropping" in record.getMessage()] From 812f0054a404fbf1b1fa1ced7311a3ef4612a1f2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 16:44:47 -0700 Subject: [PATCH 8/8] fix(reasoning): suggest reasoning_effort_levels and base_model only where the route reads them The chat gate suggested model_info.reasoning_effort_levels on routes that never pass declared levels (github_copilot, chatgpt and other OpenAIConfig delegates) and model_info.base_model on OpenAI and Azure AI chat, whose gate looks up the deployment name rather than base_model. /v1/responses/compact suggested levels it never reads either. Declared levels are now None when a caller passes none, so the advice appears only when a caller fed them, and the base_model advice is limited to the Azure chat gate, the one that looks up base_model. The openai row of the chat warning test now expects no base_model advice, the Responses warning test goes through get_optional_params_responses_api the way /v1/responses and compact call it, and the Azure direct-config test passes declared levels the way get_optional_params does. Long test lines are wrapped. --- .../chat/gpt_reasoning_transformation.py | 3 ++- litellm/llms/azure_ai/chat/transformation.py | 2 +- .../chat/gpt_reasoning_transformation.py | 10 ++++--- .../llms/openai/responses/transformation.py | 8 +++--- litellm/responses/utils.py | 6 ++--- ...test_azure_gpt_reasoning_transformation.py | 1 + .../test_bedrock_openai_responses.py | 5 +++- ...bedrock_mantle_responses_transformation.py | 4 ++- .../test_openai_responses_transformation.py | 26 +++++++++++++------ .../test_gpt_reasoning_transformation.py | 25 +++++++++++++----- 10 files changed, 60 insertions(+), 30 deletions(-) diff --git a/litellm/llms/azure/chat/gpt_reasoning_transformation.py b/litellm/llms/azure/chat/gpt_reasoning_transformation.py index 5bd03f511c5..72a0500e0af 100644 --- a/litellm/llms/azure/chat/gpt_reasoning_transformation.py +++ b/litellm/llms/azure/chat/gpt_reasoning_transformation.py @@ -24,6 +24,7 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): GPT5_SERIES_ROUTE = "gpt5_series/" OPT_IN_REASONING_EFFORTS: ClassVar[frozenset[str]] = frozenset(("xhigh", "none")) + LOOKS_UP_BASE_MODEL: ClassVar[bool] = True @classmethod def _model_map_lookup_name(cls, model: str) -> str: @@ -88,7 +89,7 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): drop_params: bool, api_version: str = "", *, - declared_reasoning_efforts: Collection[str] = (), + declared_reasoning_efforts: Collection[str] | None = None, ) -> dict: return OpenAIGPT5Config.map_openai_params( self, diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py index 6aed7eb3c54..8bb6072d0a9 100644 --- a/litellm/llms/azure_ai/chat/transformation.py +++ b/litellm/llms/azure_ai/chat/transformation.py @@ -106,7 +106,7 @@ class AzureAIStudioConfig(OpenAIConfig): model: str, drop_params: bool, *, - declared_reasoning_efforts: Collection[str] = (), + declared_reasoning_efforts: Collection[str] | None = None, ) -> dict[str, object]: # mutable-ok: OpenAIConfig.map_openai_params signature if not azureAIGPT5Config.is_model_gpt_5_model(model): return super().map_openai_params( diff --git a/litellm/llms/openai/chat/gpt_reasoning_transformation.py b/litellm/llms/openai/chat/gpt_reasoning_transformation.py index 151db5d6887..8c9551d01c0 100644 --- a/litellm/llms/openai/chat/gpt_reasoning_transformation.py +++ b/litellm/llms/openai/chat/gpt_reasoning_transformation.py @@ -114,6 +114,7 @@ class OpenAIGPT5Config(OpenAIGPTConfig): OPT_IN_REASONING_EFFORTS: ClassVar[frozenset[str]] = frozenset(("xhigh",)) ROUTE_SPECIFIC_REASONING_EFFORTS: ClassVar[frozenset[str]] = frozenset(("max",)) + LOOKS_UP_BASE_MODEL: ClassVar[bool] = False @classmethod def is_model_gpt_5_model(cls, model: str) -> bool: @@ -328,7 +329,7 @@ class OpenAIGPT5Config(OpenAIGPTConfig): model: str, drop_params: bool, *, - declared_reasoning_efforts: Collection[str] = (), + declared_reasoning_efforts: Collection[str] | None = None, ) -> dict: if self.is_model_gpt_5_search_model(model): if "max_tokens" in non_default_params: @@ -359,14 +360,15 @@ class OpenAIGPT5Config(OpenAIGPTConfig): if ( requested_effort is not None - and requested_effort not in declared_reasoning_efforts + and requested_effort not in (declared_reasoning_efforts or ()) and self.refuses_reasoning_effort_level(model, requested_effort) ): if litellm.drop_params or drop_params: verbose_logger.warning( refused_reasoning_effort_warning( - suggest_declared_levels=True, - suggest_base_model=not has_catalog_entry(self._model_map_lookup_name(model), None), + suggest_declared_levels=declared_reasoning_efforts is not None, + suggest_base_model=self.LOOKS_UP_BASE_MODEL + and not has_catalog_entry(self._model_map_lookup_name(model), None), ), "reasoning_effort", requested_effort, diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 21f19e8175c..07ba095c50c 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -382,7 +382,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): ) -> dict: return dict( self.map_openai_params_for_deployment( - response_api_optional_params, model, drop_params, declared_reasoning_efforts=() + response_api_optional_params, model, drop_params, declared_reasoning_efforts=None ) ) @@ -395,7 +395,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): model: str, drop_params: bool, *, - declared_reasoning_efforts: Collection[str], + declared_reasoning_efforts: Collection[str] | None, ) -> Mapping[str, object]: """No mapping applied since inputs are in OpenAI spec already. @@ -435,7 +435,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): if ( isinstance(reasoning, dict) and isinstance(requested_effort, str) - and requested_effort not in declared_reasoning_efforts + and requested_effort not in (declared_reasoning_efforts or ()) and self._effort_level_is_disabled(lookup_name, requested_effort) ): if not (drop_params or litellm.drop_params): @@ -445,7 +445,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): ) verbose_logger.warning( refused_reasoning_effort_warning( - suggest_declared_levels=self.honors_declared_reasoning_efforts(), suggest_base_model=False + suggest_declared_levels=declared_reasoning_efforts is not None, suggest_base_model=False ), "reasoning.effort", requested_effort, diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index 25f18a760d0..159d3ab8a65 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -127,12 +127,12 @@ def _provider_mapped_params( params: ResponsesAPIOptionalRequestParams, model: str, drop_params: bool, - declared_reasoning_efforts: Collection[str], + declared_reasoning_efforts: Collection[str] | None, ) -> Mapping[str, object]: from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig if ( - declared_reasoning_efforts + declared_reasoning_efforts is not None and isinstance(config, OpenAIResponsesAPIConfig) and config.honors_declared_reasoning_efforts() ): @@ -283,7 +283,7 @@ class ResponsesAPIRequestUtils: response_api_optional_params: ResponsesAPIOptionalRequestParams, allowed_openai_params: list[str] | None = None, drop_params: bool | None = None, - declared_reasoning_efforts: Collection[str] = (), + declared_reasoning_efforts: Collection[str] | None = None, ) -> dict: """ Get optional parameters for the responses API. diff --git a/tests/unit/llms/azure/chat/test_azure_gpt_reasoning_transformation.py b/tests/unit/llms/azure/chat/test_azure_gpt_reasoning_transformation.py index 1d5d0c808de..2d9381ec16a 100644 --- a/tests/unit/llms/azure/chat/test_azure_gpt_reasoning_transformation.py +++ b/tests/unit/llms/azure/chat/test_azure_gpt_reasoning_transformation.py @@ -291,6 +291,7 @@ def test_azure_dropped_effort_warning_suggests_base_model_only_for_a_name_the_ma model=model, drop_params=True, api_version="2025-04-01-preview", + declared_reasoning_efforts=(), ) assert "reasoning_effort" not in params diff --git a/tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py b/tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py index c598abc86e5..a1466cfdae4 100644 --- a/tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py +++ b/tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py @@ -14,7 +14,10 @@ import pytest import litellm from litellm.llms.bedrock.common_utils import bedrock_supports_openai_responses -from litellm.llms.bedrock.responses.transformation import BedrockOpenAIResponsesConfig, _without_disabled_reasoning_effort +from litellm.llms.bedrock.responses.transformation import ( + BedrockOpenAIResponsesConfig, + _without_disabled_reasoning_effort, +) from litellm.responses.file_search.emulated_handler import should_use_emulated_file_search from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders diff --git a/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py b/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py index 03b57d0c318..0ae6a73968d 100644 --- a/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py +++ b/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py @@ -591,7 +591,9 @@ class TestBedrockMantleRefusedEffort: assert "reasoning" not in params drop_warnings: Final = [ - record.getMessage() for record in caplog.records if f"Dropping reasoning.effort={refused!r}" in record.getMessage() + record.getMessage() + for record in caplog.records + if f"Dropping reasoning.effort={refused!r}" in record.getMessage() ] assert len(drop_warnings) == 1 assert "model_info" not in drop_warnings[0] diff --git a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py index d48617536b4..a528b1a7b75 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py @@ -16,6 +16,7 @@ from litellm.llms.azure.responses.transformation import AzureOpenAIResponsesAPIC from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.responses.litellm_completion_transformation.transformation import LiteLLMCompletionResponsesConfig +from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.openai import ( ImageGenerationPartialImageEvent, IncompleteDetails, @@ -2388,8 +2389,17 @@ class TestResponsesSurfaceSharesTheEffortRule: ) assert mapped["reasoning"] == {"summary": "auto"} - def test_dropping_a_refused_effort_logs_one_warning_naming_the_level_and_the_fix( - self, local_model_cost_map: None, caplog: pytest.LogCaptureFixture + @pytest.mark.parametrize( + ("declared_reasoning_efforts", "suggests_declared_levels"), + [((), True), (None, False)], + ids=["responses route reads model_info levels", "compact route reads none"], + ) + def test_dropping_a_refused_effort_logs_one_warning_naming_only_the_fix_its_route_honors( + self, + local_model_cost_map: None, + caplog: pytest.LogCaptureFixture, + declared_reasoning_efforts: tuple[str, ...] | None, + suggests_declared_levels: bool, ) -> None: refused: Final = next( level @@ -2397,19 +2407,19 @@ class TestResponsesSurfaceSharesTheEffortRule: if litellm.model_cost["gpt-6.1-sol"].get(f"supports_{level}_reasoning_effort") is False ) with caplog.at_level(logging.WARNING, logger="LiteLLM"): - mapped: Final = OpenAIResponsesAPIConfig().map_openai_params( - response_api_optional_params={"reasoning": {"effort": refused}}, + mapped: Final = ResponsesAPIRequestUtils.get_optional_params_responses_api( model="gpt-6.1-sol", + responses_api_provider_config=OpenAIResponsesAPIConfig(), + response_api_optional_params={"reasoning": {"effort": refused}}, drop_params=True, + declared_reasoning_efforts=declared_reasoning_efforts, ) assert "reasoning" not in mapped drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] assert len(drop_warnings) == 1 - assert all( - fragment in drop_warnings[0] - for fragment in (repr(refused), "gpt-6.1-sol", "model_info.reasoning_effort_levels") - ) + assert repr(refused) in drop_warnings[0] and "gpt-6.1-sol" in drop_warnings[0] + assert ("model_info.reasoning_effort_levels" in drop_warnings[0]) is suggests_declared_levels assert "model_info.base_model" not in drop_warnings[0] def test_dropping_reasoning_for_a_model_without_supports_reasoning_logs_one_warning( diff --git a/tests/unit/llms/openai/test_gpt_reasoning_transformation.py b/tests/unit/llms/openai/test_gpt_reasoning_transformation.py index 98f37cb2c4e..807a43c5d9d 100644 --- a/tests/unit/llms/openai/test_gpt_reasoning_transformation.py +++ b/tests/unit/llms/openai/test_gpt_reasoning_transformation.py @@ -1593,11 +1593,17 @@ def test_gpt_6_astra_takes_the_reasoning_series_request_shape(): @pytest.mark.parametrize( - ("custom_llm_provider", "level"), - [("azure", "none"), ("azure", "xhigh"), ("openai", "xhigh")], + ("custom_llm_provider", "level", "advice"), + [ + ("azure", "none", frozenset(("model_info.reasoning_effort_levels", "model_info.base_model"))), + ("azure", "xhigh", frozenset(("model_info.reasoning_effort_levels", "model_info.base_model"))), + ("openai", "xhigh", frozenset(("model_info.reasoning_effort_levels",))), + ("azure_ai", "xhigh", frozenset(("model_info.reasoning_effort_levels",))), + ("github_copilot", "xhigh", frozenset()), + ], ) -def test_a_reasoning_effort_the_level_gate_drops_logs_a_warning_naming_the_fix( - caplog: pytest.LogCaptureFixture, custom_llm_provider: str, level: str +def test_a_reasoning_effort_the_level_gate_drops_logs_a_warning_naming_only_the_fixes_its_route_honors( + caplog: pytest.LogCaptureFixture, custom_llm_provider: str, level: str, advice: frozenset[str] ) -> None: with caplog.at_level(logging.WARNING, logger="LiteLLM"): params: Final = litellm.get_optional_params( @@ -1614,9 +1620,14 @@ def test_a_reasoning_effort_the_level_gate_drops_logs_a_warning_naming_the_fix( if record.levelno == logging.WARNING and "reasoning_effort" in record.getMessage() ] assert len(drop_warnings) == 1 - assert all( - fragment in drop_warnings[0] - for fragment in (level, "gpt-5.6-terra-ptu", "model_info.base_model", "model_info.reasoning_effort_levels") + assert repr(level) in drop_warnings[0] and "gpt-5.6-terra-ptu" in drop_warnings[0] + assert ( + frozenset( + fragment + for fragment in ("model_info.reasoning_effort_levels", "model_info.base_model") + if fragment in drop_warnings[0] + ) + == advice )