From 9d95382d2f3821059825bc900203d80baf324fda Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 01:35:28 -0700 Subject: [PATCH 01/12] fix(anthropic): keep thinking disabled, display, between_tools, and effort on /v1/messages to OpenAI The Messages to Responses bridge only built reasoning for adaptive and enabled thinking, so disabled, between_tools, and output_config.effort alone reached OpenAI as nothing, and display never asked for a summary. Disabled now maps to effort none, between_tools thinks only after tool results, effort alone maps to reasoning.effort, and display shares the chat route's summary rule --- .../anthropic/messages/adapters/handler.py | 15 +- .../messages/responses_adapters/handler.py | 62 +++++--- .../responses_adapters/transformation.py | 112 +++++++++------ litellm/llms/anthropic/messages/utils.py | 39 ++++-- .../test_responses_adapters_handler.py | 36 ++++- .../test_responses_adapters_transformation.py | 132 ++++++++++++++++-- 6 files changed, 296 insertions(+), 100 deletions(-) diff --git a/litellm/llms/anthropic/messages/adapters/handler.py b/litellm/llms/anthropic/messages/adapters/handler.py index 71ccfbaf915..32c5ad0df4f 100644 --- a/litellm/llms/anthropic/messages/adapters/handler.py +++ b/litellm/llms/anthropic/messages/adapters/handler.py @@ -38,6 +38,7 @@ from litellm.llms.anthropic.messages.utils import ( local_model_name, proxy_sub_call_logging_kwargs, proxy_sub_call_trace_id, + requested_reasoning_summary, ) from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS from litellm.types.llms.anthropic_messages.anthropic_response import ( @@ -461,16 +462,6 @@ class LiteLLMMessagesToCompletionTransformationHandler: updated_reasoning_effort["summary"] = effective_summary completion_kwargs["reasoning_effort"] = updated_reasoning_effort - @staticmethod - def _requested_reasoning_summary(thinking: Mapping[str, object]) -> str | None: - if thinking.get("type") == "disabled": - return None - summary: Final = thinking.get("summary") - if isinstance(summary, str): - return summary - display: Final = thinking.get("display") - return "auto" if display is not None and display != "omitted" else None - @staticmethod def _chat_target_reasoning( completion_kwargs: _CompletionKwargs, @@ -481,9 +472,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: return None reasoning_effort: Final = completion_kwargs.get("reasoning_effort") wraps_summary: Final = isinstance(reasoning_effort, dict) and "summary" in reasoning_effort - requested_summary: Final = LiteLLMMessagesToCompletionTransformationHandler._requested_reasoning_summary( - thinking - ) + requested_summary: Final = requested_reasoning_summary(thinking) if not wraps_summary and requested_summary is None: return None effort: Final[object] = ( diff --git a/litellm/llms/anthropic/messages/responses_adapters/handler.py b/litellm/llms/anthropic/messages/responses_adapters/handler.py index 99a07a1e9bf..8e026b9d298 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/handler.py +++ b/litellm/llms/anthropic/messages/responses_adapters/handler.py @@ -14,7 +14,7 @@ from litellm.integrations.anthropic_cache_control_hook import ( AnthropicCacheControlHook, supports_openai_prompt_cache_breakpoint, ) -from litellm.litellm_core_utils.core_helpers import drops_unsupported_params +from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params from litellm.litellm_core_utils.prompt_templates.common_utils import cache_control_as_prompt_cache_breakpoint from litellm.responses.utils import auto_reasoning_summary from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS @@ -34,7 +34,13 @@ from litellm.utils import ProviderConfigManager from ..adapters.handler import request_safeguards_evaluator from ..convertible_blocks import reject_malformed_tool_result_text from ..safeguards import with_safeguard_results -from ..utils import litellm_logging_obj_from_kwargs, local_model_name +from ..utils import ( + REASONING_EFFORT_OFF, + litellm_logging_obj_from_kwargs, + local_model_name, + model_reasoning_efforts, + normalize_reasoning_effort_value, +) from .streaming_iterator import AnthropicResponsesStreamWrapper from .transformation import LiteLLMAnthropicToResponsesAPIAdapter @@ -119,6 +125,30 @@ def _rewritten_blocks(content: object, rewrite: BlockRewrite) -> Sequence[Mappin return () +def _reasoning_for_target( + reasoning: object, + *, + model: str, + custom_llm_provider: str | None, + deployment_model_info: object, +) -> object: + """A model the map knows does not reason already honors reasoning turned off, and its Responses API would + reject ``reasoning`` outright.""" + reasoning_params: Final = as_str_mapping(reasoning) + effort: Final = reasoning_params.get("effort") if reasoning_params is not None else None + if reasoning_params is None or not isinstance(effort, str): + return reasoning + if effort == REASONING_EFFORT_OFF and model_reasoning_efforts(model, custom_llm_provider) == (): + return None + normalized: Final = normalize_reasoning_effort_value( + effort, + model=model, + custom_llm_provider=custom_llm_provider, + deployment_model_info=deployment_model_info, + ) + return reasoning if normalized == effort else {**reasoning_params, "effort": normalized} + + def _build_responses_kwargs( *, max_tokens: int, @@ -195,25 +225,15 @@ def _build_responses_kwargs( drop_params=drops_unsupported_params(forwarded_kwargs.get("drop_params")), ) - # Normalize reasoning effort based on model capabilities - # (e.g. "max" → "xhigh"/"high", "minimal" → "low" if unsupported) - reasoning: Final = responses_kwargs.get("reasoning") - if isinstance(reasoning, dict): - effort: Final[object] = reasoning.get("effort") - if isinstance(effort, str): - from litellm.llms.anthropic.messages.utils import ( - normalize_reasoning_effort_value, - ) - - provider_hint: Final = forwarded_kwargs.get("custom_llm_provider") - normalized: Final = normalize_reasoning_effort_value( - effort, - model=model, - custom_llm_provider=provider_hint if isinstance(provider_hint, str) else None, - deployment_model_info=forwarded_kwargs.get("model_info"), - ) - if normalized != effort: - responses_kwargs["reasoning"] = {**reasoning, "effort": normalized} + provider_hint: Final = forwarded_kwargs.get("custom_llm_provider") + target_reasoning: Final = _reasoning_for_target( + responses_kwargs.pop("reasoning", None), + model=model, + custom_llm_provider=provider_hint if isinstance(provider_hint, str) else None, + deployment_model_info=forwarded_kwargs.get("model_info"), + ) + if target_reasoning is not None: + responses_kwargs["reasoning"] = target_reasoning if stream: responses_kwargs["stream"] = True diff --git a/litellm/llms/anthropic/messages/responses_adapters/transformation.py b/litellm/llms/anthropic/messages/responses_adapters/transformation.py index 1dd07c1739b..8a35e1401f7 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/messages/responses_adapters/transformation.py @@ -10,6 +10,7 @@ from collections.abc import Iterable, Mapping, Sequence from itertools import groupby from typing import Any, Final, cast +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.prompt_templates.common_utils import ( TOOL_RESULT_IMAGE_BOUNDARY, TOOL_RESULT_IMAGE_PLACEHOLDER, @@ -26,9 +27,11 @@ from litellm.llms.anthropic.messages.native.utils import ( responses_output_refusal_text, ) from litellm.llms.anthropic.messages.utils import ( + REASONING_EFFORT_OFF, is_reasoning_auto_summary_enabled, json_copy_or_original, prompt_cache_key_from_user_id, + requested_reasoning_summary, ) from litellm.types.llms.anthropic import ( AllAnthropicPassThroughMessageValues, @@ -537,47 +540,73 @@ class LiteLLMAnthropicToResponsesAPIAdapter: return result if result else None @staticmethod + def _requested_effort( + thinking: Mapping[str, object], + configured_effort: object, + *, + drop_params: bool, + between_tool_calls: bool, + ) -> object: + match thinking.get("type"): + case "adaptive": + return configured_effort or "medium" + case "enabled": + return reasoning_effort_from_thinking(thinking, drop_params=drop_params) + case "disabled": + return REASONING_EFFORT_OFF + case "between_tools": + return (configured_effort or "medium") if between_tool_calls else REASONING_EFFORT_OFF + case _: + return configured_effort or None + + @staticmethod + def _reasoning_summary(thinking: Mapping[str, object], auto_summary_level: str) -> str | None: + requested: Final = requested_reasoning_summary(thinking) + if requested: + return requested + if thinking.get("display") is not None: + return None + return auto_summary_level if is_reasoning_auto_summary_enabled() else None + + @classmethod def translate_thinking_to_reasoning( - thinking: dict[str, object], - output_config: dict[str, object] | None = None, + cls, + thinking: object, + output_config: object = None, auto_summary_level: str = "detailed", *, drop_params: bool = True, + between_tool_calls: bool = False, ) -> dict[str, object] | None: """ - Convert Anthropic thinking param to Responses API reasoning param. + Convert Anthropic thinking and ``output_config.effort`` to the Responses API reasoning param. - ``thinking.budget_tokens`` is bucketed via the shared - ``reasoning_effort_from_thinking_budget`` thresholds. For adaptive - thinking, uses ``output_config.effort`` if available, otherwise defaults - to medium. + ``between_tools`` thinking reasons only once the model has tool results to read, so a fresh turn turns + reasoning off like ``disabled`` does. """ - if not isinstance(thinking, dict): + thinking_params: Final[Mapping[str, object]] = as_str_mapping(thinking) or {} + output_params: Final = as_str_mapping(output_config) + effort: Final = cls._requested_effort( + thinking_params, + output_params.get("effort") if output_params is not None else None, + drop_params=drop_params, + between_tool_calls=between_tool_calls, + ) + if effort is None: return None + if effort == REASONING_EFFORT_OFF: + return {"effort": effort} + summary: Final = cls._reasoning_summary(thinking_params, auto_summary_level) + return {"effort": effort} if summary is None else {"effort": effort, "summary": summary} - thinking_type: Final = thinking.get("type") - - if thinking_type == "adaptive": - # Use output_config.effort if available - effort = "medium" - if isinstance(output_config, dict) and output_config.get("effort"): - effort = output_config["effort"] - elif thinking_type == "enabled": - budget_effort: Final = reasoning_effort_from_thinking(thinking, drop_params=drop_params) - if budget_effort is None: - return None - effort = budget_effort - else: - return None - - auto_summary: Final = is_reasoning_auto_summary_enabled() - result: Final[dict[str, object]] = {"effort": effort} - summary: Final = thinking.get("summary") - if summary: - result["summary"] = summary - elif auto_summary: - result["summary"] = auto_summary_level - return result + @classmethod + def _is_between_tool_calls(cls, messages: Sequence[object]) -> bool: + last_message: Final = as_str_mapping(messages[-1]) if messages else None + return ( + last_message is not None + and last_message.get("role") == "user" + and any(block.get("type") == "tool_result" for block in cls._content_blocks(last_message.get("content"))) + ) @classmethod def system_blocks_to_instructions(cls, system: object) -> str: @@ -667,18 +696,15 @@ class LiteLLMAnthropicToResponsesAPIAdapter: cast(AnthropicMessagesToolChoice, tool_choice) ) - # thinking -> reasoning - thinking: Final = anthropic_request.get("thinking") - if isinstance(thinking, dict): - output_config = anthropic_request.get("output_config") - reasoning: Final = self.translate_thinking_to_reasoning( - thinking, - output_config=cast(dict[str, object] | None, output_config), - auto_summary_level=auto_summary_level, - drop_params=drop_params, - ) - if reasoning: - responses_kwargs["reasoning"] = reasoning + reasoning: Final = self.translate_thinking_to_reasoning( + anthropic_request.get("thinking"), + output_config=anthropic_request.get("output_config"), + auto_summary_level=auto_summary_level, + drop_params=drop_params, + between_tool_calls=self._is_between_tool_calls(messages_list), + ) + if reasoning: + responses_kwargs["reasoning"] = reasoning # output_format / output_config.format -> text format # output_format: {"type": "json_schema", "schema": {...}} diff --git a/litellm/llms/anthropic/messages/utils.py b/litellm/llms/anthropic/messages/utils.py index 31619a04b74..1009ec6b34c 100644 --- a/litellm/llms/anthropic/messages/utils.py +++ b/litellm/llms/anthropic/messages/utils.py @@ -25,7 +25,7 @@ _EFFORT_DEGRADATION_CHAIN: Final[Mapping[str, tuple[str, ...]]] = MappingProxyTy "minimal": ("minimal", "low"), } ) -_THINKING_OFF: Final = "none" +REASONING_EFFORT_OFF: Final = "none" _JsonValue: Final = TypeVar("_JsonValue") @@ -189,6 +189,28 @@ def is_reasoning_auto_summary_enabled() -> bool: return litellm.reasoning_auto_summary or os.getenv("LITELLM_REASONING_AUTO_SUMMARY", "false").lower() == "true" +def requested_reasoning_summary(thinking: Mapping[str, object]) -> str | None: + if thinking.get("type") == "disabled": + return None + summary: Final = thinking.get("summary") + if isinstance(summary, str): + return summary + display: Final = thinking.get("display") + return "auto" if display is not None and display != "omitted" else None + + +def model_reasoning_efforts(model: str, custom_llm_provider: str | None) -> tuple[str, ...] | None: + """``()`` is a model the map knows takes no effort level, ``None`` a model the map does not know.""" + from litellm.router_utils.reasoning_effort_capability import resolve_supported_reasoning_efforts + from litellm.utils import get_model_info + + try: + model_info: Final[ModelInfo] = get_model_info(model=model, custom_llm_provider=custom_llm_provider) + except Exception: + return None + return resolve_supported_reasoning_efforts(model_info, deployment_is_mapped=True) + + def normalize_reasoning_effort_value( effort: str, model: str, @@ -214,23 +236,14 @@ def normalize_reasoning_effort_value( if chain is None: return effort - from litellm.router_utils.reasoning_effort_capability import ( - deployment_rounds_reasoning_effort, - resolve_supported_reasoning_efforts, - ) + from litellm.router_utils.reasoning_effort_capability import deployment_rounds_reasoning_effort if deployment_rounds_reasoning_effort(deployment_model_info): return effort - from litellm.utils import get_model_info - try: - model_info: Final[ModelInfo] = get_model_info(model=model, custom_llm_provider=custom_llm_provider) - except Exception: - return chain[-1] - - supported: Final = resolve_supported_reasoning_efforts(model_info, deployment_is_mapped=True) + supported: Final = model_reasoning_efforts(model, custom_llm_provider) if not supported: return chain[-1] - accepted_tiers: Final = tuple(level for level in supported if level != _THINKING_OFF) + accepted_tiers: Final = tuple(level for level in supported if level != REASONING_EFFORT_OFF) return next((level for level in (*chain, *accepted_tiers) if level in supported), chain[-1]) diff --git a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py index 7de722e1171..6deacde0b73 100644 --- a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py @@ -3,7 +3,7 @@ import json import os import sys from collections.abc import Mapping -from typing import Final, cast +from typing import Final, Literal, cast from unittest.mock import AsyncMock, patch import pytest @@ -21,6 +21,7 @@ from litellm.llms.anthropic.messages.responses_adapters.handler import ( _build_responses_kwargs, ) from litellm.router import Router +from litellm.router_utils.reasoning_effort_capability import resolve_supported_reasoning_efforts from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS, converted_stream_requested MESSAGES = [{"role": "user", "content": "hello"}] @@ -131,6 +132,39 @@ def test_build_responses_kwargs_auto_summary_asks_each_target_for_a_level_it_acc assert cast(Mapping[str, object], responses_kwargs["reasoning"])["summary"] == summary +@pytest.mark.parametrize("model", ["gpt-6-luna", "gpt-6.1-sol", "gpt-4.1", "gpt-bridge-unmapped"]) +def test_build_responses_kwargs_turns_reasoning_off_unless_the_map_knows_the_model_never_reasons(model: str) -> None: + responses_kwargs: Final = _build_responses_kwargs( + max_tokens=1024, + messages=MESSAGES, + model=f"openai/{model}", + thinking={"type": "disabled"}, + extra_kwargs={"custom_llm_provider": "openai"}, + ) + + map_row: Final = litellm.model_cost.get(model) + never_reasons: Final = map_row is not None and not map_row.get("supports_reasoning") + assert responses_kwargs.get("reasoning") == (None if never_reasons else {"effort": "none"}) + + +@pytest.mark.parametrize("effort", ["max", "xhigh"]) +def test_build_responses_kwargs_forwards_an_output_effort_the_model_accepts(effort: Literal["max", "xhigh"]) -> None: + responses_kwargs: Final = _build_responses_kwargs( + max_tokens=1024, + messages=MESSAGES, + model="openai/gpt-5.6-luna", + output_config={"effort": effort}, + extra_kwargs={"custom_llm_provider": "openai"}, + ) + + forwarded: Final = cast(Mapping[str, object], responses_kwargs["reasoning"])["effort"] + accepted: Final = resolve_supported_reasoning_efforts( + litellm.get_model_info(model="gpt-5.6-luna", custom_llm_provider="openai"), deployment_is_mapped=True + ) + assert accepted is not None and forwarded in accepted + assert (forwarded == effort) == (effort in accepted) + + def test_build_responses_kwargs_keeps_the_deployment_include_next_to_encrypted_reasoning(): responses_kwargs = _build_responses_kwargs( max_tokens=1024, diff --git a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py index f6fcb9e6f1e..253b55430f6 100644 --- a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py @@ -12,6 +12,7 @@ import pytest from openai.types.responses import ResponseInputImageParam from pydantic import TypeAdapter +import litellm from litellm.constants import ( DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET, @@ -29,6 +30,7 @@ from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfi from litellm.types.llms.anthropic import ( AllAnthropicToolsValues, AnthropicMessagesRequest, + AnthropicOutputConfig, ) from litellm.types.llms.openai import ResponseAPIUsage @@ -1021,10 +1023,6 @@ class TestTranslateThinkingToReasoning: assert result_low["effort"] == "low" assert "summary" not in result_low - def test_disabled_type_returns_none(self): - result = _ADAPTER.translate_thinking_to_reasoning({"type": "disabled"}) - assert result is None - def test_non_dict_returns_none(self): result = _ADAPTER.translate_thinking_to_reasoning("enabled") # type: ignore assert result is None @@ -1067,6 +1065,127 @@ class TestTranslateThinkingToReasoning: os.environ.pop("LITELLM_REASONING_AUTO_SUMMARY", None) +def _set_auto_summary(monkeypatch: pytest.MonkeyPatch, enabled: bool) -> None: + monkeypatch.setattr(litellm, "reasoning_auto_summary", enabled) + monkeypatch.delenv("LITELLM_REASONING_AUTO_SUMMARY", raising=False) + + +_FRESH_TURN: Final = ({"role": "user", "content": "What is the weather in Denver?"},) +_TOOL_CALL_TURNS: Final = ( + *_FRESH_TURN, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_1", "name": "weather", "input": {"city": "Denver"}}], + }, + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "toolu_1", "content": "Sunny"}, + {"type": "text", "text": "Keep it short."}, + ], + }, +) +_NEW_QUESTION_AFTER_TOOL_CALLS: Final = ( + *_TOOL_CALL_TURNS, + {"role": "assistant", "content": "It is sunny."}, + {"role": "user", "content": [{"type": "text", "text": "And tomorrow?"}]}, +) +_LOW_BUDGET: Final = {"type": "enabled", "budget_tokens": DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET} + + +@pytest.mark.parametrize( + "thinking, output_config, messages, reasoning", + [ + pytest.param({"type": "disabled"}, {"effort": "high"}, _FRESH_TURN, {"effort": "none"}, id="disabled"), + pytest.param( + {"type": "between_tools"}, {"effort": "high"}, _FRESH_TURN, {"effort": "none"}, id="between-fresh" + ), + pytest.param( + {"type": "between_tools"}, + {"effort": "high"}, + _NEW_QUESTION_AFTER_TOOL_CALLS, + {"effort": "none"}, + id="between-new-question-after-tool-calls", + ), + pytest.param( + {"type": "between_tools"}, + {"effort": "high"}, + _TOOL_CALL_TURNS, + {"effort": "high"}, + id="between-tool-result", + ), + pytest.param({"type": "between_tools"}, None, _TOOL_CALL_TURNS, {"effort": "medium"}, id="between-default"), + pytest.param(None, {"effort": "low"}, _FRESH_TURN, {"effort": "low"}, id="output-effort-alone"), + pytest.param({"type": "future"}, {"effort": "low"}, _FRESH_TURN, {"effort": "low"}, id="unknown-type-effort"), + pytest.param({"type": "adaptive"}, {"effort": "xhigh"}, _FRESH_TURN, {"effort": "xhigh"}, id="adaptive-effort"), + pytest.param({"type": "adaptive"}, None, _FRESH_TURN, {"effort": "medium"}, id="adaptive-default"), + pytest.param(_LOW_BUDGET, {"effort": "high"}, _FRESH_TURN, {"effort": "low"}, id="enabled-budget-wins"), + pytest.param(None, None, _FRESH_TURN, None, id="nothing-set"), + pytest.param({"type": "future"}, None, _FRESH_TURN, None, id="unknown-type-alone"), + ], +) +def test_translate_request_turns_thinking_and_output_effort_into_reasoning( + monkeypatch: pytest.MonkeyPatch, + thinking: dict[str, object] | None, + output_config: AnthropicOutputConfig | None, + messages: tuple[dict[str, object], ...], + reasoning: dict[str, object] | None, +) -> None: + _set_auto_summary(monkeypatch, False) + request: Final = _make_request(messages=list(messages), thinking=thinking, output_config=output_config) + + assert _ADAPTER.translate_request(request).get("reasoning") == reasoning + + +@pytest.mark.parametrize( + "auto_summary, thinking, output_config, reasoning", + [ + pytest.param( + False, + {"type": "adaptive", "display": "summarized"}, + None, + {"effort": "medium", "summary": "auto"}, + id="summarized", + ), + pytest.param( + False, + {"type": "adaptive", "display": "updates"}, + None, + {"effort": "medium", "summary": "auto"}, + id="updates", + ), + pytest.param( + False, {**_LOW_BUDGET, "display": "summarized"}, None, {"effort": "low", "summary": "auto"}, id="enabled" + ), + pytest.param(True, {"type": "adaptive", "display": "omitted"}, None, {"effort": "medium"}, id="omitted"), + pytest.param( + True, + {"type": "adaptive", "summary": "detailed", "display": "summarized"}, + None, + {"effort": "medium", "summary": "detailed"}, + id="summary-beats-display", + ), + pytest.param(True, {"type": "adaptive"}, None, {"effort": "medium", "summary": "concise"}, id="auto-summary"), + pytest.param(False, {"type": "adaptive"}, None, {"effort": "medium"}, id="no-summary-asked"), + pytest.param(True, None, {"effort": "low"}, {"effort": "low", "summary": "concise"}, id="output-effort-alone"), + pytest.param(True, {"type": "disabled", "display": "summarized"}, None, {"effort": "none"}, id="disabled"), + pytest.param( + True, {"type": "between_tools", "display": "summarized"}, None, {"effort": "none"}, id="between-fresh" + ), + ], +) +def test_translate_thinking_to_reasoning_asks_for_the_summary_the_client_shows( + monkeypatch: pytest.MonkeyPatch, + auto_summary: bool, + thinking: dict[str, object] | None, + output_config: dict[str, object] | None, + reasoning: dict[str, object], +) -> None: + _set_auto_summary(monkeypatch, auto_summary) + + assert _ADAPTER.translate_thinking_to_reasoning(thinking, output_config, auto_summary_level="concise") == reasoning + + # --------------------------------------------------------------------------- # translate_request – broader coverage # --------------------------------------------------------------------------- @@ -1178,11 +1297,6 @@ class TestTranslateRequestBroaderCoverage: assert kwargs["reasoning"] == {"effort": "high"} assert "summary" not in kwargs["reasoning"] - def test_disabled_thinking_not_included_in_kwargs(self): - req = _make_request(thinking={"type": "disabled"}) - kwargs = _ADAPTER.translate_request(req) - assert "reasoning" not in kwargs - def test_thinking_asks_for_the_encrypted_reasoning(self): """The documented way to get reasoning that survives store=false is to ask for it.""" req = _make_request(thinking={"type": "enabled", "budget_tokens": 12000}) From a67dc7c58d57a7936edbad452230a2ed53a8f191 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 2 Nov 2026 18:47:18 -0800 Subject: [PATCH 02/12] fix(anthropic): drop an unconfirmed reasoning off switch under drop_params and read the whole final user turn between_tools now treats every user message after the last assistant message as the final turn, so a tool_result followed by a separate text message still counts as between tool calls, and it asks for a reasoning summary there the way Anthropic returns progress updates. thinking disabled or a first-turn between_tools on a reasoning model whose cost map row does not set supports_none_reasoning_effort (o3, o4-mini) no longer sends effort none under drop_params, which OpenAI refused with a 400. --- .../messages/responses_adapters/handler.py | 41 ++++++++++++++++--- .../responses_adapters/transformation.py | 21 ++++++---- .../test_responses_adapters_handler.py | 18 +++++++- .../test_responses_adapters_transformation.py | 22 +++++++++- 4 files changed, 86 insertions(+), 16 deletions(-) diff --git a/litellm/llms/anthropic/messages/responses_adapters/handler.py b/litellm/llms/anthropic/messages/responses_adapters/handler.py index c2b378701c5..0ac8da68f5c 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/handler.py +++ b/litellm/llms/anthropic/messages/responses_adapters/handler.py @@ -39,7 +39,7 @@ from litellm.types.llms.anthropic_messages.anthropic_response import ( ) from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.utils import CountContextManagement -from litellm.utils import ProviderConfigManager +from litellm.utils import ProviderConfigManager, supports_none_reasoning_effort, supports_reasoning from ..adapters.handler import context_management_explicitly_dropped, request_safeguards_evaluator from ..context_management import AnthropicContextManagementError, PolyfillResult, apply_context_management @@ -253,21 +253,48 @@ def _rewritten_blocks(content: object, rewrite: BlockRewrite) -> Sequence[Mappin return () +def _reasoning_off_for_target( + reasoning: Mapping[str, object], + *, + model: str, + custom_llm_provider: str | None, + drop_params: bool, +) -> Mapping[str, object] | None: + """A model the map knows does not reason already honors reasoning turned off, and its Responses API would + reject ``reasoning`` outright. A reasoning model whose row does not confirm ``none`` may refuse it, so + drop_params drops it there instead of letting the provider answer 400.""" + if model_reasoning_efforts(model, custom_llm_provider) == (): + return None + if ( + not drop_params + or not supports_reasoning(model, custom_llm_provider) + or supports_none_reasoning_effort(model, custom_llm_provider) + ): + return reasoning + verbose_logger.warning( + "Dropping reasoning.effort=none for %s, its model cost map entry does not set " + "supports_none_reasoning_effort, because drop_params is set", + model, + ) + return None + + def _reasoning_for_target( reasoning: object, *, model: str, custom_llm_provider: str | None, deployment_model_info: object, + drop_params: bool, ) -> object: - """A model the map knows does not reason already honors reasoning turned off, and its Responses API would - reject ``reasoning`` outright.""" reasoning_params: Final = as_str_mapping(reasoning) effort: Final = reasoning_params.get("effort") if reasoning_params is not None else None if reasoning_params is None or not isinstance(effort, str): return reasoning - if effort == REASONING_EFFORT_OFF and model_reasoning_efforts(model, custom_llm_provider) == (): - return None + if effort == REASONING_EFFORT_OFF: + return _reasoning_off_for_target( + reasoning_params, model=model, custom_llm_provider=custom_llm_provider, drop_params=drop_params + ) normalized: Final = normalize_reasoning_effort_value( effort, model=model, @@ -344,13 +371,14 @@ def _build_responses_kwargs( request_data["output_format"] = output_format anthropic_request: Final = AnthropicMessagesRequest(**request_data) + drop_params: Final = drops_unsupported_params(forwarded_kwargs.get("drop_params")) responses_kwargs: Final = _ADAPTER.translate_request( anthropic_request, include_encrypted_reasoning=_provider_returns_encrypted_reasoning( model, forwarded_kwargs.get("custom_llm_provider") ), auto_summary_level=auto_reasoning_summary(model, forwarded_kwargs.get("custom_llm_provider")), - drop_params=drops_unsupported_params(forwarded_kwargs.get("drop_params")), + drop_params=drop_params, ) provider_hint: Final = forwarded_kwargs.get("custom_llm_provider") @@ -359,6 +387,7 @@ def _build_responses_kwargs( model=model, custom_llm_provider=provider_hint if isinstance(provider_hint, str) else None, deployment_model_info=forwarded_kwargs.get("model_info"), + drop_params=drop_params, ) if target_reasoning is not None: responses_kwargs["reasoning"] = target_reasoning diff --git a/litellm/llms/anthropic/messages/responses_adapters/transformation.py b/litellm/llms/anthropic/messages/responses_adapters/transformation.py index 5ef4cb3a8c5..22c7e055c5f 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/messages/responses_adapters/transformation.py @@ -6,8 +6,8 @@ path used for OpenAI and Azure models. """ import json -from collections.abc import Iterable, Mapping, Sequence -from itertools import groupby +from collections.abc import Iterable, Iterator, Mapping, Sequence +from itertools import chain, groupby from typing import Any, Final, cast from litellm.litellm_core_utils.core_helpers import as_str_mapping @@ -554,6 +554,8 @@ class LiteLLMAnthropicToResponsesAPIAdapter: @staticmethod def _reasoning_summary(thinking: Mapping[str, object], auto_summary_level: str) -> str | None: + if thinking.get("type") == "between_tools": + return "auto" requested: Final = requested_reasoning_summary(thinking) if requested: return requested @@ -594,14 +596,19 @@ class LiteLLMAnthropicToResponsesAPIAdapter: summary: Final = cls._reasoning_summary(thinking_params, auto_summary_level) return {"effort": effort} if summary is None else {"effort": effort, "summary": summary} + @staticmethod + def _final_user_turn(messages: Sequence[object]) -> Iterator[Mapping[str, object]]: + for message in map(as_str_mapping, reversed(messages)): + if message is None or message.get("role") != "user": + return + yield message + @classmethod def _is_between_tool_calls(cls, messages: Sequence[object]) -> bool: - last_message: Final = as_str_mapping(messages[-1]) if messages else None - return ( - last_message is not None - and last_message.get("role") == "user" - and any(block.get("type") == "tool_result" for block in cls._content_blocks(last_message.get("content"))) + blocks: Final = chain.from_iterable( + cls._content_blocks(message.get("content")) for message in cls._final_user_turn(messages) ) + return any(block.get("type") == "tool_result" for block in blocks) @classmethod def system_blocks_to_instructions(cls, system: object) -> str: diff --git a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py index d71fbef24dd..df8999a70f8 100644 --- a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py @@ -133,7 +133,7 @@ def test_build_responses_kwargs_auto_summary_asks_each_target_for_a_level_it_acc assert cast(Mapping[str, object], responses_kwargs["reasoning"])["summary"] == summary -@pytest.mark.parametrize("model", ["gpt-6-luna", "gpt-6.1-sol", "gpt-4.1", "gpt-bridge-unmapped"]) +@pytest.mark.parametrize("model", ["gpt-6-luna", "gpt-6.1-sol", "o3", "gpt-4.1", "gpt-bridge-unmapped"]) def test_build_responses_kwargs_turns_reasoning_off_unless_the_map_knows_the_model_never_reasons(model: str) -> None: responses_kwargs: Final = _build_responses_kwargs( max_tokens=1024, @@ -148,6 +148,22 @@ def test_build_responses_kwargs_turns_reasoning_off_unless_the_map_knows_the_mod assert responses_kwargs.get("reasoning") == (None if never_reasons else {"effort": "none"}) +@pytest.mark.parametrize("model", ["gpt-6-luna", "gpt-6.1-sol", "o3", "gpt-4.1", "gpt-bridge-unmapped"]) +def test_build_responses_kwargs_drops_reasoning_off_under_drop_params_unless_the_map_confirms_none(model: str) -> None: + responses_kwargs: Final = _build_responses_kwargs( + max_tokens=1024, + messages=MESSAGES, + model=f"openai/{model}", + thinking={"type": "disabled"}, + extra_kwargs={"custom_llm_provider": "openai", "drop_params": True}, + ) + + map_row: Final = litellm.model_cost.get(model) + confirms_none: Final = map_row is not None and map_row.get("supports_none_reasoning_effort") is True + keeps_none: Final = map_row is None or (bool(map_row.get("supports_reasoning")) and confirms_none) + assert responses_kwargs.get("reasoning") == ({"effort": "none"} if keeps_none else None) + + @pytest.mark.parametrize("effort", ["max", "xhigh"]) def test_build_responses_kwargs_forwards_an_output_effort_the_model_accepts(effort: Literal["max", "xhigh"]) -> None: responses_kwargs: Final = _build_responses_kwargs( diff --git a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py index fee5332fd65..e9085ad33b8 100644 --- a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py @@ -1097,6 +1097,11 @@ _NEW_QUESTION_AFTER_TOOL_CALLS: Final = ( {"role": "assistant", "content": "It is sunny."}, {"role": "user", "content": [{"type": "text", "text": "And tomorrow?"}]}, ) +_TOOL_RESULT_SPLIT_FROM_TEXT: Final = ( + *_TOOL_CALL_TURNS[:2], + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_1", "content": "Sunny"}]}, + {"role": "user", "content": "Keep it short."}, +) _LOW_BUDGET: Final = {"type": "enabled", "budget_tokens": DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET} @@ -1118,10 +1123,23 @@ _LOW_BUDGET: Final = {"type": "enabled", "budget_tokens": DEFAULT_REASONING_EFFO {"type": "between_tools"}, {"effort": "high"}, _TOOL_CALL_TURNS, - {"effort": "high"}, + {"effort": "high", "summary": "auto"}, id="between-tool-result", ), - pytest.param({"type": "between_tools"}, None, _TOOL_CALL_TURNS, {"effort": "medium"}, id="between-default"), + pytest.param( + {"type": "between_tools"}, + {"effort": "high"}, + _TOOL_RESULT_SPLIT_FROM_TEXT, + {"effort": "high", "summary": "auto"}, + id="between-tool-result-split-from-text", + ), + pytest.param( + {"type": "between_tools"}, + None, + _TOOL_CALL_TURNS, + {"effort": "medium", "summary": "auto"}, + id="between-default", + ), pytest.param(None, {"effort": "low"}, _FRESH_TURN, {"effort": "low"}, id="output-effort-alone"), pytest.param({"type": "future"}, {"effort": "low"}, _FRESH_TURN, {"effort": "low"}, id="unknown-type-effort"), pytest.param({"type": "adaptive"}, {"effort": "xhigh"}, _FRESH_TURN, {"effort": "xhigh"}, id="adaptive-effort"), From 4033448fa067002cee0e260fdce41620a013067f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 2 Nov 2026 19:51:22 -0800 Subject: [PATCH 03/12] fix(anthropic): keep reasoning off for a bare effort on models that reason only when asked A request with output_config.effort and no thinking now leaves reasoning off on a model whose cost map row sets default_reasoning_effort none (gpt-5.1, gpt-5.2, the gpt-5.4 family). Turning it on there made Claude Code with thinking off pay for reasoning, and made a temperature the model accepts at none answer 400. Models that reason by default still get the effort. between_tools now honors thinking.summary and display omitted before falling back to summary auto, and the new helpers lose their docstrings. --- .../messages/responses_adapters/handler.py | 22 +++++++++++-------- .../responses_adapters/transformation.py | 4 ++-- litellm/llms/anthropic/messages/utils.py | 19 ++++++++++++---- .../test_responses_adapters_handler.py | 21 ++++++++++++++++++ .../test_responses_adapters_transformation.py | 14 ++++++++++++ 5 files changed, 65 insertions(+), 15 deletions(-) diff --git a/litellm/llms/anthropic/messages/responses_adapters/handler.py b/litellm/llms/anthropic/messages/responses_adapters/handler.py index 0ac8da68f5c..719b3815a9d 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/handler.py +++ b/litellm/llms/anthropic/messages/responses_adapters/handler.py @@ -53,6 +53,7 @@ from ..utils import ( local_model_name, model_reasoning_efforts, normalize_reasoning_effort_value, + reasoning_defaults_off, ) from .streaming_iterator import AnthropicResponsesStreamWrapper from .transformation import LiteLLMAnthropicToResponsesAPIAdapter @@ -260,9 +261,6 @@ def _reasoning_off_for_target( custom_llm_provider: str | None, drop_params: bool, ) -> Mapping[str, object] | None: - """A model the map knows does not reason already honors reasoning turned off, and its Responses API would - reject ``reasoning`` outright. A reasoning model whose row does not confirm ``none`` may refuse it, so - drop_params drops it there instead of letting the provider answer 400.""" if model_reasoning_efforts(model, custom_llm_provider) == (): return None if ( @@ -382,12 +380,18 @@ def _build_responses_kwargs( ) provider_hint: Final = forwarded_kwargs.get("custom_llm_provider") - target_reasoning: Final = _reasoning_for_target( - responses_kwargs.pop("reasoning", None), - model=model, - custom_llm_provider=provider_hint if isinstance(provider_hint, str) else None, - deployment_model_info=forwarded_kwargs.get("model_info"), - drop_params=drop_params, + provider: Final = provider_hint if isinstance(provider_hint, str) else None + bridged_reasoning: Final = responses_kwargs.pop("reasoning", None) + target_reasoning: Final = ( + None + if not thinking and reasoning_defaults_off(model, provider) + else _reasoning_for_target( + bridged_reasoning, + model=model, + custom_llm_provider=provider, + deployment_model_info=forwarded_kwargs.get("model_info"), + drop_params=drop_params, + ) ) if target_reasoning is not None: responses_kwargs["reasoning"] = target_reasoning diff --git a/litellm/llms/anthropic/messages/responses_adapters/transformation.py b/litellm/llms/anthropic/messages/responses_adapters/transformation.py index 22c7e055c5f..9f8dd85a838 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/messages/responses_adapters/transformation.py @@ -554,13 +554,13 @@ class LiteLLMAnthropicToResponsesAPIAdapter: @staticmethod def _reasoning_summary(thinking: Mapping[str, object], auto_summary_level: str) -> str | None: - if thinking.get("type") == "between_tools": - return "auto" requested: Final = requested_reasoning_summary(thinking) if requested: return requested if thinking.get("display") is not None: return None + if thinking.get("type") == "between_tools": + return "auto" return auto_summary_level if is_reasoning_auto_summary_enabled() else None @classmethod diff --git a/litellm/llms/anthropic/messages/utils.py b/litellm/llms/anthropic/messages/utils.py index 758da138c6c..e65c11012ea 100644 --- a/litellm/llms/anthropic/messages/utils.py +++ b/litellm/llms/anthropic/messages/utils.py @@ -199,18 +199,29 @@ def requested_reasoning_summary(thinking: Mapping[str, object]) -> str | None: return "auto" if display is not None and display != "omitted" else None -def model_reasoning_efforts(model: str, custom_llm_provider: str | None) -> tuple[str, ...] | None: - """``()`` is a model the map knows takes no effort level, ``None`` a model the map does not know.""" - from litellm.litellm_core_utils.reasoning_effort_capability import resolve_supported_reasoning_efforts +def _mapped_model_info(model: str, custom_llm_provider: str | None) -> ModelInfo | None: from litellm.utils import get_model_info try: - model_info: Final[ModelInfo] = get_model_info(model=model, custom_llm_provider=custom_llm_provider) + return get_model_info(model=model, custom_llm_provider=custom_llm_provider) except Exception: return None + + +def model_reasoning_efforts(model: str, custom_llm_provider: str | None) -> tuple[str, ...] | None: + from litellm.litellm_core_utils.reasoning_effort_capability import resolve_supported_reasoning_efforts + + model_info: Final = _mapped_model_info(model, custom_llm_provider) + if model_info is None: + return None return resolve_supported_reasoning_efforts(model_info, deployment_is_mapped=True) +def reasoning_defaults_off(model: str, custom_llm_provider: str | None) -> bool: + model_info: Final = _mapped_model_info(model, custom_llm_provider) + return model_info is not None and model_info.get("default_reasoning_effort") == REASONING_EFFORT_OFF + + def normalize_reasoning_effort_value( effort: str, model: str, diff --git a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py index df8999a70f8..3638366fffd 100644 --- a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py @@ -164,6 +164,27 @@ def test_build_responses_kwargs_drops_reasoning_off_under_drop_params_unless_the assert responses_kwargs.get("reasoning") == ({"effort": "none"} if keeps_none else None) +@pytest.mark.parametrize("model", ["gpt-5.4-mini", "gpt-5.1", "gpt-6-luna", "gpt-bridge-unmapped"]) +@pytest.mark.parametrize("thinking", [None, {"type": "adaptive"}], ids=["thinking-left-out", "adaptive"]) +def test_build_responses_kwargs_never_turns_on_reasoning_the_client_left_off( + model: str, thinking: dict[str, object] | None +) -> None: + responses_kwargs: Final = _build_responses_kwargs( + max_tokens=1024, + messages=MESSAGES, + model=f"openai/{model}", + output_config={"effort": "high"}, + thinking=thinking, + extra_kwargs={"custom_llm_provider": "openai"}, + ) + + map_row: Final = litellm.model_cost.get(model) + reasons_only_when_asked: Final = map_row is not None and map_row.get("default_reasoning_effort") == "none" + reasoning: Final = responses_kwargs.get("reasoning") + assert (reasoning is None) == (thinking is None and reasons_only_when_asked) + assert reasoning is None or cast(Mapping[str, object], reasoning)["effort"] != "none" + + @pytest.mark.parametrize("effort", ["max", "xhigh"]) def test_build_responses_kwargs_forwards_an_output_effort_the_model_accepts(effort: Literal["max", "xhigh"]) -> None: responses_kwargs: Final = _build_responses_kwargs( diff --git a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py index e9085ad33b8..52ea850042b 100644 --- a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_transformation.py @@ -1140,6 +1140,20 @@ _LOW_BUDGET: Final = {"type": "enabled", "budget_tokens": DEFAULT_REASONING_EFFO {"effort": "medium", "summary": "auto"}, id="between-default", ), + pytest.param( + {"type": "between_tools", "summary": "concise"}, + {"effort": "high"}, + _TOOL_CALL_TURNS, + {"effort": "high", "summary": "concise"}, + id="between-tool-result-own-summary", + ), + pytest.param( + {"type": "between_tools", "display": "omitted"}, + {"effort": "high"}, + _TOOL_CALL_TURNS, + {"effort": "high"}, + id="between-tool-result-omitted", + ), pytest.param(None, {"effort": "low"}, _FRESH_TURN, {"effort": "low"}, id="output-effort-alone"), pytest.param({"type": "future"}, {"effort": "low"}, _FRESH_TURN, {"effort": "low"}, id="unknown-type-effort"), pytest.param({"type": "adaptive"}, {"effort": "xhigh"}, _FRESH_TURN, {"effort": "xhigh"}, id="adaptive-effort"), From f0bca7f7b0411bab4c692f25d4314c988e963863 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 2 Nov 2026 20:42:02 -0800 Subject: [PATCH 04/12] fix(anthropic): let a bare effort yield to reasoning_effort and unmapped models, drop an undefined effort under drop_params --- .../messages/responses_adapters/handler.py | 28 ++++++++++--- litellm/llms/anthropic/messages/utils.py | 4 +- .../test_responses_adapters_handler.py | 39 ++++++++++++++++++- 3 files changed, 61 insertions(+), 10 deletions(-) diff --git a/litellm/llms/anthropic/messages/responses_adapters/handler.py b/litellm/llms/anthropic/messages/responses_adapters/handler.py index 719b3815a9d..91420565f48 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/handler.py +++ b/litellm/llms/anthropic/messages/responses_adapters/handler.py @@ -5,7 +5,7 @@ Used when the target model is an OpenAI or Azure model. """ from collections.abc import AsyncIterator, Callable, Coroutine, Mapping, Sequence -from typing import Final, TypeAlias +from typing import Final, TypeAlias, get_args from pydantic import TypeAdapter, ValidationError @@ -37,7 +37,7 @@ from litellm.types.llms.anthropic import ( from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, ) -from litellm.types.llms.openai import ResponsesAPIResponse +from litellm.types.llms.openai import REASONING_EFFORT, ResponsesAPIResponse from litellm.types.utils import CountContextManagement from litellm.utils import ProviderConfigManager, supports_none_reasoning_effort, supports_reasoning @@ -51,9 +51,9 @@ from ..utils import ( REASONING_EFFORT_OFF, litellm_logging_obj_from_kwargs, local_model_name, + maps_bare_output_effort, model_reasoning_efforts, normalize_reasoning_effort_value, - reasoning_defaults_off, ) from .streaming_iterator import AnthropicResponsesStreamWrapper from .transformation import LiteLLMAnthropicToResponsesAPIAdapter @@ -67,6 +67,7 @@ _SplitEdits: TypeAlias = tuple[dict[str, object] | None, Sequence[Mapping[str, o AnthropicContextManagementSpec: TypeAlias = dict[str, object] | list[dict[str, object]] | None _ADAPTER: Final = LiteLLMAnthropicToResponsesAPIAdapter() +_KNOWN_REASONING_EFFORTS: Final = frozenset(get_args(REASONING_EFFORT)) def _split_context_management( @@ -293,6 +294,13 @@ def _reasoning_for_target( return _reasoning_off_for_target( reasoning_params, model=model, custom_llm_provider=custom_llm_provider, drop_params=drop_params ) + if drop_params and effort not in _KNOWN_REASONING_EFFORTS: + verbose_logger.warning( + "Dropping reasoning.effort=%r for %s, no reasoning API defines it, because drop_params is set", + effort, + model, + ) + return None normalized: Final = normalize_reasoning_effort_value( effort, model=model, @@ -302,6 +310,14 @@ def _reasoning_for_target( return reasoning if normalized == effort else {**reasoning_params, "effort": normalized} +def _maps_reasoning( + thinking: Mapping[str, object] | None, model: str, provider: str | None, forwarded_kwargs: Mapping[str, object] +) -> bool: + return bool(thinking) or ( + forwarded_kwargs.get("reasoning_effort") is None and maps_bare_output_effort(model, provider) + ) + + def _build_responses_kwargs( *, max_tokens: int, @@ -383,15 +399,15 @@ def _build_responses_kwargs( provider: Final = provider_hint if isinstance(provider_hint, str) else None bridged_reasoning: Final = responses_kwargs.pop("reasoning", None) target_reasoning: Final = ( - None - if not thinking and reasoning_defaults_off(model, provider) - else _reasoning_for_target( + _reasoning_for_target( bridged_reasoning, model=model, custom_llm_provider=provider, deployment_model_info=forwarded_kwargs.get("model_info"), drop_params=drop_params, ) + if _maps_reasoning(thinking, model, provider, forwarded_kwargs) + else None ) if target_reasoning is not None: responses_kwargs["reasoning"] = target_reasoning diff --git a/litellm/llms/anthropic/messages/utils.py b/litellm/llms/anthropic/messages/utils.py index e65c11012ea..bff33025295 100644 --- a/litellm/llms/anthropic/messages/utils.py +++ b/litellm/llms/anthropic/messages/utils.py @@ -217,9 +217,9 @@ def model_reasoning_efforts(model: str, custom_llm_provider: str | None) -> tupl return resolve_supported_reasoning_efforts(model_info, deployment_is_mapped=True) -def reasoning_defaults_off(model: str, custom_llm_provider: str | None) -> bool: +def maps_bare_output_effort(model: str, custom_llm_provider: str | None) -> bool: model_info: Final = _mapped_model_info(model, custom_llm_provider) - return model_info is not None and model_info.get("default_reasoning_effort") == REASONING_EFFORT_OFF + return model_info is not None and model_info.get("default_reasoning_effort") != REASONING_EFFORT_OFF def normalize_reasoning_effort_value( diff --git a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py index 3638366fffd..37d912c1f80 100644 --- a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py @@ -24,6 +24,7 @@ from litellm.llms.anthropic.messages.responses_adapters.handler import ( from litellm.router import Router from litellm.litellm_core_utils.reasoning_effort_capability import resolve_supported_reasoning_efforts from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS, converted_stream_requested +from litellm.types.llms.anthropic import AnthropicOutputConfig MESSAGES = [{"role": "user", "content": "hello"}] CLAUDE_CODE_USER_ID = json.dumps({"device_id": "d" * 64, "account_uuid": "", "session_id": "session-abc"}) @@ -179,12 +180,46 @@ def test_build_responses_kwargs_never_turns_on_reasoning_the_client_left_off( ) map_row: Final = litellm.model_cost.get(model) - reasons_only_when_asked: Final = map_row is not None and map_row.get("default_reasoning_effort") == "none" + reasons_unasked: Final = map_row is not None and map_row.get("default_reasoning_effort") != "none" reasoning: Final = responses_kwargs.get("reasoning") - assert (reasoning is None) == (thinking is None and reasons_only_when_asked) + assert (reasoning is None) == (thinking is None and not reasons_unasked) assert reasoning is None or cast(Mapping[str, object], reasoning)["effort"] != "none" +@pytest.mark.parametrize("thinking", [None, {"type": "adaptive"}], ids=["thinking-left-out", "adaptive"]) +def test_build_responses_kwargs_leaves_a_bare_output_effort_to_an_explicit_reasoning_effort( + thinking: dict[str, object] | None, +) -> None: + responses_kwargs: Final = _build_responses_kwargs( + max_tokens=1024, + messages=MESSAGES, + model="openai/gpt-5-mini", + output_config={"effort": "high"}, + thinking=thinking, + extra_kwargs={"custom_llm_provider": "openai", "reasoning_effort": "minimal"}, + ) + + assert responses_kwargs["reasoning_effort"] == "minimal" + assert ("reasoning" in responses_kwargs) == (thinking is not None) + + +@pytest.mark.parametrize("drop_params", [True, False], ids=["drop_params", "no-drop_params"]) +@pytest.mark.parametrize("thinking", [None, {"type": "adaptive"}], ids=["thinking-left-out", "adaptive"]) +def test_build_responses_kwargs_drops_an_effort_no_reasoning_api_defines_only_under_drop_params( + drop_params: bool, thinking: dict[str, object] | None +) -> None: + responses_kwargs: Final = _build_responses_kwargs( + max_tokens=1024, + messages=MESSAGES, + model="openai/gpt-6-luna", + output_config=cast(AnthropicOutputConfig, {"effort": "turbo"}), + thinking=thinking, + extra_kwargs={"custom_llm_provider": "openai", "drop_params": drop_params}, + ) + + assert responses_kwargs.get("reasoning") == (None if drop_params else {"effort": "turbo"}) + + @pytest.mark.parametrize("effort", ["max", "xhigh"]) def test_build_responses_kwargs_forwards_an_output_effort_the_model_accepts(effort: Literal["max", "xhigh"]) -> None: responses_kwargs: Final = _build_responses_kwargs( From 89c3ee44fa001c5b5d2d5b39b42232b4a00c539e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 2 Nov 2026 23:28:26 -0800 Subject: [PATCH 05/12] fix(anthropic): retry an unverified org without the reasoning summary, keep disabled reasoning off at the lowest tier, and match between_tools on the chat route Under drop_params, the Responses bridge now retries once without reasoning.summary when OpenAI refuses summaries to an unverified organization. It drops an effort the model's own levels exclude unless the deployment rounds efforts itself. It sends minimal or low for disabled when the model lists no none. The chat-completions route now maps between_tools and a bare output_config.effort for non-Claude targets the same way the bridge does, through a shared tool-turn check --- .../anthropic/messages/adapters/handler.py | 6 +- .../messages/adapters/transformation.py | 71 +++++-- .../messages/responses_adapters/handler.py | 81 ++++++- .../responses_adapters/transformation.py | 38 +--- litellm/llms/anthropic/messages/utils.py | 34 ++- ..._handler_reasoning_effort_normalization.py | 159 ++++++++++++++ .../test_responses_adapters_handler.py | 201 +++++++++++++++++- .../messages/test_reasoning_effort_fields.py | 19 ++ 8 files changed, 541 insertions(+), 68 deletions(-) diff --git a/litellm/llms/anthropic/messages/adapters/handler.py b/litellm/llms/anthropic/messages/adapters/handler.py index fafa4fbf0bc..2e590093b9c 100644 --- a/litellm/llms/anthropic/messages/adapters/handler.py +++ b/litellm/llms/anthropic/messages/adapters/handler.py @@ -32,6 +32,7 @@ from litellm.llms.anthropic.messages.safeguards import ( with_safeguard_results, ) from litellm.llms.anthropic.messages.utils import ( + REASONING_EFFORT_OFF, ProxySubCallLoggingKwargs, is_reasoning_auto_summary_enabled, litellm_logging_obj_from_kwargs, @@ -470,9 +471,9 @@ class LiteLLMMessagesToCompletionTransformationHandler: *, thinking: Mapping[str, object] | None, ) -> _ChatTargetReasoning | None: - if not thinking: - return None reasoning_effort: Final = completion_kwargs.get("reasoning_effort") + if not thinking or reasoning_effort == REASONING_EFFORT_OFF: + return None wraps_summary: Final = isinstance(reasoning_effort, dict) and "summary" in reasoning_effort requested_summary: Final = requested_reasoning_summary(thinking) if not wraps_summary and requested_summary is None: @@ -646,6 +647,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None, api_base=api_base if isinstance(api_base, str) else None, drop_params=drops_unsupported_params(extra_kwargs.get("drop_params")), + translate_bare_output_effort=extra_kwargs.get("reasoning_effort") is None, ) if openai_request is None: diff --git a/litellm/llms/anthropic/messages/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py index 4b1b5a8a11a..2b89ca31bb6 100644 --- a/litellm/llms/anthropic/messages/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -12,8 +12,11 @@ from litellm.litellm_core_utils.core_helpers import normalize_drop_params from litellm.litellm_core_utils.hidden_params import get_hidden_params from litellm.llms.anthropic.messages.convertible_blocks import tool_result_text_part from litellm.llms.anthropic.messages.utils import ( + REASONING_EFFORT_OFF, + is_between_tool_calls, is_reasoning_auto_summary_enabled, json_copy_or_original, + maps_bare_output_effort, prompt_cache_key_from_user_id, ) from litellm.responses.utils import auto_reasoning_summary @@ -265,6 +268,7 @@ class AnthropicAdapter: custom_llm_provider: str | None = None, api_base: str | None = None, drop_params: bool = True, + translate_bare_output_effort: bool = False, ) -> tuple[ChatCompletionRequest | None, dict[str, str]]: """ Translate Anthropic request params to OpenAI format, returning tool name mapping. @@ -301,6 +305,7 @@ class AnthropicAdapter: api_base=api_base, drop_params=drop_params, report_reasoning_drops=True, + translate_bare_output_effort=translate_bare_output_effort, ) return translated_body, tool_name_mapping @@ -790,7 +795,7 @@ class LiteLLMAnthropicMessagesAdapter: reasoning_effort as a plain string, not a summary dict. """ thinking_type: Final = thinking.get("type") if isinstance(thinking, dict) else None - if thinking_type == "disabled": + if thinking_type == "disabled" or reasoning_effort == REASONING_EFFORT_OFF: return reasoning_effort summary: Final = thinking.get("summary") if isinstance(thinking, dict) else None @@ -1152,6 +1157,26 @@ class LiteLLMAnthropicMessagesAdapter: new_kwargs["tools"] = translated_tools return tool_name_mapping + @classmethod + def _is_claude_target(cls, model: str, custom_llm_provider: str | None) -> bool: + return ( + cls.is_anthropic_claude_model(model) or cls.is_bedrock_arn_model(model) + ) and custom_llm_provider not in _CLAUDE_OVER_REASONING_EFFORT_PROVIDERS + + @staticmethod + def _bare_output_effort(output_effort: object, model: str, custom_llm_provider: str | None) -> str | None: + return ( + output_effort + if isinstance(output_effort, str) and maps_bare_output_effort(model, custom_llm_provider) + else None + ) + + @staticmethod + def _between_tools_reasoning_effort(anthropic_message_request: Mapping[str, object], output_effort: object) -> str: + if not is_between_tool_calls(anthropic_message_request): + return REASONING_EFFORT_OFF + return output_effort if isinstance(output_effort, str) and output_effort else "medium" + def _translate_thinking_to_openai( self, anthropic_message_request: AnthropicMessagesRequest, @@ -1160,6 +1185,7 @@ class LiteLLMAnthropicMessagesAdapter: custom_llm_provider: str | None = None, drop_params: bool = True, report_reasoning_drops: bool = False, + translate_bare_output_effort: bool = False, ) -> None: """Translate Anthropic thinking to either thinking or reasoning_effort. @@ -1185,23 +1211,16 @@ class LiteLLMAnthropicMessagesAdapter: An adaptive request with no tier stays untouched either way, so the provider's own default still applies. """ - if "thinking" not in anthropic_message_request: - return - - thinking: Final = anthropic_message_request["thinking"] - if not thinking: - return - model: Final = new_kwargs.get("model", "") + output_config: Final = anthropic_message_request.get("output_config") + output_effort: Final = output_config.get("effort") if isinstance(output_config, dict) else None + thinking: Final = anthropic_message_request.get("thinking") is_bedrock_target: Final = model.startswith(("bedrock/", "converse/", "invoke/")) or self.is_bedrock_arn_model( model ) - is_claude_target: Final = ( - self.is_anthropic_claude_model(model) or self.is_bedrock_arn_model(model) - ) and custom_llm_provider not in _CLAUDE_OVER_REASONING_EFFORT_PROVIDERS - output_config: Final = anthropic_message_request.get("output_config") + is_claude_target: Final = self._is_claude_target(model, custom_llm_provider) - if is_claude_target: + if thinking and is_claude_target: new_kwargs["thinking"] = thinking if is_bedrock_target: if isinstance(output_config, dict): @@ -1213,24 +1232,30 @@ class LiteLLMAnthropicMessagesAdapter: return thinking_type: Final = thinking.get("type") if isinstance(thinking, dict) else None - declared_effort: Final = ( - output_config.get("effort") if thinking_type == "adaptive" and isinstance(output_config, dict) else None - ) + declared_effort: Final = output_effort if thinking_type == "adaptive" else None if is_claude_target and not declared_effort: return - reasoning_effort: Final = declared_effort or self.translate_anthropic_thinking_to_reasoning_effort( - cast(AnthropicThinkingParam, thinking), - model=model, - drop_params=drop_params, - report_reasoning_drops=report_reasoning_drops, + reasoning_effort: Final = declared_effort or ( + self._between_tools_reasoning_effort(anthropic_message_request, output_effort) + if thinking_type == "between_tools" + else self.translate_anthropic_thinking_to_reasoning_effort( + cast(AnthropicThinkingParam, thinking), + model=model, + drop_params=drop_params, + report_reasoning_drops=report_reasoning_drops, + ) + if thinking + else self._bare_output_effort(output_effort, model, custom_llm_provider) + if translate_bare_output_effort + else None ) if not reasoning_effort: return new_kwargs["reasoning_effort"] = ( reasoning_effort - if is_claude_target + if is_claude_target or not thinking else self._apply_reasoning_summary_wrapping( reasoning_effort, cast(dict[str, object], thinking), model, custom_llm_provider ) @@ -1284,6 +1309,7 @@ class LiteLLMAnthropicMessagesAdapter: preserve_midturn_system: bool = False, drop_params: bool = True, report_reasoning_drops: bool = False, + translate_bare_output_effort: bool = False, ) -> tuple[ChatCompletionRequest, dict[str, str]]: """ This is used by the beta Anthropic Adapter, for translating anthropic `/v1/messages` requests to the openai format. @@ -1339,6 +1365,7 @@ class LiteLLMAnthropicMessagesAdapter: custom_llm_provider=custom_llm_provider, drop_params=drop_params, report_reasoning_drops=report_reasoning_drops, + translate_bare_output_effort=translate_bare_output_effort, ) ## CONVERT STOP_SEQUENCES self._translate_stop_sequences_to_openai( diff --git a/litellm/llms/anthropic/messages/responses_adapters/handler.py b/litellm/llms/anthropic/messages/responses_adapters/handler.py index 91420565f48..e098dd8822a 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/handler.py +++ b/litellm/llms/anthropic/messages/responses_adapters/handler.py @@ -24,6 +24,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( prompt_cache_breakpoint_for_wire, with_prompt_cache_breakpoint, ) +from litellm.litellm_core_utils.reasoning_effort_capability import deployment_rounds_reasoning_effort from litellm.responses.utils import auto_reasoning_summary from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS from litellm.types.llms.anthropic import ( @@ -68,6 +69,8 @@ AnthropicContextManagementSpec: TypeAlias = dict[str, object] | list[dict[str, o _ADAPTER: Final = LiteLLMAnthropicToResponsesAPIAdapter() _KNOWN_REASONING_EFFORTS: Final = frozenset(get_args(REASONING_EFFORT)) +_LOWEST_REASONING_TIERS: Final = ("minimal", "low") +_UNVERIFIED_ORGANIZATION_SUMMARY_REFUSAL: Final = "must be verified to generate reasoning summaries" def _split_context_management( @@ -255,6 +258,12 @@ def _rewritten_blocks(content: object, rewrite: BlockRewrite) -> Sequence[Mappin return () +def _lowest_tier_instead_of_off(levels: tuple[str, ...] | None) -> str | None: + if not levels or REASONING_EFFORT_OFF in levels: + return None + return next((tier for tier in _LOWEST_REASONING_TIERS if tier in levels), None) + + def _reasoning_off_for_target( reasoning: Mapping[str, object], *, @@ -262,8 +271,12 @@ def _reasoning_off_for_target( custom_llm_provider: str | None, drop_params: bool, ) -> Mapping[str, object] | None: - if model_reasoning_efforts(model, custom_llm_provider) == (): + levels: Final = model_reasoning_efforts(model, custom_llm_provider) + if levels == (): return None + lowest_tier: Final = _lowest_tier_instead_of_off(levels) + if lowest_tier is not None: + return {**reasoning, "effort": lowest_tier} if ( not drop_params or not supports_reasoning(model, custom_llm_provider) @@ -307,9 +320,67 @@ def _reasoning_for_target( custom_llm_provider=custom_llm_provider, deployment_model_info=deployment_model_info, ) + if drop_params and _model_excludes_effort( + normalized, model=model, custom_llm_provider=custom_llm_provider, deployment_model_info=deployment_model_info + ): + verbose_logger.warning( + "Dropping reasoning.effort=%r for %s, its model cost map entry does not list it, because drop_params is set", + normalized, + model, + ) + return None return reasoning if normalized == effort else {**reasoning_params, "effort": normalized} +def _model_excludes_effort( + effort: str, *, model: str, custom_llm_provider: str | None, deployment_model_info: object +) -> bool: + if deployment_rounds_reasoning_effort(deployment_model_info): + return False + levels: Final = model_reasoning_efforts(model, custom_llm_provider) or () + return bool(levels) and effort not in levels + + +def _retry_without_refused_summary( + responses_kwargs: Mapping[str, object], error: litellm.BadRequestError, *, model: str, drop_params: bool +) -> Mapping[str, object] | None: + reasoning: Final = as_str_mapping(responses_kwargs.get("reasoning")) + if not drop_params or reasoning is None or "summary" not in reasoning: + return None + if _UNVERIFIED_ORGANIZATION_SUMMARY_REFUSAL not in error.message: + return None + verbose_logger.warning( + "Retrying %s without reasoning.summary, its OpenAI organization cannot get reasoning summaries until it is " + "verified, because drop_params is set", + model, + ) + return {**responses_kwargs, "reasoning": {key: value for key, value in reasoning.items() if key != "summary"}} + + +async def _aresponses(responses_kwargs: Mapping[str, object], *, model: str, drop_params: bool) -> object: + try: + return await litellm.aresponses(**responses_kwargs) + except litellm.BadRequestError as error: + retry_kwargs: Final = _retry_without_refused_summary( + responses_kwargs, error, model=model, drop_params=drop_params + ) + if retry_kwargs is None: + raise + return await litellm.aresponses(**retry_kwargs) + + +def _responses(responses_kwargs: Mapping[str, object], *, model: str, drop_params: bool) -> object: + try: + return litellm.responses(**responses_kwargs) + except litellm.BadRequestError as error: + retry_kwargs: Final = _retry_without_refused_summary( + responses_kwargs, error, model=model, drop_params=drop_params + ) + if retry_kwargs is None: + raise + return litellm.responses(**retry_kwargs) + + def _maps_reasoning( thinking: Mapping[str, object] | None, model: str, provider: str | None, forwarded_kwargs: Mapping[str, object] ) -> bool: @@ -500,7 +571,9 @@ class LiteLLMMessagesToResponsesAPIHandler: ) safeguards_evaluator: Final = request_safeguards_evaluator(kwargs, messages) - result: Final = await litellm.aresponses(**responses_kwargs) + result: Final = await _aresponses( + responses_kwargs, model=model, drop_params=drops_unsupported_params(kwargs.get("drop_params")) + ) if stream: wrapper: Final = AnthropicResponsesStreamWrapper( @@ -603,7 +676,9 @@ class LiteLLMMessagesToResponsesAPIHandler: extra_kwargs=kwargs, ) - result: Final = litellm.responses(**responses_kwargs) + result: Final = _responses( + responses_kwargs, model=model, drop_params=drops_unsupported_params(kwargs.get("drop_params")) + ) if stream: wrapper: Final = AnthropicResponsesStreamWrapper( diff --git a/litellm/llms/anthropic/messages/responses_adapters/transformation.py b/litellm/llms/anthropic/messages/responses_adapters/transformation.py index 9f8dd85a838..2255cff02b1 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/messages/responses_adapters/transformation.py @@ -6,8 +6,8 @@ path used for OpenAI and Azure models. """ import json -from collections.abc import Iterable, Iterator, Mapping, Sequence -from itertools import chain, groupby +from collections.abc import Iterable, Mapping, Sequence +from itertools import groupby from typing import Any, Final, cast from litellm.litellm_core_utils.core_helpers import as_str_mapping @@ -29,6 +29,8 @@ from litellm.llms.anthropic.messages.native.utils import ( ) from litellm.llms.anthropic.messages.utils import ( REASONING_EFFORT_OFF, + content_blocks, + is_between_tool_calls, is_reasoning_auto_summary_enabled, json_copy_or_original, prompt_cache_key_from_user_id, @@ -128,16 +130,6 @@ class LiteLLMAnthropicToResponsesAPIAdapter: return {"type": "input_file", "file_url": url} return None - @staticmethod - def _content_blocks(content: object) -> tuple[Mapping[str, object], ...]: - if not isinstance(content, list): - return () - return tuple( - cast(Mapping[str, object], block) # cast-ok: untrusted client payload - for block in cast(Sequence[object], content) # cast-ok: untyped client content list - if isinstance(block, dict) - ) - @classmethod def _tool_result_image_parts( cls, blocks: Sequence[Mapping[str, object]] @@ -162,7 +154,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: block, the images Anthropic caches for the mark. The result's text and documents join into one output, which the mark covers whole. """ - inner: Final = cls._content_blocks(block.get("content")) + inner: Final = content_blocks(block.get("content")) own_breakpoint: Final = block.get("prompt_cache_breakpoint") if own_breakpoint is not None: return own_breakpoint, cls._tool_result_image_parts(inner), () @@ -559,8 +551,6 @@ class LiteLLMAnthropicToResponsesAPIAdapter: return requested if thinking.get("display") is not None: return None - if thinking.get("type") == "between_tools": - return "auto" return auto_summary_level if is_reasoning_auto_summary_enabled() else None @classmethod @@ -596,25 +586,11 @@ class LiteLLMAnthropicToResponsesAPIAdapter: summary: Final = cls._reasoning_summary(thinking_params, auto_summary_level) return {"effort": effort} if summary is None else {"effort": effort, "summary": summary} - @staticmethod - def _final_user_turn(messages: Sequence[object]) -> Iterator[Mapping[str, object]]: - for message in map(as_str_mapping, reversed(messages)): - if message is None or message.get("role") != "user": - return - yield message - - @classmethod - def _is_between_tool_calls(cls, messages: Sequence[object]) -> bool: - blocks: Final = chain.from_iterable( - cls._content_blocks(message.get("content")) for message in cls._final_user_turn(messages) - ) - return any(block.get("type") == "tool_result" for block in blocks) - @classmethod def system_blocks_to_instructions(cls, system: object) -> str: return "\n".join( text - for block in cls._content_blocks(system) + for block in content_blocks(system) if block.get("type") == "text" and isinstance(text := block.get("text"), str) and text ) @@ -704,7 +680,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: auto_summary_level=auto_summary_level, model=model, drop_params=drop_params, - between_tool_calls=self._is_between_tool_calls(messages_list), + between_tool_calls=is_between_tool_calls(anthropic_request), ) if reasoning: responses_kwargs["reasoning"] = reasoning diff --git a/litellm/llms/anthropic/messages/utils.py b/litellm/llms/anthropic/messages/utils.py index bff33025295..410c2ffcbff 100644 --- a/litellm/llms/anthropic/messages/utils.py +++ b/litellm/llms/anthropic/messages/utils.py @@ -1,6 +1,7 @@ import json import os -from collections.abc import Mapping +from collections.abc import Iterator, Mapping, Sequence +from itertools import chain from types import MappingProxyType from typing import TYPE_CHECKING, Final, TypeVar @@ -9,6 +10,7 @@ from typing_extensions import ReadOnly import litellm from litellm.constants import X_LITELLM_DISABLE_CALLBACKS +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.internal_call_metadata import sanitize_user_api_key_auth from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.utils import TRUSTED_CALLBACK_VARS_FIELD, ModelInfo, StandardCallbackDynamicParams @@ -83,6 +85,7 @@ PROXY_SUB_CALL_METADATA_KEYS: Final = ( ) _USER_API_KEY_AUTH_METADATA_KEY: Final = "user_api_key_auth" _OBJECT_MAPPING: Final = TypeAdapter(Mapping[str, object]) +_OBJECT_LIST: Final = TypeAdapter(list[object]) _NO_SETTINGS: Final[Mapping[str, object]] = MappingProxyType({}) @@ -196,7 +199,34 @@ def requested_reasoning_summary(thinking: Mapping[str, object]) -> str | None: if isinstance(summary, str): return summary display: Final = thinking.get("display") - return "auto" if display is not None and display != "omitted" else None + if display is not None: + return None if display == "omitted" else "auto" + return "auto" if thinking.get("type") == "between_tools" else None + + +def _list_items(value: object) -> Sequence[object]: + try: + return _OBJECT_LIST.validate_python(value, strict=True) + except ValidationError: + return () + + +def content_blocks(content: object) -> tuple[Mapping[str, object], ...]: + return tuple(block for block in map(as_str_mapping, _list_items(content)) if block is not None) + + +def _final_user_turn(messages: object) -> Iterator[Mapping[str, object]]: + for message in map(as_str_mapping, reversed(_list_items(messages))): + if message is None or message.get("role") != "user": + return + yield message + + +def is_between_tool_calls(request: Mapping[str, object]) -> bool: + blocks: Final = chain.from_iterable( + content_blocks(message.get("content")) for message in _final_user_turn(request.get("messages")) + ) + return any(block.get("type") == "tool_result" for block in blocks) def _mapped_model_info(model: str, custom_llm_provider: str | None) -> ModelInfo | None: diff --git a/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py b/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py index 27ed1f5ebd6..bb9c537a4fb 100644 --- a/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py @@ -20,6 +20,7 @@ import litellm from litellm.llms.anthropic.messages.adapters.handler import ( LiteLLMMessagesToCompletionTransformationHandler, ) +from litellm.llms.anthropic.messages.utils import maps_bare_output_effort from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig from litellm.router import Router from litellm.utils import reasoning_effort_with_summary_alias @@ -582,3 +583,161 @@ class TestADeclaredDeploymentRoundsTheMessagesTierOntoItsLevels: sent: Final = json.loads(route.calls.last.request.read()) assert functools.reduce(lambda body, key: body[key], effort_path, sent) == "minimal" + + +TOOL_RESULT_TURN: Final = [ + {"role": "user", "content": "multiply 6 by 7"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_1", "name": "multiply", "input": {"a": 6, "b": 7}}], + }, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_1", "content": "42"}]}, +] +CHAT_TARGETS: Final = [ + ("databricks/databricks-qwen35-122b-a10b", "databricks"), + ("fireworks_ai/kimi-k3", "fireworks_ai"), + ("gemini/gemini-3.1-pro-preview", "gemini"), +] +GEMINI: Final = "gemini/gemini-3.1-pro-preview" + + +def _chat_completion_kwargs( + model: str, + provider: str, + *, + thinking: dict[str, object] | None, + messages: list[dict[str, object]], + output_config: dict[str, str] | None = None, + reasoning_effort: str | None = None, +) -> Mapping[str, object]: + extra_kwargs: Final = { + key: value + for key, value in ( + ("custom_llm_provider", provider), + ("output_config", output_config), + ("reasoning_effort", reasoning_effort), + ) + if value is not None + } + completion_kwargs, _ = LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs( + max_tokens=1024, + messages=messages, + model=model, + thinking=thinking, + tools=[MULTIPLY_TOOL], + extra_kwargs=extra_kwargs, + ) + return completion_kwargs + + +def _gemini_includes_thoughts(sent: Mapping[str, object]) -> object: + gemini: Final = VertexGeminiConfig() + reasoning_effort: Final = reasoning_effort_with_summary_alias( + sent.get("reasoning_effort"), sent, accepts_summary=gemini.accepts_reasoning_summary_in_reasoning_effort + ) + mapped: Final = gemini.map_openai_params({"reasoning_effort": reasoning_effort}, {}, GEMINI.split("/", 1)[1], True) + return mapped["thinkingConfig"]["includeThoughts"] + + +class TestBetweenToolsAndABareEffortReachAChatTargetAsOnTheResponsesBridge: + @pytest.mark.parametrize("model, provider", CHAT_TARGETS) + @pytest.mark.parametrize("output_config, tier", [(None, "medium"), ({"effort": "high"}, "high")]) + def test_between_tools_reasons_after_a_tool_result_at_the_declared_tier_or_medium( + self, + local_model_cost_map: None, + model: str, + provider: str, + output_config: dict[str, str] | None, + tier: str, + ) -> None: + sent: Final = _chat_completion_kwargs( + model, provider, thinking={"type": "between_tools"}, messages=TOOL_RESULT_TURN, output_config=output_config + ) + + assert sent["reasoning_effort"] == tier + + @pytest.mark.parametrize("model, provider", CHAT_TARGETS) + def test_between_tools_turns_reasoning_off_on_a_turn_without_a_tool_result( + self, local_model_cost_map: None, model: str, provider: str + ) -> None: + sent: Final = _chat_completion_kwargs( + model, provider, thinking={"type": "between_tools"}, messages=MESSAGES, output_config={"effort": "high"} + ) + + assert sent["reasoning_effort"] == "none" + assert "reasoning_summary" not in sent + + @pytest.mark.parametrize( + "thinking, messages, includes_thoughts", + [ + ({"type": "between_tools"}, TOOL_RESULT_TURN, True), + ({"type": "between_tools", "display": "omitted"}, TOOL_RESULT_TURN, False), + ({"type": "between_tools", "display": "omitted", "summary": "concise"}, TOOL_RESULT_TURN, True), + ({"type": "between_tools", "summary": "concise"}, MESSAGES, False), + ], + ids=["after-a-tool-result", "display-omitted", "client-summary-wins", "first-turn"], + ) + def test_between_tools_asks_gemini_for_thoughts_by_the_bridge_summary_rule( + self, + local_model_cost_map: None, + monkeypatch: pytest.MonkeyPatch, + thinking: dict[str, object], + messages: list[dict[str, object]], + includes_thoughts: bool, + ) -> None: + monkeypatch.setattr(litellm, "gemini_include_thoughts_default", False) + + sent: Final = _chat_completion_kwargs(GEMINI, "gemini", thinking=thinking, messages=messages) + + assert _gemini_includes_thoughts(sent) is includes_thoughts + + def test_a_bridged_target_gets_the_plain_off_switch_on_a_turn_without_a_tool_result( + self, local_model_cost_map: None + ) -> None: + sent: Final = _chat_completion_kwargs( + "azure/responses/gpt-5.1", "azure", thinking={"type": "between_tools", "summary": "concise"}, messages=MESSAGES + ) + + assert sent["reasoning_effort"] == "none" + + @pytest.mark.parametrize( + "model, provider", [*CHAT_TARGETS, ("openai/gpt-5.1", "openai"), ("hosted_vllm/unmapped-model", "hosted_vllm")] + ) + def test_a_bare_output_effort_is_the_tier_when_the_model_reasons_unasked( + self, local_model_cost_map: None, model: str, provider: str + ) -> None: + sent: Final = _chat_completion_kwargs( + model, provider, thinking=None, messages=MESSAGES, output_config={"effort": "high"} + ) + + assert sent.get("reasoning_effort") == ("high" if maps_bare_output_effort(model, provider) else None) + + def test_an_explicit_reasoning_effort_beats_a_bare_output_effort(self, local_model_cost_map: None) -> None: + sent: Final = _chat_completion_kwargs( + "databricks/databricks-qwen35-122b-a10b", + "databricks", + thinking=None, + messages=MESSAGES, + output_config={"effort": "high"}, + reasoning_effort="low", + ) + + assert sent["reasoning_effort"] == "low" + + @pytest.mark.parametrize( + "thinking, messages", + [(None, MESSAGES), ({"type": "between_tools"}, TOOL_RESULT_TURN), ({"type": "between_tools"}, MESSAGES)], + ids=["bare-effort", "between_tools-after-a-tool-result", "between_tools-first-turn"], + ) + def test_a_claude_target_still_gets_no_reasoning_effort( + self, local_model_cost_map: None, thinking: dict[str, object] | None, messages: list[dict[str, object]] + ) -> None: + sent: Final = _chat_completion_kwargs( + "anthropic/claude-sonnet-4-5", + "anthropic", + thinking=thinking, + messages=messages, + output_config={"effort": "high"}, + ) + + assert "reasoning_effort" not in sent diff --git a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py index 37d912c1f80..e26480c3c31 100644 --- a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py @@ -7,6 +7,7 @@ from itertools import chain from typing import Final, Literal, cast from unittest.mock import AsyncMock, patch +import httpx import pytest import respx @@ -21,6 +22,7 @@ from litellm.llms.anthropic.messages.responses_adapters.handler import ( LiteLLMMessagesToResponsesAPIHandler, _build_responses_kwargs, ) +from litellm.llms.anthropic.messages.utils import model_reasoning_efforts from litellm.router import Router from litellm.litellm_core_utils.reasoning_effort_capability import resolve_supported_reasoning_efforts from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS, converted_stream_requested @@ -134,35 +136,63 @@ def test_build_responses_kwargs_auto_summary_asks_each_target_for_a_level_it_acc assert cast(Mapping[str, object], responses_kwargs["reasoning"])["summary"] == summary -@pytest.mark.parametrize("model", ["gpt-6-luna", "gpt-6.1-sol", "o3", "gpt-4.1", "gpt-bridge-unmapped"]) -def test_build_responses_kwargs_turns_reasoning_off_unless_the_map_knows_the_model_never_reasons(model: str) -> None: +OFF_SWITCH_MODELS: Final = [ + "gpt-5", + "gpt-5-mini", + "gpt-5-nano", + "gpt-6.1-sol", + "gpt-6-luna", + "gpt-5.1", + "chat-latest", + "o3", + "gpt-4.1", + "gpt-bridge-unmapped", +] +OFF_THINKING: Final = [{"type": "disabled"}, {"type": "between_tools"}] +OFF_THINKING_IDS: Final = ["disabled", "between_tools-first-turn"] + + +def _lowest_listed_off_switch(model: str) -> str: + levels: Final = model_reasoning_efforts(model, "openai") or () + return next((level for level in ("none", "minimal", "low") if level in levels), "none") + + +@pytest.mark.parametrize("thinking", OFF_THINKING, ids=OFF_THINKING_IDS) +@pytest.mark.parametrize("model", OFF_SWITCH_MODELS) +def test_build_responses_kwargs_turns_reasoning_off_at_the_lowest_level_the_model_lists_unless_it_never_reasons( + model: str, thinking: dict[str, object] +) -> None: responses_kwargs: Final = _build_responses_kwargs( max_tokens=1024, messages=MESSAGES, model=f"openai/{model}", - thinking={"type": "disabled"}, + thinking=thinking, extra_kwargs={"custom_llm_provider": "openai"}, ) map_row: Final = litellm.model_cost.get(model) never_reasons: Final = map_row is not None and not map_row.get("supports_reasoning") - assert responses_kwargs.get("reasoning") == (None if never_reasons else {"effort": "none"}) + assert responses_kwargs.get("reasoning") == (None if never_reasons else {"effort": _lowest_listed_off_switch(model)}) -@pytest.mark.parametrize("model", ["gpt-6-luna", "gpt-6.1-sol", "o3", "gpt-4.1", "gpt-bridge-unmapped"]) -def test_build_responses_kwargs_drops_reasoning_off_under_drop_params_unless_the_map_confirms_none(model: str) -> None: +@pytest.mark.parametrize("thinking", OFF_THINKING, ids=OFF_THINKING_IDS) +@pytest.mark.parametrize("model", OFF_SWITCH_MODELS) +def test_build_responses_kwargs_under_drop_params_keeps_a_listed_lower_tier_and_drops_an_unconfirmed_none( + model: str, thinking: dict[str, object] +) -> None: responses_kwargs: Final = _build_responses_kwargs( max_tokens=1024, messages=MESSAGES, model=f"openai/{model}", - thinking={"type": "disabled"}, + thinking=thinking, extra_kwargs={"custom_llm_provider": "openai", "drop_params": True}, ) + off_switch: Final = _lowest_listed_off_switch(model) map_row: Final = litellm.model_cost.get(model) confirms_none: Final = map_row is not None and map_row.get("supports_none_reasoning_effort") is True keeps_none: Final = map_row is None or (bool(map_row.get("supports_reasoning")) and confirms_none) - assert responses_kwargs.get("reasoning") == ({"effort": "none"} if keeps_none else None) + assert responses_kwargs.get("reasoning") == ({"effort": off_switch} if off_switch != "none" or keeps_none else None) @pytest.mark.parametrize("model", ["gpt-5.4-mini", "gpt-5.1", "gpt-6-luna", "gpt-bridge-unmapped"]) @@ -220,6 +250,42 @@ def test_build_responses_kwargs_drops_an_effort_no_reasoning_api_defines_only_un assert responses_kwargs.get("reasoning") == (None if drop_params else {"effort": "turbo"}) +@pytest.mark.parametrize("drop_params", [True, False], ids=["drop_params", "no-drop_params"]) +@pytest.mark.parametrize("thinking", [None, {"type": "adaptive"}], ids=["thinking-left-out", "adaptive"]) +@pytest.mark.parametrize("effort", ["low", "medium", "high"]) +def test_build_responses_kwargs_drops_an_effort_the_model_does_not_list_only_under_drop_params( + effort: Literal["low", "medium", "high"], thinking: dict[str, object] | None, drop_params: bool +) -> None: + responses_kwargs: Final = _build_responses_kwargs( + max_tokens=1024, + messages=MESSAGES, + model="openai/chat-latest", + output_config={"effort": effort}, + thinking=thinking, + extra_kwargs={"custom_llm_provider": "openai", "drop_params": drop_params}, + ) + + listed: Final = effort in (model_reasoning_efforts("chat-latest", "openai") or ()) + assert responses_kwargs.get("reasoning") == ({"effort": effort} if listed or not drop_params else None) + + +def test_build_responses_kwargs_under_drop_params_leaves_an_unlisted_effort_to_a_deployment_that_rounds_it() -> None: + responses_kwargs: Final = _build_responses_kwargs( + max_tokens=1024, + messages=MESSAGES, + model="openai/chat-latest", + output_config={"effort": "high"}, + thinking={"type": "adaptive"}, + extra_kwargs={ + "custom_llm_provider": "openai", + "drop_params": True, + "model_info": {"reasoning_effort_levels": ["low", "high"]}, + }, + ) + + assert responses_kwargs.get("reasoning") == {"effort": "high"} + + @pytest.mark.parametrize("effort", ["max", "xhigh"]) def test_build_responses_kwargs_forwards_an_output_effort_the_model_accepts(effort: Literal["max", "xhigh"]) -> None: responses_kwargs: Final = _build_responses_kwargs( @@ -1190,3 +1256,122 @@ async def test_the_sync_path_reports_its_edits_on_the_streamed_message_delta( ] message_delta: Final = next(event for event in events if event["type"] == "message_delta") assert message_delta["context_management"]["applied_edits"][0]["cleared_tool_uses"] == 2 + + +SUMMARY_REFUSAL: Final = { + "error": { + "message": ( + "Your organization must be verified to generate reasoning summaries. Please go to: " + "https://platform.openai.com/settings/organization/general and click on Verify Organization. " + "If you just verified, it can take up to 15 minutes for access to propagate." + ), + "type": "invalid_request_error", + "param": "reasoning.summary", + "code": "unsupported_value", + } +} +OTHER_SUMMARY_400: Final = { + "error": { + "message": "Unsupported value: 'reasoning.summary' does not support 'auto' with this model.", + "type": "invalid_request_error", + "param": "reasoning.summary", + "code": "unsupported_value", + } +} + + +def _upstream_answering_400_then_200( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, *, refusal: Mapping[str, object], stream: bool +) -> respx.Route: + monkeypatch.setenv("OPENAI_API_KEY", "sk-summary-refusal-test") + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + litellm.in_memory_llm_clients_cache.flush_cache() + served: Final = ( + httpx.Response(200, headers={"Content-Type": "text/event-stream"}, content=RESPONSES_SSE_BODY) + if stream + else httpx.Response(200, json=TEXT_RESPONSE) + ) + return respx_mock.post("https://api.openai.com/v1/responses").mock( + side_effect=[httpx.Response(400, json=refusal), served] + ) + + +async def _summarized_turn(*, sync: bool, stream: bool, drop_params: bool) -> object: + if sync: + return LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler( + max_tokens=1024, + messages=MESSAGES, + model="openai/gpt-5.6-luna", + stream=stream, + custom_llm_provider="openai", + thinking={"type": "adaptive", "display": "summarized"}, + output_config={"effort": "high"}, + drop_params=drop_params, + ) + return await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler( + max_tokens=1024, + messages=MESSAGES, + model="openai/gpt-5.6-luna", + stream=stream, + custom_llm_provider="openai", + thinking={"type": "adaptive", "display": "summarized"}, + output_config={"effort": "high"}, + drop_params=drop_params, + ) + + +async def _reply_types(served: object, *, stream: bool) -> list[object]: + if not stream: + return [cast(Mapping[str, object], served)["type"]] + return [ + json.loads(chunk.decode().split("data: ", 1)[1])["type"] + async for chunk in cast(AsyncIterator[bytes], served) + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"]) +@pytest.mark.parametrize("sync", [False, True], ids=["async", "sync"]) +async def test_under_drop_params_an_unverified_organization_gets_the_turn_again_without_the_summary( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, sync: bool, stream: bool +) -> None: + upstream: Final = _upstream_answering_400_then_200(respx_mock, monkeypatch, refusal=SUMMARY_REFUSAL, stream=stream) + + reply_types: Final = await _reply_types( + await _summarized_turn(sync=sync, stream=stream, drop_params=True), stream=stream + ) + + first, second = (json.loads(call.request.content) for call in upstream.calls) + assert first["reasoning"]["summary"] + assert second["reasoning"] == {"effort": first["reasoning"]["effort"]} + assert {key: value for key, value in second.items() if key != "reasoning"} == { + key: value for key, value in first.items() if key != "reasoning" + } + assert (reply_types[0], reply_types[-1]) == (("message_start", "message_stop") if stream else ("message",) * 2) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"]) +@pytest.mark.parametrize("sync", [False, True], ids=["async", "sync"]) +async def test_without_drop_params_an_unverified_organization_gets_the_400_after_one_request( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, sync: bool, stream: bool +) -> None: + upstream: Final = _upstream_answering_400_then_200(respx_mock, monkeypatch, refusal=SUMMARY_REFUSAL, stream=stream) + + with pytest.raises(litellm.BadRequestError, match="must be verified to generate reasoning summaries"): + await _reply_types(await _summarized_turn(sync=sync, stream=stream, drop_params=False), stream=stream) + + assert upstream.call_count == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("sync", [False, True], ids=["async", "sync"]) +async def test_under_drop_params_another_400_about_the_summary_is_not_retried( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, sync: bool +) -> None: + upstream: Final = _upstream_answering_400_then_200(respx_mock, monkeypatch, refusal=OTHER_SUMMARY_400, stream=False) + + with pytest.raises(litellm.BadRequestError, match="does not support 'auto' with this model"): + await _summarized_turn(sync=sync, stream=False, drop_params=True) + + assert upstream.call_count == 1 diff --git a/tests/unit/llms/anthropic/messages/test_reasoning_effort_fields.py b/tests/unit/llms/anthropic/messages/test_reasoning_effort_fields.py index 7a26f403408..3e1d55dd358 100644 --- a/tests/unit/llms/anthropic/messages/test_reasoning_effort_fields.py +++ b/tests/unit/llms/anthropic/messages/test_reasoning_effort_fields.py @@ -19,6 +19,7 @@ from litellm.litellm_core_utils.reasoning_effort_capability import ( ) from litellm.llms.anthropic.messages.utils import ( normalize_reasoning_effort_value, + requested_reasoning_summary, ) from litellm.utils import get_model_info @@ -318,3 +319,21 @@ class TestADeclarationDisjointFromTheChain: what every deployment got before the resolver was consulted. Dropping the parameter outright is the real answer and belongs with the callers that build the request.""" assert normalize_reasoning_effort_value(effort, declared_effort_entry, "synthetic") == expected + + +@pytest.mark.parametrize( + "thinking, summary", + [ + ({"type": "disabled", "display": "summarized", "summary": "detailed"}, None), + ({"type": "adaptive", "display": "omitted", "summary": "concise"}, "concise"), + ({"type": "adaptive", "display": "omitted"}, None), + ({"type": "adaptive", "display": "summarized"}, "auto"), + ({"type": "adaptive"}, None), + ({"type": "between_tools"}, "auto"), + ({"type": "between_tools", "display": "omitted"}, None), + ], +) +def test_requested_reasoning_summary_takes_the_client_summary_then_the_display_and_never_one_for_disabled( + thinking: dict[str, object], summary: str | None +) -> None: + assert requested_reasoning_summary(thinking) == summary From 01cb5e813cfe92e3d42533ce3c371157694763e6 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 3 Nov 2026 00:14:06 -0800 Subject: [PATCH 06/12] fix(anthropic): limit the chat route's between_tools and bare effort mapping to OpenAI, keep reasoning off at the lowest level the model takes, and ignore a bare effort on a model that cannot reason --- .../messages/adapters/transformation.py | 25 +++- .../llms/anthropic/messages/native/handler.py | 6 +- .../messages/responses_adapters/handler.py | 10 +- litellm/llms/anthropic/messages/utils.py | 16 ++- ..._handler_reasoning_effort_normalization.py | 116 ++++++++---------- .../messages/test_reasoning_effort_fields.py | 28 +++++ 6 files changed, 119 insertions(+), 82 deletions(-) diff --git a/litellm/llms/anthropic/messages/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py index 2b89ca31bb6..11f35c9f04e 100644 --- a/litellm/llms/anthropic/messages/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -13,10 +13,13 @@ from litellm.litellm_core_utils.hidden_params import get_hidden_params from litellm.llms.anthropic.messages.convertible_blocks import tool_result_text_part from litellm.llms.anthropic.messages.utils import ( REASONING_EFFORT_OFF, + RESPONSES_API_PROVIDERS, is_between_tool_calls, is_reasoning_auto_summary_enabled, json_copy_or_original, + lowest_tier_instead_of_off, maps_bare_output_effort, + model_reasoning_efforts, prompt_cache_key_from_user_id, ) from litellm.responses.utils import auto_reasoning_summary @@ -1171,6 +1174,13 @@ class LiteLLMAnthropicMessagesAdapter: else None ) + @staticmethod + def _reasoning_off_for_target(model: str, custom_llm_provider: str | None) -> str | None: + levels: Final = model_reasoning_efforts(model, custom_llm_provider) + if levels == (): + return None + return lowest_tier_instead_of_off(levels) or REASONING_EFFORT_OFF + @staticmethod def _between_tools_reasoning_effort(anthropic_message_request: Mapping[str, object], output_effort: object) -> str: if not is_between_tool_calls(anthropic_message_request): @@ -1236,9 +1246,10 @@ class LiteLLMAnthropicMessagesAdapter: if is_claude_target and not declared_effort: return + mirrors_responses_bridge: Final = custom_llm_provider in RESPONSES_API_PROVIDERS reasoning_effort: Final = declared_effort or ( self._between_tools_reasoning_effort(anthropic_message_request, output_effort) - if thinking_type == "between_tools" + if thinking_type == "between_tools" and mirrors_responses_bridge else self.translate_anthropic_thinking_to_reasoning_effort( cast(AnthropicThinkingParam, thinking), model=model, @@ -1247,19 +1258,27 @@ class LiteLLMAnthropicMessagesAdapter: ) if thinking else self._bare_output_effort(output_effort, model, custom_llm_provider) - if translate_bare_output_effort + if translate_bare_output_effort and mirrors_responses_bridge else None ) if not reasoning_effort: return - new_kwargs["reasoning_effort"] = ( + effort_sent: Final = ( reasoning_effort if is_claude_target or not thinking else self._apply_reasoning_summary_wrapping( reasoning_effort, cast(dict[str, object], thinking), model, custom_llm_provider ) ) + target_effort: Final = ( + self._reasoning_off_for_target(model, custom_llm_provider) + if mirrors_responses_bridge and effort_sent == REASONING_EFFORT_OFF + else effort_sent + ) + if target_effort is None: + return + new_kwargs["reasoning_effort"] = target_effort def _translate_output_format_to_openai( self, diff --git a/litellm/llms/anthropic/messages/native/handler.py b/litellm/llms/anthropic/messages/native/handler.py index 1bf00cbdb2f..539207731d6 100644 --- a/litellm/llms/anthropic/messages/native/handler.py +++ b/litellm/llms/anthropic/messages/native/handler.py @@ -55,14 +55,12 @@ from ..safeguards import ( proxy_serves_safeguards, with_safeguard_results, ) +from ..utils import RESPONSES_API_PROVIDERS from .interceptors import get_messages_interceptors from .utils import AnthropicMessagesRequestUtils, mock_response, prepare_native_messages __all__ = ("anthropic_messages", "anthropic_messages_handler") -# Providers that are routed directly to the OpenAI Responses API instead of -# going through chat/completions. -_RESPONSES_API_PROVIDERS: Final = frozenset({"openai"}) _SAFEGUARD_KWARGS: Final = TypeAdapter(Mapping[str, object]) _SAFEGUARD_MESSAGES: Final = TypeAdapter(Sequence[Mapping[str, object]]) @@ -110,7 +108,7 @@ def _should_route_to_responses_api( return False if stop_sequences: return False - if custom_llm_provider in _RESPONSES_API_PROVIDERS: + if custom_llm_provider in RESPONSES_API_PROVIDERS: return True if custom_llm_provider is None or requested_model is None or resolved_model is None: return False diff --git a/litellm/llms/anthropic/messages/responses_adapters/handler.py b/litellm/llms/anthropic/messages/responses_adapters/handler.py index e098dd8822a..ce4c0f67aff 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/handler.py +++ b/litellm/llms/anthropic/messages/responses_adapters/handler.py @@ -52,6 +52,7 @@ from ..utils import ( REASONING_EFFORT_OFF, litellm_logging_obj_from_kwargs, local_model_name, + lowest_tier_instead_of_off, maps_bare_output_effort, model_reasoning_efforts, normalize_reasoning_effort_value, @@ -69,7 +70,6 @@ AnthropicContextManagementSpec: TypeAlias = dict[str, object] | list[dict[str, o _ADAPTER: Final = LiteLLMAnthropicToResponsesAPIAdapter() _KNOWN_REASONING_EFFORTS: Final = frozenset(get_args(REASONING_EFFORT)) -_LOWEST_REASONING_TIERS: Final = ("minimal", "low") _UNVERIFIED_ORGANIZATION_SUMMARY_REFUSAL: Final = "must be verified to generate reasoning summaries" @@ -258,12 +258,6 @@ def _rewritten_blocks(content: object, rewrite: BlockRewrite) -> Sequence[Mappin return () -def _lowest_tier_instead_of_off(levels: tuple[str, ...] | None) -> str | None: - if not levels or REASONING_EFFORT_OFF in levels: - return None - return next((tier for tier in _LOWEST_REASONING_TIERS if tier in levels), None) - - def _reasoning_off_for_target( reasoning: Mapping[str, object], *, @@ -274,7 +268,7 @@ def _reasoning_off_for_target( levels: Final = model_reasoning_efforts(model, custom_llm_provider) if levels == (): return None - lowest_tier: Final = _lowest_tier_instead_of_off(levels) + lowest_tier: Final = lowest_tier_instead_of_off(levels) if lowest_tier is not None: return {**reasoning, "effort": lowest_tier} if ( diff --git a/litellm/llms/anthropic/messages/utils.py b/litellm/llms/anthropic/messages/utils.py index 410c2ffcbff..55ec4d8b010 100644 --- a/litellm/llms/anthropic/messages/utils.py +++ b/litellm/llms/anthropic/messages/utils.py @@ -28,6 +28,8 @@ _EFFORT_DEGRADATION_CHAIN: Final[Mapping[str, tuple[str, ...]]] = MappingProxyTy } ) REASONING_EFFORT_OFF: Final = "none" +RESPONSES_API_PROVIDERS: Final = frozenset({"openai"}) +_LOWEST_REASONING_TIERS: Final = ("minimal", "low") _JsonValue: Final = TypeVar("_JsonValue") @@ -247,9 +249,21 @@ def model_reasoning_efforts(model: str, custom_llm_provider: str | None) -> tupl return resolve_supported_reasoning_efforts(model_info, deployment_is_mapped=True) +def lowest_tier_instead_of_off(levels: tuple[str, ...] | None) -> str | None: + if not levels or REASONING_EFFORT_OFF in levels: + return None + return next((tier for tier in _LOWEST_REASONING_TIERS if tier in levels), None) + + def maps_bare_output_effort(model: str, custom_llm_provider: str | None) -> bool: + from litellm.litellm_core_utils.reasoning_effort_capability import resolve_supported_reasoning_efforts + model_info: Final = _mapped_model_info(model, custom_llm_provider) - return model_info is not None and model_info.get("default_reasoning_effort") != REASONING_EFFORT_OFF + return ( + model_info is not None + and model_info.get("default_reasoning_effort") != REASONING_EFFORT_OFF + and resolve_supported_reasoning_efforts(model_info, deployment_is_mapped=True) != () + ) def normalize_reasoning_effort_value( diff --git a/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py b/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py index bb9c537a4fb..6f457fe8794 100644 --- a/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py @@ -20,7 +20,7 @@ import litellm from litellm.llms.anthropic.messages.adapters.handler import ( LiteLLMMessagesToCompletionTransformationHandler, ) -from litellm.llms.anthropic.messages.utils import maps_bare_output_effort +from litellm.llms.anthropic.messages.utils import maps_bare_output_effort, model_reasoning_efforts from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig from litellm.router import Router from litellm.utils import reasoning_effort_with_summary_alias @@ -593,12 +593,13 @@ TOOL_RESULT_TURN: Final = [ }, {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_1", "content": "42"}]}, ] -CHAT_TARGETS: Final = [ +OPENAI_CHAT_TARGETS: Final = ["openai/gpt-5.1", "openai/gpt-5-mini"] +OTHER_PROVIDER_CHAT_TARGETS: Final = [ ("databricks/databricks-qwen35-122b-a10b", "databricks"), ("fireworks_ai/kimi-k3", "fireworks_ai"), ("gemini/gemini-3.1-pro-preview", "gemini"), + ("hosted_vllm/unmapped-model", "hosted_vllm"), ] -GEMINI: Final = "gemini/gemini-3.1-pro-preview" def _chat_completion_kwargs( @@ -630,92 +631,55 @@ def _chat_completion_kwargs( return completion_kwargs -def _gemini_includes_thoughts(sent: Mapping[str, object]) -> object: - gemini: Final = VertexGeminiConfig() - reasoning_effort: Final = reasoning_effort_with_summary_alias( - sent.get("reasoning_effort"), sent, accepts_summary=gemini.accepts_reasoning_summary_in_reasoning_effort - ) - mapped: Final = gemini.map_openai_params({"reasoning_effort": reasoning_effort}, {}, GEMINI.split("/", 1)[1], True) - return mapped["thinkingConfig"]["includeThoughts"] +def _lowest_accepted_level(levels: tuple[str, ...] | None) -> str | None: + if levels is None: + return "none" + return levels[0] if levels else None -class TestBetweenToolsAndABareEffortReachAChatTargetAsOnTheResponsesBridge: - @pytest.mark.parametrize("model, provider", CHAT_TARGETS) +class TestAnOpenAIChatRouteRequestGetsTheResponsesBridgeThinkingControls: + @pytest.mark.parametrize("model", OPENAI_CHAT_TARGETS) @pytest.mark.parametrize("output_config, tier", [(None, "medium"), ({"effort": "high"}, "high")]) def test_between_tools_reasons_after_a_tool_result_at_the_declared_tier_or_medium( - self, - local_model_cost_map: None, - model: str, - provider: str, - output_config: dict[str, str] | None, - tier: str, + self, local_model_cost_map: None, model: str, output_config: dict[str, str] | None, tier: str ) -> None: sent: Final = _chat_completion_kwargs( - model, provider, thinking={"type": "between_tools"}, messages=TOOL_RESULT_TURN, output_config=output_config + model, "openai", thinking={"type": "between_tools"}, messages=TOOL_RESULT_TURN, output_config=output_config ) assert sent["reasoning_effort"] == tier - @pytest.mark.parametrize("model, provider", CHAT_TARGETS) - def test_between_tools_turns_reasoning_off_on_a_turn_without_a_tool_result( - self, local_model_cost_map: None, model: str, provider: str + @pytest.mark.parametrize( + "model", [*OPENAI_CHAT_TARGETS, "openai/gpt-6.1-sol", "openai/gpt-4.1", "openai/o3"] + ) + @pytest.mark.parametrize( + "thinking", + [{"type": "disabled"}, {"type": "between_tools", "summary": "concise"}], + ids=["disabled", "between_tools-first-turn"], + ) + def test_reasoning_off_is_the_lowest_level_the_model_accepts( + self, local_model_cost_map: None, model: str, thinking: dict[str, object] ) -> None: sent: Final = _chat_completion_kwargs( - model, provider, thinking={"type": "between_tools"}, messages=MESSAGES, output_config={"effort": "high"} + model, "openai", thinking=thinking, messages=MESSAGES, output_config={"effort": "high"} ) - assert sent["reasoning_effort"] == "none" - assert "reasoning_summary" not in sent + assert sent.get("reasoning_effort") == _lowest_accepted_level(model_reasoning_efforts(model, "openai")) - @pytest.mark.parametrize( - "thinking, messages, includes_thoughts", - [ - ({"type": "between_tools"}, TOOL_RESULT_TURN, True), - ({"type": "between_tools", "display": "omitted"}, TOOL_RESULT_TURN, False), - ({"type": "between_tools", "display": "omitted", "summary": "concise"}, TOOL_RESULT_TURN, True), - ({"type": "between_tools", "summary": "concise"}, MESSAGES, False), - ], - ids=["after-a-tool-result", "display-omitted", "client-summary-wins", "first-turn"], - ) - def test_between_tools_asks_gemini_for_thoughts_by_the_bridge_summary_rule( - self, - local_model_cost_map: None, - monkeypatch: pytest.MonkeyPatch, - thinking: dict[str, object], - messages: list[dict[str, object]], - includes_thoughts: bool, - ) -> None: - monkeypatch.setattr(litellm, "gemini_include_thoughts_default", False) - - sent: Final = _chat_completion_kwargs(GEMINI, "gemini", thinking=thinking, messages=messages) - - assert _gemini_includes_thoughts(sent) is includes_thoughts - - def test_a_bridged_target_gets_the_plain_off_switch_on_a_turn_without_a_tool_result( - self, local_model_cost_map: None - ) -> None: - sent: Final = _chat_completion_kwargs( - "azure/responses/gpt-5.1", "azure", thinking={"type": "between_tools", "summary": "concise"}, messages=MESSAGES - ) - - assert sent["reasoning_effort"] == "none" - - @pytest.mark.parametrize( - "model, provider", [*CHAT_TARGETS, ("openai/gpt-5.1", "openai"), ("hosted_vllm/unmapped-model", "hosted_vllm")] - ) + @pytest.mark.parametrize("model", [*OPENAI_CHAT_TARGETS, "openai/gpt-4.1", "openai/o3"]) def test_a_bare_output_effort_is_the_tier_when_the_model_reasons_unasked( - self, local_model_cost_map: None, model: str, provider: str + self, local_model_cost_map: None, model: str ) -> None: sent: Final = _chat_completion_kwargs( - model, provider, thinking=None, messages=MESSAGES, output_config={"effort": "high"} + model, "openai", thinking=None, messages=MESSAGES, output_config={"effort": "high"} ) - assert sent.get("reasoning_effort") == ("high" if maps_bare_output_effort(model, provider) else None) + assert sent.get("reasoning_effort") == ("high" if maps_bare_output_effort(model, "openai") else None) def test_an_explicit_reasoning_effort_beats_a_bare_output_effort(self, local_model_cost_map: None) -> None: sent: Final = _chat_completion_kwargs( - "databricks/databricks-qwen35-122b-a10b", - "databricks", + "openai/gpt-5-mini", + "openai", thinking=None, messages=MESSAGES, output_config={"effort": "high"}, @@ -724,6 +688,26 @@ class TestBetweenToolsAndABareEffortReachAChatTargetAsOnTheResponsesBridge: assert sent["reasoning_effort"] == "low" + @pytest.mark.parametrize("model, provider", OTHER_PROVIDER_CHAT_TARGETS) + @pytest.mark.parametrize( + "thinking, messages", + [(None, MESSAGES), ({"type": "between_tools"}, TOOL_RESULT_TURN), ({"type": "between_tools"}, MESSAGES)], + ids=["bare-effort", "between_tools-after-a-tool-result", "between_tools-first-turn"], + ) + def test_another_provider_keeps_ignoring_between_tools_and_a_bare_effort( + self, + local_model_cost_map: None, + model: str, + provider: str, + thinking: dict[str, object] | None, + messages: list[dict[str, object]], + ) -> None: + sent: Final = _chat_completion_kwargs( + model, provider, thinking=thinking, messages=messages, output_config={"effort": "high"} + ) + + assert "reasoning_effort" not in sent + @pytest.mark.parametrize( "thinking, messages", [(None, MESSAGES), ({"type": "between_tools"}, TOOL_RESULT_TURN), ({"type": "between_tools"}, MESSAGES)], diff --git a/tests/unit/llms/anthropic/messages/test_reasoning_effort_fields.py b/tests/unit/llms/anthropic/messages/test_reasoning_effort_fields.py index 3e1d55dd358..1de1772032f 100644 --- a/tests/unit/llms/anthropic/messages/test_reasoning_effort_fields.py +++ b/tests/unit/llms/anthropic/messages/test_reasoning_effort_fields.py @@ -18,6 +18,7 @@ from litellm.litellm_core_utils.reasoning_effort_capability import ( resolve_supported_reasoning_efforts, ) from litellm.llms.anthropic.messages.utils import ( + maps_bare_output_effort, normalize_reasoning_effort_value, requested_reasoning_summary, ) @@ -321,6 +322,33 @@ class TestADeclarationDisjointFromTheChain: assert normalize_reasoning_effort_value(effort, declared_effort_entry, "synthetic") == expected +@pytest.fixture +def synthetic_model_entry(local_model_cost_map, request): + key = f"synthetic/{request.node.name}" + litellm.model_cost[key] = {"litellm_provider": "synthetic", "mode": "chat", **request.param} + litellm.get_model_info.cache_clear() + try: + yield key.removeprefix("synthetic/") + finally: + litellm.model_cost.pop(key, None) + litellm.get_model_info.cache_clear() + + +@pytest.mark.parametrize( + "synthetic_model_entry, maps", + [ + ({"supports_reasoning": True}, True), + ({"supports_reasoning": True, "default_reasoning_effort": "none"}, False), + ({"supports_reasoning": False}, False), + ({}, False), + ], + indirect=["synthetic_model_entry"], + ids=["reasons-by-default", "reasoning-off-by-default", "cannot-reason", "no-reasoning-flag"], +) +def test_a_bare_output_effort_maps_only_onto_a_model_that_reasons_by_default(synthetic_model_entry, maps): + assert maps_bare_output_effort(synthetic_model_entry, "synthetic") is maps + + @pytest.mark.parametrize( "thinking, summary", [ From 9dd5e9a0082dc6f7346805b10c9be5b11a647e8d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 3 Nov 2026 01:39:29 -0800 Subject: [PATCH 07/12] fix(anthropic): drop an unconfirmed reasoning off switch under drop_params on the OpenAI chat route too, through one helper both routes share --- .../messages/adapters/transformation.py | 12 ++----- .../messages/responses_adapters/handler.py | 36 +++---------------- litellm/llms/anthropic/messages/utils.py | 26 +++++++++++++- ..._handler_reasoning_effort_normalization.py | 28 ++++++++++----- 4 files changed, 51 insertions(+), 51 deletions(-) diff --git a/litellm/llms/anthropic/messages/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py index 11f35c9f04e..a8dffc2528f 100644 --- a/litellm/llms/anthropic/messages/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -17,10 +17,9 @@ from litellm.llms.anthropic.messages.utils import ( is_between_tool_calls, is_reasoning_auto_summary_enabled, json_copy_or_original, - lowest_tier_instead_of_off, maps_bare_output_effort, - model_reasoning_efforts, prompt_cache_key_from_user_id, + reasoning_off_level, ) from litellm.responses.utils import auto_reasoning_summary @@ -1174,13 +1173,6 @@ class LiteLLMAnthropicMessagesAdapter: else None ) - @staticmethod - def _reasoning_off_for_target(model: str, custom_llm_provider: str | None) -> str | None: - levels: Final = model_reasoning_efforts(model, custom_llm_provider) - if levels == (): - return None - return lowest_tier_instead_of_off(levels) or REASONING_EFFORT_OFF - @staticmethod def _between_tools_reasoning_effort(anthropic_message_request: Mapping[str, object], output_effort: object) -> str: if not is_between_tool_calls(anthropic_message_request): @@ -1272,7 +1264,7 @@ class LiteLLMAnthropicMessagesAdapter: ) ) target_effort: Final = ( - self._reasoning_off_for_target(model, custom_llm_provider) + reasoning_off_level(model, custom_llm_provider, drop_params=drop_params) if mirrors_responses_bridge and effort_sent == REASONING_EFFORT_OFF else effort_sent ) diff --git a/litellm/llms/anthropic/messages/responses_adapters/handler.py b/litellm/llms/anthropic/messages/responses_adapters/handler.py index 725b2a7276d..bf6f114f198 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/handler.py +++ b/litellm/llms/anthropic/messages/responses_adapters/handler.py @@ -42,7 +42,7 @@ from litellm.types.llms.anthropic_messages.anthropic_response import ( ) from litellm.types.llms.openai import REASONING_EFFORT, ResponsesAPIResponse from litellm.types.utils import CountContextManagement -from litellm.utils import ProviderConfigManager, supports_none_reasoning_effort, supports_reasoning +from litellm.utils import ProviderConfigManager from ..adapters.handler import context_management_explicitly_dropped, request_safeguards_evaluator from ..context_management import AnthropicContextManagementError, PolyfillResult, apply_context_management @@ -54,10 +54,10 @@ from ..utils import ( REASONING_EFFORT_OFF, litellm_logging_obj_from_kwargs, local_model_name, - lowest_tier_instead_of_off, maps_bare_output_effort, model_reasoning_efforts, normalize_reasoning_effort_value, + reasoning_off_level, ) from .streaming_iterator import AnthropicResponsesStreamWrapper from .transformation import LiteLLMAnthropicToResponsesAPIAdapter @@ -262,33 +262,6 @@ def _rewritten_blocks(content: object, rewrite: BlockRewrite) -> Sequence[Mappin return () -def _reasoning_off_for_target( - reasoning: Mapping[str, object], - *, - model: str, - custom_llm_provider: str | None, - drop_params: bool, -) -> Mapping[str, object] | None: - levels: Final = model_reasoning_efforts(model, custom_llm_provider) - if levels == (): - return None - lowest_tier: Final = lowest_tier_instead_of_off(levels) - if lowest_tier is not None: - return {**reasoning, "effort": lowest_tier} - if ( - not drop_params - or not supports_reasoning(model, custom_llm_provider) - or supports_none_reasoning_effort(model, custom_llm_provider) - ): - return reasoning - verbose_logger.warning( - "Dropping reasoning.effort=none for %s, its model cost map entry does not set " - "supports_none_reasoning_effort, because drop_params is set", - model, - ) - return None - - def _reasoning_for_target( reasoning: object, *, @@ -302,9 +275,8 @@ def _reasoning_for_target( if reasoning_params is None or not isinstance(effort, str): return reasoning if effort == REASONING_EFFORT_OFF: - return _reasoning_off_for_target( - reasoning_params, model=model, custom_llm_provider=custom_llm_provider, drop_params=drop_params - ) + off_level: Final = reasoning_off_level(model, custom_llm_provider, drop_params=drop_params) + return None if off_level is None else {**reasoning_params, "effort": off_level} if drop_params and effort not in _KNOWN_REASONING_EFFORTS: verbose_logger.warning( "Dropping reasoning.effort=%r for %s, no reasoning API defines it, because drop_params is set", diff --git a/litellm/llms/anthropic/messages/utils.py b/litellm/llms/anthropic/messages/utils.py index 55ec4d8b010..02a0c9c70c7 100644 --- a/litellm/llms/anthropic/messages/utils.py +++ b/litellm/llms/anthropic/messages/utils.py @@ -9,6 +9,7 @@ from pydantic import ConfigDict, TypeAdapter, ValidationError from typing_extensions import ReadOnly import litellm +from litellm._logging import verbose_logger from litellm.constants import X_LITELLM_DISABLE_CALLBACKS from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.internal_call_metadata import sanitize_user_api_key_auth @@ -249,12 +250,35 @@ def model_reasoning_efforts(model: str, custom_llm_provider: str | None) -> tupl return resolve_supported_reasoning_efforts(model_info, deployment_is_mapped=True) -def lowest_tier_instead_of_off(levels: tuple[str, ...] | None) -> str | None: +def _lowest_tier_instead_of_off(levels: tuple[str, ...] | None) -> str | None: if not levels or REASONING_EFFORT_OFF in levels: return None return next((tier for tier in _LOWEST_REASONING_TIERS if tier in levels), None) +def reasoning_off_level(model: str, custom_llm_provider: str | None, *, drop_params: bool) -> str | None: + from litellm.utils import supports_none_reasoning_effort, supports_reasoning + + levels: Final = model_reasoning_efforts(model, custom_llm_provider) + if levels == (): + return None + lowest_tier: Final = _lowest_tier_instead_of_off(levels) + if lowest_tier is not None: + return lowest_tier + if ( + not drop_params + or not supports_reasoning(model, custom_llm_provider) + or supports_none_reasoning_effort(model, custom_llm_provider) + ): + return REASONING_EFFORT_OFF + verbose_logger.warning( + "Dropping reasoning_effort=none for %s, its model cost map entry does not set " + "supports_none_reasoning_effort, because drop_params is set", + model, + ) + return None + + def maps_bare_output_effort(model: str, custom_llm_provider: str | None) -> bool: from litellm.litellm_core_utils.reasoning_effort_capability import resolve_supported_reasoning_efforts diff --git a/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py b/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py index 6f457fe8794..c313a79e902 100644 --- a/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py @@ -23,7 +23,7 @@ from litellm.llms.anthropic.messages.adapters.handler import ( from litellm.llms.anthropic.messages.utils import maps_bare_output_effort, model_reasoning_efforts from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig from litellm.router import Router -from litellm.utils import reasoning_effort_with_summary_alias +from litellm.utils import reasoning_effort_with_summary_alias, supports_none_reasoning_effort, supports_reasoning MESSAGES = [{"role": "user", "content": "hello"}] @@ -610,6 +610,7 @@ def _chat_completion_kwargs( messages: list[dict[str, object]], output_config: dict[str, str] | None = None, reasoning_effort: str | None = None, + drop_params: bool | None = None, ) -> Mapping[str, object]: extra_kwargs: Final = { key: value @@ -617,6 +618,7 @@ def _chat_completion_kwargs( ("custom_llm_provider", provider), ("output_config", output_config), ("reasoning_effort", reasoning_effort), + ("drop_params", drop_params), ) if value is not None } @@ -631,10 +633,14 @@ def _chat_completion_kwargs( return completion_kwargs -def _lowest_accepted_level(levels: tuple[str, ...] | None) -> str | None: - if levels is None: - return "none" - return levels[0] if levels else None +def _lowest_accepted_level(model: str, *, drop_params: bool) -> str | None: + levels: Final = model_reasoning_efforts(model, "openai") + if levels is not None: + return levels[0] if levels else None + drops_unconfirmed_off: Final = ( + drop_params and supports_reasoning(model, "openai") and not supports_none_reasoning_effort(model, "openai") + ) + return None if drops_unconfirmed_off else "none" class TestAnOpenAIChatRouteRequestGetsTheResponsesBridgeThinkingControls: @@ -657,14 +663,20 @@ class TestAnOpenAIChatRouteRequestGetsTheResponsesBridgeThinkingControls: [{"type": "disabled"}, {"type": "between_tools", "summary": "concise"}], ids=["disabled", "between_tools-first-turn"], ) + @pytest.mark.parametrize("drop_params", [False, True], ids=["no-drop_params", "drop_params"]) def test_reasoning_off_is_the_lowest_level_the_model_accepts( - self, local_model_cost_map: None, model: str, thinking: dict[str, object] + self, local_model_cost_map: None, model: str, thinking: dict[str, object], drop_params: bool ) -> None: sent: Final = _chat_completion_kwargs( - model, "openai", thinking=thinking, messages=MESSAGES, output_config={"effort": "high"} + model, + "openai", + thinking=thinking, + messages=MESSAGES, + output_config={"effort": "high"}, + drop_params=drop_params, ) - assert sent.get("reasoning_effort") == _lowest_accepted_level(model_reasoning_efforts(model, "openai")) + assert sent.get("reasoning_effort") == _lowest_accepted_level(model, drop_params=drop_params) @pytest.mark.parametrize("model", [*OPENAI_CHAT_TARGETS, "openai/gpt-4.1", "openai/o3"]) def test_a_bare_output_effort_is_the_tier_when_the_model_reasons_unasked( From 927601ac0f2b6524092e1f375977df28f71ec4f3 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 3 Nov 2026 07:50:22 -0800 Subject: [PATCH 08/12] fix(anthropic): keep sending reasoning_effort none on the chat route to an openai/ deployment the cost map does not list The router registers an unlisted deployment as a stub row with no mode, which made its reasoning levels read as known-empty, so thinking disabled stopped sending none on the chat-completions route. The levels now read as unknown for such a row. The Responses bridge keeps sending no reasoning there, since the Responses transformation answers 400 to reasoning.effort on a row without supports_reasoning. --- .../messages/responses_adapters/handler.py | 5 ++- litellm/llms/anthropic/messages/utils.py | 15 ++++++- ..._handler_reasoning_effort_normalization.py | 44 +++++++++++++++++++ 3 files changed, 61 insertions(+), 3 deletions(-) diff --git a/litellm/llms/anthropic/messages/responses_adapters/handler.py b/litellm/llms/anthropic/messages/responses_adapters/handler.py index bf6f114f198..cb78b47a2b7 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/handler.py +++ b/litellm/llms/anthropic/messages/responses_adapters/handler.py @@ -52,6 +52,7 @@ from ..convertible_blocks import reject_malformed_tool_result_text from ..safeguards import with_safeguard_results from ..utils import ( REASONING_EFFORT_OFF, + cost_map_row_lacks_reasoning, litellm_logging_obj_from_kwargs, local_model_name, maps_bare_output_effort, @@ -276,7 +277,9 @@ def _reasoning_for_target( return reasoning if effort == REASONING_EFFORT_OFF: off_level: Final = reasoning_off_level(model, custom_llm_provider, drop_params=drop_params) - return None if off_level is None else {**reasoning_params, "effort": off_level} + if off_level is None or cost_map_row_lacks_reasoning(model, custom_llm_provider): + return None + return {**reasoning_params, "effort": off_level} if drop_params and effort not in _KNOWN_REASONING_EFFORTS: verbose_logger.warning( "Dropping reasoning.effort=%r for %s, no reasoning API defines it, because drop_params is set", diff --git a/litellm/llms/anthropic/messages/utils.py b/litellm/llms/anthropic/messages/utils.py index 02a0c9c70c7..87f7089cfb4 100644 --- a/litellm/llms/anthropic/messages/utils.py +++ b/litellm/llms/anthropic/messages/utils.py @@ -92,6 +92,7 @@ _OBJECT_LIST: Final = TypeAdapter(list[object]) _NO_SETTINGS: Final[Mapping[str, object]] = MappingProxyType({}) +_NO_OPERATOR_MODEL_INFO: Final[Mapping[str, object]] = MappingProxyType({}) def proxy_general_settings() -> Mapping[str, object]: @@ -242,12 +243,17 @@ def _mapped_model_info(model: str, custom_llm_provider: str | None) -> ModelInfo def model_reasoning_efforts(model: str, custom_llm_provider: str | None) -> tuple[str, ...] | None: - from litellm.litellm_core_utils.reasoning_effort_capability import resolve_supported_reasoning_efforts + from litellm.litellm_core_utils.reasoning_effort_capability import ( + deployment_is_catalog_mapped, + resolve_supported_reasoning_efforts, + ) model_info: Final = _mapped_model_info(model, custom_llm_provider) if model_info is None: return None - return resolve_supported_reasoning_efforts(model_info, deployment_is_mapped=True) + return resolve_supported_reasoning_efforts( + model_info, deployment_is_mapped=deployment_is_catalog_mapped(model_info, _NO_OPERATOR_MODEL_INFO) + ) def _lowest_tier_instead_of_off(levels: tuple[str, ...] | None) -> str | None: @@ -256,6 +262,11 @@ def _lowest_tier_instead_of_off(levels: tuple[str, ...] | None) -> str | None: return next((tier for tier in _LOWEST_REASONING_TIERS if tier in levels), None) +def cost_map_row_lacks_reasoning(model: str, custom_llm_provider: str | None) -> bool: + model_info: Final = _mapped_model_info(model, custom_llm_provider) + return model_info is not None and not model_info.get("supports_reasoning") + + def reasoning_off_level(model: str, custom_llm_provider: str | None, *, drop_params: bool) -> str | None: from litellm.utils import supports_none_reasoning_effort, supports_reasoning diff --git a/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py b/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py index c313a79e902..fe091e67a5a 100644 --- a/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py @@ -737,3 +737,47 @@ class TestAnOpenAIChatRouteRequestGetsTheResponsesBridgeThinkingControls: ) assert "reasoning_effort" not in sent + + +_OFF_MAP_OPENAI_DEPLOYMENT: Final = { + "model": "openai/off-map-hybrid-reasoner", + "api_base": "https://compatible.example.test/v1", + "api_key": "fake-key", +} + + +class TestDisabledOnAnOffMapOpenAIDeployment: + @pytest.fixture(autouse=True) + def _httpx_transport(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + @pytest.mark.asyncio + async def test_the_chat_route_sends_reasoning_effort_none( + self, respx_mock: respx.MockRouter, local_model_cost_map: None + ) -> None: + sent: Final = await _body_sent_upstream( + respx_mock, + _OFF_MAP_OPENAI_DEPLOYMENT, + {}, + r"https://compatible\.example\.test/v1/chat/completions", + _CHAT_COMPLETION, + thinking={"type": "disabled"}, + stop_sequences=["STOP"], + ) + + assert sent["reasoning_effort"] == "none" + + @pytest.mark.asyncio + async def test_the_responses_bridge_sends_no_reasoning_its_transformation_would_refuse( + self, respx_mock: respx.MockRouter, local_model_cost_map: None + ) -> None: + sent: Final = await _body_sent_upstream( + respx_mock, + _OFF_MAP_OPENAI_DEPLOYMENT, + {}, + r"https://compatible\.example\.test/v1/responses", + _RESPONSE, + thinking={"type": "disabled"}, + ) + + assert "reasoning" not in sent From 476755238fdf6c709f98a237541cd97308c8769a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 3 Nov 2026 10:00:15 -0800 Subject: [PATCH 09/12] fix(anthropic): send reasoning effort none wherever the Responses transformation accepts it An openai/ deployment the cost map does not list now gets effort none for disabled thinking on the chat route even when its model_info sets a mode, since the off level now asks whether the loaded cost map lists the model instead of reading the row the router registers. On the Responses bridge the off switch is dropped only where the Responses transformation itself would refuse reasoning.effort, so a slash-named model it accepts gets none again for adaptive effort none and disabled thinking. --- .../messages/responses_adapters/handler.py | 12 +++++-- litellm/llms/anthropic/messages/utils.py | 14 ++------ .../llms/openai/responses/transformation.py | 4 +-- ..._handler_reasoning_effort_normalization.py | 36 +++++++++++++++++++ 4 files changed, 50 insertions(+), 16 deletions(-) diff --git a/litellm/llms/anthropic/messages/responses_adapters/handler.py b/litellm/llms/anthropic/messages/responses_adapters/handler.py index cb78b47a2b7..77d50bc4a16 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/handler.py +++ b/litellm/llms/anthropic/messages/responses_adapters/handler.py @@ -27,6 +27,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( ) from litellm.litellm_core_utils.reasoning_effort_capability import deployment_rounds_reasoning_effort from litellm.litellm_core_utils.tool_fields import drops_mcp_servers, tools_without_mcp_servers +from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.responses.utils import auto_reasoning_summary from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS from litellm.types.llms.anthropic import ( @@ -41,7 +42,7 @@ from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, ) from litellm.types.llms.openai import REASONING_EFFORT, ResponsesAPIResponse -from litellm.types.utils import CountContextManagement +from litellm.types.utils import CountContextManagement, LlmProviders from litellm.utils import ProviderConfigManager from ..adapters.handler import context_management_explicitly_dropped, request_safeguards_evaluator @@ -52,7 +53,6 @@ from ..convertible_blocks import reject_malformed_tool_result_text from ..safeguards import with_safeguard_results from ..utils import ( REASONING_EFFORT_OFF, - cost_map_row_lacks_reasoning, litellm_logging_obj_from_kwargs, local_model_name, maps_bare_output_effort, @@ -277,7 +277,7 @@ def _reasoning_for_target( return reasoning if effort == REASONING_EFFORT_OFF: off_level: Final = reasoning_off_level(model, custom_llm_provider, drop_params=drop_params) - if off_level is None or cost_map_row_lacks_reasoning(model, custom_llm_provider): + if off_level is None or _responses_api_refuses_reasoning_effort(model, custom_llm_provider): return None return {**reasoning_params, "effort": off_level} if drop_params and effort not in _KNOWN_REASONING_EFFORTS: @@ -305,6 +305,12 @@ def _reasoning_for_target( return reasoning if normalized == effort else {**reasoning_params, "effort": normalized} +def _responses_api_refuses_reasoning_effort(model: str, custom_llm_provider: str | None) -> bool: + return custom_llm_provider == LlmProviders.OPENAI.value and not OpenAIResponsesAPIConfig.supports_reasoning_param( + model + ) + + def _model_excludes_effort( effort: str, *, model: str, custom_llm_provider: str | None, deployment_model_info: object ) -> bool: diff --git a/litellm/llms/anthropic/messages/utils.py b/litellm/llms/anthropic/messages/utils.py index 87f7089cfb4..05e9cc19d11 100644 --- a/litellm/llms/anthropic/messages/utils.py +++ b/litellm/llms/anthropic/messages/utils.py @@ -92,7 +92,6 @@ _OBJECT_LIST: Final = TypeAdapter(list[object]) _NO_SETTINGS: Final[Mapping[str, object]] = MappingProxyType({}) -_NO_OPERATOR_MODEL_INFO: Final[Mapping[str, object]] = MappingProxyType({}) def proxy_general_settings() -> Mapping[str, object]: @@ -243,16 +242,14 @@ def _mapped_model_info(model: str, custom_llm_provider: str | None) -> ModelInfo def model_reasoning_efforts(model: str, custom_llm_provider: str | None) -> tuple[str, ...] | None: - from litellm.litellm_core_utils.reasoning_effort_capability import ( - deployment_is_catalog_mapped, - resolve_supported_reasoning_efforts, - ) + from litellm.litellm_core_utils.reasoning_effort_capability import resolve_supported_reasoning_efforts + from litellm.utils import has_catalog_entry model_info: Final = _mapped_model_info(model, custom_llm_provider) if model_info is None: return None return resolve_supported_reasoning_efforts( - model_info, deployment_is_mapped=deployment_is_catalog_mapped(model_info, _NO_OPERATOR_MODEL_INFO) + model_info, deployment_is_mapped=has_catalog_entry(model, custom_llm_provider) ) @@ -262,11 +259,6 @@ def _lowest_tier_instead_of_off(levels: tuple[str, ...] | None) -> str | None: return next((tier for tier in _LOWEST_REASONING_TIERS if tier in levels), None) -def cost_map_row_lacks_reasoning(model: str, custom_llm_provider: str | None) -> bool: - model_info: Final = _mapped_model_info(model, custom_llm_provider) - return model_info is not None and not model_info.get("supports_reasoning") - - def reasoning_off_level(model: str, custom_llm_provider: str | None, *, drop_params: bool) -> str | None: from litellm.utils import supports_none_reasoning_effort, supports_reasoning diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 6bbf3598bd8..8bdb96c62c3 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -405,7 +405,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): return OpenAIGPT5Config.effort_resolves_to_none(model, effort) @staticmethod - def _supports_reasoning_param(model: str) -> bool: + def supports_reasoning_param(model: str) -> bool: from litellm.utils import get_model_info_helper try: @@ -503,7 +503,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): if ( self.custom_llm_provider == LlmProviders.OPENAI and self._requests_reasoning_effort(params.get("reasoning")) - and not self._supports_reasoning_param(model=model) + and not self.supports_reasoning_param(model=model) ): if drop_params or litellm.drop_params: dropped_reasoning: Final = params.pop("reasoning", None) diff --git a/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py b/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py index fe091e67a5a..bedf95e8a95 100644 --- a/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py @@ -781,3 +781,39 @@ class TestDisabledOnAnOffMapOpenAIDeployment: ) assert "reasoning" not in sent + + @pytest.mark.asyncio + async def test_the_chat_route_sends_none_when_the_operator_sets_the_deployment_mode( + self, respx_mock: respx.MockRouter, local_model_cost_map: None + ) -> None: + sent: Final = await _body_sent_upstream( + respx_mock, + {**_OFF_MAP_OPENAI_DEPLOYMENT, "model": "openai/vendor/off-map-hybrid-reasoner"}, + {"mode": "responses"}, + r"https://compatible\.example\.test/v1/responses", + _RESPONSE, + thinking={"type": "disabled"}, + stop_sequences=["STOP"], + ) + + assert sent["reasoning"]["effort"] == "none" + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "messages_params", + [{"thinking": {"type": "disabled"}}, {"thinking": {"type": "adaptive"}, "output_config": {"effort": "none"}}], + ids=["disabled", "adaptive-effort-none"], + ) + async def test_the_responses_bridge_sends_none_where_its_transformation_accepts_it( + self, respx_mock: respx.MockRouter, local_model_cost_map: None, messages_params: Mapping[str, object] + ) -> None: + sent: Final = await _body_sent_upstream( + respx_mock, + {**_OFF_MAP_OPENAI_DEPLOYMENT, "model": "openai/vendor/off-map-hybrid-reasoner"}, + {}, + r"https://compatible\.example\.test/v1/responses", + _RESPONSE, + **messages_params, + ) + + assert sent["reasoning"] == {"effort": "none"} From c53486f37e9b3415b5d3fb2b3ce3ffad69c05665 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 3 Nov 2026 13:06:01 -0800 Subject: [PATCH 10/12] fix(anthropic): leave reasoning to the model on a first-turn between_tools when its levels are unlisted A first-turn between_tools request sent reasoning effort none to every openai/ deployment whose cost map row lists no effort levels, so a backend that requires reasoning answered 400 where main answered 200. That turn now sends no reasoning unless the row or the deployment's model_info lists the model's levels. The OpenAI integration fixture now expects the summary that display summarized asks for. --- .../messages/adapters/transformation.py | 10 +++++-- .../messages/responses_adapters/handler.py | 6 +++- litellm/llms/anthropic/messages/utils.py | 6 ++-- .../translation/templates/openai.py | 2 +- ..._handler_reasoning_effort_normalization.py | 29 +++++++++++++++++-- .../test_responses_adapters_handler.py | 14 +++++++-- 6 files changed, 56 insertions(+), 11 deletions(-) diff --git a/litellm/llms/anthropic/messages/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py index a8dffc2528f..9b92320a2b7 100644 --- a/litellm/llms/anthropic/messages/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -1234,6 +1234,7 @@ class LiteLLMAnthropicMessagesAdapter: return thinking_type: Final = thinking.get("type") if isinstance(thinking, dict) else None + is_between_tools: Final = isinstance(thinking_type, str) and thinking_type == "between_tools" declared_effort: Final = output_effort if thinking_type == "adaptive" else None if is_claude_target and not declared_effort: return @@ -1241,7 +1242,7 @@ class LiteLLMAnthropicMessagesAdapter: mirrors_responses_bridge: Final = custom_llm_provider in RESPONSES_API_PROVIDERS reasoning_effort: Final = declared_effort or ( self._between_tools_reasoning_effort(anthropic_message_request, output_effort) - if thinking_type == "between_tools" and mirrors_responses_bridge + if is_between_tools and mirrors_responses_bridge else self.translate_anthropic_thinking_to_reasoning_effort( cast(AnthropicThinkingParam, thinking), model=model, @@ -1264,7 +1265,12 @@ class LiteLLMAnthropicMessagesAdapter: ) ) target_effort: Final = ( - reasoning_off_level(model, custom_llm_provider, drop_params=drop_params) + reasoning_off_level( + model, + custom_llm_provider, + drop_params=drop_params, + between_tools=is_between_tools, + ) if mirrors_responses_bridge and effort_sent == REASONING_EFFORT_OFF else effort_sent ) diff --git a/litellm/llms/anthropic/messages/responses_adapters/handler.py b/litellm/llms/anthropic/messages/responses_adapters/handler.py index 77d50bc4a16..be6444a51ae 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/handler.py +++ b/litellm/llms/anthropic/messages/responses_adapters/handler.py @@ -270,13 +270,16 @@ def _reasoning_for_target( custom_llm_provider: str | None, deployment_model_info: object, drop_params: bool, + between_tools: bool, ) -> object: reasoning_params: Final = as_str_mapping(reasoning) effort: Final = reasoning_params.get("effort") if reasoning_params is not None else None if reasoning_params is None or not isinstance(effort, str): return reasoning if effort == REASONING_EFFORT_OFF: - off_level: Final = reasoning_off_level(model, custom_llm_provider, drop_params=drop_params) + off_level: Final = reasoning_off_level( + model, custom_llm_provider, drop_params=drop_params, between_tools=between_tools + ) if off_level is None or _responses_api_refuses_reasoning_effort(model, custom_llm_provider): return None return {**reasoning_params, "effort": off_level} @@ -463,6 +466,7 @@ def _build_responses_kwargs( custom_llm_provider=provider, deployment_model_info=forwarded_kwargs.get("model_info"), drop_params=drop_params, + between_tools=thinking is not None and thinking.get("type") == "between_tools", ) if _maps_reasoning(thinking, model, provider, forwarded_kwargs) else None diff --git a/litellm/llms/anthropic/messages/utils.py b/litellm/llms/anthropic/messages/utils.py index 05e9cc19d11..aff6ccea066 100644 --- a/litellm/llms/anthropic/messages/utils.py +++ b/litellm/llms/anthropic/messages/utils.py @@ -259,11 +259,13 @@ def _lowest_tier_instead_of_off(levels: tuple[str, ...] | None) -> str | None: return next((tier for tier in _LOWEST_REASONING_TIERS if tier in levels), None) -def reasoning_off_level(model: str, custom_llm_provider: str | None, *, drop_params: bool) -> str | None: +def reasoning_off_level( + model: str, custom_llm_provider: str | None, *, drop_params: bool, between_tools: bool +) -> str | None: from litellm.utils import supports_none_reasoning_effort, supports_reasoning levels: Final = model_reasoning_efforts(model, custom_llm_provider) - if levels == (): + if levels == () or (between_tools and levels is None): return None lowest_tier: Final = _lowest_tier_instead_of_off(levels) if lowest_tier is not None: diff --git a/tests/integration/translation/templates/openai.py b/tests/integration/translation/templates/openai.py index b3f0d90362c..3de8fe28112 100644 --- a/tests/integration/translation/templates/openai.py +++ b/tests/integration/translation/templates/openai.py @@ -140,7 +140,7 @@ OPENAI_RESPONSES_REQUEST_FROM_MESSAGES: Final[Mapping[str, JsonValue]] = { "zero, negative and non-integer quantities before choosing the next file to " "inspect.", "max_output_tokens": 1024, - "reasoning": {"effort": "high"}, + "reasoning": {"effort": "high", "summary": "auto"}, "tools": [ { "type": "function", diff --git a/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py b/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py index bedf95e8a95..f6890ecf669 100644 --- a/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py @@ -388,7 +388,7 @@ _RESPONSE: Final = { "id": "msg_1", "status": "completed", "role": "assistant", - "content": [{"type": "output_text", "text": "ok", "annotations": []}], + "content": [{"type": "output_text", "text": "ok", "annotations": ()}], } ], "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, @@ -633,10 +633,12 @@ def _chat_completion_kwargs( return completion_kwargs -def _lowest_accepted_level(model: str, *, drop_params: bool) -> str | None: +def _lowest_accepted_level(model: str, *, drop_params: bool, between_tools: bool) -> str | None: levels: Final = model_reasoning_efforts(model, "openai") if levels is not None: return levels[0] if levels else None + if between_tools: + return None drops_unconfirmed_off: Final = ( drop_params and supports_reasoning(model, "openai") and not supports_none_reasoning_effort(model, "openai") ) @@ -676,7 +678,9 @@ class TestAnOpenAIChatRouteRequestGetsTheResponsesBridgeThinkingControls: drop_params=drop_params, ) - assert sent.get("reasoning_effort") == _lowest_accepted_level(model, drop_params=drop_params) + assert sent.get("reasoning_effort") == _lowest_accepted_level( + model, drop_params=drop_params, between_tools=thinking["type"] == "between_tools" + ) @pytest.mark.parametrize("model", [*OPENAI_CHAT_TARGETS, "openai/gpt-4.1", "openai/o3"]) def test_a_bare_output_effort_is_the_tier_when_the_model_reasons_unasked( @@ -817,3 +821,22 @@ class TestDisabledOnAnOffMapOpenAIDeployment: ) assert sent["reasoning"] == {"effort": "none"} + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "uses_chat_route, reasoning_key", [(True, "reasoning_effort"), (False, "reasoning")], ids=["chat-route", "responses-bridge"] + ) + async def test_a_first_turn_between_tools_leaves_reasoning_to_the_model( + self, respx_mock: respx.MockRouter, local_model_cost_map: None, uses_chat_route: bool, reasoning_key: str + ) -> None: + sent: Final = await _body_sent_upstream( + respx_mock, + {**_OFF_MAP_OPENAI_DEPLOYMENT, "model": "openai/vendor/off-map-hybrid-reasoner"}, + {}, + rf"https://compatible\.example\.test/v1/{'chat/completions' if uses_chat_route else 'responses'}", + _CHAT_COMPLETION if uses_chat_route else _RESPONSE, + thinking={"type": "between_tools"}, + stop_sequences=["STOP"] if uses_chat_route else None, + ) + + assert reasoning_key not in sent diff --git a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py index 77cf124bc13..606b5ae31e3 100644 --- a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py @@ -152,6 +152,10 @@ OFF_THINKING: Final = [{"type": "disabled"}, {"type": "between_tools"}] OFF_THINKING_IDS: Final = ["disabled", "between_tools-first-turn"] +def _leaves_reasoning_to_the_model(model: str, thinking: dict[str, object]) -> bool: + return thinking["type"] == "between_tools" and model_reasoning_efforts(model, "openai") is None + + def _lowest_listed_off_switch(model: str) -> str: levels: Final = model_reasoning_efforts(model, "openai") or () return next((level for level in ("none", "minimal", "low") if level in levels), "none") @@ -172,7 +176,10 @@ def test_build_responses_kwargs_turns_reasoning_off_at_the_lowest_level_the_mode map_row: Final = litellm.model_cost.get(model) never_reasons: Final = map_row is not None and not map_row.get("supports_reasoning") - assert responses_kwargs.get("reasoning") == (None if never_reasons else {"effort": _lowest_listed_off_switch(model)}) + sends_no_reasoning: Final = never_reasons or _leaves_reasoning_to_the_model(model, thinking) + assert responses_kwargs.get("reasoning") == ( + None if sends_no_reasoning else {"effort": _lowest_listed_off_switch(model)} + ) @pytest.mark.parametrize("thinking", OFF_THINKING, ids=OFF_THINKING_IDS) @@ -192,7 +199,10 @@ def test_build_responses_kwargs_under_drop_params_keeps_a_listed_lower_tier_and_ map_row: Final = litellm.model_cost.get(model) confirms_none: Final = map_row is not None and map_row.get("supports_none_reasoning_effort") is True keeps_none: Final = map_row is None or (bool(map_row.get("supports_reasoning")) and confirms_none) - assert responses_kwargs.get("reasoning") == ({"effort": off_switch} if off_switch != "none" or keeps_none else None) + sends_off_switch: Final = (off_switch != "none" or keeps_none) and not _leaves_reasoning_to_the_model( + model, thinking + ) + assert responses_kwargs.get("reasoning") == ({"effort": off_switch} if sends_off_switch else None) @pytest.mark.parametrize("model", ["gpt-5.4-mini", "gpt-5.1", "gpt-6-luna", "gpt-bridge-unmapped"]) From 42fdea3b91497aba7831db89434824462e1cd511 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 3 Nov 2026 17:00:11 -0800 Subject: [PATCH 11/12] fix(anthropic): read each deployment's own model_info for the off switch and guard a non-object thinking --- .../anthropic/messages/adapters/handler.py | 1 + .../messages/adapters/transformation.py | 15 ++- .../messages/responses_adapters/handler.py | 15 ++- litellm/llms/anthropic/messages/utils.py | 53 +++++++++-- litellm/utils.py | 8 +- ..._handler_reasoning_effort_normalization.py | 94 ++++++++++++++++++- .../messages/test_reasoning_effort_fields.py | 2 +- 7 files changed, 166 insertions(+), 22 deletions(-) diff --git a/litellm/llms/anthropic/messages/adapters/handler.py b/litellm/llms/anthropic/messages/adapters/handler.py index 29716b7104c..9c6585b4a13 100644 --- a/litellm/llms/anthropic/messages/adapters/handler.py +++ b/litellm/llms/anthropic/messages/adapters/handler.py @@ -669,6 +669,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: api_base=api_base if isinstance(api_base, str) else None, drop_params=drop_params, translate_bare_output_effort=extra_kwargs.get("reasoning_effort") is None, + deployment_model_info=extra_kwargs.get("model_info"), ) if openai_request is None: diff --git a/litellm/llms/anthropic/messages/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py index 9b92320a2b7..c81bb995a24 100644 --- a/litellm/llms/anthropic/messages/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -271,6 +271,7 @@ class AnthropicAdapter: api_base: str | None = None, drop_params: bool = True, translate_bare_output_effort: bool = False, + deployment_model_info: object = None, ) -> tuple[ChatCompletionRequest | None, dict[str, str]]: """ Translate Anthropic request params to OpenAI format, returning tool name mapping. @@ -308,6 +309,7 @@ class AnthropicAdapter: drop_params=drop_params, report_reasoning_drops=True, translate_bare_output_effort=translate_bare_output_effort, + deployment_model_info=deployment_model_info, ) return translated_body, tool_name_mapping @@ -1166,10 +1168,13 @@ class LiteLLMAnthropicMessagesAdapter: ) and custom_llm_provider not in _CLAUDE_OVER_REASONING_EFFORT_PROVIDERS @staticmethod - def _bare_output_effort(output_effort: object, model: str, custom_llm_provider: str | None) -> str | None: + def _bare_output_effort( + output_effort: object, model: str, custom_llm_provider: str | None, deployment_model_info: object + ) -> str | None: return ( output_effort - if isinstance(output_effort, str) and maps_bare_output_effort(model, custom_llm_provider) + if isinstance(output_effort, str) + and maps_bare_output_effort(model, custom_llm_provider, deployment_model_info=deployment_model_info) else None ) @@ -1188,6 +1193,7 @@ class LiteLLMAnthropicMessagesAdapter: drop_params: bool = True, report_reasoning_drops: bool = False, translate_bare_output_effort: bool = False, + deployment_model_info: object = None, ) -> None: """Translate Anthropic thinking to either thinking or reasoning_effort. @@ -1250,7 +1256,7 @@ class LiteLLMAnthropicMessagesAdapter: report_reasoning_drops=report_reasoning_drops, ) if thinking - else self._bare_output_effort(output_effort, model, custom_llm_provider) + else self._bare_output_effort(output_effort, model, custom_llm_provider, deployment_model_info) if translate_bare_output_effort and mirrors_responses_bridge else None ) @@ -1268,6 +1274,7 @@ class LiteLLMAnthropicMessagesAdapter: reasoning_off_level( model, custom_llm_provider, + deployment_model_info=deployment_model_info, drop_params=drop_params, between_tools=is_between_tools, ) @@ -1327,6 +1334,7 @@ class LiteLLMAnthropicMessagesAdapter: drop_params: bool = True, report_reasoning_drops: bool = False, translate_bare_output_effort: bool = False, + deployment_model_info: object = None, ) -> tuple[ChatCompletionRequest, dict[str, str]]: """ This is used by the beta Anthropic Adapter, for translating anthropic `/v1/messages` requests to the openai format. @@ -1383,6 +1391,7 @@ class LiteLLMAnthropicMessagesAdapter: drop_params=drop_params, report_reasoning_drops=report_reasoning_drops, translate_bare_output_effort=translate_bare_output_effort, + deployment_model_info=deployment_model_info, ) ## CONVERT STOP_SEQUENCES self._translate_stop_sequences_to_openai( diff --git a/litellm/llms/anthropic/messages/responses_adapters/handler.py b/litellm/llms/anthropic/messages/responses_adapters/handler.py index be6444a51ae..6cc5d2959da 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/handler.py +++ b/litellm/llms/anthropic/messages/responses_adapters/handler.py @@ -53,10 +53,10 @@ from ..convertible_blocks import reject_malformed_tool_result_text from ..safeguards import with_safeguard_results from ..utils import ( REASONING_EFFORT_OFF, + deployment_reasoning_efforts, litellm_logging_obj_from_kwargs, local_model_name, maps_bare_output_effort, - model_reasoning_efforts, normalize_reasoning_effort_value, reasoning_off_level, ) @@ -278,7 +278,11 @@ def _reasoning_for_target( return reasoning if effort == REASONING_EFFORT_OFF: off_level: Final = reasoning_off_level( - model, custom_llm_provider, drop_params=drop_params, between_tools=between_tools + model, + custom_llm_provider, + deployment_model_info=deployment_model_info, + drop_params=drop_params, + between_tools=between_tools, ) if off_level is None or _responses_api_refuses_reasoning_effort(model, custom_llm_provider): return None @@ -319,7 +323,7 @@ def _model_excludes_effort( ) -> bool: if deployment_rounds_reasoning_effort(deployment_model_info): return False - levels: Final = model_reasoning_efforts(model, custom_llm_provider) or () + levels: Final = deployment_reasoning_efforts(model, custom_llm_provider, deployment_model_info) or () return bool(levels) and effort not in levels @@ -367,7 +371,8 @@ def _maps_reasoning( thinking: Mapping[str, object] | None, model: str, provider: str | None, forwarded_kwargs: Mapping[str, object] ) -> bool: return bool(thinking) or ( - forwarded_kwargs.get("reasoning_effort") is None and maps_bare_output_effort(model, provider) + forwarded_kwargs.get("reasoning_effort") is None + and maps_bare_output_effort(model, provider, deployment_model_info=forwarded_kwargs.get("model_info")) ) @@ -466,7 +471,7 @@ def _build_responses_kwargs( custom_llm_provider=provider, deployment_model_info=forwarded_kwargs.get("model_info"), drop_params=drop_params, - between_tools=thinking is not None and thinking.get("type") == "between_tools", + between_tools=isinstance(thinking, dict) and thinking.get("type") == "between_tools", ) if _maps_reasoning(thinking, model, provider, forwarded_kwargs) else None diff --git a/litellm/llms/anthropic/messages/utils.py b/litellm/llms/anthropic/messages/utils.py index aff6ccea066..1bc1430731b 100644 --- a/litellm/llms/anthropic/messages/utils.py +++ b/litellm/llms/anthropic/messages/utils.py @@ -241,11 +241,24 @@ def _mapped_model_info(model: str, custom_llm_provider: str | None) -> ModelInfo return None -def model_reasoning_efforts(model: str, custom_llm_provider: str | None) -> tuple[str, ...] | None: +def _deployment_model_info( + model: str, custom_llm_provider: str | None, deployment_model_info: object +) -> Mapping[str, object] | None: + from litellm.utils import catalog_entry + + own_info: Final = as_str_mapping(deployment_model_info) + if own_info is None: + return _mapped_model_info(model, custom_llm_provider) + declared: Final = {key: value for key, value in own_info.items() if value is not None} + return {**(catalog_entry(model, custom_llm_provider) or {}), **declared} + + +def _resolved_reasoning_efforts( + model_info: Mapping[str, object] | None, model: str, custom_llm_provider: str | None +) -> tuple[str, ...] | None: from litellm.litellm_core_utils.reasoning_effort_capability import resolve_supported_reasoning_efforts from litellm.utils import has_catalog_entry - model_info: Final = _mapped_model_info(model, custom_llm_provider) if model_info is None: return None return resolve_supported_reasoning_efforts( @@ -253,6 +266,22 @@ def model_reasoning_efforts(model: str, custom_llm_provider: str | None) -> tupl ) +def model_reasoning_efforts(model: str, custom_llm_provider: str | None) -> tuple[str, ...] | None: + return _resolved_reasoning_efforts(_mapped_model_info(model, custom_llm_provider), model, custom_llm_provider) + + +def deployment_reasoning_efforts( + model: str, custom_llm_provider: str | None, deployment_model_info: object +) -> tuple[str, ...] | None: + return _resolved_reasoning_efforts( + _deployment_model_info(model, custom_llm_provider, deployment_model_info), model, custom_llm_provider + ) + + +def _declares(model_info: Mapping[str, object] | None, capability: str) -> bool: + return model_info is not None and model_info.get(capability) is True + + def _lowest_tier_instead_of_off(levels: tuple[str, ...] | None) -> str | None: if not levels or REASONING_EFFORT_OFF in levels: return None @@ -260,11 +289,15 @@ def _lowest_tier_instead_of_off(levels: tuple[str, ...] | None) -> str | None: def reasoning_off_level( - model: str, custom_llm_provider: str | None, *, drop_params: bool, between_tools: bool + model: str, + custom_llm_provider: str | None, + *, + deployment_model_info: object, + drop_params: bool, + between_tools: bool, ) -> str | None: - from litellm.utils import supports_none_reasoning_effort, supports_reasoning - - levels: Final = model_reasoning_efforts(model, custom_llm_provider) + model_info: Final = _deployment_model_info(model, custom_llm_provider, deployment_model_info) + levels: Final = _resolved_reasoning_efforts(model_info, model, custom_llm_provider) if levels == () or (between_tools and levels is None): return None lowest_tier: Final = _lowest_tier_instead_of_off(levels) @@ -272,8 +305,8 @@ def reasoning_off_level( return lowest_tier if ( not drop_params - or not supports_reasoning(model, custom_llm_provider) - or supports_none_reasoning_effort(model, custom_llm_provider) + or not _declares(model_info, "supports_reasoning") + or _declares(model_info, "supports_none_reasoning_effort") ): return REASONING_EFFORT_OFF verbose_logger.warning( @@ -284,10 +317,10 @@ def reasoning_off_level( return None -def maps_bare_output_effort(model: str, custom_llm_provider: str | None) -> bool: +def maps_bare_output_effort(model: str, custom_llm_provider: str | None, *, deployment_model_info: object) -> bool: from litellm.litellm_core_utils.reasoning_effort_capability import resolve_supported_reasoning_efforts - model_info: Final = _mapped_model_info(model, custom_llm_provider) + model_info: Final = _deployment_model_info(model, custom_llm_provider, deployment_model_info) return ( model_info is not None and model_info.get("default_reasoning_effort") != REASONING_EFFORT_OFF diff --git a/litellm/utils.py b/litellm/utils.py index 108c6fae982..de4754e461a 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5471,9 +5471,13 @@ def is_known_non_reasoning_model(model: str, custom_llm_provider: str | None) -> return info.get("supports_reasoning") is False or info["key"] in GetModelCostMap.loaded_model_cost_map() -def has_catalog_entry(model: str, custom_llm_provider: str | None) -> bool: +def catalog_entry(model: str, custom_llm_provider: str | None) -> Mapping[str, object] | None: info: Final = lookup_catalog_model_info(model, custom_llm_provider) - return not isinstance(info, ModelInfoLookupMiss) and info["key"] in GetModelCostMap.loaded_model_cost_map() + return None if isinstance(info, ModelInfoLookupMiss) else GetModelCostMap.loaded_model_cost_map().get(info["key"]) + + +def has_catalog_entry(model: str, custom_llm_provider: str | None) -> bool: + return catalog_entry(model, custom_llm_provider) is not None def _dropped_reasoning_warning(model: str, custom_llm_provider: str | None) -> str: diff --git a/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py b/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py index f6890ecf669..45bd0b96c8e 100644 --- a/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py @@ -690,7 +690,9 @@ class TestAnOpenAIChatRouteRequestGetsTheResponsesBridgeThinkingControls: model, "openai", thinking=None, messages=MESSAGES, output_config={"effort": "high"} ) - assert sent.get("reasoning_effort") == ("high" if maps_bare_output_effort(model, "openai") else None) + assert sent.get("reasoning_effort") == ( + "high" if maps_bare_output_effort(model, "openai", deployment_model_info=None) else None + ) def test_an_explicit_reasoning_effort_beats_a_bare_output_effort(self, local_model_cost_map: None) -> None: sent: Final = _chat_completion_kwargs( @@ -840,3 +842,93 @@ class TestDisabledOnAnOffMapOpenAIDeployment: ) assert reasoning_key not in sent + + +async def _bodies_sent_alone_and_beside_a_sibling( + respx_mock: respx.MockRouter, + litellm_params: Mapping[str, object], + sibling_model_info: Mapping[str, object], + upstream_url: str, + upstream_body: Mapping[str, object], + **messages_params: object, +) -> tuple[Mapping[str, object], Mapping[str, object]]: + route: Final = respx_mock.post(url__regex=upstream_url).mock(return_value=httpx.Response(200, json=upstream_body)) + deployment: Final = {"model_name": "group", "litellm_params": {**litellm_params}, "model_info": {}} + sibling: Final = {"model_name": "sibling", "litellm_params": {**litellm_params}, "model_info": {**sibling_model_info}} + await Router(model_list=[deployment]).aanthropic_messages( + model="group", messages=MESSAGES, max_tokens=4096, **messages_params + ) + alone: Final = json.loads(route.calls.last.request.read()) + await Router(model_list=[sibling, deployment]).aanthropic_messages( + model="group", messages=MESSAGES, max_tokens=4096, **messages_params + ) + return alone, json.loads(route.calls.last.request.read()) + + +class TestASiblingDeploymentLeavesTheOffSwitchAlone: + @pytest.fixture(autouse=True) + def _httpx_transport(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "litellm_params, api_base", + [ + ({**_OFF_MAP_OPENAI_DEPLOYMENT, "model": "openai/vendor/off-map-hybrid-reasoner"}, r"https://compatible\.example\.test/v1"), + ({"model": "openai/gpt-5-mini", "api_key": "fake-key"}, r"https://api\.openai\.com/v1"), + ], + ids=["off-map", "catalog"], + ) + @pytest.mark.parametrize( + "uses_chat_route, reasoning_key", [(True, "reasoning_effort"), (False, "reasoning")], ids=["chat-route", "responses-bridge"] + ) + async def test_a_first_turn_between_tools_sends_what_the_deployment_sends_alone( + self, + respx_mock: respx.MockRouter, + local_model_cost_map: None, + monkeypatch: pytest.MonkeyPatch, + litellm_params: Mapping[str, object], + api_base: str, + uses_chat_route: bool, + reasoning_key: str, + ) -> None: + upstream_url: Final = f"{api_base}/{'chat/completions' if uses_chat_route else 'responses'}" + upstream_body: Final = _CHAT_COMPLETION if uses_chat_route else _RESPONSE + monkeypatch.setattr(litellm, "use_chat_completions_url_for_anthropic_messages", uses_chat_route) + + alone, beside = await _bodies_sent_alone_and_beside_a_sibling( + respx_mock, + litellm_params, + {"supports_none_reasoning_effort": True}, + upstream_url, + upstream_body, + thinking={"type": "between_tools"}, + ) + + assert beside.get(reasoning_key) == alone.get(reasoning_key) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "messages_params", + [{"thinking": "enabled"}, {"thinking": "", "output_config": {"effort": "low"}}], + ids=["truthy", "falsy-with-effort"], +) +async def test_a_non_object_thinking_still_reaches_the_responses_bridge( + respx_mock: respx.MockRouter, + local_model_cost_map: None, + monkeypatch: pytest.MonkeyPatch, + messages_params: Mapping[str, object], +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + sent: Final = await _body_sent_upstream( + respx_mock, + {"model": "openai/gpt-5-mini", "api_key": "fake-key"}, + {}, + r"https://api\.openai\.com/v1/responses", + _RESPONSE, + **messages_params, + ) + + assert sent["model"] == "gpt-5-mini" diff --git a/tests/unit/llms/anthropic/messages/test_reasoning_effort_fields.py b/tests/unit/llms/anthropic/messages/test_reasoning_effort_fields.py index 1de1772032f..6de958eb1d9 100644 --- a/tests/unit/llms/anthropic/messages/test_reasoning_effort_fields.py +++ b/tests/unit/llms/anthropic/messages/test_reasoning_effort_fields.py @@ -346,7 +346,7 @@ def synthetic_model_entry(local_model_cost_map, request): ids=["reasons-by-default", "reasoning-off-by-default", "cannot-reason", "no-reasoning-flag"], ) def test_a_bare_output_effort_maps_only_onto_a_model_that_reasons_by_default(synthetic_model_entry, maps): - assert maps_bare_output_effort(synthetic_model_entry, "synthetic") is maps + assert maps_bare_output_effort(synthetic_model_entry, "synthetic", deployment_model_info=None) is maps @pytest.mark.parametrize( From 9db0487383d44ca99cf1fb08927261e42edc7dc8 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 3 Nov 2026 19:53:01 -0800 Subject: [PATCH 12/12] fix(anthropic): keep a client's effort none on the unsupported-value path and send minimal as the off level only where confirmed --- .../messages/adapters/transformation.py | 6 +- .../messages/responses_adapters/handler.py | 10 +- litellm/llms/anthropic/messages/utils.py | 30 ++++- ..._handler_reasoning_effort_normalization.py | 104 ++++++++++++++++++ 4 files changed, 140 insertions(+), 10 deletions(-) diff --git a/litellm/llms/anthropic/messages/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py index c81bb995a24..00946a8b946 100644 --- a/litellm/llms/anthropic/messages/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -20,6 +20,7 @@ from litellm.llms.anthropic.messages.utils import ( maps_bare_output_effort, prompt_cache_key_from_user_id, reasoning_off_level, + thinking_off_switch, ) from litellm.responses.utils import auto_reasoning_summary @@ -1270,15 +1271,16 @@ class LiteLLMAnthropicMessagesAdapter: reasoning_effort, cast(dict[str, object], thinking), model, custom_llm_provider ) ) + off_switch: Final = thinking_off_switch(anthropic_message_request) target_effort: Final = ( reasoning_off_level( model, custom_llm_provider, deployment_model_info=deployment_model_info, drop_params=drop_params, - between_tools=is_between_tools, + between_tools=off_switch == "between_tools", ) - if mirrors_responses_bridge and effort_sent == REASONING_EFFORT_OFF + if mirrors_responses_bridge and off_switch is not None and effort_sent == REASONING_EFFORT_OFF else effort_sent ) if target_effort is None: diff --git a/litellm/llms/anthropic/messages/responses_adapters/handler.py b/litellm/llms/anthropic/messages/responses_adapters/handler.py index 6cc5d2959da..4baaf3cc2b0 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/handler.py +++ b/litellm/llms/anthropic/messages/responses_adapters/handler.py @@ -53,12 +53,14 @@ from ..convertible_blocks import reject_malformed_tool_result_text from ..safeguards import with_safeguard_results from ..utils import ( REASONING_EFFORT_OFF, + ThinkingOffSwitch, deployment_reasoning_efforts, litellm_logging_obj_from_kwargs, local_model_name, maps_bare_output_effort, normalize_reasoning_effort_value, reasoning_off_level, + thinking_off_switch, ) from .streaming_iterator import AnthropicResponsesStreamWrapper from .transformation import LiteLLMAnthropicToResponsesAPIAdapter @@ -270,19 +272,19 @@ def _reasoning_for_target( custom_llm_provider: str | None, deployment_model_info: object, drop_params: bool, - between_tools: bool, + off_switch: ThinkingOffSwitch | None, ) -> object: reasoning_params: Final = as_str_mapping(reasoning) effort: Final = reasoning_params.get("effort") if reasoning_params is not None else None if reasoning_params is None or not isinstance(effort, str): return reasoning - if effort == REASONING_EFFORT_OFF: + if effort == REASONING_EFFORT_OFF and off_switch is not None: off_level: Final = reasoning_off_level( model, custom_llm_provider, deployment_model_info=deployment_model_info, drop_params=drop_params, - between_tools=between_tools, + between_tools=off_switch == "between_tools", ) if off_level is None or _responses_api_refuses_reasoning_effort(model, custom_llm_provider): return None @@ -471,7 +473,7 @@ def _build_responses_kwargs( custom_llm_provider=provider, deployment_model_info=forwarded_kwargs.get("model_info"), drop_params=drop_params, - between_tools=isinstance(thinking, dict) and thinking.get("type") == "between_tools", + off_switch=thinking_off_switch(anthropic_request), ) if _maps_reasoning(thinking, model, provider, forwarded_kwargs) else None diff --git a/litellm/llms/anthropic/messages/utils.py b/litellm/llms/anthropic/messages/utils.py index 1bc1430731b..96aa077abea 100644 --- a/litellm/llms/anthropic/messages/utils.py +++ b/litellm/llms/anthropic/messages/utils.py @@ -3,7 +3,7 @@ import os from collections.abc import Iterator, Mapping, Sequence from itertools import chain from types import MappingProxyType -from typing import TYPE_CHECKING, Final, TypeVar +from typing import TYPE_CHECKING, Final, Literal, TypeVar from pydantic import ConfigDict, TypeAdapter, ValidationError from typing_extensions import ReadOnly @@ -31,6 +31,9 @@ _EFFORT_DEGRADATION_CHAIN: Final[Mapping[str, tuple[str, ...]]] = MappingProxyTy REASONING_EFFORT_OFF: Final = "none" RESPONSES_API_PROVIDERS: Final = frozenset({"openai"}) _LOWEST_REASONING_TIERS: Final = ("minimal", "low") +_UNCONFIRMED_MINIMAL_LOWEST_TIERS: Final = ("low",) + +ThinkingOffSwitch = Literal["disabled", "between_tools"] _JsonValue: Final = TypeVar("_JsonValue") @@ -232,6 +235,16 @@ def is_between_tool_calls(request: Mapping[str, object]) -> bool: return any(block.get("type") == "tool_result" for block in blocks) +def thinking_off_switch(request: Mapping[str, object]) -> ThinkingOffSwitch | None: + thinking: Final = as_str_mapping(request.get("thinking")) + thinking_type: Final = thinking.get("type") if thinking is not None else None + if thinking_type == "disabled": + return "disabled" + if thinking_type == "between_tools" and not is_between_tool_calls(request): + return "between_tools" + return None + + def _mapped_model_info(model: str, custom_llm_provider: str | None) -> ModelInfo | None: from litellm.utils import get_model_info @@ -282,10 +295,19 @@ def _declares(model_info: Mapping[str, object] | None, capability: str) -> bool: return model_info is not None and model_info.get(capability) is True -def _lowest_tier_instead_of_off(levels: tuple[str, ...] | None) -> str | None: +def _confirms_minimal(model_info: Mapping[str, object] | None) -> bool: + from litellm.litellm_core_utils.reasoning_effort_capability import declared_reasoning_efforts + + return model_info is not None and ( + declared_reasoning_efforts(model_info) is not None or _declares(model_info, "supports_minimal_reasoning_effort") + ) + + +def _lowest_tier_instead_of_off(levels: tuple[str, ...] | None, model_info: Mapping[str, object] | None) -> str | None: if not levels or REASONING_EFFORT_OFF in levels: return None - return next((tier for tier in _LOWEST_REASONING_TIERS if tier in levels), None) + tiers: Final = _LOWEST_REASONING_TIERS if _confirms_minimal(model_info) else _UNCONFIRMED_MINIMAL_LOWEST_TIERS + return next((tier for tier in tiers if tier in levels), None) def reasoning_off_level( @@ -300,7 +322,7 @@ def reasoning_off_level( levels: Final = _resolved_reasoning_efforts(model_info, model, custom_llm_provider) if levels == () or (between_tools and levels is None): return None - lowest_tier: Final = _lowest_tier_instead_of_off(levels) + lowest_tier: Final = _lowest_tier_instead_of_off(levels, model_info) if lowest_tier is not None: return lowest_tier if ( diff --git a/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py b/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py index 45bd0b96c8e..09b2de0642a 100644 --- a/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py @@ -932,3 +932,107 @@ async def test_a_non_object_thinking_still_reaches_the_responses_bridge( ) assert sent["model"] == "gpt-5-mini" + + +_CLIENT_EFFORT_NONE: Final = [ + {"thinking": {"type": "adaptive"}, "output_config": {"effort": "none"}}, + {"output_config": {"effort": "none"}}, +] + + +class TestAClientEffortNoneFollowsTheUnsupportedValueRule: + @pytest.fixture(autouse=True) + def _httpx_transport(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + @pytest.mark.asyncio + @pytest.mark.parametrize("messages_params", _CLIENT_EFFORT_NONE, ids=["adaptive", "bare"]) + @pytest.mark.parametrize( + "uses_chat_route, reasoning_key", [(True, "reasoning_effort"), (False, "reasoning")], ids=["chat-route", "responses-bridge"] + ) + async def test_drop_params_drops_it_on_a_model_that_refuses_none( + self, + respx_mock: respx.MockRouter, + local_model_cost_map: None, + monkeypatch: pytest.MonkeyPatch, + messages_params: Mapping[str, object], + uses_chat_route: bool, + reasoning_key: str, + ) -> None: + monkeypatch.setattr(litellm, "use_chat_completions_url_for_anthropic_messages", uses_chat_route) + + sent: Final = await _body_sent_upstream( + respx_mock, + {"model": "openai/gpt-5-mini", "api_key": "fake-key", "drop_params": True}, + {}, + rf"https://api\.openai\.com/v1/{'chat/completions' if uses_chat_route else 'responses'}", + _CHAT_COMPLETION if uses_chat_route else _RESPONSE, + **messages_params, + ) + + assert reasoning_key not in sent + + @pytest.mark.asyncio + @pytest.mark.parametrize("messages_params", _CLIENT_EFFORT_NONE, ids=["adaptive", "bare"]) + @pytest.mark.parametrize("uses_chat_route", [True, False], ids=["chat-route", "responses-bridge"]) + async def test_without_drop_params_it_answers_400( + self, + respx_mock: respx.MockRouter, + local_model_cost_map: None, + monkeypatch: pytest.MonkeyPatch, + messages_params: Mapping[str, object], + uses_chat_route: bool, + ) -> None: + monkeypatch.setattr(litellm, "use_chat_completions_url_for_anthropic_messages", uses_chat_route) + + with pytest.raises(litellm.UnsupportedParamsError, match="effort=none is not supported for gpt-5-mini"): + await _body_sent_upstream( + respx_mock, + {"model": "openai/gpt-5-mini", "api_key": "fake-key"}, + {}, + rf"https://api\.openai\.com/v1/{'chat/completions' if uses_chat_route else 'responses'}", + _CHAT_COMPLETION if uses_chat_route else _RESPONSE, + **messages_params, + ) + + +class TestTheOffSwitchUsesMinimalOnlyWhereTheDeploymentConfirmsIt: + @pytest.fixture(autouse=True) + def _httpx_transport(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "model_info, expected", + [ + ({"supports_none_reasoning_effort": False}, "low"), + ({"reasoning_effort_levels": ["minimal", "low", "medium", "high"]}, "minimal"), + ], + ids=["none-flag-only", "declared-levels"], + ) + @pytest.mark.parametrize("thinking", [{"type": "disabled"}, {"type": "between_tools"}], ids=["disabled", "between_tools"]) + @pytest.mark.parametrize("uses_chat_route", [True, False], ids=["chat-route", "responses-bridge"]) + @pytest.mark.parametrize("drop_params", [False, True], ids=["no-drop_params", "drop_params"]) + async def test_an_o3_deployment_gets_the_lowest_level_it_confirms( + self, + respx_mock: respx.MockRouter, + local_model_cost_map: None, + monkeypatch: pytest.MonkeyPatch, + model_info: Mapping[str, object], + expected: str, + thinking: Mapping[str, object], + uses_chat_route: bool, + drop_params: bool, + ) -> None: + monkeypatch.setattr(litellm, "use_chat_completions_url_for_anthropic_messages", uses_chat_route) + + sent: Final = await _body_sent_upstream( + respx_mock, + {"model": "openai/o3", "api_key": "fake-key", "drop_params": drop_params}, + model_info, + rf"https://api\.openai\.com/v1/{'chat/completions' if uses_chat_route else 'responses'}", + _CHAT_COMPLETION if uses_chat_route else _RESPONSE, + thinking=thinking, + ) + + assert (sent["reasoning_effort"] if uses_chat_route else sent["reasoning"]["effort"]) == expected