diff --git a/litellm/llms/anthropic/messages/adapters/handler.py b/litellm/llms/anthropic/messages/adapters/handler.py index 29716b7104c..9c6585b4a13 100644 --- a/litellm/llms/anthropic/messages/adapters/handler.py +++ b/litellm/llms/anthropic/messages/adapters/handler.py @@ -669,6 +669,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: api_base=api_base if isinstance(api_base, str) else None, drop_params=drop_params, translate_bare_output_effort=extra_kwargs.get("reasoning_effort") is None, + deployment_model_info=extra_kwargs.get("model_info"), ) if openai_request is None: diff --git a/litellm/llms/anthropic/messages/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py index 9b92320a2b7..c81bb995a24 100644 --- a/litellm/llms/anthropic/messages/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -271,6 +271,7 @@ class AnthropicAdapter: api_base: str | None = None, drop_params: bool = True, translate_bare_output_effort: bool = False, + deployment_model_info: object = None, ) -> tuple[ChatCompletionRequest | None, dict[str, str]]: """ Translate Anthropic request params to OpenAI format, returning tool name mapping. @@ -308,6 +309,7 @@ class AnthropicAdapter: drop_params=drop_params, report_reasoning_drops=True, translate_bare_output_effort=translate_bare_output_effort, + deployment_model_info=deployment_model_info, ) return translated_body, tool_name_mapping @@ -1166,10 +1168,13 @@ class LiteLLMAnthropicMessagesAdapter: ) and custom_llm_provider not in _CLAUDE_OVER_REASONING_EFFORT_PROVIDERS @staticmethod - def _bare_output_effort(output_effort: object, model: str, custom_llm_provider: str | None) -> str | None: + def _bare_output_effort( + output_effort: object, model: str, custom_llm_provider: str | None, deployment_model_info: object + ) -> str | None: return ( output_effort - if isinstance(output_effort, str) and maps_bare_output_effort(model, custom_llm_provider) + if isinstance(output_effort, str) + and maps_bare_output_effort(model, custom_llm_provider, deployment_model_info=deployment_model_info) else None ) @@ -1188,6 +1193,7 @@ class LiteLLMAnthropicMessagesAdapter: drop_params: bool = True, report_reasoning_drops: bool = False, translate_bare_output_effort: bool = False, + deployment_model_info: object = None, ) -> None: """Translate Anthropic thinking to either thinking or reasoning_effort. @@ -1250,7 +1256,7 @@ class LiteLLMAnthropicMessagesAdapter: report_reasoning_drops=report_reasoning_drops, ) if thinking - else self._bare_output_effort(output_effort, model, custom_llm_provider) + else self._bare_output_effort(output_effort, model, custom_llm_provider, deployment_model_info) if translate_bare_output_effort and mirrors_responses_bridge else None ) @@ -1268,6 +1274,7 @@ class LiteLLMAnthropicMessagesAdapter: reasoning_off_level( model, custom_llm_provider, + deployment_model_info=deployment_model_info, drop_params=drop_params, between_tools=is_between_tools, ) @@ -1327,6 +1334,7 @@ class LiteLLMAnthropicMessagesAdapter: drop_params: bool = True, report_reasoning_drops: bool = False, translate_bare_output_effort: bool = False, + deployment_model_info: object = None, ) -> tuple[ChatCompletionRequest, dict[str, str]]: """ This is used by the beta Anthropic Adapter, for translating anthropic `/v1/messages` requests to the openai format. @@ -1383,6 +1391,7 @@ class LiteLLMAnthropicMessagesAdapter: drop_params=drop_params, report_reasoning_drops=report_reasoning_drops, translate_bare_output_effort=translate_bare_output_effort, + deployment_model_info=deployment_model_info, ) ## CONVERT STOP_SEQUENCES self._translate_stop_sequences_to_openai( diff --git a/litellm/llms/anthropic/messages/responses_adapters/handler.py b/litellm/llms/anthropic/messages/responses_adapters/handler.py index be6444a51ae..6cc5d2959da 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/handler.py +++ b/litellm/llms/anthropic/messages/responses_adapters/handler.py @@ -53,10 +53,10 @@ from ..convertible_blocks import reject_malformed_tool_result_text from ..safeguards import with_safeguard_results from ..utils import ( REASONING_EFFORT_OFF, + deployment_reasoning_efforts, litellm_logging_obj_from_kwargs, local_model_name, maps_bare_output_effort, - model_reasoning_efforts, normalize_reasoning_effort_value, reasoning_off_level, ) @@ -278,7 +278,11 @@ def _reasoning_for_target( return reasoning if effort == REASONING_EFFORT_OFF: off_level: Final = reasoning_off_level( - model, custom_llm_provider, drop_params=drop_params, between_tools=between_tools + model, + custom_llm_provider, + deployment_model_info=deployment_model_info, + drop_params=drop_params, + between_tools=between_tools, ) if off_level is None or _responses_api_refuses_reasoning_effort(model, custom_llm_provider): return None @@ -319,7 +323,7 @@ def _model_excludes_effort( ) -> bool: if deployment_rounds_reasoning_effort(deployment_model_info): return False - levels: Final = model_reasoning_efforts(model, custom_llm_provider) or () + levels: Final = deployment_reasoning_efforts(model, custom_llm_provider, deployment_model_info) or () return bool(levels) and effort not in levels @@ -367,7 +371,8 @@ def _maps_reasoning( thinking: Mapping[str, object] | None, model: str, provider: str | None, forwarded_kwargs: Mapping[str, object] ) -> bool: return bool(thinking) or ( - forwarded_kwargs.get("reasoning_effort") is None and maps_bare_output_effort(model, provider) + forwarded_kwargs.get("reasoning_effort") is None + and maps_bare_output_effort(model, provider, deployment_model_info=forwarded_kwargs.get("model_info")) ) @@ -466,7 +471,7 @@ def _build_responses_kwargs( custom_llm_provider=provider, deployment_model_info=forwarded_kwargs.get("model_info"), drop_params=drop_params, - between_tools=thinking is not None and thinking.get("type") == "between_tools", + between_tools=isinstance(thinking, dict) and thinking.get("type") == "between_tools", ) if _maps_reasoning(thinking, model, provider, forwarded_kwargs) else None diff --git a/litellm/llms/anthropic/messages/utils.py b/litellm/llms/anthropic/messages/utils.py index aff6ccea066..1bc1430731b 100644 --- a/litellm/llms/anthropic/messages/utils.py +++ b/litellm/llms/anthropic/messages/utils.py @@ -241,11 +241,24 @@ def _mapped_model_info(model: str, custom_llm_provider: str | None) -> ModelInfo return None -def model_reasoning_efforts(model: str, custom_llm_provider: str | None) -> tuple[str, ...] | None: +def _deployment_model_info( + model: str, custom_llm_provider: str | None, deployment_model_info: object +) -> Mapping[str, object] | None: + from litellm.utils import catalog_entry + + own_info: Final = as_str_mapping(deployment_model_info) + if own_info is None: + return _mapped_model_info(model, custom_llm_provider) + declared: Final = {key: value for key, value in own_info.items() if value is not None} + return {**(catalog_entry(model, custom_llm_provider) or {}), **declared} + + +def _resolved_reasoning_efforts( + model_info: Mapping[str, object] | None, model: str, custom_llm_provider: str | None +) -> tuple[str, ...] | None: from litellm.litellm_core_utils.reasoning_effort_capability import resolve_supported_reasoning_efforts from litellm.utils import has_catalog_entry - model_info: Final = _mapped_model_info(model, custom_llm_provider) if model_info is None: return None return resolve_supported_reasoning_efforts( @@ -253,6 +266,22 @@ def model_reasoning_efforts(model: str, custom_llm_provider: str | None) -> tupl ) +def model_reasoning_efforts(model: str, custom_llm_provider: str | None) -> tuple[str, ...] | None: + return _resolved_reasoning_efforts(_mapped_model_info(model, custom_llm_provider), model, custom_llm_provider) + + +def deployment_reasoning_efforts( + model: str, custom_llm_provider: str | None, deployment_model_info: object +) -> tuple[str, ...] | None: + return _resolved_reasoning_efforts( + _deployment_model_info(model, custom_llm_provider, deployment_model_info), model, custom_llm_provider + ) + + +def _declares(model_info: Mapping[str, object] | None, capability: str) -> bool: + return model_info is not None and model_info.get(capability) is True + + def _lowest_tier_instead_of_off(levels: tuple[str, ...] | None) -> str | None: if not levels or REASONING_EFFORT_OFF in levels: return None @@ -260,11 +289,15 @@ def _lowest_tier_instead_of_off(levels: tuple[str, ...] | None) -> str | None: def reasoning_off_level( - model: str, custom_llm_provider: str | None, *, drop_params: bool, between_tools: bool + model: str, + custom_llm_provider: str | None, + *, + deployment_model_info: object, + drop_params: bool, + between_tools: bool, ) -> str | None: - from litellm.utils import supports_none_reasoning_effort, supports_reasoning - - levels: Final = model_reasoning_efforts(model, custom_llm_provider) + model_info: Final = _deployment_model_info(model, custom_llm_provider, deployment_model_info) + levels: Final = _resolved_reasoning_efforts(model_info, model, custom_llm_provider) if levels == () or (between_tools and levels is None): return None lowest_tier: Final = _lowest_tier_instead_of_off(levels) @@ -272,8 +305,8 @@ def reasoning_off_level( return lowest_tier if ( not drop_params - or not supports_reasoning(model, custom_llm_provider) - or supports_none_reasoning_effort(model, custom_llm_provider) + or not _declares(model_info, "supports_reasoning") + or _declares(model_info, "supports_none_reasoning_effort") ): return REASONING_EFFORT_OFF verbose_logger.warning( @@ -284,10 +317,10 @@ def reasoning_off_level( return None -def maps_bare_output_effort(model: str, custom_llm_provider: str | None) -> bool: +def maps_bare_output_effort(model: str, custom_llm_provider: str | None, *, deployment_model_info: object) -> bool: from litellm.litellm_core_utils.reasoning_effort_capability import resolve_supported_reasoning_efforts - model_info: Final = _mapped_model_info(model, custom_llm_provider) + model_info: Final = _deployment_model_info(model, custom_llm_provider, deployment_model_info) return ( model_info is not None and model_info.get("default_reasoning_effort") != REASONING_EFFORT_OFF diff --git a/litellm/utils.py b/litellm/utils.py index 108c6fae982..de4754e461a 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5471,9 +5471,13 @@ def is_known_non_reasoning_model(model: str, custom_llm_provider: str | None) -> return info.get("supports_reasoning") is False or info["key"] in GetModelCostMap.loaded_model_cost_map() -def has_catalog_entry(model: str, custom_llm_provider: str | None) -> bool: +def catalog_entry(model: str, custom_llm_provider: str | None) -> Mapping[str, object] | None: info: Final = lookup_catalog_model_info(model, custom_llm_provider) - return not isinstance(info, ModelInfoLookupMiss) and info["key"] in GetModelCostMap.loaded_model_cost_map() + return None if isinstance(info, ModelInfoLookupMiss) else GetModelCostMap.loaded_model_cost_map().get(info["key"]) + + +def has_catalog_entry(model: str, custom_llm_provider: str | None) -> bool: + return catalog_entry(model, custom_llm_provider) is not None def _dropped_reasoning_warning(model: str, custom_llm_provider: str | None) -> str: diff --git a/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py b/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py index f6890ecf669..45bd0b96c8e 100644 --- a/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_handler_reasoning_effort_normalization.py @@ -690,7 +690,9 @@ class TestAnOpenAIChatRouteRequestGetsTheResponsesBridgeThinkingControls: model, "openai", thinking=None, messages=MESSAGES, output_config={"effort": "high"} ) - assert sent.get("reasoning_effort") == ("high" if maps_bare_output_effort(model, "openai") else None) + assert sent.get("reasoning_effort") == ( + "high" if maps_bare_output_effort(model, "openai", deployment_model_info=None) else None + ) def test_an_explicit_reasoning_effort_beats_a_bare_output_effort(self, local_model_cost_map: None) -> None: sent: Final = _chat_completion_kwargs( @@ -840,3 +842,93 @@ class TestDisabledOnAnOffMapOpenAIDeployment: ) assert reasoning_key not in sent + + +async def _bodies_sent_alone_and_beside_a_sibling( + respx_mock: respx.MockRouter, + litellm_params: Mapping[str, object], + sibling_model_info: Mapping[str, object], + upstream_url: str, + upstream_body: Mapping[str, object], + **messages_params: object, +) -> tuple[Mapping[str, object], Mapping[str, object]]: + route: Final = respx_mock.post(url__regex=upstream_url).mock(return_value=httpx.Response(200, json=upstream_body)) + deployment: Final = {"model_name": "group", "litellm_params": {**litellm_params}, "model_info": {}} + sibling: Final = {"model_name": "sibling", "litellm_params": {**litellm_params}, "model_info": {**sibling_model_info}} + await Router(model_list=[deployment]).aanthropic_messages( + model="group", messages=MESSAGES, max_tokens=4096, **messages_params + ) + alone: Final = json.loads(route.calls.last.request.read()) + await Router(model_list=[sibling, deployment]).aanthropic_messages( + model="group", messages=MESSAGES, max_tokens=4096, **messages_params + ) + return alone, json.loads(route.calls.last.request.read()) + + +class TestASiblingDeploymentLeavesTheOffSwitchAlone: + @pytest.fixture(autouse=True) + def _httpx_transport(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "litellm_params, api_base", + [ + ({**_OFF_MAP_OPENAI_DEPLOYMENT, "model": "openai/vendor/off-map-hybrid-reasoner"}, r"https://compatible\.example\.test/v1"), + ({"model": "openai/gpt-5-mini", "api_key": "fake-key"}, r"https://api\.openai\.com/v1"), + ], + ids=["off-map", "catalog"], + ) + @pytest.mark.parametrize( + "uses_chat_route, reasoning_key", [(True, "reasoning_effort"), (False, "reasoning")], ids=["chat-route", "responses-bridge"] + ) + async def test_a_first_turn_between_tools_sends_what_the_deployment_sends_alone( + self, + respx_mock: respx.MockRouter, + local_model_cost_map: None, + monkeypatch: pytest.MonkeyPatch, + litellm_params: Mapping[str, object], + api_base: str, + uses_chat_route: bool, + reasoning_key: str, + ) -> None: + upstream_url: Final = f"{api_base}/{'chat/completions' if uses_chat_route else 'responses'}" + upstream_body: Final = _CHAT_COMPLETION if uses_chat_route else _RESPONSE + monkeypatch.setattr(litellm, "use_chat_completions_url_for_anthropic_messages", uses_chat_route) + + alone, beside = await _bodies_sent_alone_and_beside_a_sibling( + respx_mock, + litellm_params, + {"supports_none_reasoning_effort": True}, + upstream_url, + upstream_body, + thinking={"type": "between_tools"}, + ) + + assert beside.get(reasoning_key) == alone.get(reasoning_key) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "messages_params", + [{"thinking": "enabled"}, {"thinking": "", "output_config": {"effort": "low"}}], + ids=["truthy", "falsy-with-effort"], +) +async def test_a_non_object_thinking_still_reaches_the_responses_bridge( + respx_mock: respx.MockRouter, + local_model_cost_map: None, + monkeypatch: pytest.MonkeyPatch, + messages_params: Mapping[str, object], +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + sent: Final = await _body_sent_upstream( + respx_mock, + {"model": "openai/gpt-5-mini", "api_key": "fake-key"}, + {}, + r"https://api\.openai\.com/v1/responses", + _RESPONSE, + **messages_params, + ) + + assert sent["model"] == "gpt-5-mini" diff --git a/tests/unit/llms/anthropic/messages/test_reasoning_effort_fields.py b/tests/unit/llms/anthropic/messages/test_reasoning_effort_fields.py index 1de1772032f..6de958eb1d9 100644 --- a/tests/unit/llms/anthropic/messages/test_reasoning_effort_fields.py +++ b/tests/unit/llms/anthropic/messages/test_reasoning_effort_fields.py @@ -346,7 +346,7 @@ def synthetic_model_entry(local_model_cost_map, request): ids=["reasons-by-default", "reasoning-off-by-default", "cannot-reason", "no-reasoning-flag"], ) def test_a_bare_output_effort_maps_only_onto_a_model_that_reasons_by_default(synthetic_model_entry, maps): - assert maps_bare_output_effort(synthetic_model_entry, "synthetic") is maps + assert maps_bare_output_effort(synthetic_model_entry, "synthetic", deployment_model_info=None) is maps @pytest.mark.parametrize(