diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 54a8780ca5b..1da94f59668 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -97,6 +97,7 @@ from litellm.utils import ( ) from ..common_utils import ( + THINKING_BLOCK_BINDING_PROVIDERS, AnthropicError, AnthropicModelInfo, eager_input_streaming_flag, @@ -329,6 +330,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): def _resolved_provider(self) -> str: return self.custom_llm_provider or "anthropic" + @property + def _carries_thinking_block_binding(self) -> bool: + return self._resolved_provider in THINKING_BLOCK_BINDING_PROVIDERS + @classmethod def get_config(cls, *, model: str | None = None): config: Final = super().get_config() @@ -1656,7 +1661,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): else: optional_params["thinking"] = value AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model( - model=model, optional_params=optional_params, custom_llm_provider=self._resolved_provider + model=model, + optional_params=optional_params, + custom_llm_provider=self._resolved_provider, + carry_block_binding=self._carries_thinking_block_binding, ) elif param == "reasoning_effort": # Accept both string ("low") and dict ({"effort": "low", diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 12d77112b08..2e184ee82fe 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -108,6 +108,7 @@ _DOTTED_VERSION_RE: Final = re.compile(r"(\d)\.(\d)") _CLAUDE_CODE_BILLING_HEADER_PREFIX: Final = "x-anthropic-billing-header:" _OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object]) _CLAUDE_CODE_OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object]) +THINKING_BLOCK_BINDING_PROVIDERS: Final = frozenset({"anthropic", "azure_ai"}) _CLAUDE_CODE_USER_AGENT_PREFIXES: Final = ("claude-cli/", "claude-code/") @@ -162,6 +163,11 @@ def validated_object_mapping(value: object) -> dict[object, object] | None: return None +def thinking_block_binding(thinking: Mapping[object, object] | None) -> dict[str, object]: + block_binding: Final = thinking.get("block_binding") if thinking is not None else None + return {} if block_binding is None else {"block_binding": block_binding} + + def _validated_claude_code_list(value: object) -> list[object] | None: try: return _CLAUDE_CODE_OBJECT_LIST_ADAPTER.validate_python(value) @@ -914,6 +920,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): model: str, optional_params: MutableMapping[str, object], # mutable-ok: in-place out-param like the sibling helpers custom_llm_provider: str, + carry_block_binding: bool = False, ) -> None: """Translate legacy ``thinking.type=enabled`` to adaptive for the adaptive-thinking models that reject it (4.7+ and the 5 families). @@ -939,10 +946,8 @@ class AnthropicModelInfo(BaseLLMModelInfo): existing_output_config: Final = optional_params.get("output_config") display: Final = thinking.get("display") supported_display: Final = {"display": display} if display in ("summarized", "omitted") else {} - carried_thinking_keys: Final = { - key: value for key, value in thinking.items() if key not in ("type", "budget_tokens", "display") - } - optional_params["thinking"] = {**carried_thinking_keys, "type": "adaptive", **supported_display} + carried_block_binding: Final = thinking_block_binding(thinking) if carry_block_binding else {} + optional_params["thinking"] = {**carried_block_binding, "type": "adaptive", **supported_display} optional_params["output_config"] = { "effort": effort, **(existing_output_config if isinstance(existing_output_config, dict) else MappingProxyType({})), diff --git a/litellm/llms/anthropic/pass_through/messages/transformation.py b/litellm/llms/anthropic/pass_through/messages/transformation.py index f1eb30b63c1..9558a0012a5 100644 --- a/litellm/llms/anthropic/pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/pass_through/messages/transformation.py @@ -23,6 +23,7 @@ from litellm.types.llms.anthropic_tool_search import get_tool_search_beta_header from litellm.types.router import GenericLiteLLMParams from ...common_utils import ( + THINKING_BLOCK_BINDING_PROVIDERS, AnthropicError, AnthropicModelInfo, merge_anthropic_beta_headers, @@ -80,6 +81,10 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): def _resolved_provider(self) -> str: return self.custom_llm_provider or "anthropic" + @property + def _carries_thinking_block_binding(self) -> bool: + return self._resolved_provider in THINKING_BLOCK_BINDING_PROVIDERS + def get_supported_anthropic_messages_params(self, model: str) -> list: return [ "messages", @@ -607,6 +612,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): model=model, optional_params=anthropic_messages_optional_request_params, custom_llm_provider=self._resolved_provider, + carry_block_binding=self._carries_thinking_block_binding, ) self._translate_adaptive_effort_for_non_adaptive_model( diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py index b8bc276de0d..24c219e9837 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py @@ -12,7 +12,7 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import ( convert_url_to_base64, ) from litellm.llms.anthropic.chat.transformation import AnthropicConfig -from litellm.llms.anthropic.common_utils import AnthropicModelInfo +from litellm.llms.anthropic.common_utils import AnthropicModelInfo, thinking_block_binding from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import ( AmazonInvokeConfig, ) @@ -67,6 +67,10 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): def custom_llm_provider(self) -> str | None: return "bedrock" + @property + def _carries_thinking_block_binding(self) -> bool: + return True + def should_strip_billing_metadata(self) -> bool: return True @@ -115,7 +119,10 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): model = original_model AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model( - model=original_model, optional_params=optional_params, custom_llm_provider="bedrock" + model=original_model, + optional_params=optional_params, + custom_llm_provider="bedrock", + carry_block_binding=self._carries_thinking_block_binding, ) translated_thinking: Final = optional_params.get("thinking") if ( @@ -123,7 +130,11 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): and isinstance(translated_thinking, dict) and translated_thinking.get("type") == "adaptive" ): - optional_params["thinking"] = {**translated_thinking, "display": "updates"} + optional_params["thinking"] = { + **thinking_block_binding(translated_thinking), + "type": "adaptive", + "display": "updates", + } # The stub model hides the original model from the parent's forced-tool-use backstop response_format_tool_choice: Final = optional_params.get("tool_choice") diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py index eb6ec66bb09..1b857e49505 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py @@ -17,7 +17,11 @@ from litellm.llms.anthropic.chat.transformation import ( DROP_UNSUPPORTED_OUTPUT_CONFIG_WARNING, AnthropicConfig, ) -from litellm.llms.anthropic.common_utils import AnthropicModelInfo, validated_object_mapping +from litellm.llms.anthropic.common_utils import ( + AnthropicModelInfo, + thinking_block_binding, + validated_object_mapping, +) from litellm.llms.anthropic.pass_through.messages.transformation import ( AnthropicMessagesConfig, ) @@ -81,6 +85,10 @@ class AmazonAnthropicClaudeMessagesConfig( def custom_llm_provider(self) -> str | None: return "bedrock" + @property + def _carries_thinking_block_binding(self) -> bool: + return True + @property def beta_headers_provider(self) -> str: return self.custom_llm_provider or "bedrock" @@ -349,9 +357,13 @@ class AmazonAnthropicClaudeMessagesConfig( output_config = {} output_config.setdefault("effort", self._effort_from_thinking_budget(budget_tokens)) thinking: Final = validated_object_mapping(anthropic_messages_request.get("thinking")) - carried: Final = {key: value for key, value in (thinking or {}).items() if key not in ("type", "budget_tokens")} + display: Final = thinking.get("display") if thinking is not None else None anthropic_messages_request["output_config"] = output_config - anthropic_messages_request["thinking"] = {**carried, "type": "adaptive"} + anthropic_messages_request["thinking"] = { + **thinking_block_binding(thinking), + "type": "adaptive", + **({} if display is None else {"display": display}), + } verbose_logger.debug( "Bedrock clear_thinking_20251015: injected adaptive thinking with effort=%s for model=%s", output_config.get("effort"), @@ -692,7 +704,11 @@ class AmazonAnthropicClaudeMessagesConfig( and isinstance(translated_thinking, dict) and translated_thinking.get("type") == "adaptive" ): - anthropic_messages_request["thinking"] = {**translated_thinking, "display": "updates"} + anthropic_messages_request["thinking"] = { + **thinking_block_binding(translated_thinking), + "type": "adaptive", + "display": "updates", + } self._normalize_system_role_messages(anthropic_messages_request, model=model) ######################################################### ############## BEDROCK Invoke SPECIFIC TRANSFORMATION ### diff --git a/tests/unit/llms/anthropic/test_anthropic_common_utils.py b/tests/unit/llms/anthropic/test_anthropic_common_utils.py index dcf0d58d675..05aa3bfe5ef 100644 --- a/tests/unit/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/unit/llms/anthropic/test_anthropic_common_utils.py @@ -16,7 +16,7 @@ import sys import threading from collections.abc import Mapping from pathlib import Path -from types import SimpleNamespace +from types import MappingProxyType, SimpleNamespace from typing import Final from unittest.mock import patch @@ -4172,28 +4172,81 @@ def test_shared_legacy_thinking_translation_preserves_supported_display( assert optional_params["thinking"] == expected_thinking -@pytest.mark.usefixtures("local_model_cost_map") -def test_shared_legacy_thinking_translation_carries_block_binding() -> None: - from litellm.llms.anthropic.common_utils import AnthropicModelInfo - - optional_params: Final = { - "thinking": { - "type": "enabled", - "budget_tokens": 2048, - "block_binding": {"prefix_mismatch_behavior": "error"}, - }, +LEGACY_THINKING_WITH_BLOCK_BINDING: Final = MappingProxyType( + { + "type": "enabled", + "budget_tokens": 2048, + "block_binding": {"prefix_mismatch_behavior": "error"}, + "future_key": 1, } +) +ADAPTIVE_THINKING_WITH_BLOCK_BINDING: Final = MappingProxyType( + {"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "error"}} +) - AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model( - model="claude-opus-5", - optional_params=optional_params, - custom_llm_provider="azure_ai", + +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize( + ("model", "custom_llm_provider", "expected_thinking"), + ( + ("claude-opus-4-8", "anthropic", ADAPTIVE_THINKING_WITH_BLOCK_BINDING), + ("claude-opus-4-8", "azure_ai", ADAPTIVE_THINKING_WITH_BLOCK_BINDING), + ("invoke/us.anthropic.claude-opus-4-8", "bedrock", ADAPTIVE_THINKING_WITH_BLOCK_BINDING), + ("converse/us.anthropic.claude-opus-4-8", "bedrock", {"type": "adaptive"}), + ("claude-opus-4-8", "vertex_ai", {"type": "adaptive"}), + ("databricks-claude-opus-4-8", "databricks", {"type": "adaptive"}), + ), + ids=("anthropic", "azure_ai", "bedrock_invoke", "bedrock_converse", "vertex_ai", "databricks"), +) +def test_chat_legacy_thinking_translation_carries_only_block_binding_where_supported( + model: str, custom_llm_provider: str, expected_thinking: Mapping[str, object] +) -> None: + from litellm.utils import get_optional_params + + optional_params: Final = get_optional_params( + model=model, + custom_llm_provider=custom_llm_provider, + thinking=dict(LEGACY_THINKING_WITH_BLOCK_BINDING), ) - assert optional_params["thinking"] == { - "type": "adaptive", - "block_binding": {"prefix_mismatch_behavior": "error"}, - } + assert optional_params["thinking"] == expected_thinking + + +@pytest.mark.usefixtures("local_model_cost_map", "local_beta_headers_config") +@pytest.mark.parametrize( + ("model", "custom_llm_provider", "expected_thinking"), + ( + ("claude-opus-4-8", "anthropic", ADAPTIVE_THINKING_WITH_BLOCK_BINDING), + ("claude-opus-4-8", "azure_ai", ADAPTIVE_THINKING_WITH_BLOCK_BINDING), + ("us.anthropic.claude-opus-4-8", "bedrock", ADAPTIVE_THINKING_WITH_BLOCK_BINDING), + ("claude-opus-4-8", "vertex_ai", {"type": "adaptive"}), + ), + ids=("anthropic", "azure_ai", "bedrock_invoke", "vertex_ai"), +) +def test_messages_legacy_thinking_translation_carries_only_block_binding_where_supported( + model: str, custom_llm_provider: str, expected_thinking: Mapping[str, object] +) -> None: + from litellm.types.router import GenericLiteLLMParams + from litellm.types.utils import LlmProviders + from litellm.utils import ProviderConfigManager + + config: Final = ProviderConfigManager.get_provider_anthropic_messages_config( + model=model, provider=LlmProviders(custom_llm_provider) + ) + assert config is not None + + result: Final = config.transform_anthropic_messages_request( + model=model, + messages=[{"role": "user", "content": "Reply with OK"}], + anthropic_messages_optional_request_params={ + "max_tokens": 512, + "thinking": dict(LEGACY_THINKING_WITH_BLOCK_BINDING), + }, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert result.get("thinking") == expected_thinking @pytest.mark.usefixtures("local_model_cost_map", "local_beta_headers_config") diff --git a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py index 1abebaaa8c5..cc154c3d026 100644 --- a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py +++ b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py @@ -1004,8 +1004,14 @@ BLOCK_BINDING: Final = {"prefix_mismatch_behavior": "error"} @pytest.mark.parametrize( "requested_thinking", ( - {"type": "adaptive", "display": "updates", "block_binding": BLOCK_BINDING}, - {"type": "enabled", "budget_tokens": 2048, "display": "updates", "block_binding": BLOCK_BINDING}, + {"type": "adaptive", "display": "updates", "block_binding": BLOCK_BINDING, "future_key": 1}, + { + "type": "enabled", + "budget_tokens": 2048, + "display": "updates", + "block_binding": BLOCK_BINDING, + "future_key": 1, + }, ), ids=("adaptive", "legacy"), ) diff --git a/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py index 579972f8258..059ef765990 100644 --- a/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py +++ b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py @@ -3536,15 +3536,15 @@ BLOCK_BINDING: Final = {"prefix_mismatch_behavior": "error"} @pytest.mark.parametrize( ("model", "requested_thinking", "extra_params"), ( - ("eu.anthropic.claude-opus-5", {"type": "adaptive", "display": "updates", "block_binding": BLOCK_BINDING}, {}), + ("eu.anthropic.claude-opus-5", {"type": "adaptive", "display": "updates", "block_binding": BLOCK_BINDING, "future_key": 1}, {}), ( "eu.anthropic.claude-opus-5", - {"type": "enabled", "budget_tokens": 24000, "display": "updates", "block_binding": BLOCK_BINDING}, + {"type": "enabled", "budget_tokens": 24000, "display": "updates", "block_binding": BLOCK_BINDING, "future_key": 1}, {}, ), ( "us.anthropic.claude-opus-4-6", - {"type": "enabled", "budget_tokens": 2048, "display": "updates", "block_binding": BLOCK_BINDING}, + {"type": "enabled", "budget_tokens": 2048, "display": "updates", "block_binding": BLOCK_BINDING, "future_key": 1}, {"context_management": {"edits": [{"type": "clear_thinking_20251015"}]}}, ), ),