diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 2f1a4147544..74d18e6f783 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -1993,10 +1993,8 @@ def is_unsignable_thinking_block(block: object) -> bool: Anthropic verifies the signature cryptographically, so a block with a null, empty, or missing signature (e.g. from an open-source reasoning model) is rejected with a 400, and so is a block whose signature or data carries - another provider's encrypted reasoning. It also rejects a `thinking` block - whose text is empty or whitespace-only ("each thinking block must contain - thinking"), regardless of signature, e.g. when a `thinking_blocks` history - item from a non-Anthropic reasoning provider is replayed through this path. + another provider's encrypted reasoning. A signed block is kept whatever its + text, since ``display: omitted`` returns signed blocks with empty ``thinking``. `redacted_thinking` blocks carry no signature and are always kept. """ if is_encrypted_reasoning_block(block): @@ -2007,10 +2005,7 @@ def is_unsignable_thinking_block(block: object) -> bool: if mapping.get("type") != "thinking": return False signature: Final = mapping.get("signature") - if not (isinstance(signature, str) and len(signature) > 0): - return True - thinking_text: Final = mapping.get("thinking") - return not (isinstance(thinking_text, str) and len(thinking_text.strip()) > 0) + return not (isinstance(signature, str) and len(signature) > 0) def strip_encrypted_reasoning_from_messages( diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 0c48b7c1c2a..84a3da288f3 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -2684,13 +2684,8 @@ def anthropic_messages_pt( if not isinstance(m, dict): continue # handle thinking blocks - thinking_block = cast(str, m.get("thinking", "")) text_block = cast(str, m.get("text", "")) - if ( - m.get("type", "") == "thinking" - and len(thinking_block) > 0 - and not is_unsignable_thinking_block(m) - ): # don't pass empty text blocks. anthropic api raises errors. + if m.get("type", "") == "thinking" and not is_unsignable_thinking_block(m): anthropic_message: ChatCompletionThinkingBlock | AnthropicMessagesTextParam = cast( ChatCompletionThinkingBlock, m ) diff --git a/litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py b/litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py index 2169dfcad39..2a214f11a8b 100644 --- a/litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py +++ b/litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py @@ -257,8 +257,7 @@ def _assistant_part_renders(part: object) -> bool: """A text part always renders: the converter pads empty text with a placeholder.""" part_type: Final = message_field(part, "type") if part_type == "thinking": - thinking: Final = message_field(part, "thinking") - return isinstance(thinking, str) and bool(thinking) and _thinking_block_renders(part) + return _thinking_block_renders(part) return part_type in _RENDERED_ASSISTANT_PART_TYPES or ( isinstance(part_type, str) and part_type.endswith("_tool_result") ) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 490912d42eb..18fdbbba487 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -327,6 +327,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): def _resolved_provider(self) -> str: return self.custom_llm_provider or "anthropic" + @property + def _carries_thinking_block_binding(self) -> bool: + return self._resolved_provider in ("anthropic", "azure_ai") + @classmethod def get_config(cls, *, model: str | None = None): config: Final = super().get_config() @@ -1588,7 +1592,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): else: optional_params["thinking"] = value AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model( - model=model, optional_params=optional_params, custom_llm_provider=self._resolved_provider + model=model, + optional_params=optional_params, + custom_llm_provider=self._resolved_provider, + carries_block_binding=self._carries_thinking_block_binding, ) elif param == "reasoning_effort": # Accept both string ("low") and dict ({"effort": "low", diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index fb2480cf0ac..4448c67a212 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -78,7 +78,7 @@ _INFERENCE_PROFILE_MINOR_RE: Final = re.compile(r":\d+$") _DATED_RELEASE_SUFFIX_RE: Final = re.compile(r"-\d{8}$") _DOTTED_VERSION_RE: Final = re.compile(r"(\d)\.(\d)") _CLAUDE_CODE_BILLING_HEADER_PREFIX: Final = "x-anthropic-billing-header:" -_CLAUDE_CODE_OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object]) +_OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object]) _CLAUDE_CODE_OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object]) @@ -127,9 +127,9 @@ def is_claude_code_user_agent(user_agent: str) -> bool: return user_agent.startswith(_CLAUDE_CODE_USER_AGENT_PREFIXES) -def _validated_claude_code_mapping(value: object) -> dict[object, object] | None: +def validated_object_mapping(value: object) -> dict[object, object] | None: try: - return _CLAUDE_CODE_OBJECT_MAPPING_ADAPTER.validate_python(value) + return _OBJECT_MAPPING_ADAPTER.validate_python(value) except ValidationError: return None @@ -166,7 +166,7 @@ def _claude_code_billing_texts(system: object) -> tuple[str, ...] | None: blocks: Final = _validated_claude_code_list(system) if blocks is None: return None - block_mappings: Final = tuple(_validated_claude_code_mapping(block) for block in blocks) + block_mappings: Final = tuple(validated_object_mapping(block) for block in blocks) if any(block is None for block in block_mappings): return None text_values: Final = tuple( @@ -199,7 +199,7 @@ def is_claude_code_one_shot_subagent_request( tools: object, user_agent: str | None, ) -> bool: - only_message: Final = _validated_claude_code_mapping(messages[0]) if len(messages) == 1 else None + only_message: Final = validated_object_mapping(messages[0]) if len(messages) == 1 else None return ( user_agent is not None and is_claude_code_user_agent(user_agent) @@ -748,6 +748,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): model: str, optional_params: MutableMapping[str, object], # mutable-ok: in-place out-param like the sibling helpers custom_llm_provider: str, + carries_block_binding: bool = False, ) -> None: """Translate legacy ``thinking.type=enabled`` to adaptive for the adaptive-thinking models that reject it (4.7+ and the 5 families). @@ -760,21 +761,25 @@ class AnthropicModelInfo(BaseLLMModelInfo): return if AnthropicModelInfo._supports_legacy_thinking(model, custom_llm_provider): return - thinking: Final = optional_params.get("thinking") - if not isinstance(thinking, dict) or thinking.get("type") != "enabled": + thinking: Final = validated_object_mapping(optional_params.get("thinking")) + if thinking is None or thinking.get("type") != "enabled": return + budget_tokens: Final = thinking.get("budget_tokens") or 0 effort: Final = AnthropicModelInfo._legacy_budget_to_effort( model=model, - budget_tokens=int(thinking.get("budget_tokens") or 0), + budget_tokens=int(budget_tokens) if isinstance(budget_tokens, (int, float, str)) else 0, custom_llm_provider=custom_llm_provider, ) existing_output_config: Final = optional_params.get("output_config") display: Final = thinking.get("display") - if display in ("summarized", "omitted"): - optional_params["thinking"] = {"type": "adaptive", "display": display} - else: - optional_params["thinking"] = {"type": "adaptive"} + supported_display: Final = {"display": display} if display in ("summarized", "omitted") else {} + carried_block_binding: Final = ( + {"block_binding": thinking["block_binding"]} + if carries_block_binding and "block_binding" in thinking + else {} + ) + optional_params["thinking"] = {**carried_block_binding, "type": "adaptive", **supported_display} optional_params["output_config"] = { "effort": effort, **(existing_output_config if isinstance(existing_output_config, dict) else MappingProxyType({})), @@ -1320,6 +1325,20 @@ def is_anthropic_invalid_thinking_block_error(error_text: str) -> bool: return "must contain thinking" in lower +def is_anthropic_thinking_binding_mismatch_error(error_text: str) -> bool: + """messages.N.content.M: Invalid `signature` in `thinking` block. The block is bound to a different conversation.""" + return "bound to a different conversation" in error_text.lower() + + +def anthropic_request_errors_on_thinking_binding_mismatch(request_body: object) -> bool: + """``thinking.block_binding.prefix_mismatch_behavior: error`` is the caller asking + for the 400, so a retry that strips thinking must not override it.""" + body: Final = validated_object_mapping(request_body) + thinking: Final = validated_object_mapping(body.get("thinking")) if body is not None else None + binding: Final = validated_object_mapping(thinking.get("block_binding")) if thinking is not None else None + return binding is not None and binding.get("prefix_mismatch_behavior") == "error" + + def strip_thinking_blocks_from_anthropic_messages(messages: Sequence[object]) -> list[object]: """ Return a new message list with thinking / redacted_thinking content blocks removed diff --git a/litellm/llms/anthropic/pass_through/messages/transformation.py b/litellm/llms/anthropic/pass_through/messages/transformation.py index 2fbb51ec949..b3a83e60bcc 100644 --- a/litellm/llms/anthropic/pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/pass_through/messages/transformation.py @@ -63,6 +63,10 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): def _resolved_provider(self) -> str: return self.custom_llm_provider or "anthropic" + @property + def _carries_thinking_block_binding(self) -> bool: + return self._resolved_provider in ("anthropic", "azure_ai") + def get_supported_anthropic_messages_params(self, model: str) -> list: return [ "messages", @@ -514,6 +518,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): model=model, optional_params=anthropic_messages_optional_request_params, custom_llm_provider=self._resolved_provider, + carries_block_binding=self._carries_thinking_block_binding, ) self._translate_adaptive_effort_for_non_adaptive_model( diff --git a/litellm/llms/base_llm/anthropic_messages/transformation.py b/litellm/llms/base_llm/anthropic_messages/transformation.py index 101a5e6c58c..d87ccb50af4 100644 --- a/litellm/llms/base_llm/anthropic_messages/transformation.py +++ b/litellm/llms/base_llm/anthropic_messages/transformation.py @@ -1,6 +1,6 @@ from abc import ABC, abstractmethod from collections.abc import AsyncIterator -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Final import httpx @@ -156,16 +156,28 @@ class BaseAnthropicMessagesConfig(ABC): """ return 2 - def should_retry_anthropic_messages_on_http_error(self, e: httpx.HTTPStatusError, litellm_params: dict) -> bool: + def should_retry_anthropic_messages_on_http_error( + self, e: httpx.HTTPStatusError, litellm_params: dict, request_body: object = None + ) -> bool: """ When True, async_anthropic_messages_handler will transform the request body and issue one more attempt (bounded by max_retry_on_anthropic_messages_http_error). + A binding-mismatch 400 is not retried when the body's ``thinking.block_binding`` + asked for it, since the retry would strip the very blocks the caller pinned. """ from litellm.llms.anthropic.common_utils import ( + anthropic_request_errors_on_thinking_binding_mismatch, is_anthropic_invalid_thinking_block_error, + is_anthropic_thinking_binding_mismatch_error, ) - return e.response.status_code == 400 and is_anthropic_invalid_thinking_block_error(e.response.text) + if e.response.status_code != 400: + return False + error_text: Final[str] = e.response.text + caller_asked_for_this_error: Final = anthropic_request_errors_on_thinking_binding_mismatch( + request_body + ) and is_anthropic_thinking_binding_mismatch_error(error_text) + return not caller_asked_for_this_error and is_anthropic_invalid_thinking_block_error(error_text) def transform_anthropic_messages_request_on_http_error(self, e: httpx.HTTPStatusError, request_data: dict) -> dict: """ diff --git a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py index 0be087d2dd4..b3673eb1e6c 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py @@ -67,6 +67,10 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): def custom_llm_provider(self) -> str | None: return "bedrock" + @property + def _carries_thinking_block_binding(self) -> bool: + return True + def should_strip_billing_metadata(self) -> bool: return True @@ -115,7 +119,10 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): model = original_model AnthropicModelInfo.translate_legacy_thinking_for_adaptive_model( - model=original_model, optional_params=optional_params, custom_llm_provider="bedrock" + model=original_model, + optional_params=optional_params, + custom_llm_provider="bedrock", + carries_block_binding=self._carries_thinking_block_binding, ) translated_thinking: Final = optional_params.get("thinking") if ( @@ -123,8 +130,7 @@ class AmazonAnthropicClaudeConfig(AmazonInvokeConfig, AnthropicConfig): and isinstance(translated_thinking, dict) and translated_thinking.get("type") == "adaptive" ): - thinking_with_display: Final[AnthropicThinkingParam] = {"type": "adaptive", "display": "updates"} - optional_params["thinking"] = thinking_with_display + optional_params["thinking"] = {**translated_thinking, "display": "updates"} # The stub model hides the original model from the parent's forced-tool-use backstop response_format_tool_choice: Final = optional_params.get("tool_choice") diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py index 6234ca3a9c3..cf733f5605c 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py @@ -17,7 +17,7 @@ from litellm.llms.anthropic.chat.transformation import ( DROP_UNSUPPORTED_OUTPUT_CONFIG_WARNING, AnthropicConfig, ) -from litellm.llms.anthropic.common_utils import AnthropicModelInfo +from litellm.llms.anthropic.common_utils import AnthropicModelInfo, validated_object_mapping from litellm.llms.anthropic.pass_through.messages.transformation import ( AnthropicMessagesConfig, ) @@ -49,7 +49,6 @@ from litellm.types.llms.anthropic import ( ANTHROPIC_BETA_HEADER_VALUES, ANTHROPIC_FINE_GRAINED_TOOL_STREAMING_BETA_HEADER, ANTHROPIC_TOOL_SEARCH_BETA_HEADER, - AnthropicThinkingParam, ) from litellm.types.llms.bedrock import BedrockInvokeAnthropicMessagesRequest from litellm.types.llms.openai import AllMessageValues @@ -82,6 +81,10 @@ class AmazonAnthropicClaudeMessagesConfig( def custom_llm_provider(self) -> str | None: return "bedrock" + @property + def _carries_thinking_block_binding(self) -> bool: + return True + @property def beta_headers_provider(self) -> str: return self.custom_llm_provider or "bedrock" @@ -349,18 +352,10 @@ class AmazonAnthropicClaudeMessagesConfig( if not isinstance(output_config, dict): output_config = {} output_config.setdefault("effort", self._effort_from_thinking_budget(budget_tokens)) - thinking: Final = anthropic_messages_request.get("thinking") - display: Final = thinking.get("display") if isinstance(thinking, dict) else None + thinking: Final = validated_object_mapping(anthropic_messages_request.get("thinking")) + carried: Final = {key: value for key, value in (thinking or {}).items() if key not in ("type", "budget_tokens")} anthropic_messages_request["output_config"] = output_config - if display is None: - adaptive_thinking: Final[AnthropicThinkingParam] = {"type": "adaptive"} - anthropic_messages_request["thinking"] = adaptive_thinking - else: - adaptive_thinking_with_display: Final[AnthropicThinkingParam] = { - "type": "adaptive", - "display": display, - } - anthropic_messages_request["thinking"] = adaptive_thinking_with_display + anthropic_messages_request["thinking"] = {**carried, "type": "adaptive"} verbose_logger.debug( "Bedrock clear_thinking_20251015: injected adaptive thinking with effort=%s for model=%s", output_config.get("effort"), @@ -701,8 +696,7 @@ class AmazonAnthropicClaudeMessagesConfig( and isinstance(translated_thinking, dict) and translated_thinking.get("type") == "adaptive" ): - thinking_with_display: Final[AnthropicThinkingParam] = {"type": "adaptive", "display": "updates"} - anthropic_messages_request["thinking"] = thinking_with_display + anthropic_messages_request["thinking"] = {**translated_thinking, "display": "updates"} self._normalize_system_role_messages(anthropic_messages_request, model=model) ######################################################### ############## BEDROCK Invoke SPECIFIC TRANSFORMATION ### diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 8383078b7ba..97aa7bbd7be 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -2018,7 +2018,7 @@ class BaseLLMHTTPHandler: # str when the caller passes a pre-serialized (unsigned) body to avoid # re-dumping; bytes when a provider signed the request (e.g. Bedrock). signed_json_body: str | bytes | None, - request_body: dict, + request_body: dict[str, object], stream: bool, logging_obj: LiteLLMLoggingObj, provider_config: BaseAnthropicMessagesConfig, @@ -2045,7 +2045,7 @@ class BaseLLMHTTPHandler: except httpx.HTTPStatusError as e: hit_max_attempt = attempt_idx + 1 == max_attempts should_retry = provider_config.should_retry_anthropic_messages_on_http_error( - e=e, litellm_params=litellm_params_dict + e=e, litellm_params=litellm_params_dict, request_body=request_body ) if should_retry and not hit_max_attempt: if logging_obj.baseline_cache_context is not None: diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index ee357cd6581..4f147437b38 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -730,10 +730,15 @@ ANTHROPIC_API_ONLY_HEADERS: Final = { # fails if calling anthropic on vertex ai } +class AnthropicThinkingBlockBinding(TypedDict, total=False): + prefix_mismatch_behavior: ReadOnly[Literal["error", "drop_block"]] + + class AnthropicThinkingParam(TypedDict, total=False): type: ReadOnly[Literal["enabled", "adaptive", "disabled"]] budget_tokens: int display: ReadOnly[Literal["summarized", "omitted", "updates"]] + block_binding: ReadOnly[AnthropicThinkingBlockBinding] class ANTHROPIC_HOSTED_TOOLS(str, Enum): diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index 8d2e6b9fd0c..a8adc33bf1f 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -27,7 +27,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( ollama_pt, sanitize_messages_for_tool_calling, ) -from litellm.types.llms.openai import ChatCompletionToolMessage +from litellm.types.llms.openai import ChatCompletionAssistantMessage, ChatCompletionToolMessage from litellm.utils import validate_and_fix_openai_messages @@ -3922,44 +3922,42 @@ def test_anthropic_messages_pt_drops_a_system_message_with_no_text(): assert [m["role"] for m in result] == ["user", "assistant"] -def test_anthropic_messages_pt_drops_empty_but_signed_thinking_block(): - """ - Anthropic rejects a `thinking` block whose `thinking` text is empty, even - when it carries a valid-looking signature, with: - 400 messages.N.content.M.thinking: each thinking block must contain thinking - This shape is reachable via cross-provider replay of a `thinking_blocks` - history item (see PR #36033), so `is_unsignable_thinking_block()` must - also check the thinking text, not just the signature. - """ +@pytest.mark.parametrize( + "assistant_message", + ( + { + "role": "assistant", + "content": "4", + "thinking_blocks": [{"type": "thinking", "thinking": "", "signature": "sig_abc123_looks_valid"}], + }, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "", "signature": "sig_abc123_looks_valid"}, + {"type": "text", "text": "4"}, + ], + }, + ), + ids=("thinking_blocks", "inline_content"), +) +def test_anthropic_messages_pt_keeps_empty_but_signed_thinking_block( + assistant_message: ChatCompletionAssistantMessage, +) -> None: + """``display: omitted`` returns signed thinking blocks with empty text, and they + must reach the next turn unchanged.""" from litellm.litellm_core_utils.prompt_templates.factory import ( anthropic_messages_pt, ) - messages = [ - {"role": "user", "content": "What's 2+2?"}, - { - "role": "assistant", - "content": "4", - "thinking_blocks": [ - { - "type": "thinking", - "thinking": "", - "signature": "sig_abc123_looks_valid", - } - ], - }, - ] - result = anthropic_messages_pt( - messages=messages, - model="claude-sonnet-4-5-20250929", + messages=[{"role": "user", "content": "What's 2+2?"}, assistant_message], + model="claude-opus-5-5", llm_provider="anthropic", ) assistant_msg = result[1] assert isinstance(assistant_msg["content"], list) - content_types = [block.get("type") for block in assistant_msg["content"]] - assert "thinking" not in content_types, "empty-text thinking block must be dropped even though it has a signature" + assert assistant_msg["content"][0] == {"type": "thinking", "thinking": "", "signature": "sig_abc123_looks_valid"} def test_anthropic_messages_pt_keeps_non_empty_signed_thinking_block(): @@ -4073,29 +4071,31 @@ def test_anthropic_messages_pt_drops_unsigned_thinking_block(): assert "thinking" not in content_types, "unsigned thinking block must still be dropped" -def test_is_unsignable_thinking_block_treats_whitespace_only_as_empty(): +def test_anthropic_messages_pt_drops_unsigned_empty_thinking_block(): """ - Edge case: a `thinking` field that is present but whitespace-only (e.g. - a single trailing newline forwarded from another provider's empty - reasoning summary) is functionally empty and Anthropic's API will still - reject it with "each thinking block must contain thinking". We treat it - the same as a fully empty string and drop the block. - - The check lives in the shared `is_unsignable_thinking_block` helper, which - `_drop_unsignable_thinking_blocks` calls standalone, so the whitespace-aware - test has to hold there rather than only at the factory call site. + An empty block without a signature is the shape Claude Code replays after a + non-Anthropic turn, and Anthropic rejects it, so only the missing signature + decides whether a block is dropped. """ - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - is_unsignable_thinking_block, + from litellm.litellm_core_utils.prompt_templates.factory import ( + anthropic_messages_pt, ) - whitespace_only_block = { - "type": "thinking", - "thinking": " \n\t ", - "signature": "sig_abc123_looks_valid", - } + result = anthropic_messages_pt( + messages=[ + {"role": "user", "content": "What's 2+2?"}, + { + "role": "assistant", + "content": [{"type": "thinking", "thinking": "", "signature": ""}, {"type": "text", "text": "4"}], + }, + ], + model="claude-opus-5-5", + llm_provider="anthropic", + ) - assert is_unsignable_thinking_block(whitespace_only_block) is True + assistant_msg = result[1] + assert isinstance(assistant_msg["content"], list) + assert [block.get("type") for block in assistant_msg["content"]] == ["text"] _CONTENT_LESS_USER_MESSAGES: Final = ({"role": "user"}, {"role": "user", "content": None}) diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_mid_conversation_system.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_mid_conversation_system.py index d1e23a17747..2d9c4c99c3e 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_mid_conversation_system.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_mid_conversation_system.py @@ -336,6 +336,7 @@ def test_flagged_placement_keeps_a_system_whose_empty_assistant_follower_ends_th }, {"role": "assistant", "content": None, "thinking_blocks": [{"type": "redacted_thinking", "data": "x"}]}, {"role": "assistant", "content": [{"type": "thinking", "thinking": "hm", "signature": "s"}]}, + {"role": "assistant", "content": [{"type": "thinking", "thinking": "", "signature": "s"}]}, {"role": "assistant", "content": None, "function_call": {"name": "f", "arguments": "{}"}}, litellm.Message( role="assistant", @@ -348,6 +349,7 @@ def test_flagged_placement_keeps_a_system_whose_empty_assistant_follower_ends_th "signed-thinking-block", "redacted-thinking-block", "signed-thinking-part", + "signed-empty-thinking-part", "function-call", "pydantic-tool-calls", ], diff --git a/tests/unit/llms/anthropic/test_anthropic_common_utils.py b/tests/unit/llms/anthropic/test_anthropic_common_utils.py index e3cb138b508..677ef16fc64 100644 --- a/tests/unit/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/unit/llms/anthropic/test_anthropic_common_utils.py @@ -13,10 +13,12 @@ Verifies that: import json import os import sys +from collections.abc import Mapping from types import SimpleNamespace from typing import Final from unittest.mock import patch +import httpx import pytest sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../.."))) @@ -1319,6 +1321,12 @@ class TestPassthroughAuthToken: assert url == "https://custom.example.com/v1/messages" +_BOUND_THINKING_ERROR: Final = ( + "messages.1.content.0: Invalid `signature` in `thinking` block. The block is bound to a different conversation." +) +_NULL_SIGNATURE_ERROR: Final = "messages.1.content.0.thinking.signature.str: Input should be a valid string" + + class TestAnthropicThinkingSignatureSelfHeal: """Helpers for retrying after invalid thinking blocks in replayed history: invalid encrypted signatures, and blocks with empty thinking text.""" @@ -1984,6 +1992,49 @@ class TestAnthropicThinkingSignatureSelfHeal: assert "thinking" not in data assert data["messages"] == [] + @pytest.mark.parametrize( + ("thinking", "error_message", "expected"), + ( + ( + {"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "error"}}, + _BOUND_THINKING_ERROR, + False, + ), + ( + {"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "error"}}, + _NULL_SIGNATURE_ERROR, + True, + ), + ( + {"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "drop_block"}}, + _BOUND_THINKING_ERROR, + True, + ), + ({"type": "adaptive"}, _BOUND_THINKING_ERROR, True), + ), + ids=("error", "error_on_unrelated_bad_block", "drop_block", "unbound"), + ) + def test_anthropic_messages_retry_honors_thinking_block_binding( + self, thinking: Mapping[str, object], error_message: str, expected: bool + ) -> None: + from litellm.llms.anthropic.pass_through.messages.transformation import ( + AnthropicMessagesConfig, + ) + + req: Final = httpx.Request("POST", "https://api.anthropic.com/v1/messages") + resp: Final = httpx.Response( + 400, + request=req, + text=json.dumps({"type": "error", "error": {"type": "invalid_request_error", "message": error_message}}), + ) + err: Final = httpx.HTTPStatusError("bad", request=req, response=resp) + + should_retry = AnthropicMessagesConfig().should_retry_anthropic_messages_on_http_error( + err, {}, request_body={"model": "claude-opus-5-5", "thinking": thinking} + ) + + assert should_retry is expected + class TestClaudeOpus48AdaptiveThinking: """Opus 4.8 requires adaptive thinking (``thinking.type='adaptive'`` + @@ -2482,6 +2533,100 @@ def test_shared_legacy_thinking_translation_preserves_supported_display( assert optional_params["thinking"] == expected_thinking +LEGACY_THINKING_WITH_BINDING: Final = { + "type": "enabled", + "budget_tokens": 2048, + "block_binding": {"prefix_mismatch_behavior": "error"}, + "future_key": 1, +} + + +def _rebuilt_thinking(carries_block_binding: bool) -> dict[str, object]: + return { + "type": "adaptive", + **({"block_binding": LEGACY_THINKING_WITH_BINDING["block_binding"]} if carries_block_binding else {}), + } + + +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize( + ("provider", "carries_block_binding"), + ( + ("anthropic", True), + ("azure_ai", True), + ("bedrock_invoke", True), + ("vertex_ai", False), + ("databricks", False), + ("bedrock_converse", False), + ), +) +def test_chat_legacy_thinking_rebuild_carries_only_block_binding_and_only_where_supported( + provider: str, carries_block_binding: bool +) -> None: + from litellm.llms.anthropic.chat.transformation import AnthropicConfig + from litellm.llms.azure_ai.anthropic.transformation import AzureAnthropicConfig + from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig + from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import ( + AmazonAnthropicClaudeConfig, + ) + from litellm.llms.databricks.chat.transformation import DatabricksConfig + from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import VertexAIAnthropicConfig + + config, model = { + "anthropic": (AnthropicConfig(), "claude-opus-4-8"), + "azure_ai": (AzureAnthropicConfig(), "claude-opus-4-8"), + "bedrock_invoke": (AmazonAnthropicClaudeConfig(), "us.anthropic.claude-opus-4-8"), + "vertex_ai": (VertexAIAnthropicConfig(), "claude-opus-4-8"), + "databricks": (DatabricksConfig(), "databricks-claude-opus-4-8"), + "bedrock_converse": (AmazonConverseConfig(), "us.anthropic.claude-opus-4-8"), + }[provider] + + mapped: Final = config.map_openai_params( + non_default_params={"max_tokens": 4096, "thinking": dict(LEGACY_THINKING_WITH_BINDING)}, + optional_params={}, + model=model, + drop_params=False, + ) + + assert mapped["thinking"] == _rebuilt_thinking(carries_block_binding) + + +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize( + ("provider", "carries_block_binding"), + (("anthropic", True), ("azure_ai", True), ("bedrock_invoke", True), ("vertex_ai", False)), +) +def test_messages_legacy_thinking_rebuild_carries_only_block_binding_and_only_where_supported( + provider: str, carries_block_binding: bool +) -> None: + from litellm.llms.anthropic.pass_through.messages.transformation import AnthropicMessagesConfig + from litellm.llms.azure_ai.anthropic.messages_transformation import AzureAnthropicMessagesConfig + from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import ( + AmazonAnthropicClaudeMessagesConfig, + ) + from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation import ( + VertexAIPartnerModelsAnthropicMessagesConfig, + ) + from litellm.types.router import GenericLiteLLMParams + + config, model = { + "anthropic": (AnthropicMessagesConfig(), "claude-opus-4-8"), + "azure_ai": (AzureAnthropicMessagesConfig(), "claude-opus-4-8"), + "bedrock_invoke": (AmazonAnthropicClaudeMessagesConfig(), "us.anthropic.claude-opus-4-8"), + "vertex_ai": (VertexAIPartnerModelsAnthropicMessagesConfig(), "claude-opus-4-8"), + }[provider] + + request: Final = config.transform_anthropic_messages_request( + model=model, + messages=[{"role": "user", "content": "Reply with OK"}], + anthropic_messages_optional_request_params={"max_tokens": 4096, "thinking": dict(LEGACY_THINKING_WITH_BINDING)}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert request["thinking"] == _rebuilt_thinking(carries_block_binding) + + @pytest.mark.usefixtures("local_model_cost_map", "local_beta_headers_config") @pytest.mark.parametrize("action", (None, "tool_addition", "tool_removal")) @pytest.mark.parametrize("explicit_beta", (False, True)) diff --git a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py index 2c82ba1c5b8..ce5c5ad1b27 100644 --- a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py +++ b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py @@ -759,6 +759,30 @@ def test_bedrock_chat_invoke_tool_based_response_format_still_upgrades_legacy_th assert result["output_config"] == {"effort": "high"} +def test_bedrock_chat_invoke_adaptive_thinking_keeps_the_forced_response_format_tool(local_model_cost_map): + """Anthropic accepts a forced tool_choice next to adaptive thinking, and dropping + it lets the model answer in prose instead of the requested JSON.""" + result = AmazonAnthropicClaudeConfig().map_openai_params( + non_default_params={ + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "test_schema", + "schema": {"type": "object", "properties": {"result": {"type": "string"}}}, + }, + }, + "thinking": {"type": "adaptive"}, + "max_tokens": 8192, + }, + optional_params={}, + model="us.anthropic.claude-opus-5", + drop_params=False, + ) + + assert result["tool_choice"] == {"type": "tool", "name": result["tools"][0]["name"]} + assert result["thinking"] == {"type": "adaptive"} + + def test_bedrock_chat_invoke_response_format_stub_still_upgrades_legacy_thinking(local_model_cost_map): """Regression: the tool-based ``response_format`` path swaps in a Claude 3 stub model before the shared Anthropic mapping, which hid the adaptive-only model @@ -973,6 +997,39 @@ def test_bedrock_chat_invoke_preserves_display_when_translating_legacy_thinking( assert ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER in result.get("anthropic_beta", []) +BLOCK_BINDING: Final = {"prefix_mismatch_behavior": "error"} + + +@pytest.mark.parametrize( + "requested_thinking", + ( + {"type": "adaptive", "display": "updates", "block_binding": BLOCK_BINDING}, + {"type": "enabled", "budget_tokens": 2048, "display": "updates", "block_binding": BLOCK_BINDING}, + ), + ids=("adaptive", "legacy"), +) +def test_bedrock_chat_invoke_keeps_block_binding_with_display_updates( + local_model_cost_map, local_beta_headers_config, requested_thinking +) -> None: + config: Final = AmazonAnthropicClaudeConfig() + model: Final = "us.anthropic.claude-opus-5" + optional_params: Final = config.map_openai_params( + non_default_params={"max_tokens": 512, "thinking": requested_thinking}, + optional_params={}, + model=model, + drop_params=False, + ) + result: Final = config.transform_request( + model=model, + messages=[{"role": "user", "content": "Reply with OK"}], + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert result.get("thinking") == {"type": "adaptive", "display": "updates", "block_binding": BLOCK_BINDING} + + FINE_GRAINED_TOOL_STREAMING_BETA: Final = "fine-grained-tool-streaming-2025-05-14" EAGER_TOOL_SCHEMA: Final = {"type": "object", "properties": {"path": {"type": "string"}}, "required": ["path"]} diff --git a/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py index a269d556262..f303c899898 100644 --- a/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py +++ b/tests/unit/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py @@ -3529,6 +3529,43 @@ def test_bedrock_clear_thinking_preserves_display_updates() -> None: assert ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER in result.get("anthropic_beta", []) +BLOCK_BINDING: Final = {"prefix_mismatch_behavior": "error"} + + +@pytest.mark.usefixtures("local_model_cost_map", "local_beta_headers_config") +@pytest.mark.parametrize( + ("model", "requested_thinking", "extra_params"), + ( + ("eu.anthropic.claude-opus-5", {"type": "adaptive", "display": "updates", "block_binding": BLOCK_BINDING}, {}), + ( + "eu.anthropic.claude-opus-5", + {"type": "enabled", "budget_tokens": 24000, "display": "updates", "block_binding": BLOCK_BINDING}, + {}, + ), + ( + "us.anthropic.claude-opus-4-6", + {"type": "enabled", "budget_tokens": 2048, "display": "updates", "block_binding": BLOCK_BINDING}, + {"context_management": {"edits": [{"type": "clear_thinking_20251015"}]}}, + ), + ), + ids=("adaptive", "legacy", "legacy_clear_thinking"), +) +def test_bedrock_messages_keeps_block_binding_with_display_updates( + model: str, requested_thinking: dict[str, object], extra_params: dict[str, object] +) -> None: + from litellm.types.router import GenericLiteLLMParams + + result: Final = AmazonAnthropicClaudeMessagesConfig().transform_anthropic_messages_request( + model=model, + messages=[{"role": "user", "content": "Reply with OK"}], + anthropic_messages_optional_request_params={"max_tokens": 512, "thinking": requested_thinking, **extra_params}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert result.get("thinking") == {"type": "adaptive", "display": "updates", "block_binding": BLOCK_BINDING} + + @pytest.mark.usefixtures("local_model_cost_map", "local_beta_headers_config") @pytest.mark.parametrize("action", (None, "tool_addition", "tool_removal")) @pytest.mark.parametrize("explicit_beta", (False, True)) diff --git a/tests/unit/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py index f3332cb513c..a6b6f5196d0 100644 --- a/tests/unit/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py @@ -1636,6 +1636,71 @@ async def test_anthropic_post_retry_reserializes_mutated_body(): assert "MUTATED" in second_sent # ... the mutated body +_BOUND_THINKING_ERROR: Final = ( + '{"type":"error","error":{"type":"invalid_request_error","message":"messages.1.content.0: ' + 'Invalid `signature` in `thinking` block. The block is bound to a different conversation."}}' +) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("prefix_mismatch_behavior", "expected_posts"), + (("error", 1), ("drop_block", 2)), +) +async def test_anthropic_post_skips_the_thinking_strip_retry_when_block_binding_errors( + prefix_mismatch_behavior: str, expected_posts: int +) -> None: + """A caller that pinned ``prefix_mismatch_behavior: error`` asked for the 400, + so the handler must surface it instead of stripping the bound blocks and retrying.""" + from litellm.llms.anthropic.pass_through.messages.transformation import AnthropicMessagesConfig + + handler = BaseLLMHTTPHandler() + thinking_block = {"type": "thinking", "thinking": "", "signature": "sig"} + request_body = { + "model": "claude-opus-5-5", + "messages": [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": [thinking_block, {"type": "text", "text": "hello"}]}, + ], + "thinking": {"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": prefix_mismatch_behavior}}, + } + request = httpx.Request("POST", "https://api.anthropic.com/v1/messages") + http_client = Mock() + http_client.post = AsyncMock( + side_effect=[ + httpx.Response(400, request=request, text=_BOUND_THINKING_ERROR), + httpx.Response(200, request=request, json={"type": "message", "content": []}), + ] + ) + logging_obj: Final = Mock(baseline_cache_context=None) + logging_obj.model_call_details = {} + + async def _post() -> httpx.Response: + return await handler._async_post_anthropic_messages_with_http_error_retry( + async_httpx_client=http_client, + request_url="https://api.anthropic.com/v1/messages", + headers={}, + signed_json_body=None, + request_body=request_body, + stream=False, + logging_obj=logging_obj, + provider_config=AnthropicMessagesConfig(), + litellm_params=GenericLiteLLMParams(), + api_key="k", + model="claude-opus-5-5", + ) + + if expected_posts == 1: + with pytest.raises(BaseLLMException) as raised: + await _post() + assert raised.value.status_code == 400 + assert "bound to a different conversation" in raised.value.message + assert request_body["messages"][1]["content"][0] == thinking_block + else: + assert (await _post()).status_code == 200 + assert http_client.post.await_count == expected_posts + + def test_base_responses_config_sign_request_is_noop_by_default(): """Default responses sign_request must be a no-op: unchanged headers, no signed body.