diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index a0da14bcc2b..3be5a0c816f 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -2,6 +2,7 @@ This file contains common utils for anthropic calls. """ +import copy from typing import Any, Dict, List, Optional, Union import httpx @@ -736,6 +737,69 @@ def strip_advisor_blocks_from_messages( return messages +def is_anthropic_invalid_thinking_signature_error(error_text: str) -> bool: + """ + Detect Anthropic 400 when encrypted thinking signatures in history do not match + the current deployment (e.g. user rotated API key or switched model endpoint). + + Example API message: + messages.N.content.M: Invalid `signature` in `thinking` block + """ + if not error_text: + return False + lower = error_text.lower() + return ( + "invalid" in lower + and "signature" in lower + and "thinking" in lower + and "block" in lower + ) + + +def strip_thinking_blocks_from_anthropic_messages(messages: List[Any]) -> List[Any]: + """ + Return a new message list with thinking / redacted_thinking content blocks removed + from each message. Used to recover from invalid thinking signatures on retry. + + Messages whose content is a list and becomes empty after stripping are omitted, + since Anthropic rejects empty content arrays. + """ + out: List[Any] = [] + for m in messages: + if not isinstance(m, dict): + out.append(m) + continue + mm = copy.deepcopy(m) + content = mm.get("content") + if isinstance(content, list): + filtered = [ + b + for b in content + if not ( + isinstance(b, dict) + and b.get("type") in ("thinking", "redacted_thinking") + ) + ] + if not filtered: + continue + mm["content"] = filtered + out.append(mm) + return out + + +def strip_thinking_blocks_from_anthropic_messages_request_dict( + data: Dict[str, Any], +) -> None: + """ + Mutate an Anthropic Messages-style request dict: strip thinking blocks from + ``messages`` and remove the top-level ``thinking`` extended-thinking param. + """ + msgs = data.get("messages") + if isinstance(msgs, list): + data["messages"] = strip_thinking_blocks_from_anthropic_messages(msgs) + data.pop("thinking", None) + + def process_anthropic_headers(headers: Union[httpx.Headers, dict]) -> dict: openai_headers = {} if "anthropic-ratelimit-requests-limit" in headers: diff --git a/litellm/llms/base_llm/anthropic_messages/transformation.py b/litellm/llms/base_llm/anthropic_messages/transformation.py index fdad1633e8f..733faca8532 100644 --- a/litellm/llms/base_llm/anthropic_messages/transformation.py +++ b/litellm/llms/base_llm/anthropic_messages/transformation.py @@ -120,3 +120,44 @@ class BaseAnthropicMessagesConfig(ABC): return BaseLLMException( message=error_message, status_code=status_code, headers=headers ) + + @property + def max_retry_on_anthropic_messages_http_error(self) -> int: + """ + Max HTTP attempts for /v1/messages when the handler may mutate the body and + retry (e.g. strip invalid encrypted thinking signatures after a deployment or + credential change). + """ + return 2 + + def should_retry_anthropic_messages_on_http_error( + self, e: httpx.HTTPStatusError, litellm_params: dict + ) -> bool: + """ + When True, async_anthropic_messages_handler will transform the request body + and issue one more attempt (bounded by max_retry_on_anthropic_messages_http_error). + """ + from litellm.llms.anthropic.common_utils import ( + is_anthropic_invalid_thinking_signature_error, + ) + + return e.response.status_code == 400 and is_anthropic_invalid_thinking_signature_error( + e.response.text + ) + + def transform_anthropic_messages_request_on_http_error( + self, e: httpx.HTTPStatusError, request_data: dict + ) -> dict: + """ + Mutates request_data in place when retrying after a recoverable HTTP error. + """ + from litellm.llms.anthropic.common_utils import ( + is_anthropic_invalid_thinking_signature_error, + strip_thinking_blocks_from_anthropic_messages_request_dict, + ) + + if e.response.status_code == 400 and is_anthropic_invalid_thinking_signature_error( + e.response.text + ): + strip_thinking_blocks_from_anthropic_messages_request_dict(request_data) + return request_data diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 7a8820a8785..60efa45df62 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -1816,6 +1816,69 @@ class BaseLLMHTTPHandler: logging_obj=logging_obj, ) + async def _async_post_anthropic_messages_with_http_error_retry( + self, + async_httpx_client: AsyncHTTPHandler, + request_url: str, + headers: dict, + signed_json_body: Optional[bytes], + request_body: dict, + stream: bool, + logging_obj: LiteLLMLoggingObj, + provider_config: BaseAnthropicMessagesConfig, + litellm_params: GenericLiteLLMParams, + api_key: Optional[str], + model: str, + ) -> httpx.Response: + max_attempts = max(provider_config.max_retry_on_anthropic_messages_http_error, 1) + litellm_params_dict = dict(litellm_params) + optional_params_dict = dict(litellm_params) + for attempt_idx in range(max_attempts): + try: + response = await async_httpx_client.post( + url=request_url, + headers=headers, + data=signed_json_body or json.dumps(request_body), + stream=stream or False, + logging_obj=logging_obj, + ) + response.raise_for_status() + return response + except httpx.HTTPStatusError as e: + hit_max_attempt = attempt_idx + 1 == max_attempts + should_retry = provider_config.should_retry_anthropic_messages_on_http_error( + e=e, litellm_params=litellm_params_dict + ) + if should_retry and not hit_max_attempt: + verbose_logger.debug( + "Anthropic /v1/messages: invalid thinking signature; " + "stripping thinking blocks and retrying (attempt %s/%s).", + attempt_idx + 2, + max_attempts, + ) + provider_config.transform_anthropic_messages_request_on_http_error( + e=e, request_data=request_body + ) + headers, signed_json_body = provider_config.sign_request( + headers=headers, + optional_params=optional_params_dict, + request_data=request_body, + api_base=request_url, + api_key=api_key, + stream=stream, + fake_stream=False, + model=model, + ) + logging_obj.model_call_details.update(request_body) + continue + raise self._handle_error(e=e, provider_config=provider_config) + except Exception as e: + raise self._handle_error(e=e, provider_config=provider_config) + + raise RuntimeError( + "unreachable: anthropic messages HTTP retry loop exited without return" + ) + async def async_anthropic_messages_handler( self, model: str, @@ -1955,19 +2018,19 @@ class BaseLLMHTTPHandler: }, ) - try: - response = await async_httpx_client.post( - url=request_url, - headers=headers, - data=signed_json_body or json.dumps(request_body), - stream=stream or False, - logging_obj=logging_obj, - ) - response.raise_for_status() - except Exception as e: - raise self._handle_error( - e=e, provider_config=anthropic_messages_provider_config - ) + response = await self._async_post_anthropic_messages_with_http_error_retry( + async_httpx_client=async_httpx_client, + request_url=request_url, + headers=headers, + signed_json_body=signed_json_body, + request_body=request_body, + stream=stream or False, + logging_obj=logging_obj, + provider_config=anthropic_messages_provider_config, + litellm_params=litellm_params, + api_key=api_key, + model=model, + ) # used for logging + cost tracking logging_obj.model_call_details["httpx_response"] = response diff --git a/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py b/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py index 22470b93540..d48d7716a8e 100644 --- a/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py @@ -1131,3 +1131,143 @@ class TestPassthroughAuthToken: ) assert url == "https://custom.example.com/v1/messages" + + +class TestAnthropicThinkingSignatureSelfHeal: + """Helpers for retrying after invalid encrypted thinking signatures.""" + + def test_is_anthropic_invalid_thinking_signature_error_positive(self): + from litellm.llms.anthropic.common_utils import ( + is_anthropic_invalid_thinking_signature_error, + ) + + raw = ( + '{"type":"error","error":{"type":"invalid_request_error",' + '"message":"messages.3.content.3: Invalid `signature` in `thinking` block"},' + '"request_id":"req_011Ca2EtQDxp7x6RGUY2jVn9"}' + ) + assert is_anthropic_invalid_thinking_signature_error(raw) is True + + def test_is_anthropic_invalid_thinking_signature_error_negative(self): + from litellm.llms.anthropic.common_utils import ( + is_anthropic_invalid_thinking_signature_error, + ) + + assert is_anthropic_invalid_thinking_signature_error("") is False + assert ( + is_anthropic_invalid_thinking_signature_error("rate limit exceeded") + is False + ) + + def test_strip_thinking_blocks_from_anthropic_messages(self): + from litellm.llms.anthropic.common_utils import ( + strip_thinking_blocks_from_anthropic_messages, + ) + + messages = [ + {"role": "user", "content": "hi"}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "plan", "signature": "sig"}, + {"type": "text", "text": "hello"}, + ], + }, + ] + out = strip_thinking_blocks_from_anthropic_messages(messages) + assert len(out) == 2 + assert out[0] == messages[0] + assert len(out[1]["content"]) == 1 + assert out[1]["content"][0]["type"] == "text" + assert messages[1]["content"][0]["type"] == "thinking" + + def test_strip_thinking_blocks_drops_message_when_only_thinking_blocks(self): + from litellm.llms.anthropic.common_utils import ( + strip_thinking_blocks_from_anthropic_messages, + ) + + messages = [ + {"role": "user", "content": "hi"}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "plan", "signature": "sig"}, + ], + }, + ] + out = strip_thinking_blocks_from_anthropic_messages(messages) + assert len(out) == 1 + assert out[0]["role"] == "user" + + def test_strip_thinking_blocks_from_anthropic_messages_request_dict(self): + from litellm.llms.anthropic.common_utils import ( + strip_thinking_blocks_from_anthropic_messages_request_dict, + ) + + data = { + "model": "claude-sonnet-4-20250514", + "messages": [ + { + "role": "assistant", + "content": [ + { + "type": "thinking", + "thinking": "x", + "signature": "y", + }, + ], + } + ], + "thinking": {"type": "enabled", "budget_tokens": 1024}, + } + strip_thinking_blocks_from_anthropic_messages_request_dict(data) + assert "thinking" not in data + assert data["messages"] == [] + + def test_anthropic_messages_config_http_retry_helpers(self): + import httpx + + from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( + AnthropicMessagesConfig, + ) + + config = AnthropicMessagesConfig() + assert config.max_retry_on_anthropic_messages_http_error == 2 + + req = httpx.Request("POST", "https://api.anthropic.com/v1/messages") + err_text = ( + '{"type":"error","error":{"type":"invalid_request_error",' + '"message":"messages.3.content.3: Invalid `signature` in `thinking` block"},' + '"request_id":"req_011Ca2EtQDxp7x6RGUY2jVn9"}' + ) + resp = httpx.Response(400, request=req, text=err_text) + err = httpx.HTTPStatusError("bad", request=req, response=resp) + assert config.should_retry_anthropic_messages_on_http_error(err, {}) is True + + resp_bad = httpx.Response(400, request=req, text="rate limit exceeded") + err_bad = httpx.HTTPStatusError("bad", request=req, response=resp_bad) + assert config.should_retry_anthropic_messages_on_http_error(err_bad, {}) is False + + resp_500 = httpx.Response(500, request=req, text=err_text) + err_500 = httpx.HTTPStatusError("bad", request=req, response=resp_500) + assert config.should_retry_anthropic_messages_on_http_error(err_500, {}) is False + + data = { + "model": "claude-sonnet-4-20250514", + "messages": [ + { + "role": "assistant", + "content": [ + { + "type": "thinking", + "thinking": "x", + "signature": "y", + }, + ], + } + ], + "thinking": {"type": "enabled", "budget_tokens": 1024}, + } + config.transform_anthropic_messages_request_on_http_error(err, data) + assert "thinking" not in data + assert data["messages"] == []