From 15e60170124e399fb5830c31e2c88bdc0da3b1ee Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 7 Nov 2026 01:51:08 -0800 Subject: [PATCH] fix(anthropic): send openai/ Messages turns again once without encrypted reasoning a backend rejects The openai/ Responses path on /v1/messages had no retry, so a chatgpt/ block left in history answered 400 on every later openai/ turn. That path now sends the turn once more without encrypted reasoning, the same way the chat bridge does. The retry matches only invalid_encrypted_content or a "could not be verified" message naming the content, so the exception mapper's own hint no longer triggers it. The xAI web search keep path is backed out, since xAI sends no encrypted reasoning over the bridge. --- .../anthropic/messages/adapters/handler.py | 51 ++++-- .../messages/adapters/transformation.py | 4 - .../messages/responses_adapters/handler.py | 158 +++++++++++++----- litellm/responses/bridge_check.py | 2 - ...hropic_messages_adapters_transformation.py | 39 +---- .../test_responses_adapters_handler.py | 117 +++++++++++++ 6 files changed, 279 insertions(+), 92 deletions(-) diff --git a/litellm/llms/anthropic/messages/adapters/handler.py b/litellm/llms/anthropic/messages/adapters/handler.py index bde1fc1f502..abdf7924266 100644 --- a/litellm/llms/anthropic/messages/adapters/handler.py +++ b/litellm/llms/anthropic/messages/adapters/handler.py @@ -1,3 +1,4 @@ +import re from collections.abc import AsyncIterator, Callable, Coroutine, Iterator, Mapping, Sequence from dataclasses import dataclass from functools import partial @@ -78,6 +79,7 @@ class _CompletionKwargs(TypedDict, total=False, extra_items=object): _ANTHROPIC_MESSAGES: Final = TypeAdapter(list[dict[str, object]]) +_UNVERIFIED_ENCRYPTED_CONTENT: Final = re.compile(r"encrypted content \S+ could not be verified", re.IGNORECASE) _PrepareReplay: TypeAlias = "Callable[[_AnthropicMessages], _CompletionKwargs]" @@ -125,24 +127,49 @@ def _sends_encrypted_reasoning(completion_kwargs: _CompletionKwargs) -> bool: return any(_message_carries_encrypted_reasoning(message) for message in messages) +def _rejects_encrypted_reasoning(error: litellm.BadRequestError) -> bool: + error_message: Final = str(error) + return ( + "invalid_encrypted_content" in error_message or _UNVERIFIED_ENCRYPTED_CONTENT.search(error_message) is not None + ) + + +def messages_without_rejected_reasoning( + messages: Sequence[Mapping[str, object]], + error: litellm.BadRequestError, + *, + sent_encrypted_reasoning: bool, + provider: str, + model: str, +) -> "_AnthropicMessages | None": + if not sent_encrypted_reasoning or not _rejects_encrypted_reasoning(error): + return None + verbose_logger.warning( + "Backend rejected replayed encrypted reasoning, retrying once without it. " + "Enable the encrypted_content_affinity pre-call check to drop such blocks before the call. " + "provider=%s model=%s status=%s code=%s", + provider, + model, + error.status_code, + _logged_rejection_code(error), + ) + return _ANTHROPIC_MESSAGES.validate_python( + strip_encrypted_reasoning_blocks_from_anthropic_messages([dict(message) for message in messages]) + ) + + def _messages_without_rejected_reasoning( messages: _AnthropicMessages, error: litellm.BadRequestError, completion_kwargs: _CompletionKwargs, ) -> "_AnthropicMessages | None": - error_message: Final = str(error) - rejected: Final = "invalid_encrypted_content" in error_message or "could not be verified" in error_message - if not rejected or not _sends_encrypted_reasoning(completion_kwargs): - return None - verbose_logger.warning( - "Backend rejected replayed encrypted reasoning, retrying once without it. " - "provider=%s model=%s status=%s code=%s", - _completion_text(completion_kwargs, "custom_llm_provider"), - _completion_text(completion_kwargs, "model"), - error.status_code, - _logged_rejection_code(error), + return messages_without_rejected_reasoning( + messages, + error, + sent_encrypted_reasoning=_sends_encrypted_reasoning(completion_kwargs), + provider=_completion_text(completion_kwargs, "custom_llm_provider"), + model=_completion_text(completion_kwargs, "model"), ) - return _ANTHROPIC_MESSAGES.validate_python(strip_encrypted_reasoning_blocks_from_anthropic_messages(messages)) async def _acompletion_without_rejected_reasoning( diff --git a/litellm/llms/anthropic/messages/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py index 5b96cacca59..be97946399d 100644 --- a/litellm/llms/anthropic/messages/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -239,7 +239,6 @@ from litellm.types.llms.openai import ( ChatCompletionToolParamFunctionChunk, ChatCompletionToolReferenceObject, ChatCompletionUserMessage, - OpenAIWebSearchOptions, ToolMessageContentPart, ) from litellm.types.utils import Choices, LlmProviders, ModelResponse, StreamingChoices, Usage @@ -267,7 +266,6 @@ class ChatRouteInputs: reasoning_effort: str | Mapping[str, object] | None = None api_base: str | None = None thinking: Mapping[str, object] | None = None - web_search: bool = False _NO_ROUTE_INPUTS: Final = ChatRouteInputs() @@ -314,7 +312,6 @@ def _keeps_encrypted_reasoning_for_responses( tools=route.tools, reasoning_effort=route.reasoning_effort, api_base=route.api_base, - web_search_options=OpenAIWebSearchOptions() if route.web_search else None, ) @@ -1447,7 +1444,6 @@ class LiteLLMAnthropicMessagesAdapter: reasoning_effort=new_kwargs.get("reasoning_effort"), api_base=api_base, thinking=anthropic_message_request.get("thinking"), - web_search="web_search_options" in new_kwargs, ), ) ## ADD SYSTEM MESSAGE TO MESSAGES diff --git a/litellm/llms/anthropic/messages/responses_adapters/handler.py b/litellm/llms/anthropic/messages/responses_adapters/handler.py index 85f042f621c..04755fd4e34 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/handler.py +++ b/litellm/llms/anthropic/messages/responses_adapters/handler.py @@ -19,13 +19,14 @@ from litellm.integrations.anthropic_cache_control_hook import ( from litellm.litellm_core_utils.asyncify import ( run_async_function, # pyright: ignore[reportUnknownVariableType] # untyped helper ) -from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params +from litellm.litellm_core_utils.core_helpers import as_object_list, as_str_mapping, drops_unsupported_params from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( cache_control_as_prompt_cache_breakpoint, prompt_cache_breakpoint_with_readable_mode, with_prompt_cache_breakpoint, ) from litellm.litellm_core_utils.tool_fields import drops_mcp_servers, tools_without_mcp_servers +from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator from litellm.responses.utils import auto_reasoning_summary from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS from litellm.types.llms.anthropic import ( @@ -43,7 +44,11 @@ from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.utils import CountContextManagement from litellm.utils import ProviderConfigManager -from ..adapters.handler import context_management_explicitly_dropped, request_safeguards_evaluator +from ..adapters.handler import ( + context_management_explicitly_dropped, + messages_without_rejected_reasoning, + request_safeguards_evaluator, +) from ..context_management import AnthropicContextManagementError, PolyfillResult, apply_context_management from ..context_management.constants import COMPACT_EDIT_TYPE from ..context_management.dispatcher import normalize_spec @@ -391,6 +396,67 @@ def _build_responses_kwargs( return responses_kwargs +_ResponsesKwargsFor: TypeAlias = Callable[[AnthropicRequestMessages], Mapping[str, object]] +_ResponsesResult: TypeAlias = ResponsesAPIResponse | BaseResponsesAPIStreamingIterator +_SyncResponsesResult: TypeAlias = _ResponsesResult | Coroutine[object, object, _ResponsesResult] + + +def _is_encrypted_reasoning_item(item: object) -> bool: + mapping: Final = as_str_mapping(item) + return mapping is not None and mapping.get("type") == "reasoning" and bool(mapping.get("encrypted_content")) + + +def _sends_encrypted_reasoning(responses_kwargs: Mapping[str, object]) -> bool: + return any(map(_is_encrypted_reasoning_item, as_object_list(responses_kwargs.get("input")) or ())) + + +def _kwarg_text(responses_kwargs: Mapping[str, object], key: str) -> str: + value: Final = responses_kwargs.get(key) + return value if isinstance(value, str) else "" + + +def _messages_without_rejected_reasoning( + messages: AnthropicRequestMessages, + error: litellm.BadRequestError, + responses_kwargs: Mapping[str, object], +) -> AnthropicRequestMessages | None: + return messages_without_rejected_reasoning( + messages, + error, + sent_encrypted_reasoning=_sends_encrypted_reasoning(responses_kwargs), + provider=_kwarg_text(responses_kwargs, "custom_llm_provider"), + model=_kwarg_text(responses_kwargs, "model"), + ) + + +async def _aresponses_without_rejected_reasoning( + messages: AnthropicRequestMessages, responses_kwargs_for: _ResponsesKwargsFor +) -> tuple[Mapping[str, object], _ResponsesResult]: + responses_kwargs: Final = responses_kwargs_for(messages) + try: + return responses_kwargs, await litellm.aresponses(**responses_kwargs) + except litellm.BadRequestError as error: + replayed_messages: Final = _messages_without_rejected_reasoning(messages, error, responses_kwargs) + if replayed_messages is None: + raise + replay_kwargs: Final = responses_kwargs_for(replayed_messages) + return replay_kwargs, await litellm.aresponses(**replay_kwargs) + + +def _responses_without_rejected_reasoning( + messages: AnthropicRequestMessages, responses_kwargs_for: _ResponsesKwargsFor +) -> tuple[Mapping[str, object], _SyncResponsesResult]: + responses_kwargs: Final = responses_kwargs_for(messages) + try: + return responses_kwargs, litellm.responses(**responses_kwargs) + except litellm.BadRequestError as error: + replayed_messages: Final = _messages_without_rejected_reasoning(messages, error, responses_kwargs) + if replayed_messages is None: + raise + replay_kwargs: Final = responses_kwargs_for(replayed_messages) + return replay_kwargs, litellm.responses(**replay_kwargs) + + class LiteLLMMessagesToResponsesAPIHandler: """ Handles Anthropic /v1/messages requests for OpenAI / Azure models by @@ -422,28 +488,32 @@ class LiteLLMMessagesToResponsesAPIHandler: polyfill_result: Final = await _polyfilled( model=model, messages=messages, tools=tools, system=system, edits=polyfilled_edits, kwargs=kwargs ) - responses_kwargs: Final = _build_responses_kwargs( - max_tokens=max_tokens, - messages=polyfill_result.messages if polyfill_result is not None else messages, - model=model, - context_management=native_context_management, - metadata=metadata, - output_config=output_config, - stop_sequences=stop_sequences, - stream=stream, - system=system, - temperature=temperature, - thinking=thinking, - tool_choice=tool_choice, - tools=tools, - top_k=top_k, - top_p=top_p, - output_format=output_format, - extra_kwargs=kwargs, - ) + + def responses_kwargs_for(sent_messages: AnthropicRequestMessages) -> Mapping[str, object]: + return _build_responses_kwargs( + max_tokens=max_tokens, + messages=sent_messages, + model=model, + context_management=native_context_management, + metadata=metadata, + output_config=output_config, + stop_sequences=stop_sequences, + stream=stream, + system=system, + temperature=temperature, + thinking=thinking, + tool_choice=tool_choice, + tools=tools, + top_k=top_k, + top_p=top_p, + output_format=output_format, + extra_kwargs=kwargs, + ) safeguards_evaluator: Final = request_safeguards_evaluator(kwargs, messages) - result: Final = await litellm.aresponses(**responses_kwargs) + responses_kwargs, result = await _aresponses_without_rejected_reasoning( + polyfill_result.messages if polyfill_result is not None else messages, responses_kwargs_for + ) if stream: wrapper: Final = AnthropicResponsesStreamWrapper( @@ -526,27 +596,31 @@ class LiteLLMMessagesToResponsesAPIHandler: else None ) polyfill_result: Final = ran if isinstance(ran, PolyfillResult) else None - responses_kwargs: Final = _build_responses_kwargs( - max_tokens=max_tokens, - messages=polyfill_result.messages if polyfill_result is not None else messages, - model=model, - context_management=native_context_management, - metadata=metadata, - output_config=output_config, - stop_sequences=stop_sequences, - stream=stream, - system=system, - temperature=temperature, - thinking=thinking, - tool_choice=tool_choice, - tools=tools, - top_k=top_k, - top_p=top_p, - output_format=output_format, - extra_kwargs=kwargs, - ) - result: Final = litellm.responses(**responses_kwargs) + def responses_kwargs_for(sent_messages: AnthropicRequestMessages) -> Mapping[str, object]: + return _build_responses_kwargs( + max_tokens=max_tokens, + messages=sent_messages, + model=model, + context_management=native_context_management, + metadata=metadata, + output_config=output_config, + stop_sequences=stop_sequences, + stream=stream, + system=system, + temperature=temperature, + thinking=thinking, + tool_choice=tool_choice, + tools=tools, + top_k=top_k, + top_p=top_p, + output_format=output_format, + extra_kwargs=kwargs, + ) + + responses_kwargs, result = _responses_without_rejected_reasoning( + polyfill_result.messages if polyfill_result is not None else messages, responses_kwargs_for + ) if stream: wrapper: Final = AnthropicResponsesStreamWrapper( diff --git a/litellm/responses/bridge_check.py b/litellm/responses/bridge_check.py index b284b381130..74e04bb51ae 100644 --- a/litellm/responses/bridge_check.py +++ b/litellm/responses/bridge_check.py @@ -304,7 +304,6 @@ def chat_completion_ends_in_responses_api( tools: Sequence[Mapping[str, object]] | None = None, reasoning_effort: str | Mapping[str, object] | None = None, api_base: str | None = None, - web_search_options: OpenAIWebSearchOptions | None = None, ) -> bool: """Whether chat completion for this model is served by a Responses API. @@ -323,7 +322,6 @@ def chat_completion_ends_in_responses_api( model_info, _resolved = responses_api_bridge_check( model=resolved_model, custom_llm_provider=provider, - web_search_options=web_search_options, tools=tools, reasoning_effort=reasoning_effort, api_base=api_base, diff --git a/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py b/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py index d6f74fab0cd..eddec1f9d0a 100644 --- a/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py @@ -5782,6 +5782,9 @@ _RETRY_REJECTION: Final = httpx.Response( _UNRELATED_REJECTION: Final = httpx.Response( 400, json={"error": {"message": "max_output_tokens is too small", "type": "invalid_request_error"}} ) +_UNRELATED_UNVERIFIED_REJECTION: Final = httpx.Response( + 400, json={"error": {"message": "The organization could not be verified.", "type": "invalid_request_error"}} +) _RETRY_REPLY: Final = { "id": "resp_retry", "object": "response", @@ -5856,6 +5859,7 @@ async def test_async_messages_retry_once_without_reasoning_the_backend_cannot_de assert [block["text"] for block in response["content"] if block["type"] == "text"] == ["Oslo"] assert retry_warnings == ( "Backend rejected replayed encrypted reasoning, retrying once without it. " + "Enable the encrypted_content_affinity pre-call check to drop such blocks before the call. " "provider=azure model=azure/gpt-5.4-nano status=400 code=invalid_encrypted_content", ) assert all( @@ -5897,6 +5901,9 @@ def test_sync_messages_retry_once_without_reasoning_the_backend_cannot_decrypt( id="no_encrypted_reasoning_to_drop", ), pytest.param(_history_with_foreign_reasoning(), _UNRELATED_REJECTION, id="unrelated_rejection"), + pytest.param( + _history_with_foreign_reasoning(), _UNRELATED_UNVERIFIED_REJECTION, id="unrelated_unverified_rejection" + ), ], ) async def test_messages_do_not_retry_other_rejections( @@ -5970,38 +5977,6 @@ async def test_openai_thinking_on_the_chat_adapter_replays_the_reasoning_its_res assert [item["encrypted_content"] for item in _sent_reasoning_items(route.calls[0].request)] == ["gAAA_same_org"] -@pytest.mark.asyncio -@pytest.mark.usefixtures("_chat_bridge_httpx") -async def test_xai_web_search_turn_replays_the_reasoning_its_responses_route_minted( - respx_mock: respx.MockRouter, -) -> None: - route: Final = respx_mock.post(url__regex=r".*xai-search\.example/v1/responses").mock( - return_value=httpx.Response(200, json=_RETRY_REPLY) - ) - - await litellm.anthropic.messages.acreate( - model="xai/grok-4.7", - api_base="https://xai-search.example/v1", - api_key="fake-key", - max_tokens=2048, - tools=[{"type": "web_search_20250305", "name": "web_search"}], - messages=[ - {"role": "user", "content": "Which city is colder today?"}, - { - "role": "assistant", - "content": [ - {"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAA_same_org")}, - {"type": "text", "text": "Oslo."}, - ], - }, - {"role": "user", "content": "And by how much?"}, - ], - ) - - assert route.call_count == 1 - assert [item["encrypted_content"] for item in _sent_reasoning_items(route.calls[0].request)] == ["gAAA_same_org"] - - def _responses_sse(events: Sequence[Mapping[str, object]]) -> bytes: return b"".join( f"event: {event['type']}\ndata: {json.dumps({**event, 'sequence_number': number})}\n\n".encode() diff --git a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py index c06c9d721ac..97c6e4c081c 100644 --- a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py @@ -7,6 +7,7 @@ from itertools import chain from typing import Final, cast from unittest.mock import AsyncMock, patch +import httpx import pytest import respx @@ -17,6 +18,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( TOOL_RESULT_IMAGE_BOUNDARY, TOOL_RESULT_IMAGE_PLACEHOLDER, ) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import encrypted_reasoning_signature from litellm.llms.anthropic.messages.responses_adapters.handler import ( LiteLLMMessagesToResponsesAPIHandler, _build_responses_kwargs, @@ -1135,3 +1137,118 @@ async def test_the_sync_path_reports_its_edits_on_the_streamed_message_delta( ] message_delta: Final = next(event for event in events if event["type"] == "message_delta") assert message_delta["context_management"]["applied_edits"][0]["cleared_tool_uses"] == 2 + + +REJECTED_REPLAYED_REASONING: Final = { + "error": { + "message": "The encrypted content gAAA_foreign could not be verified.", + "type": "invalid_request_error", + "param": None, + "code": "invalid_encrypted_content", + } +} +UNVERIFIED_ORGANIZATION: Final = { + "error": {"message": "The organization could not be verified.", "type": "invalid_request_error"} +} + + +def _history(*, replayed_reasoning: bool) -> list[dict[str, object]]: + reasoning: Final = ( + ({"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAA_foreign")},) + if replayed_reasoning + else () + ) + return [ + {"role": "user", "content": "List the logs."}, + {"role": "assistant", "content": [*reasoning, {"type": "tool_use", "id": "call_1", "name": "Bash", "input": {}}]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "call_1", "content": "app.log"}]}, + ] + + +def _replayed_reasoning_counts(upstream: respx.Route) -> list[int]: + return [ + sum(item.get("type") == "reasoning" for item in json.loads(call.request.content)["input"]) + for call in upstream.calls + ] + + +def _upstream(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, first: httpx.Response, stream: bool): + monkeypatch.setenv("OPENAI_API_KEY", "sk-replay-test") + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + litellm.in_memory_llm_clients_cache.flush_cache() + served: Final = ( + httpx.Response(200, headers={"Content-Type": "text/event-stream"}, content=RESPONSES_SSE_BODY) + if stream + else httpx.Response(200, json=TEXT_RESPONSE) + ) + return respx_mock.post("https://api.openai.com/v1/responses").mock(side_effect=[first, served]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True]) +async def test_openai_turn_is_sent_again_once_without_replayed_reasoning_the_backend_cannot_decrypt( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, stream: bool +) -> None: + upstream: Final = _upstream(respx_mock, monkeypatch, httpx.Response(400, json=REJECTED_REPLAYED_REASONING), stream) + + response: Final = await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler( + max_tokens=64, + messages=_history(replayed_reasoning=True), + model="openai/gpt-5.6-luna", + tools=[dict(BASH_TOOL)], + stream=stream, + custom_llm_provider="openai", + ) + events: Final = ( + [json.loads(chunk.decode().split("data: ", 1)[1]) async for chunk in cast(AsyncIterator[bytes], response)] + if stream + else [cast(Mapping[str, object], response)] + ) + + assert _replayed_reasoning_counts(upstream) == [1, 0] + assert events[-1]["type"] in ("message", "message_stop") + + +def test_sync_openai_turn_is_sent_again_once_without_replayed_reasoning_the_backend_cannot_decrypt( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + upstream: Final = _upstream(respx_mock, monkeypatch, httpx.Response(400, json=REJECTED_REPLAYED_REASONING), False) + + response: Final = LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler( + max_tokens=64, + messages=_history(replayed_reasoning=True), + model="openai/gpt-5.6-luna", + tools=[dict(BASH_TOOL)], + custom_llm_provider="openai", + ) + + assert _replayed_reasoning_counts(upstream) == [1, 0] + assert cast(Mapping[str, object], response)["content"] == [{"type": "text", "text": "done"}] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "replayed_reasoning, rejection", + [ + pytest.param(False, REJECTED_REPLAYED_REASONING, id="no_replayed_reasoning"), + pytest.param(True, UNVERIFIED_ORGANIZATION, id="unrelated_rejection"), + ], +) +async def test_openai_turn_is_not_sent_again_for_other_rejections( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + replayed_reasoning: bool, + rejection: Mapping[str, object], +) -> None: + upstream: Final = _upstream(respx_mock, monkeypatch, httpx.Response(400, json=rejection), False) + + with pytest.raises(litellm.BadRequestError): + await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler( + max_tokens=64, + messages=_history(replayed_reasoning=replayed_reasoning), + model="openai/gpt-5.6-luna", + tools=[dict(BASH_TOOL)], + custom_llm_provider="openai", + ) + + assert upstream.call_count == 1