diff --git a/litellm/llms/anthropic/messages/adapters/handler.py b/litellm/llms/anthropic/messages/adapters/handler.py index bde1fc1f502..abdf7924266 100644 --- a/litellm/llms/anthropic/messages/adapters/handler.py +++ b/litellm/llms/anthropic/messages/adapters/handler.py @@ -1,3 +1,4 @@ +import re from collections.abc import AsyncIterator, Callable, Coroutine, Iterator, Mapping, Sequence from dataclasses import dataclass from functools import partial @@ -78,6 +79,7 @@ class _CompletionKwargs(TypedDict, total=False, extra_items=object): _ANTHROPIC_MESSAGES: Final = TypeAdapter(list[dict[str, object]]) +_UNVERIFIED_ENCRYPTED_CONTENT: Final = re.compile(r"encrypted content \S+ could not be verified", re.IGNORECASE) _PrepareReplay: TypeAlias = "Callable[[_AnthropicMessages], _CompletionKwargs]" @@ -125,24 +127,49 @@ def _sends_encrypted_reasoning(completion_kwargs: _CompletionKwargs) -> bool: return any(_message_carries_encrypted_reasoning(message) for message in messages) +def _rejects_encrypted_reasoning(error: litellm.BadRequestError) -> bool: + error_message: Final = str(error) + return ( + "invalid_encrypted_content" in error_message or _UNVERIFIED_ENCRYPTED_CONTENT.search(error_message) is not None + ) + + +def messages_without_rejected_reasoning( + messages: Sequence[Mapping[str, object]], + error: litellm.BadRequestError, + *, + sent_encrypted_reasoning: bool, + provider: str, + model: str, +) -> "_AnthropicMessages | None": + if not sent_encrypted_reasoning or not _rejects_encrypted_reasoning(error): + return None + verbose_logger.warning( + "Backend rejected replayed encrypted reasoning, retrying once without it. " + "Enable the encrypted_content_affinity pre-call check to drop such blocks before the call. " + "provider=%s model=%s status=%s code=%s", + provider, + model, + error.status_code, + _logged_rejection_code(error), + ) + return _ANTHROPIC_MESSAGES.validate_python( + strip_encrypted_reasoning_blocks_from_anthropic_messages([dict(message) for message in messages]) + ) + + def _messages_without_rejected_reasoning( messages: _AnthropicMessages, error: litellm.BadRequestError, completion_kwargs: _CompletionKwargs, ) -> "_AnthropicMessages | None": - error_message: Final = str(error) - rejected: Final = "invalid_encrypted_content" in error_message or "could not be verified" in error_message - if not rejected or not _sends_encrypted_reasoning(completion_kwargs): - return None - verbose_logger.warning( - "Backend rejected replayed encrypted reasoning, retrying once without it. " - "provider=%s model=%s status=%s code=%s", - _completion_text(completion_kwargs, "custom_llm_provider"), - _completion_text(completion_kwargs, "model"), - error.status_code, - _logged_rejection_code(error), + return messages_without_rejected_reasoning( + messages, + error, + sent_encrypted_reasoning=_sends_encrypted_reasoning(completion_kwargs), + provider=_completion_text(completion_kwargs, "custom_llm_provider"), + model=_completion_text(completion_kwargs, "model"), ) - return _ANTHROPIC_MESSAGES.validate_python(strip_encrypted_reasoning_blocks_from_anthropic_messages(messages)) async def _acompletion_without_rejected_reasoning( diff --git a/litellm/llms/anthropic/messages/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py index 5b96cacca59..be97946399d 100644 --- a/litellm/llms/anthropic/messages/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -239,7 +239,6 @@ from litellm.types.llms.openai import ( ChatCompletionToolParamFunctionChunk, ChatCompletionToolReferenceObject, ChatCompletionUserMessage, - OpenAIWebSearchOptions, ToolMessageContentPart, ) from litellm.types.utils import Choices, LlmProviders, ModelResponse, StreamingChoices, Usage @@ -267,7 +266,6 @@ class ChatRouteInputs: reasoning_effort: str | Mapping[str, object] | None = None api_base: str | None = None thinking: Mapping[str, object] | None = None - web_search: bool = False _NO_ROUTE_INPUTS: Final = ChatRouteInputs() @@ -314,7 +312,6 @@ def _keeps_encrypted_reasoning_for_responses( tools=route.tools, reasoning_effort=route.reasoning_effort, api_base=route.api_base, - web_search_options=OpenAIWebSearchOptions() if route.web_search else None, ) @@ -1447,7 +1444,6 @@ class LiteLLMAnthropicMessagesAdapter: reasoning_effort=new_kwargs.get("reasoning_effort"), api_base=api_base, thinking=anthropic_message_request.get("thinking"), - web_search="web_search_options" in new_kwargs, ), ) ## ADD SYSTEM MESSAGE TO MESSAGES diff --git a/litellm/llms/anthropic/messages/responses_adapters/handler.py b/litellm/llms/anthropic/messages/responses_adapters/handler.py index 85f042f621c..04755fd4e34 100644 --- a/litellm/llms/anthropic/messages/responses_adapters/handler.py +++ b/litellm/llms/anthropic/messages/responses_adapters/handler.py @@ -19,13 +19,14 @@ from litellm.integrations.anthropic_cache_control_hook import ( from litellm.litellm_core_utils.asyncify import ( run_async_function, # pyright: ignore[reportUnknownVariableType] # untyped helper ) -from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params +from litellm.litellm_core_utils.core_helpers import as_object_list, as_str_mapping, drops_unsupported_params from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( cache_control_as_prompt_cache_breakpoint, prompt_cache_breakpoint_with_readable_mode, with_prompt_cache_breakpoint, ) from litellm.litellm_core_utils.tool_fields import drops_mcp_servers, tools_without_mcp_servers +from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator from litellm.responses.utils import auto_reasoning_summary from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS from litellm.types.llms.anthropic import ( @@ -43,7 +44,11 @@ from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.utils import CountContextManagement from litellm.utils import ProviderConfigManager -from ..adapters.handler import context_management_explicitly_dropped, request_safeguards_evaluator +from ..adapters.handler import ( + context_management_explicitly_dropped, + messages_without_rejected_reasoning, + request_safeguards_evaluator, +) from ..context_management import AnthropicContextManagementError, PolyfillResult, apply_context_management from ..context_management.constants import COMPACT_EDIT_TYPE from ..context_management.dispatcher import normalize_spec @@ -391,6 +396,67 @@ def _build_responses_kwargs( return responses_kwargs +_ResponsesKwargsFor: TypeAlias = Callable[[AnthropicRequestMessages], Mapping[str, object]] +_ResponsesResult: TypeAlias = ResponsesAPIResponse | BaseResponsesAPIStreamingIterator +_SyncResponsesResult: TypeAlias = _ResponsesResult | Coroutine[object, object, _ResponsesResult] + + +def _is_encrypted_reasoning_item(item: object) -> bool: + mapping: Final = as_str_mapping(item) + return mapping is not None and mapping.get("type") == "reasoning" and bool(mapping.get("encrypted_content")) + + +def _sends_encrypted_reasoning(responses_kwargs: Mapping[str, object]) -> bool: + return any(map(_is_encrypted_reasoning_item, as_object_list(responses_kwargs.get("input")) or ())) + + +def _kwarg_text(responses_kwargs: Mapping[str, object], key: str) -> str: + value: Final = responses_kwargs.get(key) + return value if isinstance(value, str) else "" + + +def _messages_without_rejected_reasoning( + messages: AnthropicRequestMessages, + error: litellm.BadRequestError, + responses_kwargs: Mapping[str, object], +) -> AnthropicRequestMessages | None: + return messages_without_rejected_reasoning( + messages, + error, + sent_encrypted_reasoning=_sends_encrypted_reasoning(responses_kwargs), + provider=_kwarg_text(responses_kwargs, "custom_llm_provider"), + model=_kwarg_text(responses_kwargs, "model"), + ) + + +async def _aresponses_without_rejected_reasoning( + messages: AnthropicRequestMessages, responses_kwargs_for: _ResponsesKwargsFor +) -> tuple[Mapping[str, object], _ResponsesResult]: + responses_kwargs: Final = responses_kwargs_for(messages) + try: + return responses_kwargs, await litellm.aresponses(**responses_kwargs) + except litellm.BadRequestError as error: + replayed_messages: Final = _messages_without_rejected_reasoning(messages, error, responses_kwargs) + if replayed_messages is None: + raise + replay_kwargs: Final = responses_kwargs_for(replayed_messages) + return replay_kwargs, await litellm.aresponses(**replay_kwargs) + + +def _responses_without_rejected_reasoning( + messages: AnthropicRequestMessages, responses_kwargs_for: _ResponsesKwargsFor +) -> tuple[Mapping[str, object], _SyncResponsesResult]: + responses_kwargs: Final = responses_kwargs_for(messages) + try: + return responses_kwargs, litellm.responses(**responses_kwargs) + except litellm.BadRequestError as error: + replayed_messages: Final = _messages_without_rejected_reasoning(messages, error, responses_kwargs) + if replayed_messages is None: + raise + replay_kwargs: Final = responses_kwargs_for(replayed_messages) + return replay_kwargs, litellm.responses(**replay_kwargs) + + class LiteLLMMessagesToResponsesAPIHandler: """ Handles Anthropic /v1/messages requests for OpenAI / Azure models by @@ -422,28 +488,32 @@ class LiteLLMMessagesToResponsesAPIHandler: polyfill_result: Final = await _polyfilled( model=model, messages=messages, tools=tools, system=system, edits=polyfilled_edits, kwargs=kwargs ) - responses_kwargs: Final = _build_responses_kwargs( - max_tokens=max_tokens, - messages=polyfill_result.messages if polyfill_result is not None else messages, - model=model, - context_management=native_context_management, - metadata=metadata, - output_config=output_config, - stop_sequences=stop_sequences, - stream=stream, - system=system, - temperature=temperature, - thinking=thinking, - tool_choice=tool_choice, - tools=tools, - top_k=top_k, - top_p=top_p, - output_format=output_format, - extra_kwargs=kwargs, - ) + + def responses_kwargs_for(sent_messages: AnthropicRequestMessages) -> Mapping[str, object]: + return _build_responses_kwargs( + max_tokens=max_tokens, + messages=sent_messages, + model=model, + context_management=native_context_management, + metadata=metadata, + output_config=output_config, + stop_sequences=stop_sequences, + stream=stream, + system=system, + temperature=temperature, + thinking=thinking, + tool_choice=tool_choice, + tools=tools, + top_k=top_k, + top_p=top_p, + output_format=output_format, + extra_kwargs=kwargs, + ) safeguards_evaluator: Final = request_safeguards_evaluator(kwargs, messages) - result: Final = await litellm.aresponses(**responses_kwargs) + responses_kwargs, result = await _aresponses_without_rejected_reasoning( + polyfill_result.messages if polyfill_result is not None else messages, responses_kwargs_for + ) if stream: wrapper: Final = AnthropicResponsesStreamWrapper( @@ -526,27 +596,31 @@ class LiteLLMMessagesToResponsesAPIHandler: else None ) polyfill_result: Final = ran if isinstance(ran, PolyfillResult) else None - responses_kwargs: Final = _build_responses_kwargs( - max_tokens=max_tokens, - messages=polyfill_result.messages if polyfill_result is not None else messages, - model=model, - context_management=native_context_management, - metadata=metadata, - output_config=output_config, - stop_sequences=stop_sequences, - stream=stream, - system=system, - temperature=temperature, - thinking=thinking, - tool_choice=tool_choice, - tools=tools, - top_k=top_k, - top_p=top_p, - output_format=output_format, - extra_kwargs=kwargs, - ) - result: Final = litellm.responses(**responses_kwargs) + def responses_kwargs_for(sent_messages: AnthropicRequestMessages) -> Mapping[str, object]: + return _build_responses_kwargs( + max_tokens=max_tokens, + messages=sent_messages, + model=model, + context_management=native_context_management, + metadata=metadata, + output_config=output_config, + stop_sequences=stop_sequences, + stream=stream, + system=system, + temperature=temperature, + thinking=thinking, + tool_choice=tool_choice, + tools=tools, + top_k=top_k, + top_p=top_p, + output_format=output_format, + extra_kwargs=kwargs, + ) + + responses_kwargs, result = _responses_without_rejected_reasoning( + polyfill_result.messages if polyfill_result is not None else messages, responses_kwargs_for + ) if stream: wrapper: Final = AnthropicResponsesStreamWrapper( diff --git a/litellm/responses/bridge_check.py b/litellm/responses/bridge_check.py index b284b381130..74e04bb51ae 100644 --- a/litellm/responses/bridge_check.py +++ b/litellm/responses/bridge_check.py @@ -304,7 +304,6 @@ def chat_completion_ends_in_responses_api( tools: Sequence[Mapping[str, object]] | None = None, reasoning_effort: str | Mapping[str, object] | None = None, api_base: str | None = None, - web_search_options: OpenAIWebSearchOptions | None = None, ) -> bool: """Whether chat completion for this model is served by a Responses API. @@ -323,7 +322,6 @@ def chat_completion_ends_in_responses_api( model_info, _resolved = responses_api_bridge_check( model=resolved_model, custom_llm_provider=provider, - web_search_options=web_search_options, tools=tools, reasoning_effort=reasoning_effort, api_base=api_base, diff --git a/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py b/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py index d6f74fab0cd..eddec1f9d0a 100644 --- a/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py @@ -5782,6 +5782,9 @@ _RETRY_REJECTION: Final = httpx.Response( _UNRELATED_REJECTION: Final = httpx.Response( 400, json={"error": {"message": "max_output_tokens is too small", "type": "invalid_request_error"}} ) +_UNRELATED_UNVERIFIED_REJECTION: Final = httpx.Response( + 400, json={"error": {"message": "The organization could not be verified.", "type": "invalid_request_error"}} +) _RETRY_REPLY: Final = { "id": "resp_retry", "object": "response", @@ -5856,6 +5859,7 @@ async def test_async_messages_retry_once_without_reasoning_the_backend_cannot_de assert [block["text"] for block in response["content"] if block["type"] == "text"] == ["Oslo"] assert retry_warnings == ( "Backend rejected replayed encrypted reasoning, retrying once without it. " + "Enable the encrypted_content_affinity pre-call check to drop such blocks before the call. " "provider=azure model=azure/gpt-5.4-nano status=400 code=invalid_encrypted_content", ) assert all( @@ -5897,6 +5901,9 @@ def test_sync_messages_retry_once_without_reasoning_the_backend_cannot_decrypt( id="no_encrypted_reasoning_to_drop", ), pytest.param(_history_with_foreign_reasoning(), _UNRELATED_REJECTION, id="unrelated_rejection"), + pytest.param( + _history_with_foreign_reasoning(), _UNRELATED_UNVERIFIED_REJECTION, id="unrelated_unverified_rejection" + ), ], ) async def test_messages_do_not_retry_other_rejections( @@ -5970,38 +5977,6 @@ async def test_openai_thinking_on_the_chat_adapter_replays_the_reasoning_its_res assert [item["encrypted_content"] for item in _sent_reasoning_items(route.calls[0].request)] == ["gAAA_same_org"] -@pytest.mark.asyncio -@pytest.mark.usefixtures("_chat_bridge_httpx") -async def test_xai_web_search_turn_replays_the_reasoning_its_responses_route_minted( - respx_mock: respx.MockRouter, -) -> None: - route: Final = respx_mock.post(url__regex=r".*xai-search\.example/v1/responses").mock( - return_value=httpx.Response(200, json=_RETRY_REPLY) - ) - - await litellm.anthropic.messages.acreate( - model="xai/grok-4.7", - api_base="https://xai-search.example/v1", - api_key="fake-key", - max_tokens=2048, - tools=[{"type": "web_search_20250305", "name": "web_search"}], - messages=[ - {"role": "user", "content": "Which city is colder today?"}, - { - "role": "assistant", - "content": [ - {"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAA_same_org")}, - {"type": "text", "text": "Oslo."}, - ], - }, - {"role": "user", "content": "And by how much?"}, - ], - ) - - assert route.call_count == 1 - assert [item["encrypted_content"] for item in _sent_reasoning_items(route.calls[0].request)] == ["gAAA_same_org"] - - def _responses_sse(events: Sequence[Mapping[str, object]]) -> bytes: return b"".join( f"event: {event['type']}\ndata: {json.dumps({**event, 'sequence_number': number})}\n\n".encode() diff --git a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py index c06c9d721ac..97c6e4c081c 100644 --- a/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py +++ b/tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py @@ -7,6 +7,7 @@ from itertools import chain from typing import Final, cast from unittest.mock import AsyncMock, patch +import httpx import pytest import respx @@ -17,6 +18,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( TOOL_RESULT_IMAGE_BOUNDARY, TOOL_RESULT_IMAGE_PLACEHOLDER, ) +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import encrypted_reasoning_signature from litellm.llms.anthropic.messages.responses_adapters.handler import ( LiteLLMMessagesToResponsesAPIHandler, _build_responses_kwargs, @@ -1135,3 +1137,118 @@ async def test_the_sync_path_reports_its_edits_on_the_streamed_message_delta( ] message_delta: Final = next(event for event in events if event["type"] == "message_delta") assert message_delta["context_management"]["applied_edits"][0]["cleared_tool_uses"] == 2 + + +REJECTED_REPLAYED_REASONING: Final = { + "error": { + "message": "The encrypted content gAAA_foreign could not be verified.", + "type": "invalid_request_error", + "param": None, + "code": "invalid_encrypted_content", + } +} +UNVERIFIED_ORGANIZATION: Final = { + "error": {"message": "The organization could not be verified.", "type": "invalid_request_error"} +} + + +def _history(*, replayed_reasoning: bool) -> list[dict[str, object]]: + reasoning: Final = ( + ({"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAA_foreign")},) + if replayed_reasoning + else () + ) + return [ + {"role": "user", "content": "List the logs."}, + {"role": "assistant", "content": [*reasoning, {"type": "tool_use", "id": "call_1", "name": "Bash", "input": {}}]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "call_1", "content": "app.log"}]}, + ] + + +def _replayed_reasoning_counts(upstream: respx.Route) -> list[int]: + return [ + sum(item.get("type") == "reasoning" for item in json.loads(call.request.content)["input"]) + for call in upstream.calls + ] + + +def _upstream(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, first: httpx.Response, stream: bool): + monkeypatch.setenv("OPENAI_API_KEY", "sk-replay-test") + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + litellm.in_memory_llm_clients_cache.flush_cache() + served: Final = ( + httpx.Response(200, headers={"Content-Type": "text/event-stream"}, content=RESPONSES_SSE_BODY) + if stream + else httpx.Response(200, json=TEXT_RESPONSE) + ) + return respx_mock.post("https://api.openai.com/v1/responses").mock(side_effect=[first, served]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True]) +async def test_openai_turn_is_sent_again_once_without_replayed_reasoning_the_backend_cannot_decrypt( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, stream: bool +) -> None: + upstream: Final = _upstream(respx_mock, monkeypatch, httpx.Response(400, json=REJECTED_REPLAYED_REASONING), stream) + + response: Final = await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler( + max_tokens=64, + messages=_history(replayed_reasoning=True), + model="openai/gpt-5.6-luna", + tools=[dict(BASH_TOOL)], + stream=stream, + custom_llm_provider="openai", + ) + events: Final = ( + [json.loads(chunk.decode().split("data: ", 1)[1]) async for chunk in cast(AsyncIterator[bytes], response)] + if stream + else [cast(Mapping[str, object], response)] + ) + + assert _replayed_reasoning_counts(upstream) == [1, 0] + assert events[-1]["type"] in ("message", "message_stop") + + +def test_sync_openai_turn_is_sent_again_once_without_replayed_reasoning_the_backend_cannot_decrypt( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + upstream: Final = _upstream(respx_mock, monkeypatch, httpx.Response(400, json=REJECTED_REPLAYED_REASONING), False) + + response: Final = LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler( + max_tokens=64, + messages=_history(replayed_reasoning=True), + model="openai/gpt-5.6-luna", + tools=[dict(BASH_TOOL)], + custom_llm_provider="openai", + ) + + assert _replayed_reasoning_counts(upstream) == [1, 0] + assert cast(Mapping[str, object], response)["content"] == [{"type": "text", "text": "done"}] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "replayed_reasoning, rejection", + [ + pytest.param(False, REJECTED_REPLAYED_REASONING, id="no_replayed_reasoning"), + pytest.param(True, UNVERIFIED_ORGANIZATION, id="unrelated_rejection"), + ], +) +async def test_openai_turn_is_not_sent_again_for_other_rejections( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + replayed_reasoning: bool, + rejection: Mapping[str, object], +) -> None: + upstream: Final = _upstream(respx_mock, monkeypatch, httpx.Response(400, json=rejection), False) + + with pytest.raises(litellm.BadRequestError): + await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler( + max_tokens=64, + messages=_history(replayed_reasoning=replayed_reasoning), + model="openai/gpt-5.6-luna", + tools=[dict(BASH_TOOL)], + custom_llm_provider="openai", + ) + + assert upstream.call_count == 1