scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-13 08:52:02 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 174

fix(anthropic): send openai/ Messages turns again once without encrypted reasoning a backend rejects

Browse source 
The openai/ Responses path on /v1/messages had no retry, so a chatgpt/ block left in history answered 400 on every later openai/ turn. That path now sends the turn once more without encrypted reasoning, the same way the chat bridge does. The retry matches only invalid_encrypted_content or a "could not be verified" message naming the content, so the exception mapper's own hint no longer triggers it. The xAI web search keep path is backed out, since xAI sends no encrypted reasoning over the bridge.
This commit is contained in:
mateo-berri 2026-11-07 01:51:08 -08:00
parent e021b42087
commit 15e6017012
6 changed files with 279 additions and 92 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

51
litellm/llms/anthropic/messages/adapters/handler.py
View file

@ -1,3 +1,4 @@
import re
from collections.abc import AsyncIterator, Callable, Coroutine, Iterator, Mapping, Sequence
from dataclasses import dataclass
from functools import partial
@ -78,6 +79,7 @@ class _CompletionKwargs(TypedDict, total=False, extra_items=object):
_ANTHROPIC_MESSAGES: Final = TypeAdapter(list[dict[str, object]])
_UNVERIFIED_ENCRYPTED_CONTENT: Final = re.compile(r"encrypted content \S+ could not be verified", re.IGNORECASE)
_PrepareReplay: TypeAlias = "Callable[[_AnthropicMessages], _CompletionKwargs]"
@ -125,24 +127,49 @@ def _sends_encrypted_reasoning(completion_kwargs: _CompletionKwargs) -> bool:
return any(_message_carries_encrypted_reasoning(message) for message in messages)
def _rejects_encrypted_reasoning(error: litellm.BadRequestError) -> bool:
error_message: Final = str(error)
return (
"invalid_encrypted_content" in error_message or _UNVERIFIED_ENCRYPTED_CONTENT.search(error_message) is not None
)
def messages_without_rejected_reasoning(
messages: Sequence[Mapping[str, object]],
error: litellm.BadRequestError,
*,
sent_encrypted_reasoning: bool,
provider: str,
model: str,
) -> "_AnthropicMessages | None":
if not sent_encrypted_reasoning or not _rejects_encrypted_reasoning(error):
return None
verbose_logger.warning(
"Backend rejected replayed encrypted reasoning, retrying once without it. "
"Enable the encrypted_content_affinity pre-call check to drop such blocks before the call. "
"provider=%s model=%s status=%s code=%s",
provider,
model,
error.status_code,
_logged_rejection_code(error),
)
return _ANTHROPIC_MESSAGES.validate_python(
strip_encrypted_reasoning_blocks_from_anthropic_messages([dict(message) for message in messages])
)
def _messages_without_rejected_reasoning(
messages: _AnthropicMessages,
error: litellm.BadRequestError,
completion_kwargs: _CompletionKwargs,
) -> "_AnthropicMessages | None":
error_message: Final = str(error)
rejected: Final = "invalid_encrypted_content" in error_message or "could not be verified" in error_message
if not rejected or not _sends_encrypted_reasoning(completion_kwargs):
return None
verbose_logger.warning(
"Backend rejected replayed encrypted reasoning, retrying once without it. "
"provider=%s model=%s status=%s code=%s",
_completion_text(completion_kwargs, "custom_llm_provider"),
_completion_text(completion_kwargs, "model"),
error.status_code,
_logged_rejection_code(error),
return messages_without_rejected_reasoning(
messages,
error,
sent_encrypted_reasoning=_sends_encrypted_reasoning(completion_kwargs),
provider=_completion_text(completion_kwargs, "custom_llm_provider"),
model=_completion_text(completion_kwargs, "model"),
)
return _ANTHROPIC_MESSAGES.validate_python(strip_encrypted_reasoning_blocks_from_anthropic_messages(messages))
async def _acompletion_without_rejected_reasoning(

4
litellm/llms/anthropic/messages/adapters/transformation.py
View file

@ -239,7 +239,6 @@ from litellm.types.llms.openai import (
ChatCompletionToolParamFunctionChunk,
ChatCompletionToolReferenceObject,
ChatCompletionUserMessage,
OpenAIWebSearchOptions,
ToolMessageContentPart,
)
from litellm.types.utils import Choices, LlmProviders, ModelResponse, StreamingChoices, Usage
@ -267,7 +266,6 @@ class ChatRouteInputs:
reasoning_effort: str | Mapping[str, object] | None = None
api_base: str | None = None
thinking: Mapping[str, object] | None = None
web_search: bool = False
_NO_ROUTE_INPUTS: Final = ChatRouteInputs()
@ -314,7 +312,6 @@ def _keeps_encrypted_reasoning_for_responses(
tools=route.tools,
reasoning_effort=route.reasoning_effort,
api_base=route.api_base,
web_search_options=OpenAIWebSearchOptions() if route.web_search else None,
)
@ -1447,7 +1444,6 @@ class LiteLLMAnthropicMessagesAdapter:
reasoning_effort=new_kwargs.get("reasoning_effort"),
api_base=api_base,
thinking=anthropic_message_request.get("thinking"),
web_search="web_search_options" in new_kwargs,
),
)
## ADD SYSTEM MESSAGE TO MESSAGES

158
litellm/llms/anthropic/messages/responses_adapters/handler.py
View file

@ -19,13 +19,14 @@ from litellm.integrations.anthropic_cache_control_hook import (
from litellm.litellm_core_utils.asyncify import (
run_async_function, # pyright: ignore[reportUnknownVariableType] # untyped helper
)
from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params
from litellm.litellm_core_utils.core_helpers import as_object_list, as_str_mapping, drops_unsupported_params
from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import (
cache_control_as_prompt_cache_breakpoint,
prompt_cache_breakpoint_with_readable_mode,
with_prompt_cache_breakpoint,
)
from litellm.litellm_core_utils.tool_fields import drops_mcp_servers, tools_without_mcp_servers
from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator
from litellm.responses.utils import auto_reasoning_summary
from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS
from litellm.types.llms.anthropic import (
@ -43,7 +44,11 @@ from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.utils import CountContextManagement
from litellm.utils import ProviderConfigManager
from ..adapters.handler import context_management_explicitly_dropped, request_safeguards_evaluator
from ..adapters.handler import (
context_management_explicitly_dropped,
messages_without_rejected_reasoning,
request_safeguards_evaluator,
)
from ..context_management import AnthropicContextManagementError, PolyfillResult, apply_context_management
from ..context_management.constants import COMPACT_EDIT_TYPE
from ..context_management.dispatcher import normalize_spec
@ -391,6 +396,67 @@ def _build_responses_kwargs(
return responses_kwargs
_ResponsesKwargsFor: TypeAlias = Callable[[AnthropicRequestMessages], Mapping[str, object]]
_ResponsesResult: TypeAlias = ResponsesAPIResponse | BaseResponsesAPIStreamingIterator
_SyncResponsesResult: TypeAlias = _ResponsesResult | Coroutine[object, object, _ResponsesResult]
def _is_encrypted_reasoning_item(item: object) -> bool:
mapping: Final = as_str_mapping(item)
return mapping is not None and mapping.get("type") == "reasoning" and bool(mapping.get("encrypted_content"))
def _sends_encrypted_reasoning(responses_kwargs: Mapping[str, object]) -> bool:
return any(map(_is_encrypted_reasoning_item, as_object_list(responses_kwargs.get("input")) or ()))
def _kwarg_text(responses_kwargs: Mapping[str, object], key: str) -> str:
value: Final = responses_kwargs.get(key)
return value if isinstance(value, str) else ""
def _messages_without_rejected_reasoning(
messages: AnthropicRequestMessages,
error: litellm.BadRequestError,
responses_kwargs: Mapping[str, object],
) -> AnthropicRequestMessages | None:
return messages_without_rejected_reasoning(
messages,
error,
sent_encrypted_reasoning=_sends_encrypted_reasoning(responses_kwargs),
provider=_kwarg_text(responses_kwargs, "custom_llm_provider"),
model=_kwarg_text(responses_kwargs, "model"),
)
async def _aresponses_without_rejected_reasoning(
messages: AnthropicRequestMessages, responses_kwargs_for: _ResponsesKwargsFor
) -> tuple[Mapping[str, object], _ResponsesResult]:
responses_kwargs: Final = responses_kwargs_for(messages)
try:
return responses_kwargs, await litellm.aresponses(**responses_kwargs)
except litellm.BadRequestError as error:
replayed_messages: Final = _messages_without_rejected_reasoning(messages, error, responses_kwargs)
if replayed_messages is None:
raise
replay_kwargs: Final = responses_kwargs_for(replayed_messages)
return replay_kwargs, await litellm.aresponses(**replay_kwargs)
def _responses_without_rejected_reasoning(
messages: AnthropicRequestMessages, responses_kwargs_for: _ResponsesKwargsFor
) -> tuple[Mapping[str, object], _SyncResponsesResult]:
responses_kwargs: Final = responses_kwargs_for(messages)
try:
return responses_kwargs, litellm.responses(**responses_kwargs)
except litellm.BadRequestError as error:
replayed_messages: Final = _messages_without_rejected_reasoning(messages, error, responses_kwargs)
if replayed_messages is None:
raise
replay_kwargs: Final = responses_kwargs_for(replayed_messages)
return replay_kwargs, litellm.responses(**replay_kwargs)
class LiteLLMMessagesToResponsesAPIHandler:
"""
Handles Anthropic /v1/messages requests for OpenAI / Azure models by
@ -422,28 +488,32 @@ class LiteLLMMessagesToResponsesAPIHandler:
polyfill_result: Final = await _polyfilled(
model=model, messages=messages, tools=tools, system=system, edits=polyfilled_edits, kwargs=kwargs
)
responses_kwargs: Final = _build_responses_kwargs(
max_tokens=max_tokens,
messages=polyfill_result.messages if polyfill_result is not None else messages,
model=model,
context_management=native_context_management,
metadata=metadata,
output_config=output_config,
stop_sequences=stop_sequences,
stream=stream,
system=system,
temperature=temperature,
thinking=thinking,
tool_choice=tool_choice,
tools=tools,
top_k=top_k,
top_p=top_p,
output_format=output_format,
extra_kwargs=kwargs,
)
def responses_kwargs_for(sent_messages: AnthropicRequestMessages) -> Mapping[str, object]:
return _build_responses_kwargs(
max_tokens=max_tokens,
messages=sent_messages,
model=model,
context_management=native_context_management,
metadata=metadata,
output_config=output_config,
stop_sequences=stop_sequences,
stream=stream,
system=system,
temperature=temperature,
thinking=thinking,
tool_choice=tool_choice,
tools=tools,
top_k=top_k,
top_p=top_p,
output_format=output_format,
extra_kwargs=kwargs,
)
safeguards_evaluator: Final = request_safeguards_evaluator(kwargs, messages)
result: Final = await litellm.aresponses(**responses_kwargs)
responses_kwargs, result = await _aresponses_without_rejected_reasoning(
polyfill_result.messages if polyfill_result is not None else messages, responses_kwargs_for
)
if stream:
wrapper: Final = AnthropicResponsesStreamWrapper(
@ -526,27 +596,31 @@ class LiteLLMMessagesToResponsesAPIHandler:
else None
)
polyfill_result: Final = ran if isinstance(ran, PolyfillResult) else None
responses_kwargs: Final = _build_responses_kwargs(
max_tokens=max_tokens,
messages=polyfill_result.messages if polyfill_result is not None else messages,
model=model,
context_management=native_context_management,
metadata=metadata,
output_config=output_config,
stop_sequences=stop_sequences,
stream=stream,
system=system,
temperature=temperature,
thinking=thinking,
tool_choice=tool_choice,
tools=tools,
top_k=top_k,
top_p=top_p,
output_format=output_format,
extra_kwargs=kwargs,
)
result: Final = litellm.responses(**responses_kwargs)
def responses_kwargs_for(sent_messages: AnthropicRequestMessages) -> Mapping[str, object]:
return _build_responses_kwargs(
max_tokens=max_tokens,
messages=sent_messages,
model=model,
context_management=native_context_management,
metadata=metadata,
output_config=output_config,
stop_sequences=stop_sequences,
stream=stream,
system=system,
temperature=temperature,
thinking=thinking,
tool_choice=tool_choice,
tools=tools,
top_k=top_k,
top_p=top_p,
output_format=output_format,
extra_kwargs=kwargs,
)
responses_kwargs, result = _responses_without_rejected_reasoning(
polyfill_result.messages if polyfill_result is not None else messages, responses_kwargs_for
)
if stream:
wrapper: Final = AnthropicResponsesStreamWrapper(

2
litellm/responses/bridge_check.py
View file

@ -304,7 +304,6 @@ def chat_completion_ends_in_responses_api(
tools: Sequence[Mapping[str, object]] | None = None,
reasoning_effort: str | Mapping[str, object] | None = None,
api_base: str | None = None,
web_search_options: OpenAIWebSearchOptions | None = None,
) -> bool:
"""Whether chat completion for this model is served by a Responses API.
@ -323,7 +322,6 @@ def chat_completion_ends_in_responses_api(
model_info, _resolved = responses_api_bridge_check(
model=resolved_model,
custom_llm_provider=provider,
web_search_options=web_search_options,
tools=tools,
reasoning_effort=reasoning_effort,
api_base=api_base,

39
tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py
View file

@ -5782,6 +5782,9 @@ _RETRY_REJECTION: Final = httpx.Response(
_UNRELATED_REJECTION: Final = httpx.Response(
400, json={"error": {"message": "max_output_tokens is too small", "type": "invalid_request_error"}}
)
_UNRELATED_UNVERIFIED_REJECTION: Final = httpx.Response(
400, json={"error": {"message": "The organization could not be verified.", "type": "invalid_request_error"}}
)
_RETRY_REPLY: Final = {
"id": "resp_retry",
"object": "response",
@ -5856,6 +5859,7 @@ async def test_async_messages_retry_once_without_reasoning_the_backend_cannot_de
assert [block["text"] for block in response["content"] if block["type"] == "text"] == ["Oslo"]
assert retry_warnings == (
"Backend rejected replayed encrypted reasoning, retrying once without it. "
"Enable the encrypted_content_affinity pre-call check to drop such blocks before the call. "
"provider=azure model=azure/gpt-5.4-nano status=400 code=invalid_encrypted_content",
)
assert all(
@ -5897,6 +5901,9 @@ def test_sync_messages_retry_once_without_reasoning_the_backend_cannot_decrypt(
id="no_encrypted_reasoning_to_drop",
),
pytest.param(_history_with_foreign_reasoning(), _UNRELATED_REJECTION, id="unrelated_rejection"),
pytest.param(
_history_with_foreign_reasoning(), _UNRELATED_UNVERIFIED_REJECTION, id="unrelated_unverified_rejection"
),
],
)
async def test_messages_do_not_retry_other_rejections(
@ -5970,38 +5977,6 @@ async def test_openai_thinking_on_the_chat_adapter_replays_the_reasoning_its_res
assert [item["encrypted_content"] for item in _sent_reasoning_items(route.calls[0].request)] == ["gAAA_same_org"]
@pytest.mark.asyncio
@pytest.mark.usefixtures("_chat_bridge_httpx")
async def test_xai_web_search_turn_replays_the_reasoning_its_responses_route_minted(
respx_mock: respx.MockRouter,
) -> None:
route: Final = respx_mock.post(url__regex=r".*xai-search\.example/v1/responses").mock(
return_value=httpx.Response(200, json=_RETRY_REPLY)
)
await litellm.anthropic.messages.acreate(
model="xai/grok-4.7",
api_base="https://xai-search.example/v1",
api_key="fake-key",
max_tokens=2048,
tools=[{"type": "web_search_20250305", "name": "web_search"}],
messages=[
{"role": "user", "content": "Which city is colder today?"},
{
"role": "assistant",
"content": [
{"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAA_same_org")},
{"type": "text", "text": "Oslo."},
],
},
{"role": "user", "content": "And by how much?"},
],
)
assert route.call_count == 1
assert [item["encrypted_content"] for item in _sent_reasoning_items(route.calls[0].request)] == ["gAAA_same_org"]
def _responses_sse(events: Sequence[Mapping[str, object]]) -> bytes:
return b"".join(
f"event: {event['type']}\ndata: {json.dumps({**event, 'sequence_number': number})}\n\n".encode()

117
tests/unit/llms/anthropic/messages/responses_adapters/test_responses_adapters_handler.py
View file

@ -7,6 +7,7 @@ from itertools import chain
from typing import Final, cast
from unittest.mock import AsyncMock, patch
import httpx
import pytest
import respx
@ -17,6 +18,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
TOOL_RESULT_IMAGE_BOUNDARY,
TOOL_RESULT_IMAGE_PLACEHOLDER,
)
from litellm.litellm_core_utils.prompt_templates.reasoning_utils import encrypted_reasoning_signature
from litellm.llms.anthropic.messages.responses_adapters.handler import (
LiteLLMMessagesToResponsesAPIHandler,
_build_responses_kwargs,
@ -1135,3 +1137,118 @@ async def test_the_sync_path_reports_its_edits_on_the_streamed_message_delta(
]
message_delta: Final = next(event for event in events if event["type"] == "message_delta")
assert message_delta["context_management"]["applied_edits"][0]["cleared_tool_uses"] == 2
REJECTED_REPLAYED_REASONING: Final = {
"error": {
"message": "The encrypted content gAAA_foreign could not be verified.",
"type": "invalid_request_error",
"param": None,
"code": "invalid_encrypted_content",
}
}
UNVERIFIED_ORGANIZATION: Final = {
"error": {"message": "The organization could not be verified.", "type": "invalid_request_error"}
}
def _history(*, replayed_reasoning: bool) -> list[dict[str, object]]:
reasoning: Final = (
({"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAA_foreign")},)
if replayed_reasoning
else ()
)
return [
{"role": "user", "content": "List the logs."},
{"role": "assistant", "content": [*reasoning, {"type": "tool_use", "id": "call_1", "name": "Bash", "input": {}}]},
{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "call_1", "content": "app.log"}]},
]
def _replayed_reasoning_counts(upstream: respx.Route) -> list[int]:
return [
sum(item.get("type") == "reasoning" for item in json.loads(call.request.content)["input"])
for call in upstream.calls
]
def _upstream(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, first: httpx.Response, stream: bool):
monkeypatch.setenv("OPENAI_API_KEY", "sk-replay-test")
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
litellm.in_memory_llm_clients_cache.flush_cache()
served: Final = (
httpx.Response(200, headers={"Content-Type": "text/event-stream"}, content=RESPONSES_SSE_BODY)
if stream
else httpx.Response(200, json=TEXT_RESPONSE)
)
return respx_mock.post("https://api.openai.com/v1/responses").mock(side_effect=[first, served])
@pytest.mark.asyncio
@pytest.mark.parametrize("stream", [False, True])
async def test_openai_turn_is_sent_again_once_without_replayed_reasoning_the_backend_cannot_decrypt(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, stream: bool
) -> None:
upstream: Final = _upstream(respx_mock, monkeypatch, httpx.Response(400, json=REJECTED_REPLAYED_REASONING), stream)
response: Final = await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler(
max_tokens=64,
messages=_history(replayed_reasoning=True),
model="openai/gpt-5.6-luna",
tools=[dict(BASH_TOOL)],
stream=stream,
custom_llm_provider="openai",
)
events: Final = (
[json.loads(chunk.decode().split("data: ", 1)[1]) async for chunk in cast(AsyncIterator[bytes], response)]
if stream
else [cast(Mapping[str, object], response)]
)
assert _replayed_reasoning_counts(upstream) == [1, 0]
assert events[-1]["type"] in ("message", "message_stop")
def test_sync_openai_turn_is_sent_again_once_without_replayed_reasoning_the_backend_cannot_decrypt(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
upstream: Final = _upstream(respx_mock, monkeypatch, httpx.Response(400, json=REJECTED_REPLAYED_REASONING), False)
response: Final = LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler(
max_tokens=64,
messages=_history(replayed_reasoning=True),
model="openai/gpt-5.6-luna",
tools=[dict(BASH_TOOL)],
custom_llm_provider="openai",
)
assert _replayed_reasoning_counts(upstream) == [1, 0]
assert cast(Mapping[str, object], response)["content"] == [{"type": "text", "text": "done"}]
@pytest.mark.asyncio
@pytest.mark.parametrize(
"replayed_reasoning, rejection",
[
pytest.param(False, REJECTED_REPLAYED_REASONING, id="no_replayed_reasoning"),
pytest.param(True, UNVERIFIED_ORGANIZATION, id="unrelated_rejection"),
],
)
async def test_openai_turn_is_not_sent_again_for_other_rejections(
respx_mock: respx.MockRouter,
monkeypatch: pytest.MonkeyPatch,
replayed_reasoning: bool,
rejection: Mapping[str, object],
) -> None:
upstream: Final = _upstream(respx_mock, monkeypatch, httpx.Response(400, json=rejection), False)
with pytest.raises(litellm.BadRequestError):
await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler(
max_tokens=64,
messages=_history(replayed_reasoning=replayed_reasoning),
model="openai/gpt-5.6-luna",
tools=[dict(BASH_TOOL)],
custom_llm_provider="openai",
)
assert upstream.call_count == 1