scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-14 09:03:16 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 214

fix(anthropic): send openai/ Messages turns again once without encrypted reasoning a backend rejects

Browse source 
The openai/ Responses path on /v1/messages had no retry, so a chatgpt/ block left in history answered 400 on every later openai/ turn. That path now sends the turn once more without encrypted reasoning, the same way the chat bridge does. The retry matches only invalid_encrypted_content or a "could not be verified" message naming the content, so the exception mapper's own hint no longer triggers it. The xAI web search keep path is backed out, since xAI sends no encrypted reasoning over the bridge.

Backport of 15e6017012 from #49076 to lts/1.108.x, ported by hand onto the pass_through modules.

Co-authored-by: DDT <141296842+Hugo-DDT@users.noreply.github.com>
This commit is contained in:
mateo-berri 2026-11-07 01:53:19 -08:00
parent f86222be48
commit d6625245b4
6 changed files with 294 additions and 95 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

51
litellm/llms/anthropic/pass_through/adapters/handler.py
View file

@ -1,3 +1,4 @@
import re
from collections.abc import AsyncIterator, Callable, Coroutine, Iterator, Mapping, Sequence
from dataclasses import dataclass
from functools import partial
@ -77,6 +78,7 @@ class _CompletionKwargs(TypedDict, total=False, extra_items=object):
_ANTHROPIC_MESSAGES: Final = TypeAdapter(list[dict[str, object]])
_UNVERIFIED_ENCRYPTED_CONTENT: Final = re.compile(r"encrypted content \S+ could not be verified", re.IGNORECASE)
_PrepareReplay: TypeAlias = "Callable[[_AnthropicMessages], _CompletionKwargs]"
@ -125,24 +127,49 @@ def _sends_encrypted_reasoning(completion_kwargs: _CompletionKwargs) -> bool:
return any(_message_carries_encrypted_reasoning(message) for message in sent)
def _rejects_encrypted_reasoning(error: litellm.BadRequestError) -> bool:
error_message: Final = str(error)
return (
"invalid_encrypted_content" in error_message or _UNVERIFIED_ENCRYPTED_CONTENT.search(error_message) is not None
)
def messages_without_rejected_reasoning(
messages: Sequence[Mapping[str, object]],
error: litellm.BadRequestError,
*,
sent_encrypted_reasoning: bool,
provider: str,
model: str,
) -> "_AnthropicMessages | None":
if not sent_encrypted_reasoning or not _rejects_encrypted_reasoning(error):
return None
verbose_logger.warning(
"Backend rejected replayed encrypted reasoning, retrying once without it. "
"Enable the encrypted_content_affinity pre-call check to drop such blocks before the call. "
"provider=%s model=%s status=%s code=%s",
provider,
model,
error.status_code,
_logged_rejection_code(error),
)
return _ANTHROPIC_MESSAGES.validate_python(
strip_encrypted_reasoning_blocks_from_anthropic_messages([dict(message) for message in messages])
)
def _messages_without_rejected_reasoning(
messages: _AnthropicMessages,
error: litellm.BadRequestError,
completion_kwargs: _CompletionKwargs,
) -> "_AnthropicMessages | None":
error_message: Final = str(error)
rejected: Final = "invalid_encrypted_content" in error_message or "could not be verified" in error_message
if not rejected or not _sends_encrypted_reasoning(completion_kwargs):
return None
verbose_logger.warning(
"Backend rejected replayed encrypted reasoning, retrying once without it. "
"provider=%s model=%s status=%s code=%s",
_completion_text(completion_kwargs, "custom_llm_provider"),
_completion_text(completion_kwargs, "model"),
error.status_code,
_logged_rejection_code(error),
return messages_without_rejected_reasoning(
messages,
error,
sent_encrypted_reasoning=_sends_encrypted_reasoning(completion_kwargs),
provider=_completion_text(completion_kwargs, "custom_llm_provider"),
model=_completion_text(completion_kwargs, "model"),
)
return _ANTHROPIC_MESSAGES.validate_python(strip_encrypted_reasoning_blocks_from_anthropic_messages(messages))
async def _acompletion_without_rejected_reasoning(

4
litellm/llms/anthropic/pass_through/adapters/transformation.py
View file

@ -201,7 +201,6 @@ from litellm.types.llms.openai import (
ChatCompletionToolParamFunctionChunk,
ChatCompletionToolReferenceObject,
ChatCompletionUserMessage,
OpenAIWebSearchOptions,
ToolMessageContentPart,
)
from litellm.types.utils import Choices, ModelResponse, StreamingChoices, Usage
@ -229,7 +228,6 @@ class ChatRouteInputs:
reasoning_effort: str | Mapping[str, object] | None = None
api_base: str | None = None
thinking: Mapping[str, object] | None = None
web_search: bool = False
_NO_ROUTE_INPUTS: Final = ChatRouteInputs()
@ -274,7 +272,6 @@ def _keeps_encrypted_reasoning_for_responses(
tools=route.tools,
reasoning_effort=route.reasoning_effort,
api_base=route.api_base,
web_search_options=OpenAIWebSearchOptions() if route.web_search else None,
)
@ -1352,7 +1349,6 @@ class LiteLLMAnthropicMessagesAdapter:
reasoning_effort=new_kwargs.get("reasoning_effort"),
api_base=api_base,
thinking=anthropic_message_request.get("thinking"),
web_search="web_search_options" in new_kwargs,
),
)
## ADD SYSTEM MESSAGE TO MESSAGES

152
litellm/llms/anthropic/pass_through/responses_adapters/handler.py
View file

@ -14,9 +14,10 @@ from litellm.integrations.anthropic_cache_control_hook import (
AnthropicCacheControlHook,
supports_openai_prompt_cache_breakpoint,
)
from litellm.litellm_core_utils.core_helpers import normalize_drop_params
from litellm.litellm_core_utils.core_helpers import as_str_mapping, iter_items, normalize_drop_params
from litellm.litellm_core_utils.prompt_templates.common_utils import cache_control_as_prompt_cache_breakpoint
from litellm.litellm_core_utils.tool_fields import drops_mcp_servers, tools_without_mcp_servers
from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator
from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS
from litellm.types.llms.anthropic import (
AllAnthropicMessageValues,
@ -31,7 +32,7 @@ from litellm.types.llms.anthropic_messages.anthropic_response import (
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.utils import ProviderConfigManager
from ..adapters.handler import request_safeguards_evaluator
from ..adapters.handler import messages_without_rejected_reasoning, request_safeguards_evaluator
from ..safeguards import with_safeguard_results
from ..utils import litellm_logging_obj_from_kwargs, local_model_name
from .streaming_iterator import AnthropicResponsesStreamWrapper
@ -254,6 +255,69 @@ def _build_responses_kwargs(
return responses_kwargs
_ResponsesKwargsFor: TypeAlias = Callable[[AnthropicRequestMessages], Mapping[str, object]]
_ResponsesResult: TypeAlias = ResponsesAPIResponse | BaseResponsesAPIStreamingIterator
_SyncResponsesResult: TypeAlias = _ResponsesResult | Coroutine[object, object, _ResponsesResult]
def _is_encrypted_reasoning_item(item: object) -> bool:
mapping: Final = as_str_mapping(item)
return mapping is not None and mapping.get("type") == "reasoning" and bool(mapping.get("encrypted_content"))
def _sends_encrypted_reasoning(responses_kwargs: Mapping[str, object]) -> bool:
input_items: Final = responses_kwargs.get("input")
sent: Final = iter_items(input_items) if input_items is not None else ()
return any(map(_is_encrypted_reasoning_item, sent))
def _kwarg_text(responses_kwargs: Mapping[str, object], key: str) -> str:
value: Final = responses_kwargs.get(key)
return value if isinstance(value, str) else ""
def _messages_without_rejected_reasoning(
messages: AnthropicRequestMessages,
error: litellm.BadRequestError,
responses_kwargs: Mapping[str, object],
) -> AnthropicRequestMessages | None:
return messages_without_rejected_reasoning(
messages,
error,
sent_encrypted_reasoning=_sends_encrypted_reasoning(responses_kwargs),
provider=_kwarg_text(responses_kwargs, "custom_llm_provider"),
model=_kwarg_text(responses_kwargs, "model"),
)
async def _aresponses_without_rejected_reasoning(
messages: AnthropicRequestMessages, responses_kwargs_for: _ResponsesKwargsFor
) -> tuple[Mapping[str, object], _ResponsesResult]:
responses_kwargs: Final = responses_kwargs_for(messages)
try:
return responses_kwargs, await litellm.aresponses(**responses_kwargs)
except litellm.BadRequestError as error:
replayed_messages: Final = _messages_without_rejected_reasoning(messages, error, responses_kwargs)
if replayed_messages is None:
raise
replay_kwargs: Final = responses_kwargs_for(replayed_messages)
return replay_kwargs, await litellm.aresponses(**replay_kwargs)
def _responses_without_rejected_reasoning(
messages: AnthropicRequestMessages, responses_kwargs_for: _ResponsesKwargsFor
) -> tuple[Mapping[str, object], _SyncResponsesResult]:
responses_kwargs: Final = responses_kwargs_for(messages)
try:
return responses_kwargs, litellm.responses(**responses_kwargs)
except litellm.BadRequestError as error:
replayed_messages: Final = _messages_without_rejected_reasoning(messages, error, responses_kwargs)
if replayed_messages is None:
raise
replay_kwargs: Final = responses_kwargs_for(replayed_messages)
return replay_kwargs, litellm.responses(**replay_kwargs)
class LiteLLMMessagesToResponsesAPIHandler:
"""
Handles Anthropic /v1/messages requests for OpenAI / Azure models by
@ -281,28 +345,30 @@ class LiteLLMMessagesToResponsesAPIHandler:
output_format: AnthropicOutputSchema | None = None,
**kwargs: object,
) -> AnthropicMessagesResponse | AsyncIterator[bytes]:
responses_kwargs: Final = _build_responses_kwargs(
max_tokens=max_tokens,
messages=messages,
model=model,
context_management=context_management,
metadata=metadata,
output_config=output_config,
stop_sequences=stop_sequences,
stream=stream,
system=system,
temperature=temperature,
thinking=thinking,
tool_choice=tool_choice,
tools=tools,
top_k=top_k,
top_p=top_p,
output_format=output_format,
extra_kwargs=kwargs,
)
def responses_kwargs_for(sent_messages: AnthropicRequestMessages) -> Mapping[str, object]:
return _build_responses_kwargs(
max_tokens=max_tokens,
messages=sent_messages,
model=model,
context_management=context_management,
metadata=metadata,
output_config=output_config,
stop_sequences=stop_sequences,
stream=stream,
system=system,
temperature=temperature,
thinking=thinking,
tool_choice=tool_choice,
tools=tools,
top_k=top_k,
top_p=top_p,
output_format=output_format,
extra_kwargs=kwargs,
)
safeguards_evaluator: Final = request_safeguards_evaluator(kwargs, messages)
result: Final = await litellm.aresponses(**responses_kwargs)
responses_kwargs, result = await _aresponses_without_rejected_reasoning(messages, responses_kwargs_for)
if stream:
wrapper: Final = AnthropicResponsesStreamWrapper(
@ -366,28 +432,28 @@ class LiteLLMMessagesToResponsesAPIHandler:
**kwargs,
)
# Sync path
responses_kwargs: Final = _build_responses_kwargs(
max_tokens=max_tokens,
messages=messages,
model=model,
context_management=context_management,
metadata=metadata,
output_config=output_config,
stop_sequences=stop_sequences,
stream=stream,
system=system,
temperature=temperature,
thinking=thinking,
tool_choice=tool_choice,
tools=tools,
top_k=top_k,
top_p=top_p,
output_format=output_format,
extra_kwargs=kwargs,
)
def responses_kwargs_for(sent_messages: AnthropicRequestMessages) -> Mapping[str, object]:
return _build_responses_kwargs(
max_tokens=max_tokens,
messages=sent_messages,
model=model,
context_management=context_management,
metadata=metadata,
output_config=output_config,
stop_sequences=stop_sequences,
stream=stream,
system=system,
temperature=temperature,
thinking=thinking,
tool_choice=tool_choice,
tools=tools,
top_k=top_k,
top_p=top_p,
output_format=output_format,
extra_kwargs=kwargs,
)
result: Final = litellm.responses(**responses_kwargs)
responses_kwargs, result = _responses_without_rejected_reasoning(messages, responses_kwargs_for)
if stream:
wrapper: Final = AnthropicResponsesStreamWrapper(

2
litellm/responses/bridge_check.py
View file

@ -198,7 +198,6 @@ def chat_completion_ends_in_responses_api(
tools: Sequence[Mapping[str, object]] | None = None,
reasoning_effort: str | Mapping[str, object] | None = None,
api_base: str | None = None,
web_search_options: OpenAIWebSearchOptions | None = None,
) -> bool:
"""Whether chat completion for this model is served by a Responses API.
@ -218,7 +217,6 @@ def chat_completion_ends_in_responses_api(
model_info, _resolved = responses_api_bridge_check(
model=resolved_model,
custom_llm_provider=provider,
web_search_options=web_search_options,
tools=tools,
reasoning_effort=reasoning_effort,
api_base=api_base,

39
tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py
View file

@ -5449,6 +5449,9 @@ _RETRY_REJECTION: Final = httpx.Response(
_UNRELATED_REJECTION: Final = httpx.Response(
400, json={"error": {"message": "max_output_tokens is too small", "type": "invalid_request_error"}}
)
_UNRELATED_UNVERIFIED_REJECTION: Final = httpx.Response(
400, json={"error": {"message": "The organization could not be verified.", "type": "invalid_request_error"}}
)
_RETRY_REPLY: Final = {
"id": "resp_retry",
"object": "response",
@ -5523,6 +5526,7 @@ async def test_async_messages_retry_once_without_reasoning_the_backend_cannot_de
assert [block["text"] for block in response["content"] if block["type"] == "text"] == ["Oslo"]
assert retry_warnings == (
"Backend rejected replayed encrypted reasoning, retrying once without it. "
"Enable the encrypted_content_affinity pre-call check to drop such blocks before the call. "
"provider=azure model=azure/gpt-5.4-nano status=400 code=invalid_encrypted_content",
)
assert all(
@ -5564,6 +5568,9 @@ def test_sync_messages_retry_once_without_reasoning_the_backend_cannot_decrypt(
id="no_encrypted_reasoning_to_drop",
),
pytest.param(_history_with_foreign_reasoning(), _UNRELATED_REJECTION, id="unrelated_rejection"),
pytest.param(
_history_with_foreign_reasoning(), _UNRELATED_UNVERIFIED_REJECTION, id="unrelated_unverified_rejection"
),
],
)
async def test_messages_do_not_retry_other_rejections(
@ -5609,38 +5616,6 @@ async def test_messages_do_not_retry_when_the_translated_request_already_dropped
assert route.call_count == 1
@pytest.mark.asyncio
@pytest.mark.usefixtures("_chat_bridge_httpx")
async def test_xai_web_search_turn_replays_the_reasoning_its_responses_route_minted(
respx_mock: respx.MockRouter,
) -> None:
route: Final = respx_mock.post(url__regex=r".*xai-search\.example/v1/responses").mock(
return_value=httpx.Response(200, json=_RETRY_REPLY)
)
await litellm.anthropic.messages.acreate(
model="xai/grok-4.7",
api_base="https://xai-search.example/v1",
api_key="fake-key",
max_tokens=2048,
tools=[{"type": "web_search_20250305", "name": "web_search"}],
messages=[
{"role": "user", "content": "Which city is colder today?"},
{
"role": "assistant",
"content": [
{"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAA_same_org")},
{"type": "text", "text": "Oslo."},
],
},
{"role": "user", "content": "And by how much?"},
],
)
assert route.call_count == 1
assert [item["encrypted_content"] for item in _sent_reasoning_items(route.calls[0].request)] == ["gAAA_same_org"]
@pytest.mark.asyncio
@pytest.mark.usefixtures("_chat_bridge_httpx")
async def test_openai_thinking_on_the_chat_adapter_keeps_reasoning_its_responses_route_replays(

141
tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py
View file

@ -2,10 +2,11 @@ import datetime
import json
import os
import sys
from collections.abc import Mapping
from typing import Final
from collections.abc import AsyncIterator, Mapping
from typing import Final, cast
from unittest.mock import AsyncMock, patch
import httpx
import pytest
import respx
@ -15,6 +16,7 @@ import litellm
from litellm.litellm_core_utils.prompt_templates.common_utils import (
TOOL_RESULT_IMAGE_BOUNDARY,
TOOL_RESULT_IMAGE_PLACEHOLDER,
encrypted_reasoning_signature,
)
from litellm.llms.anthropic.pass_through.responses_adapters.handler import (
LiteLLMMessagesToResponsesAPIHandler,
@ -684,3 +686,138 @@ def test_drop_params_drops_the_mcp_toolset_tools_of_dropped_mcp_servers(monkeypa
)
assert [tool["name"] for tool in responses_kwargs["tools"]] == ["get_weather"]
TEXT_RESPONSE: Final = {
"id": "resp_replayed",
"object": "response",
"created_at": 1,
"status": "completed",
"model": "gpt-5.6-luna",
"output": [
{
"type": "message",
"id": "msg_1",
"role": "assistant",
"status": "completed",
"content": [{"type": "output_text", "text": "done", "annotations": []}],
}
],
"parallel_tool_calls": True,
"tool_choice": "auto",
"tools": [],
"usage": {"input_tokens": 3, "output_tokens": 4, "total_tokens": 7},
}
REJECTED_REPLAYED_REASONING: Final = {
"error": {
"message": "The encrypted content gAAA_foreign could not be verified.",
"type": "invalid_request_error",
"param": None,
"code": "invalid_encrypted_content",
}
}
UNVERIFIED_ORGANIZATION: Final = {
"error": {"message": "The organization could not be verified.", "type": "invalid_request_error"}
}
def _history(*, replayed_reasoning: bool) -> list[dict[str, object]]:
reasoning: Final = (
({"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAA_foreign")},)
if replayed_reasoning
else ()
)
return [
{"role": "user", "content": "List the logs."},
{"role": "assistant", "content": [*reasoning, {"type": "tool_use", "id": "call_1", "name": "Bash", "input": {}}]},
{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "call_1", "content": "app.log"}]},
]
def _replayed_reasoning_counts(upstream: respx.Route) -> list[int]:
return [
sum(item.get("type") == "reasoning" for item in json.loads(call.request.content)["input"])
for call in upstream.calls
]
def _upstream(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, first: httpx.Response, stream: bool):
monkeypatch.setenv("OPENAI_API_KEY", "sk-replay-test")
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
litellm.in_memory_llm_clients_cache.flush_cache()
served: Final = (
httpx.Response(200, headers={"Content-Type": "text/event-stream"}, content=RESPONSES_SSE_BODY)
if stream
else httpx.Response(200, json=TEXT_RESPONSE)
)
return respx_mock.post("https://api.openai.com/v1/responses").mock(side_effect=[first, served])
@pytest.mark.asyncio
@pytest.mark.parametrize("stream", [False, True])
async def test_openai_turn_is_sent_again_once_without_replayed_reasoning_the_backend_cannot_decrypt(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, stream: bool
) -> None:
upstream: Final = _upstream(respx_mock, monkeypatch, httpx.Response(400, json=REJECTED_REPLAYED_REASONING), stream)
response: Final = await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler(
max_tokens=64,
messages=_history(replayed_reasoning=True),
model="openai/gpt-5.6-luna",
tools=[dict(BASH_TOOL)],
stream=stream,
custom_llm_provider="openai",
)
events: Final = (
[json.loads(chunk.decode().split("data: ", 1)[1]) async for chunk in cast(AsyncIterator[bytes], response)]
if stream
else [cast(Mapping[str, object], response)]
)
assert _replayed_reasoning_counts(upstream) == [1, 0]
assert events[-1]["type"] in ("message", "message_stop")
def test_sync_openai_turn_is_sent_again_once_without_replayed_reasoning_the_backend_cannot_decrypt(
respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch
) -> None:
upstream: Final = _upstream(respx_mock, monkeypatch, httpx.Response(400, json=REJECTED_REPLAYED_REASONING), False)
response: Final = LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler(
max_tokens=64,
messages=_history(replayed_reasoning=True),
model="openai/gpt-5.6-luna",
tools=[dict(BASH_TOOL)],
custom_llm_provider="openai",
)
assert _replayed_reasoning_counts(upstream) == [1, 0]
assert cast(Mapping[str, object], response)["content"] == [{"type": "text", "text": "done"}]
@pytest.mark.asyncio
@pytest.mark.parametrize(
"replayed_reasoning, rejection",
[
pytest.param(False, REJECTED_REPLAYED_REASONING, id="no_replayed_reasoning"),
pytest.param(True, UNVERIFIED_ORGANIZATION, id="unrelated_rejection"),
],
)
async def test_openai_turn_is_not_sent_again_for_other_rejections(
respx_mock: respx.MockRouter,
monkeypatch: pytest.MonkeyPatch,
replayed_reasoning: bool,
rejection: Mapping[str, object],
) -> None:
upstream: Final = _upstream(respx_mock, monkeypatch, httpx.Response(400, json=rejection), False)
with pytest.raises(litellm.BadRequestError):
await LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler(
max_tokens=64,
messages=_history(replayed_reasoning=replayed_reasoning),
model="openai/gpt-5.6-luna",
tools=[dict(BASH_TOOL)],
custom_llm_provider="openai",
)
assert upstream.call_count == 1