Merge pull request #40451 from BerriAI/litellm_responses_bridge_replay_encrypted_reasoning

fix(anthropic): replay OpenAI encrypted reasoning byte for byte through the /v1/messages bridge
This commit is contained in:
Mateo Wang 2026-09-09 19:45:11 -07:00 committed by GitHub
commit 8ec2f00955
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
19 changed files with 1004 additions and 83 deletions

View file

@ -25,7 +25,7 @@ import litellm
from litellm import ModelResponse
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.prompt_templates.common_utils import (
responses_reasoning_item_from_thinking_blocks,
responses_reasoning_items_from_thinking_blocks,
)
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
from litellm.llms.base_llm.bridges.completion_transformation import (
@ -129,8 +129,8 @@ def _reasoning_input_items(msg: "AllMessageValues") -> list[dict[str, object]]:
return stored
raw_blocks: Final = msg.get("thinking_blocks") or ()
blocks: Final = cast("Iterable[ChatCompletionThinkingBlock]", raw_blocks) # cast-ok: untyped client json
from_thinking: Final = responses_reasoning_item_from_thinking_blocks(blocks)
return [] if from_thinking is None else [dict(from_thinking)] # mutable-ok: API message payload
replayed: Final = responses_reasoning_items_from_thinking_blocks(blocks)
return [dict(item) for item in replayed] # mutable-ok: API message payload
def _build_reasoning_item(

View file

@ -6,7 +6,7 @@ import io
import json
import mimetypes
import re
from collections.abc import Iterable, Mapping, Sequence
from collections.abc import Iterable, Iterator, Mapping, Sequence
from itertools import groupby
from os import PathLike
from pathlib import Path
@ -1823,14 +1823,11 @@ def _extract_reasoning_content(message: dict) -> tuple[str | None, str | None]:
return None, message_content
def _readable_thinking_text(
block: ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock,
) -> str:
def _readable_thinking_text(block: Mapping[str, object]) -> str:
"""The text a chat model can read back, empty for redacted blocks and malformed ones."""
if block.get("type") != "thinking":
return ""
thinking: Final = cast(ChatCompletionThinkingBlock, block).get("thinking") # cast-ok: narrowed by the type tag
return str(thinking or "")
return str(block.get("thinking") or "")
def reasoning_content_from_thinking_blocks(
@ -1843,24 +1840,125 @@ def reasoning_content_from_thinking_blocks(
return "\n".join(text for block in thinking_blocks if (text := _readable_thinking_text(block)))
def responses_reasoning_item_from_thinking_blocks(
thinking_blocks: Iterable[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock],
) -> ChatCompletionReasoningItem | None:
"""Build a Responses API `reasoning` input item from Anthropic thinking blocks.
ENCRYPTED_REASONING_SIGNATURE_PREFIX: Final = "litellm_encrypted_reasoning:"
The item carries no `id`: the Responses API rejects an empty one and 404s on any id it
did not mint itself, while an item without an id is always accepted.
def encrypted_reasoning_signature(encrypted_content: str) -> str:
"""The opaque value a Responses API reasoning item's `encrypted_content` travels in.
Anthropic clients echo a thinking block's `signature` and a redacted block's `data`
back verbatim, so either field can carry the encrypted reasoning across turns; the
prefix tells the two apart from a signature Anthropic minted.
"""
return f"{ENCRYPTED_REASONING_SIGNATURE_PREFIX}{encrypted_content}"
def _carries_encrypted_reasoning(signature: object) -> bool:
return isinstance(signature, str) and signature.startswith(ENCRYPTED_REASONING_SIGNATURE_PREFIX)
def encrypted_content_from_signature(signature: object) -> str | None:
if not isinstance(signature, str) or not _carries_encrypted_reasoning(signature):
return None
return signature.removeprefix(ENCRYPTED_REASONING_SIGNATURE_PREFIX) or None
def _encrypted_reasoning_field(block: Mapping[str, object]) -> object:
match block.get("type"):
case "thinking":
return block.get("signature")
case "redacted_thinking":
return block.get("data")
case _:
return None
def encrypted_content_of_block(block: Mapping[str, object]) -> str | None:
return encrypted_content_from_signature(_encrypted_reasoning_field(block))
def is_encrypted_reasoning_block(block: object) -> bool:
"""A thinking or redacted_thinking block carrying Responses API encrypted reasoning.
Only the Responses API that minted the content can read it back, so an Anthropic
backend has to drop such a block rather than fail signature verification on it.
"""
if not isinstance(block, Mapping):
return False
mapping: Final = cast(Mapping[str, object], block) # cast-ok: narrowed by isinstance
return _carries_encrypted_reasoning(_encrypted_reasoning_field(mapping))
def strip_encrypted_reasoning_from_messages(messages: object) -> None:
"""Drop the bridge-tagged reasoning blocks a routed deployment cannot decrypt from
Anthropic-shaped history.
The whole block goes, the way #40280 drops undecryptable Responses ``input`` items: a
provider that did not mint the block rejects it signed (a foreign signature) and unsigned
(a missing signature) alike, so keeping its text as an unsigned thinking block only moves
the 400 from the router to the provider.
Mutates the content lists in place: the router's fallback snapshot shares these
message objects, so a rebound list would replay the stripped blocks on the fallback hop.
"""
if not isinstance(messages, list):
return
for content in _anthropic_content_lists(cast(list[object], messages)): # cast-ok: untyped client json
_strip_encrypted_reasoning_from_blocks(content)
def _anthropic_content_lists(messages: Sequence[object]) -> Iterator[object]:
return (
cast(list[object], content) # cast-ok: narrowed by isinstance
for message in messages
if isinstance(message, Mapping)
for content in (cast(Mapping[str, object], message).get("content"),) # cast-ok: narrowed by isinstance
if isinstance(content, list)
)
def _strip_encrypted_reasoning_from_blocks(content: object) -> None:
blocks: Final = cast(list[object], content) # cast-ok: narrowed by the caller's isinstance
kept: Final = tuple(block for block in blocks if not is_encrypted_reasoning_block(block))
blocks[:] = kept # rebind-ok: shared with fallback snapshot
def _reasoning_replay_group_key(indexed_block: tuple[int, Mapping[str, object]]) -> str:
index, block = indexed_block
return f"encrypted:{index}" if is_encrypted_reasoning_block(block) else "summary"
def _reasoning_item_from_block_group(group: tuple[Mapping[str, object], ...]) -> ChatCompletionReasoningItem | None:
summary: Final[list[ChatCompletionReasoningSummaryTextBlock]] = [ # mutable-ok: API message payload
ChatCompletionReasoningSummaryTextBlock(type="summary_text", text=text)
for block in thinking_blocks
for block in group
if (text := _readable_thinking_text(block))
]
encrypted_content: Final = encrypted_content_of_block(group[0])
if encrypted_content is not None:
return ChatCompletionReasoningItem(type="reasoning", summary=summary, encrypted_content=encrypted_content)
if not summary:
return None
return ChatCompletionReasoningItem(type="reasoning", summary=summary)
def responses_reasoning_items_from_thinking_blocks(
thinking_blocks: Iterable[Mapping[str, object]],
) -> tuple[ChatCompletionReasoningItem, ...]:
"""Build Responses API `reasoning` input items from Anthropic thinking blocks.
A block carrying encrypted reasoning replays the item it came from byte for byte;
a run of plain thinking blocks collapses into one summary-only item. No item carries
an `id`: the Responses API 404s on any id it did not mint itself and rejects an empty
one, while an item without an id is always accepted.
"""
return tuple(
item
for _, group in groupby(enumerate(thinking_blocks), key=_reasoning_replay_group_key)
if (item := _reasoning_item_from_block_group(tuple(block for _, block in group))) is not None
)
def _parse_content_for_reasoning(
message_text: str | None,
) -> tuple[str | None, str | None]:

View file

@ -46,6 +46,7 @@ from litellm.types.utils import GenericImageParsingChunk
from .common_utils import (
convert_content_list_to_str,
infer_content_type_from_url_and_content,
is_encrypted_reasoning_block,
is_non_content_values_set,
parse_tool_call_arguments,
)
@ -2299,13 +2300,16 @@ def sanitize_messages_for_tool_calling(
def _is_unsignable_thinking_block(block: object) -> bool:
"""A `thinking` block that Anthropic cannot accept on input.
"""A thinking block that Anthropic cannot accept on input.
Anthropic verifies the thinking signature cryptographically, so a block whose
signature is null, empty, or missing (e.g. from an open-source reasoning model)
is rejected with a 400 and must be dropped rather than blanked or repaired.
`redacted_thinking` blocks carry no signature and are always kept.
is rejected with a 400 and must be dropped rather than blanked or repaired, and
so is a block whose signature or data carries another provider's encrypted
reasoning. A `redacted_thinking` block Anthropic minted is always kept.
"""
if is_encrypted_reasoning_block(block):
return True
if not isinstance(block, dict) or block.get("type") != "thinking":
return False
signature: Final = block.get("signature")

View file

@ -21,6 +21,7 @@ from litellm.constants import (
)
from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_file_ids_from_messages,
is_encrypted_reasoning_block,
)
from litellm.litellm_core_utils.prompt_templates.factory import (
THOUGHT_SIGNATURE_SEPARATOR,
@ -1201,6 +1202,32 @@ def strip_thinking_blocks_from_anthropic_messages(messages: list[Any]) -> list[A
return out
def _without_encrypted_reasoning_blocks(message: dict) -> dict | None: # mutable-ok: Anthropic message payload shape
if not isinstance(message, Mapping):
return message
content: Final = message.get("content")
if not isinstance(content, list):
return message
kept: Final = [b for b in content if not is_encrypted_reasoning_block(b)] # mutable-ok: API message payload
if len(kept) == len(content):
return message
if not kept:
return None
return {**message, "content": kept} # mutable-ok: API message payload
def strip_encrypted_reasoning_blocks_from_anthropic_messages(
messages: Sequence[dict], # mutable-ok: Anthropic message payload shape
) -> list[dict]: # mutable-ok: AnthropicMessagesRequest.messages is typed list[dict]
"""
Drop thinking / redacted_thinking blocks that carry another provider's encrypted
reasoning (a turn the Responses API bridge served) before the request reaches
Anthropic, which cannot verify them. Anthropic's own signed blocks are kept.
"""
stripped: Final = (_without_encrypted_reasoning_blocks(m) for m in messages)
return [m for m in stripped if m is not None] # mutable-ok: API message payload
def strip_thinking_blocks_from_anthropic_messages_request_dict(
data: dict[str, Any],
) -> None:

View file

@ -113,6 +113,7 @@ from litellm.litellm_core_utils.reasoning_effort_utils import (
from litellm.llms.anthropic.common_utils import (
is_empty_unsigned_thinking_block,
normalize_anthropic_tool_use_id,
strip_encrypted_reasoning_blocks_from_anthropic_messages,
)
from litellm.llms.anthropic.experimental_pass_through.context_management import (
PolyfillResult,
@ -417,7 +418,8 @@ class LiteLLMAnthropicMessagesAdapter:
model: str | None = None,
) -> list:
new_messages: Final[list[AllMessageValues]] = []
for m in messages:
replayable_messages: Final = strip_encrypted_reasoning_blocks_from_anthropic_messages(messages)
for m in replayable_messages:
user_message: ChatCompletionUserMessage | None = None
tool_message_list: list[ChatCompletionToolMessage] = []
new_user_content_list: list[ChatCompletionTextObject | ChatCompletionImageObject] = []

View file

@ -25,6 +25,7 @@ from ...common_utils import (
AnthropicModelInfo,
optionally_handle_anthropic_oauth,
strip_advisor_blocks_from_messages,
strip_encrypted_reasoning_blocks_from_anthropic_messages,
)
DEFAULT_ANTHROPIC_API_VERSION: Final = "2023-06-01"
@ -613,7 +614,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
messages = strip_advisor_blocks_from_messages(messages)
anthropic_messages_request: Final[AnthropicMessagesRequest] = AnthropicMessagesRequest(
messages=messages,
messages=strip_encrypted_reasoning_blocks_from_anthropic_messages(messages),
max_tokens=max_tokens,
model=model,
**anthropic_messages_optional_request_params,

View file

@ -19,6 +19,7 @@ from litellm.types.llms.anthropic_messages.anthropic_response import (
AnthropicMessagesResponse,
)
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.utils import ProviderConfigManager
from ..utils import litellm_logging_obj_from_kwargs, local_model_name
from .streaming_iterator import AnthropicResponsesStreamWrapper
@ -34,6 +35,15 @@ def _forwarded_kwargs(extra_kwargs: Mapping[str, object] | None) -> Mapping[str,
return extra_kwargs or {}
def _provider_returns_encrypted_reasoning(model: str, custom_llm_provider: object) -> bool:
provider: Final = (
custom_llm_provider if isinstance(custom_llm_provider, str) else litellm.get_llm_provider(model=model)[1]
)
provider_model: Final = local_model_name(model, provider)
responses_config: Final = ProviderConfigManager.get_provider_responses_api_config(provider, provider_model)
return responses_config is not None and "include" in responses_config.get_supported_openai_params(provider_model)
def _build_responses_kwargs(
*,
max_tokens: int,
@ -85,8 +95,13 @@ def _build_responses_kwargs(
request_data["output_format"] = output_format
anthropic_request: Final = AnthropicMessagesRequest(**request_data)
responses_kwargs: Final = _ADAPTER.translate_request(anthropic_request)
forwarded_kwargs: Final = _forwarded_kwargs(extra_kwargs)
responses_kwargs: Final = _ADAPTER.translate_request(
anthropic_request,
include_encrypted_reasoning=_provider_returns_encrypted_reasoning(
model, forwarded_kwargs.get("custom_llm_provider")
),
)
# Normalize reasoning effort based on model capabilities
# (e.g. "max" → "xhigh"/"high", "minimal" → "low" if unsupported)
@ -111,7 +126,7 @@ def _build_responses_kwargs(
responses_kwargs["stream"] = True
# Forward litellm-specific kwargs (api_key, api_base, logging obj, etc.)
excluded: Final = {"anthropic_messages"}
excluded: Final = frozenset(("anthropic_messages",))
for key, value in forwarded_kwargs.items():
if key == "litellm_logging_obj" and value is not None:
from litellm.litellm_core_utils.litellm_logging import (
@ -132,6 +147,14 @@ def _build_responses_kwargs(
if explicit_prompt_cache_key is not None:
responses_kwargs["prompt_cache_key"] = explicit_prompt_cache_key
deployment_include: Final = forwarded_kwargs.get("include")
bridge_include: Final = responses_kwargs.get("include")
if isinstance(deployment_include, list) and isinstance(bridge_include, list):
responses_kwargs["include"] = [
*bridge_include,
*(item for item in deployment_include if item not in bridge_include),
]
return responses_kwargs

View file

@ -9,13 +9,19 @@ from typing import TYPE_CHECKING, Any, Final
from litellm import verbose_logger
from litellm._uuid import uuid
from litellm.litellm_core_utils.prompt_templates.common_utils import (
encrypted_reasoning_signature,
)
from litellm.llms.anthropic.experimental_pass_through.messages.utils import (
refusal_stop_details,
responses_output_refusal_text,
)
from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage
from .transformation import LiteLLMAnthropicToResponsesAPIAdapter
from .transformation import (
REASONING_SUMMARY_PART_SEPARATOR,
LiteLLMAnthropicToResponsesAPIAdapter,
)
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
@ -29,9 +35,10 @@ class AnthropicResponsesStreamWrapper:
response.created -> message_start
response.output_item.added -> content_block_start (if message/function_call)
response.output_text.delta -> content_block_delta (text_delta)
response.reasoning_summary_part.added -> content_block_delta (thinking_delta separator)
response.reasoning_summary_text.delta -> content_block_delta (thinking_delta)
response.function_call_arguments.delta -> content_block_delta (input_json_delta)
response.output_item.done -> content_block_stop
response.output_item.done -> content_block_delta (signature_delta) + content_block_stop
response.completed -> message_delta + message_stop
"""
@ -94,6 +101,38 @@ class AnthropicResponsesStreamWrapper:
)
return block_idx
@staticmethod
def _field(source: object, name: str) -> object:
return source.get(name) if isinstance(source, dict) else getattr(source, name, None)
def _close_reasoning_item(self, item: object, item_id: str | None) -> None:
block_idx: Final = self._item_id_to_block_index.get(item_id, -1) if item_id else self._current_block_index
encrypted_content: Final = self._field(item, "encrypted_content")
signature: Final = (
encrypted_reasoning_signature(encrypted_content)
if isinstance(encrypted_content, str) and encrypted_content
else None
)
if block_idx < 0 and signature is None:
return
if block_idx < 0:
redacted_idx: Final = self._open_block(
item_id,
{"type": "redacted_thinking", "data": signature}, # mutable-ok: API message payload
)
stop: Final = {"type": "content_block_stop", "index": redacted_idx} # mutable-ok: API message payload
self._chunk_queue.append(stop)
return
if signature is not None:
self._chunk_queue.append(
{ # mutable-ok: API message payload
"type": "content_block_delta",
"index": block_idx,
"delta": {"type": "signature_delta", "signature": signature}, # mutable-ok: API message payload
}
)
self._chunk_queue.append({"type": "content_block_stop", "index": block_idx}) # mutable-ok: API message payload
def _process_event(self, event: object) -> None:
"""Convert one Responses API event into zero or more Anthropic chunks queued for emission."""
event_type = getattr(event, "type", None)
@ -175,6 +214,26 @@ class AnthropicResponsesStreamWrapper:
)
return
if event_type == "response.reasoning_summary_part.added":
part_item_id: Final = self._field(event, "item_id")
summary_index: Final = self._field(event, "summary_index")
part_block_idx: Final = (
self._item_id_to_block_index.get(part_item_id, -1) if isinstance(part_item_id, str) else -1
)
if part_block_idx < 0 or not isinstance(summary_index, int) or summary_index == 0:
return
self._chunk_queue.append(
{ # mutable-ok: API message payload
"type": "content_block_delta",
"index": part_block_idx,
"delta": { # mutable-ok: API message payload
"type": "thinking_delta",
"thinking": REASONING_SUMMARY_PART_SEPARATOR,
},
}
)
return
# ---- reasoning summary text delta ----
if event_type == "response.reasoning_summary_text.delta":
item_id = getattr(event, "item_id", None) or (event.get("item_id") if isinstance(event, dict) else None)
@ -220,6 +279,9 @@ class AnthropicResponsesStreamWrapper:
item_id = (
getattr(item, "id", None) or (item.get("id") if isinstance(item, dict) else None) if item else None
)
if self._field(item, "type") == "reasoning":
self._close_reasoning_item(item, item_id)
return
block_idx = self._item_id_to_block_index.get(item_id, -1) if item_id else self._current_block_index
if block_idx < 0:
return

View file

@ -13,7 +13,8 @@ from typing import Any, Final, cast
from litellm.litellm_core_utils.prompt_templates.common_utils import (
TOOL_RESULT_IMAGE_BOUNDARY,
TOOL_RESULT_IMAGE_PLACEHOLDER,
responses_reasoning_item_from_thinking_blocks,
encrypted_reasoning_signature,
responses_reasoning_items_from_thinking_blocks,
with_prompt_cache_breakpoint,
)
from litellm.litellm_core_utils.reasoning_effort_utils import (
@ -33,6 +34,7 @@ from litellm.types.llms.anthropic import (
AnthropicFinishReason,
AnthropicMessagesRequest,
AnthropicMessagesToolChoice,
AnthropicResponseContentBlockRedactedThinking,
AnthropicResponseContentBlockText,
AnthropicResponseContentBlockThinking,
AnthropicResponseContentBlockToolUse,
@ -43,11 +45,13 @@ from litellm.types.llms.anthropic_messages.anthropic_response import (
AnthropicUsage,
)
from litellm.types.llms.openai import (
ChatCompletionThinkingBlock,
ResponseAPIUsage,
ResponsesAPIResponse,
)
REASONING_SUMMARY_PART_SEPARATOR: Final = "\n\n"
RESPONSES_INCLUDE_ENCRYPTED_REASONING: Final = "reasoning.encrypted_content"
class LiteLLMAnthropicToResponsesAPIAdapter:
"""
@ -163,49 +167,55 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
return str(getattr(part, "text", None) or "")
@classmethod
def _thinking_blocks_from_reasoning_item(
def _thinking_block_from_reasoning_item(
cls,
summary: Iterable[object],
) -> tuple[dict[str, Any], ...]: # mutable-ok: API message payload
"""Anthropic thinking blocks for one Responses reasoning item.
encrypted_content: object,
) -> dict[str, Any] | None: # mutable-ok: API message payload
"""The one Anthropic block for a Responses reasoning item.
The signature stays empty: only Anthropic can sign a thinking block, and a stand-in
value would be replayed as a real one and rejected by every backend that verifies it.
The item's encrypted reasoning rides the block's opaque field (`signature`, or
`data` when there is no summary text) so the client echoes it back and the next
turn replays the very item OpenAI produced; without it the signature stays empty,
since only Anthropic can sign a thinking block.
"""
return tuple(
AnthropicResponseContentBlockThinking(
type="thinking",
thinking=text,
signature=None,
).model_dump()
for part in summary
if (text := cls._summary_part_text(part))
text: Final = REASONING_SUMMARY_PART_SEPARATOR.join(
part_text for part in summary if (part_text := cls._summary_part_text(part))
)
if not isinstance(encrypted_content, str) or not encrypted_content:
if not text:
return None
return AnthropicResponseContentBlockThinking(type="thinking", thinking=text, signature=None).model_dump()
signature: Final = encrypted_reasoning_signature(encrypted_content)
if not text:
return AnthropicResponseContentBlockRedactedThinking(type="redacted_thinking", data=signature).model_dump()
return AnthropicResponseContentBlockThinking(type="thinking", thinking=text, signature=signature).model_dump()
@staticmethod
def _assistant_block_group_key(indexed_block: tuple[int, Mapping[str, object]]) -> str:
"""Group a run of consecutive thinking blocks together; keep every other block alone."""
index, block = indexed_block
return "thinking" if block.get("type") == "thinking" else f"block:{index}"
return "thinking" if block.get("type") in ("thinking", "redacted_thinking") else f"block:{index}"
@classmethod
def _assistant_group_to_input_item(
def _assistant_group_to_input_items(
cls, group: tuple[Mapping[str, object], ...]
) -> dict[str, Any] | None: # mutable-ok: API message payload
) -> tuple[dict[str, Any], ...]: # mutable-ok: API message payload
first: Final = group[0]
btype: Final = first.get("type")
if btype == "thinking":
blocks: Final = cast(tuple[ChatCompletionThinkingBlock, ...], group) # cast-ok: untrusted client payload
reasoning_item: Final = responses_reasoning_item_from_thinking_blocks(blocks)
return None if reasoning_item is None else dict(reasoning_item) # mutable-ok: API message payload
if btype in ("thinking", "redacted_thinking"):
replayed: Final = responses_reasoning_items_from_thinking_blocks(group)
return tuple(dict(item) for item in replayed) # mutable-ok: API message payload
if btype == "tool_use":
return { # mutable-ok: API message payload
"type": "function_call",
"call_id": first.get("id", ""),
"name": first.get("name", ""),
"arguments": json.dumps(first.get("input", {})), # mutable-ok: API message payload
}
return None
return (
{ # mutable-ok: API message payload
"type": "function_call",
"call_id": first.get("id", ""),
"name": first.get("name", ""),
"arguments": json.dumps(first.get("input", {})), # mutable-ok: API message payload
},
)
return ()
def translate_messages_to_responses_input(
self,
@ -362,7 +372,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
input_items.extend(
item
for _, group in groupby(enumerate(blocks), key=self._assistant_block_group_key)
if (item := self._assistant_group_to_input_item(tuple(block for _, block in group))) is not None
for item in self._assistant_group_to_input_items(tuple(block for _, block in group))
)
asst_parts: list[dict[str, Any]] = [ # mutable-ok: API message payload
{"type": "output_text", "text": block.get("text", "")} # mutable-ok: API message payload
@ -495,10 +505,16 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
def translate_request(
self,
anthropic_request: AnthropicMessagesRequest,
include_encrypted_reasoning: bool = True,
) -> dict[str, Any]:
"""
Translate a full Anthropic /v1/messages request dict to
litellm.responses() / litellm.aresponses() kwargs.
``include_encrypted_reasoning`` asks the provider for ``reasoning.encrypted_content``
on every call, so a reasoning model's items can be replayed intact next turn even
when the client sent no ``thinking`` block; pass False for a provider whose
Responses API rejects ``include``.
"""
model: Final[str] = anthropic_request["model"]
messages_list: Final = cast(
@ -528,6 +544,8 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
"model": model,
"input": input_items,
}
if include_encrypted_reasoning:
responses_kwargs["include"] = [RESPONSES_INCLUDE_ENCRYPTED_REASONING] # mutable-ok: API request payload
if system and not developer_parts:
if isinstance(system, str):
@ -634,7 +652,9 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
for item in response.output:
if isinstance(item, ResponseReasoningItem):
content.extend(self._thinking_blocks_from_reasoning_item(item.summary))
reasoning_block = self._thinking_block_from_reasoning_item(item.summary, item.encrypted_content)
if reasoning_block is not None:
content.append(reasoning_block)
elif isinstance(item, ResponseOutputMessage):
for part in item.content:
@ -684,11 +704,12 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
).model_dump()
)
elif item_type == "reasoning":
content.extend(
self._thinking_blocks_from_reasoning_item(
cast(Iterable[object], item.get("summary") or ()), # cast-ok: untyped provider json
)
reasoning_block = self._thinking_block_from_reasoning_item(
cast(Iterable[object], item.get("summary") or ()), # cast-ok: untyped provider json
item.get("encrypted_content"),
)
if reasoning_block is not None:
content.append(reasoning_block)
elif item_type == "function_call":
try:
input_data = json.loads(item.get("arguments", "{}"))

View file

@ -37,7 +37,7 @@ Safe to enable globally:
"""
import time
from collections.abc import Mapping
from collections.abc import Iterator, Mapping
from typing import TYPE_CHECKING, Final, Optional, Protocol, cast
import httpx
@ -48,6 +48,10 @@ from litellm.exceptions import (
ServiceUnavailableError,
)
from litellm.integrations.custom_logger import CustomLogger, Span
from litellm.litellm_core_utils.prompt_templates.common_utils import (
encrypted_content_of_block,
strip_encrypted_reasoning_from_messages,
)
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.router_utils.cooldown_cache import CooldownCacheValue
from litellm.types.llms.openai import AllMessageValues
@ -138,15 +142,48 @@ class EncryptedContentAffinityCheck(CustomLogger):
# If no encoded ID, check if encrypted_content itself is wrapped
encrypted_content = item.get("encrypted_content")
if encrypted_content and isinstance(encrypted_content, str):
(
model_id,
_,
) = ResponsesAPIRequestUtils._unwrap_encrypted_content_with_model_id(encrypted_content)
model_id = EncryptedContentAffinityCheck._model_id_from_wrapped_encrypted_content(encrypted_content)
if model_id:
return model_id
return None
@staticmethod
def _anthropic_content_blocks(messages: object) -> Iterator[Mapping[str, object]]:
if not isinstance(messages, list):
return iter(())
return (
cast(Mapping[str, object], block) # cast-ok: narrowed by isinstance
for message in cast(list[object], messages) # cast-ok: narrowed by isinstance
if isinstance(message, Mapping)
for content in (cast(Mapping[str, object], message).get("content"),) # cast-ok: narrowed by isinstance
if isinstance(content, list)
for block in cast(list[object], content) # cast-ok: narrowed by isinstance
if isinstance(block, Mapping)
)
@staticmethod
def _model_id_from_wrapped_encrypted_content(encrypted_content: str) -> str | None:
model_id, _ = ResponsesAPIRequestUtils._unwrap_encrypted_content_with_model_id(encrypted_content)
return model_id or None
@staticmethod
def _extract_model_id_from_anthropic_messages(messages: object) -> str | None:
return next(
(
model_id
for block in EncryptedContentAffinityCheck._anthropic_content_blocks(messages)
if (encrypted_content := encrypted_content_of_block(block)) is not None
if (
model_id := EncryptedContentAffinityCheck._model_id_from_wrapped_encrypted_content(
encrypted_content
)
)
is not None
),
None,
)
@staticmethod
def _find_deployment_by_model_id(healthy_deployments: list[dict], model_id: str) -> dict | None:
for deployment in healthy_deployments:
@ -240,8 +277,9 @@ class EncryptedContentAffinityCheck(CustomLogger):
parent_otel_span: Span | None = None,
) -> list[dict]:
"""
If the request ``input`` contains litellm-encoded item IDs, decode the
embedded ``model_id`` and pin the request to that deployment. Raises
If the request ``input`` contains litellm-encoded item IDs, or its Anthropic
``messages`` replay a bridge-tagged thinking block, decode the embedded
``model_id`` and pin the request to that deployment. Raises
``RateLimitError`` / ``ServiceUnavailableError`` when the originating
deployment is a member of the routed model group but currently unavailable
and no encryption-boundary peer exists, rather than dispatching a doomed
@ -270,12 +308,15 @@ class EncryptedContentAffinityCheck(CustomLogger):
request_kwargs["litellm_metadata"]["encrypted_content_affinity_enabled"] = True
request_input: Final = request_kwargs.get("input")
model_id: Final = self._extract_model_id_from_input(request_input)
anthropic_messages: Final = messages or request_kwargs.get("messages")
model_id: Final = self._extract_model_id_from_input(
request_input
) or self._extract_model_id_from_anthropic_messages(anthropic_messages)
if not model_id:
return typed_healthy_deployments
verbose_router_logger.debug(
"EncryptedContentAffinityCheck: decoded model_id=%s from input item IDs",
"EncryptedContentAffinityCheck: decoded model_id=%s from the request's encrypted content markers",
model_id,
)
@ -327,6 +368,7 @@ class EncryptedContentAffinityCheck(CustomLogger):
model,
)
ResponsesAPIRequestUtils.strip_encrypted_reasoning_from_input(request_input)
strip_encrypted_reasoning_from_messages(anthropic_messages)
return typed_healthy_deployments
# The origin is a member of the routed group but currently unavailable (cooled down); fail fast

View file

@ -1,3 +1,4 @@
import copy
import functools
import json
import os
@ -7,14 +8,20 @@ import pytest
from litellm.litellm_core_utils.prompt_templates.common_utils import (
ENCRYPTED_REASONING_SIGNATURE_PREFIX,
TOOL_RESULT_IMAGE_BOUNDARY,
TOOL_RESULT_IMAGE_PLACEHOLDER,
add_system_prompt_to_messages,
encrypted_content_from_signature,
encrypted_reasoning_signature,
get_file_ids_from_messages,
get_format_from_file_id,
handle_any_messages_to_chat_completion_str_messages_conversion,
hoist_images_from_tool_messages,
is_encrypted_reasoning_block,
responses_reasoning_items_from_thinking_blocks,
split_concatenated_json_objects,
strip_encrypted_reasoning_from_messages,
update_messages_with_model_file_ids,
)
@ -1554,3 +1561,117 @@ class TestRequestContainsImageContent:
for _ in range(50):
nested = {"type": "tool_result", "content": [nested]}
assert request_contains_image_content([{"role": "user", "content": [nested]}]) is False
class TestEncryptedReasoningReplay:
"""Regression for https://github.com/BerriAI/litellm/issues/40288."""
def test_signature_round_trips_the_encrypted_content(self):
assert encrypted_content_from_signature(encrypted_reasoning_signature("gAAAA_bytes")) == "gAAAA_bytes"
@pytest.mark.parametrize(
"signature", [None, "", "ErcBCkgIValidAnthropicSignature", "litellm_encrypted_reasoning:", 7]
)
def test_anything_else_is_not_encrypted_content(self, signature):
assert encrypted_content_from_signature(signature) is None
def test_encrypted_thinking_block_replays_its_own_item(self):
items = responses_reasoning_items_from_thinking_blocks(
[{"type": "thinking", "thinking": "Plan.", "signature": encrypted_reasoning_signature("gAAAA_1")}]
)
assert items == (
{
"type": "reasoning",
"summary": [{"type": "summary_text", "text": "Plan."}],
"encrypted_content": "gAAAA_1",
},
)
def test_encrypted_redacted_block_replays_with_an_empty_summary(self):
items = responses_reasoning_items_from_thinking_blocks(
[{"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAAA_1")}]
)
assert items == ({"type": "reasoning", "summary": [], "encrypted_content": "gAAAA_1"},)
def test_plain_blocks_collapse_into_one_summary_item_around_encrypted_ones(self):
items = responses_reasoning_items_from_thinking_blocks(
[
{"type": "thinking", "thinking": "A.", "signature": None},
{"type": "thinking", "thinking": "B.", "signature": ""},
{"type": "thinking", "thinking": "C.", "signature": encrypted_reasoning_signature("gAAAA_c")},
{"type": "redacted_thinking", "data": "anthropic-minted-opaque-data"},
{"type": "thinking", "thinking": "D."},
]
)
assert items == (
{
"type": "reasoning",
"summary": [{"type": "summary_text", "text": "A."}, {"type": "summary_text", "text": "B."}],
},
{"type": "reasoning", "summary": [{"type": "summary_text", "text": "C."}], "encrypted_content": "gAAAA_c"},
{"type": "reasoning", "summary": [{"type": "summary_text", "text": "D."}]},
)
assert all("id" not in item for item in items)
def test_blocks_without_text_or_encrypted_content_produce_nothing(self):
assert responses_reasoning_items_from_thinking_blocks([{"type": "thinking", "thinking": ""}]) == ()
assert responses_reasoning_items_from_thinking_blocks([]) == ()
@pytest.mark.parametrize(
("block", "expected"),
[
({"type": "thinking", "thinking": "x", "signature": encrypted_reasoning_signature("g")}, True),
({"type": "redacted_thinking", "data": encrypted_reasoning_signature("g")}, True),
({"type": "thinking", "thinking": "x", "signature": ENCRYPTED_REASONING_SIGNATURE_PREFIX}, True),
({"type": "redacted_thinking", "data": ENCRYPTED_REASONING_SIGNATURE_PREFIX}, True),
({"type": "thinking", "thinking": "x", "signature": "ErcBCkgIValid"}, False),
({"type": "redacted_thinking", "data": "EmwKAhgBEgy"}, False),
({"type": "text", "text": encrypted_reasoning_signature("g")}, False),
("not a block", False),
],
)
def test_is_encrypted_reasoning_block(self, block, expected):
assert is_encrypted_reasoning_block(block) is expected
def test_strip_drops_every_bridge_tagged_block_and_leaves_no_unsigned_thinking_behind(self):
assistant_content = [
{"type": "thinking", "thinking": "minted by Anthropic", "signature": "ErcBCkgIValid"},
{"type": "thinking", "thinking": "packed by the bridge", "signature": encrypted_reasoning_signature("g1")},
{"type": "redacted_thinking", "data": encrypted_reasoning_signature("g2")},
{"type": "thinking", "thinking": "", "signature": encrypted_reasoning_signature("g3")},
{"type": "text", "text": "answer"},
]
messages = [
{"role": "user", "content": "question"},
{"role": "assistant", "content": assistant_content},
{"role": "user", "content": [{"type": "text", "text": "follow-up"}]},
]
strip_encrypted_reasoning_from_messages(messages)
assert messages[1]["content"] is assistant_content
assert assistant_content == [
{"type": "thinking", "thinking": "minted by Anthropic", "signature": "ErcBCkgIValid"},
{"type": "text", "text": "answer"},
]
assert all(block["signature"] for block in assistant_content if block["type"] == "thinking")
assert messages[0] == {"role": "user", "content": "question"}
assert messages[2] == {"role": "user", "content": [{"type": "text", "text": "follow-up"}]}
@pytest.mark.parametrize(
"messages",
[
"not a list",
None,
[{"role": "user", "content": None}],
[{"role": "user", "content": "plain string"}],
["not a message"],
[{"role": "assistant", "content": [{"type": "thinking", "thinking": "x", "signature": "ErcBCkgIValid"}]}],
],
)
def test_strip_leaves_history_without_bridge_reasoning_untouched(self, messages):
before = copy.deepcopy(messages)
strip_encrypted_reasoning_from_messages(messages)
assert messages == before

View file

@ -191,8 +191,16 @@ def test_bedrock_converse_assistant_with_empty_thinking_block_and_tool_calls():
{"type": "thinking", "thinking": "oss reasoning", "signature": None},
{"type": "thinking", "thinking": "oss reasoning", "signature": ""},
{"type": "thinking", "thinking": "oss reasoning"},
{"type": "thinking", "thinking": "openai reasoning", "signature": "litellm_encrypted_reasoning:gAAAA"},
{"type": "redacted_thinking", "data": "litellm_encrypted_reasoning:gAAAA"},
],
ids=[
"null_signature",
"empty_signature",
"missing_signature",
"encrypted_reasoning_signature",
"encrypted_reasoning_redacted_data",
],
ids=["null_signature", "empty_signature", "missing_signature"],
)
def test_anthropic_messages_pt_drops_unsignable_thinking_block(thinking_block):
"""Open-source reasoning models (DeepSeek-R1, Qwen, etc.) emit thinking blocks
@ -219,7 +227,7 @@ def test_anthropic_messages_pt_drops_unsignable_thinking_block(thinking_block):
assistant = next(m for m in result if m["role"] == "assistant")
content = assistant["content"]
assert all(
block.get("type") != "thinking" for block in content
block.get("type") not in ("thinking", "redacted_thinking") for block in content
), f"unsignable thinking block must be dropped, got {content!r}"
assert any(
block.get("type") == "text" and block.get("text") == "2+2 equals 4."

View file

@ -10,6 +10,7 @@ import litellm
from litellm.litellm_core_utils.prompt_templates.common_utils import (
TOOL_RESULT_IMAGE_PLACEHOLDER,
encrypted_reasoning_signature,
)
from litellm.litellm_core_utils.prompt_templates.factory import (
THOUGHT_SIGNATURE_SEPARATOR,
@ -423,6 +424,43 @@ def test_translate_anthropic_messages_to_openai_thinking_blocks():
assert result[1]["tool_calls"][0]["id"] == "toolu_01234"
def test_translate_anthropic_messages_to_openai_drops_bridge_encrypted_reasoning_blocks():
"""A session that moves from an OpenAI reasoning model to a chat provider replays reasoning only OpenAI can read.
Gemini rejects the whole request when such a block reaches it as a thought_signature, so the
adapter drops those blocks and keeps the provider-signed ones.
"""
anthropic_messages = [
AnthropicMessagesUserMessageParam(
role="user",
content=[{"type": "text", "text": "Who drinks water?"}],
),
AnthopicMessagesAssistantMessageParam(
role="assistant",
content=[
{"type": "thinking", "thinking": "plan", "signature": encrypted_reasoning_signature("gAAAA_1")},
{"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAAA_2")},
{"type": "text", "text": "The Norwegian."},
],
),
AnthopicMessagesAssistantMessageParam(
role="assistant",
content=[
{"type": "thinking", "thinking": "native", "signature": "EqQBCkYIAxgCIkA_signed"},
{"type": "text", "text": "Still the Norwegian."},
],
),
]
result = LiteLLMAnthropicMessagesAdapter().translate_anthropic_messages_to_openai(messages=anthropic_messages)
assert [m["role"] for m in result] == ["user", "assistant", "assistant"]
assert not result[1].get("thinking_blocks")
assert result[1]["content"] == "The Norwegian."
assert [b["signature"] for b in result[2]["thinking_blocks"]] == ["EqQBCkYIAxgCIkA_signed"]
def test_translate_anthropic_messages_to_openai_sets_reasoning_content():
"""Reasoning-aware chat providers read reasoning_content, so thinking text must land there.

View file

@ -0,0 +1,50 @@
from litellm.litellm_core_utils.prompt_templates.common_utils import (
encrypted_reasoning_signature,
)
from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
AnthropicMessagesConfig,
)
def _transform(messages):
return AnthropicMessagesConfig().transform_anthropic_messages_request(
model="claude-sonnet-4-5",
messages=messages,
anthropic_messages_optional_request_params={"max_tokens": 1024},
litellm_params={},
headers={},
)
def test_reasoning_replayed_from_the_responses_bridge_never_reaches_anthropic():
"""Claude Code resumed on a Claude model echoes the thinking blocks a gpt turn produced."""
messages = [
{"role": "user", "content": "Solve it."},
{
"role": "assistant",
"content": [
{"type": "thinking", "thinking": "plan", "signature": encrypted_reasoning_signature("gAAAA_1")},
{"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAAA_2")},
{"type": "text", "text": "The answer."},
],
},
{"role": "user", "content": "And the next one?"},
]
request = _transform(messages)
assert request["messages"][1]["content"] == [{"type": "text", "text": "The answer."}]
assert len(messages[1]["content"]) == 3
def test_anthropic_signed_thinking_blocks_are_forwarded_untouched():
messages = [
{"role": "user", "content": "Solve it."},
{
"role": "assistant",
"content": [
{"type": "thinking", "thinking": "plan", "signature": "EqQBCkYIAxgCIkA_anthropic_signed"},
{"type": "redacted_thinking", "data": "EmwKAhgBEgy_anthropic_minted"},
{"type": "text", "text": "The answer."},
],
},
]
assert _transform(messages)["messages"] == messages

View file

@ -67,6 +67,39 @@ def test_build_responses_kwargs_prefers_explicit_prompt_cache_key_over_derived()
assert responses_kwargs["prompt_cache_key"] == "explicit-key"
def test_build_responses_kwargs_asks_openai_for_encrypted_reasoning_without_thinking():
responses_kwargs = _build_responses_kwargs(
max_tokens=1024,
messages=MESSAGES,
model="openai/gpt-5.6-luna",
extra_kwargs={"custom_llm_provider": "openai"},
)
assert responses_kwargs["include"] == ["reasoning.encrypted_content"]
assert "reasoning" not in responses_kwargs
def test_build_responses_kwargs_skips_include_for_a_responses_provider_that_rejects_it():
responses_kwargs = _build_responses_kwargs(
max_tokens=1024,
messages=MESSAGES,
model="perplexity/sonar",
thinking={"type": "enabled", "budget_tokens": 4096},
extra_kwargs={"custom_llm_provider": "perplexity"},
)
assert "include" not in responses_kwargs
assert "reasoning" in responses_kwargs
def test_build_responses_kwargs_keeps_the_deployment_include_next_to_encrypted_reasoning():
responses_kwargs = _build_responses_kwargs(
max_tokens=1024,
messages=MESSAGES,
model="openai/gpt-5.6-luna",
extra_kwargs={"custom_llm_provider": "openai", "include": ["file_search_call.results"]},
)
assert responses_kwargs["include"] == ["reasoning.encrypted_content", "file_search_call.results"]
def test_build_responses_kwargs_without_metadata_sets_no_prompt_cache_key():
responses_kwargs = _build_responses_kwargs(
max_tokens=1024,

View file

@ -10,6 +10,9 @@ from types import SimpleNamespace
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../../..")))
from litellm.litellm_core_utils.prompt_templates.common_utils import (
encrypted_reasoning_signature,
)
from litellm.llms.anthropic.experimental_pass_through.responses_adapters.streaming_iterator import (
AnthropicResponsesStreamWrapper,
)
@ -114,7 +117,7 @@ class TestReasoningItemWithoutSummaryText:
"""
@staticmethod
def _gpt_turn(reasoning_summary_deltas: list) -> list:
def _gpt_turn(reasoning_summary_deltas: list, encrypted_content: str | None = None) -> list:
return [
{"type": "response.created"},
{"type": "response.output_item.added", "item": {"type": "reasoning", "id": "rs_1"}},
@ -122,7 +125,10 @@ class TestReasoningItemWithoutSummaryText:
{"type": "response.reasoning_summary_text.delta", "item_id": "rs_1", "delta": delta}
for delta in reasoning_summary_deltas
),
{"type": "response.output_item.done", "item": {"type": "reasoning", "id": "rs_1"}},
{
"type": "response.output_item.done",
"item": {"type": "reasoning", "id": "rs_1", "encrypted_content": encrypted_content},
},
{"type": "response.output_item.added", "item": {"type": "message", "id": "msg_1"}},
{"type": "response.output_text.delta", "item_id": "msg_1", "delta": "Hello"},
{"type": "response.output_item.done", "item": {"type": "message", "id": "msg_1"}},
@ -171,6 +177,70 @@ class TestReasoningItemWithoutSummaryText:
assert not [c for c in chunks if c.get("delta", {}).get("type") == "signature_delta"]
_ENCRYPTED_REASONING = "gAAAAABp_encrypted_reasoning_bytes_only_openai_can_read"
class TestEncryptedReasoningIsStreamedForReplay:
"""Regression for https://github.com/BerriAI/litellm/issues/40288.
The client echoes a thinking block's signature (or a redacted block's data) back on the
next turn, so the item's ``encrypted_content`` has to reach it through one of those.
"""
def test_encrypted_content_is_streamed_as_the_signature_before_the_block_closes(self):
chunks = _drain_async(
TestReasoningItemWithoutSummaryText._gpt_turn(
reasoning_summary_deltas=["Weighing options"], encrypted_content=_ENCRYPTED_REASONING
)
)
assert [(c["type"], c.get("index"), c.get("delta", {}).get("type")) for c in chunks[1:5]] == [
("content_block_start", 0, None),
("content_block_delta", 0, "thinking_delta"),
("content_block_delta", 0, "signature_delta"),
("content_block_stop", 0, None),
]
assert chunks[3]["delta"]["signature"] == encrypted_reasoning_signature(_ENCRYPTED_REASONING)
def test_reasoning_without_summary_streams_a_redacted_thinking_block(self):
chunks = _drain_async(
TestReasoningItemWithoutSummaryText._gpt_turn(
reasoning_summary_deltas=[], encrypted_content=_ENCRYPTED_REASONING
)
)
assert [(c["type"], c.get("index")) for c in chunks[1:]] == [
("content_block_start", 0),
("content_block_stop", 0),
("content_block_start", 1),
("content_block_delta", 1),
("content_block_stop", 1),
]
assert chunks[1]["content_block"] == {
"type": "redacted_thinking",
"data": encrypted_reasoning_signature(_ENCRYPTED_REASONING),
}
def test_summary_parts_are_separated_inside_the_one_thinking_block(self):
"""Two summary parts read as two paragraphs, not as one run-on sentence."""
events = [
{"type": "response.created"},
{"type": "response.output_item.added", "item": {"type": "reasoning", "id": "rs_1"}},
{"type": "response.reasoning_summary_part.added", "item_id": "rs_1", "summary_index": 0},
{"type": "response.reasoning_summary_text.delta", "item_id": "rs_1", "delta": "First."},
{"type": "response.reasoning_summary_part.added", "item_id": "rs_1", "summary_index": 1},
{"type": "response.reasoning_summary_text.delta", "item_id": "rs_1", "delta": "Second."},
{"type": "response.output_item.done", "item": {"type": "reasoning", "id": "rs_1"}},
]
chunks = _process_all(events)
thinking = "".join(
c["delta"]["thinking"] for c in chunks if c.get("delta", {}).get("type") == "thinking_delta"
)
assert thinking == "First.\n\nSecond."
assert [c["type"] for c in chunks].count("content_block_start") == 1
class TestToolUseBlockClosedExactlyOnce:
"""Regression for https://github.com/BerriAI/litellm/issues/37273.

View file

@ -19,6 +19,7 @@ from litellm.constants import (
from litellm.litellm_core_utils.prompt_templates.common_utils import (
TOOL_RESULT_IMAGE_BOUNDARY,
TOOL_RESULT_IMAGE_PLACEHOLDER,
encrypted_reasoning_signature,
)
from litellm.llms.anthropic.experimental_pass_through.responses_adapters.transformation import (
LiteLLMAnthropicToResponsesAPIAdapter,
@ -566,6 +567,66 @@ class TestTranslateMessagesToResponsesInput:
result = _translate_messages(messages)
assert "id" not in result[0]
def test_thinking_block_with_encrypted_signature_replays_the_encrypted_content(self):
"""Regression for https://github.com/BerriAI/litellm/issues/40288 (inbound fault site)."""
messages = [
{
"role": "assistant",
"content": [
{
"type": "thinking",
"thinking": "Private reasoning.",
"signature": encrypted_reasoning_signature("gAAAA_turn_one"),
}
],
}
]
result = _translate_messages(messages)
assert result == [
{
"type": "reasoning",
"summary": [{"type": "summary_text", "text": "Private reasoning."}],
"encrypted_content": "gAAAA_turn_one",
}
]
def test_redacted_thinking_with_encrypted_data_replays_the_encrypted_content(self):
messages = [
{
"role": "assistant",
"content": [{"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAAA_turn_one")}],
}
]
result = _translate_messages(messages)
assert result == [{"type": "reasoning", "summary": [], "encrypted_content": "gAAAA_turn_one"}]
def test_each_encrypted_thinking_block_stays_its_own_reasoning_item(self):
"""Two upstream items must not be merged into one, or the encrypted content of one is lost."""
messages = [
{
"role": "assistant",
"content": [
{"type": "thinking", "thinking": "First.", "signature": encrypted_reasoning_signature("gAAAA_1")},
{"type": "thinking", "thinking": "Second.", "signature": encrypted_reasoning_signature("gAAAA_2")},
],
}
]
result = _translate_messages(messages)
assert [item["encrypted_content"] for item in result] == ["gAAAA_1", "gAAAA_2"]
def test_anthropic_signed_thinking_block_replays_as_a_summary_only_item(self):
"""A real Anthropic signature is opaque here, so it never masquerades as encrypted content."""
messages = [
{
"role": "assistant",
"content": [{"type": "thinking", "thinking": "Private reasoning.", "signature": "ErcBCkgIValid"}],
}
]
result = _translate_messages(messages)
assert result == [
{"type": "reasoning", "summary": [{"type": "summary_text", "text": "Private reasoning."}]}
]
def test_consecutive_thinking_blocks_become_one_reasoning_item(self):
"""Summary parts of one upstream reasoning item are regrouped into that item."""
messages = [
@ -1102,6 +1163,23 @@ class TestTranslateRequestBroaderCoverage:
kwargs = _ADAPTER.translate_request(req)
assert "reasoning" not in kwargs
def test_thinking_asks_for_the_encrypted_reasoning(self):
"""The documented way to get reasoning that survives store=false is to ask for it."""
req = _make_request(thinking={"type": "enabled", "budget_tokens": 12000})
kwargs = _ADAPTER.translate_request(req)
assert kwargs["include"] == ["reasoning.encrypted_content"]
def test_encrypted_reasoning_is_asked_for_without_a_thinking_block(self):
"""A reasoning model reasons whether or not the client sent `thinking`, so the replay needs it either way."""
kwargs = _ADAPTER.translate_request(_make_request())
assert kwargs["include"] == ["reasoning.encrypted_content"]
def test_encrypted_reasoning_is_not_asked_for_when_the_provider_rejects_include(self):
req = _make_request(thinking={"type": "enabled", "budget_tokens": 12000})
kwargs = _ADAPTER.translate_request(req, include_encrypted_reasoning=False)
assert kwargs["reasoning"] == {"effort": "high"}
assert "include" not in kwargs
def test_metadata_user_id_mapped_to_user(self):
req = _make_request(metadata={"user_id": "user-42"})
kwargs = _ADAPTER.translate_request(req)
@ -1246,7 +1324,9 @@ def _make_function_call_item(call_id: str, name: str, arguments: str) -> MagicMo
return item
def _make_reasoning_item(summaries: List[str], item_id: str = "rs_test_1") -> MagicMock:
def _make_reasoning_item(
summaries: List[str], item_id: str = "rs_test_1", encrypted_content: str | None = None
) -> MagicMock:
"""Build a mock ResponseReasoningItem."""
from openai.types.responses import ResponseReasoningItem # type: ignore[import]
@ -1259,9 +1339,13 @@ def _make_reasoning_item(summaries: List[str], item_id: str = "rs_test_1") -> Ma
item = MagicMock(spec=ResponseReasoningItem)
item.id = item_id
item.summary = summary_mocks
item.encrypted_content = encrypted_content
return item
_ENCRYPTED_REASONING = "gAAAAABp_encrypted_reasoning_bytes_only_openai_can_read"
class TestTranslateResponse:
"""Responses API -> AnthropicMessagesResponse conversion."""
@ -1386,7 +1470,81 @@ class TestTranslateResponse:
reasoning = _make_reasoning_item(["Part one.", "Part two."], item_id="rs_abc123")
response = _make_mock_response(output=[reasoning])
result: Any = _ADAPTER.translate_response(response)
assert [block["signature"] for block in result["content"]] == [None, None]
assert [block["signature"] for block in result["content"]] == [None]
assert "rs_abc123" not in json.dumps(result["content"])
def test_summary_parts_join_into_one_thinking_block(self):
"""One reasoning item is one block, so its signature is echoed back exactly once."""
reasoning = _make_reasoning_item(["Part one.", "Part two."])
response = _make_mock_response(output=[reasoning])
result: Any = _ADAPTER.translate_response(response)
assert [block["thinking"] for block in result["content"]] == ["Part one.\n\nPart two."]
def test_encrypted_content_rides_the_thinking_signature(self):
"""Regression for https://github.com/BerriAI/litellm/issues/40288 (outbound fault site)."""
reasoning = _make_reasoning_item(["Part one."], item_id="rs_abc123", encrypted_content=_ENCRYPTED_REASONING)
response = _make_mock_response(output=[reasoning])
result: Any = _ADAPTER.translate_response(response)
assert result["content"] == [
{
"type": "thinking",
"thinking": "Part one.",
"signature": encrypted_reasoning_signature(_ENCRYPTED_REASONING),
}
]
def test_reasoning_without_summary_becomes_redacted_thinking(self):
"""With summaries off the encrypted reasoning still has to reach the client to be replayed."""
reasoning = _make_reasoning_item([], encrypted_content=_ENCRYPTED_REASONING)
response = _make_mock_response(output=[reasoning])
result: Any = _ADAPTER.translate_response(response)
assert result["content"] == [
{"type": "redacted_thinking", "data": encrypted_reasoning_signature(_ENCRYPTED_REASONING)}
]
def test_dict_reasoning_item_carries_its_encrypted_content(self):
response = _make_mock_response(
output=[
{
"type": "reasoning",
"id": "rs_dict_1",
"encrypted_content": _ENCRYPTED_REASONING,
"summary": [{"type": "summary_text", "text": "Weighing the options."}],
}
]
)
result: Any = _ADAPTER.translate_response(response)
assert result["content"][0]["signature"] == encrypted_reasoning_signature(_ENCRYPTED_REASONING)
def test_reasoning_item_round_trip_is_byte_stable(self):
"""Regression for https://github.com/BerriAI/litellm/issues/40288.
The reasoning item the next turn replays must be the one OpenAI produced, with its
encrypted reasoning intact, and identical on every later turn so the prompt cache
prefix keeps matching.
"""
reasoning = _make_reasoning_item(["Part one.", "Part two."], encrypted_content=_ENCRYPTED_REASONING)
turn: Any = _ADAPTER.translate_response(_make_mock_response(output=[reasoning]))
history = [{"role": "assistant", "content": turn["content"]}]
replayed_items = [_translate_messages(history) for _ in range(2)]
assert replayed_items[0] == replayed_items[1]
assert replayed_items[0] == [
{
"type": "reasoning",
"summary": [{"type": "summary_text", "text": "Part one.\n\nPart two."}],
"encrypted_content": _ENCRYPTED_REASONING,
}
]
def test_redacted_reasoning_round_trip_replays_the_encrypted_content(self):
reasoning = _make_reasoning_item([], encrypted_content=_ENCRYPTED_REASONING)
turn: Any = _ADAPTER.translate_response(_make_mock_response(output=[reasoning]))
replayed = _translate_messages([{"role": "assistant", "content": turn["content"]}])
assert replayed == [{"type": "reasoning", "summary": [], "encrypted_content": _ENCRYPTED_REASONING}]
def test_dict_reasoning_item_becomes_thinking_block(self):
"""A reasoning item arriving as a plain dict is kept, not dropped."""
@ -1402,14 +1560,26 @@ class TestTranslateResponse:
result: Any = _ADAPTER.translate_response(response)
assert result["content"] == [{"type": "thinking", "thinking": "Weighing the options.", "signature": None}]
def test_thinking_blocks_are_dropped_when_replayed_to_anthropic(self):
@pytest.mark.parametrize(
("summaries", "encrypted_content"),
[
(["Part one."], None),
(["Part one."], _ENCRYPTED_REASONING),
([], _ENCRYPTED_REASONING),
],
ids=["unsigned_thinking", "encrypted_thinking", "encrypted_redacted_thinking"],
)
def test_thinking_blocks_are_dropped_when_replayed_to_anthropic(self, summaries, encrypted_content):
"""Replaying this turn to an Anthropic model must not send a signature it cannot verify."""
from litellm.litellm_core_utils.prompt_templates.factory import (
_drop_unsignable_thinking_blocks,
)
response = _make_mock_response(output=[_make_reasoning_item(["Part one."], item_id="rs_abc123")])
response = _make_mock_response(
output=[_make_reasoning_item(summaries, item_id="rs_abc123", encrypted_content=encrypted_content)]
)
result: Any = _ADAPTER.translate_response(response)
assert len(result["content"]) == 1
assert _drop_unsignable_thinking_blocks(result["content"]) == []
def test_usage_mapped_correctly(self):

View file

@ -42,12 +42,15 @@ FAKE_AUTH_TOKEN = "sk-ant-aut01-fake-auth-token-for-testing-123456789"
def test_is_claude_code_one_shot_subagent_request(messages, system, expected):
from litellm.llms.anthropic.common_utils import is_claude_code_one_shot_subagent_request
assert is_claude_code_one_shot_subagent_request(
messages=messages,
system=system,
tools=None,
user_agent="claude-cli/2.1.263 (external, cli)",
) is expected
assert (
is_claude_code_one_shot_subagent_request(
messages=messages,
system=system,
tools=None,
user_agent="claude-cli/2.1.263 (external, cli)",
)
is expected
)
class TestOptionallyHandleAnthropicOAuth:
@ -1541,6 +1544,71 @@ class TestAnthropicThinkingSignatureSelfHeal:
out = strip_empty_content_blocks_from_anthropic_messages(msgs)
assert [b["type"] for b in out[0]["content"]] == ["thinking"]
def test_strip_keeps_encrypted_reasoning_blocks_for_the_responses_bridge(self):
"""The /v1/messages handler runs this before dispatch, so the bridge must still see the replay."""
from litellm.litellm_core_utils.prompt_templates.common_utils import (
encrypted_reasoning_signature,
)
from litellm.llms.anthropic.common_utils import (
strip_empty_content_blocks_from_anthropic_messages,
)
msgs = [
{
"role": "assistant",
"content": [
{"type": "thinking", "thinking": "plan", "signature": encrypted_reasoning_signature("gAAAA_1")},
{"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAAA_2")},
{"type": "text", "text": "The answer."},
],
}
]
assert strip_empty_content_blocks_from_anthropic_messages(msgs) == msgs
def test_strip_encrypted_reasoning_drops_only_the_bridge_tagged_blocks(self):
"""A session resumed on an Anthropic model replays reasoning only OpenAI can verify."""
from litellm.litellm_core_utils.prompt_templates.common_utils import (
encrypted_reasoning_signature,
)
from litellm.llms.anthropic.common_utils import (
strip_encrypted_reasoning_blocks_from_anthropic_messages,
)
msgs = [
{"role": "user", "content": "Solve it."},
{
"role": "assistant",
"content": [
{"type": "thinking", "thinking": "plan", "signature": encrypted_reasoning_signature("gAAAA_1")},
{"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAAA_2")},
],
},
{
"role": "assistant",
"content": [
{"type": "thinking", "thinking": "plan", "signature": encrypted_reasoning_signature("gAAAA_3")},
{"type": "thinking", "thinking": "native", "signature": "EqQBCkYIAxgCIkA_anthropic_signed"},
{"type": "redacted_thinking", "data": "EmwKAhgBEgy_anthropic_minted"},
{"type": "text", "text": "The answer."},
],
},
]
out = strip_encrypted_reasoning_blocks_from_anthropic_messages(msgs)
assert [m["role"] for m in out] == ["user", "assistant"]
assert [b["type"] for b in out[1]["content"]] == ["thinking", "redacted_thinking", "text"]
assert out[1]["content"][0]["signature"] == "EqQBCkYIAxgCIkA_anthropic_signed"
assert len(msgs[1]["content"]) == 2
assert len(msgs[2]["content"]) == 4
def test_strip_encrypted_reasoning_leaves_malformed_messages_for_the_provider_to_reject(self):
"""A bare string in messages must reach Anthropic as a 400, not die in the stripper as a 500."""
from litellm.llms.anthropic.common_utils import (
strip_encrypted_reasoning_blocks_from_anthropic_messages,
)
msgs = ["hi", {"role": "user", "content": "hello"}]
assert strip_encrypted_reasoning_blocks_from_anthropic_messages(msgs) == msgs
def test_strip_empty_text_blocks_treats_null_text_as_empty(self):
from litellm.llms.anthropic.common_utils import (
strip_empty_content_blocks_from_anthropic_messages,

View file

@ -1584,6 +1584,89 @@ async def test_model_group_encrypted_content_affinity_overrides_global_deploymen
router.discard()
@pytest.mark.asyncio
async def test_encrypted_content_affinity_pins_anthropic_messages_replayed_through_the_bridge():
"""
Claude Code behind /v1/messages replays the encrypted reasoning the bridge packed
into a thinking block's signature (or a redacted block's data). The pin has to be
read from those blocks because the bridge builds the Responses `input` only after
the router has picked a deployment.
"""
check = EncryptedContentAffinityCheck()
deployments = [
{"model_info": {"id": "openai-org-a"}, "litellm_params": {"model": "openai/gpt-5.1"}},
{"model_info": {"id": "openai-org-b"}, "litellm_params": {"model": "openai/gpt-5.1"}},
]
request_kwargs = {"model": "gpt-5.1"}
pinned = await check.async_filter_deployments(
model="gpt-5.1",
healthy_deployments=deployments,
messages=_bridge_replayed_anthropic_messages(minted_by="openai-org-b"),
request_kwargs=request_kwargs,
)
assert [d["model_info"]["id"] for d in pinned] == ["openai-org-b"]
assert request_kwargs["_encrypted_content_affinity_pinned"] is True
def _bridge_replayed_anthropic_messages(minted_by: str) -> list:
wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id("gAAAAA_turn_one", minted_by)
return [
{"role": "user", "content": "Solve the zebra puzzle"},
{
"role": "assistant",
"content": [
{"type": "thinking", "thinking": "Anthropic minted this one", "signature": "ErcCCpIBCBEYAipA"},
{"type": "redacted_thinking", "data": f"litellm_encrypted_reasoning:{wrapped}"},
{
"type": "thinking",
"thinking": "The bridge packed this one",
"signature": f"litellm_encrypted_reasoning:{wrapped}",
},
{"type": "text", "text": "The zebra owner lives in the green house."},
],
},
{"role": "user", "content": "And who drinks water?"},
]
@pytest.mark.asyncio
async def test_encrypted_content_affinity_strips_bridge_reasoning_from_messages_routed_to_another_group():
"""
The /v1/messages twin of the tier-change case: the routed group holds no deployment
of the org that minted the reasoning, so the bridge-tagged blocks are dropped whole
and the request dispatches to the routed pool. No unsigned thinking block may be left
behind: Anthropic and Bedrock reject a thinking block with a missing signature the
same way they reject a foreign one.
"""
originating = _make_originating_mock(None, "key-a", model_name="gpt-reasoning-tier")
mock_router = _make_router_mock_with_cooldown(
originating, cooldown_entries=[], routed_group_model_ids=["openai-org-b"]
)
check = EncryptedContentAffinityCheck(router=mock_router)
routed_pool = [{"model_info": {"id": "openai-org-b"}, "litellm_params": {"model": "openai/gpt-5-nano"}}]
messages = _bridge_replayed_anthropic_messages(minted_by="openai-org-a")
assistant_content = messages[1]["content"]
request_kwargs = {"model": "gpt-5.1"}
result = await check.async_filter_deployments(
model="gpt-simple-tier",
healthy_deployments=routed_pool,
messages=messages,
request_kwargs=request_kwargs,
)
assert result is routed_pool
assert "_encrypted_content_affinity_pinned" not in request_kwargs
assert messages[1]["content"] is assistant_content
assert assistant_content == [
{"type": "thinking", "thinking": "Anthropic minted this one", "signature": "ErcCCpIBCBEYAipA"},
{"type": "text", "text": "The zebra owner lives in the green house."},
]
assert all(block["signature"] for block in assistant_content if block["type"] == "thinking")
class TestStripEncryptedReasoningFromInput:
def test_keeps_summary_and_drops_encrypted_content_and_id(self):
wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id("gAAAAA-blob", "deployment-a")