diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index aa8337b6300..c3c18e3d009 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -25,7 +25,7 @@ import litellm from litellm import ModelResponse from litellm._logging import verbose_logger from litellm.litellm_core_utils.prompt_templates.common_utils import ( - responses_reasoning_item_from_thinking_blocks, + responses_reasoning_items_from_thinking_blocks, ) from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.bridges.completion_transformation import ( @@ -129,8 +129,8 @@ def _reasoning_input_items(msg: "AllMessageValues") -> list[dict[str, object]]: return stored raw_blocks: Final = msg.get("thinking_blocks") or () blocks: Final = cast("Iterable[ChatCompletionThinkingBlock]", raw_blocks) # cast-ok: untyped client json - from_thinking: Final = responses_reasoning_item_from_thinking_blocks(blocks) - return [] if from_thinking is None else [dict(from_thinking)] # mutable-ok: API message payload + replayed: Final = responses_reasoning_items_from_thinking_blocks(blocks) + return [dict(item) for item in replayed] # mutable-ok: API message payload def _build_reasoning_item( diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 00b80839dde..d70530534da 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -6,7 +6,7 @@ import io import json import mimetypes import re -from collections.abc import Iterable, Mapping, Sequence +from collections.abc import Iterable, Iterator, Mapping, Sequence from itertools import groupby from os import PathLike from pathlib import Path @@ -1823,14 +1823,11 @@ def _extract_reasoning_content(message: dict) -> tuple[str | None, str | None]: return None, message_content -def _readable_thinking_text( - block: ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock, -) -> str: +def _readable_thinking_text(block: Mapping[str, object]) -> str: """The text a chat model can read back, empty for redacted blocks and malformed ones.""" if block.get("type") != "thinking": return "" - thinking: Final = cast(ChatCompletionThinkingBlock, block).get("thinking") # cast-ok: narrowed by the type tag - return str(thinking or "") + return str(block.get("thinking") or "") def reasoning_content_from_thinking_blocks( @@ -1843,24 +1840,125 @@ def reasoning_content_from_thinking_blocks( return "\n".join(text for block in thinking_blocks if (text := _readable_thinking_text(block))) -def responses_reasoning_item_from_thinking_blocks( - thinking_blocks: Iterable[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock], -) -> ChatCompletionReasoningItem | None: - """Build a Responses API `reasoning` input item from Anthropic thinking blocks. +ENCRYPTED_REASONING_SIGNATURE_PREFIX: Final = "litellm_encrypted_reasoning:" - The item carries no `id`: the Responses API rejects an empty one and 404s on any id it - did not mint itself, while an item without an id is always accepted. + +def encrypted_reasoning_signature(encrypted_content: str) -> str: + """The opaque value a Responses API reasoning item's `encrypted_content` travels in. + + Anthropic clients echo a thinking block's `signature` and a redacted block's `data` + back verbatim, so either field can carry the encrypted reasoning across turns; the + prefix tells the two apart from a signature Anthropic minted. """ + return f"{ENCRYPTED_REASONING_SIGNATURE_PREFIX}{encrypted_content}" + + +def _carries_encrypted_reasoning(signature: object) -> bool: + return isinstance(signature, str) and signature.startswith(ENCRYPTED_REASONING_SIGNATURE_PREFIX) + + +def encrypted_content_from_signature(signature: object) -> str | None: + if not isinstance(signature, str) or not _carries_encrypted_reasoning(signature): + return None + return signature.removeprefix(ENCRYPTED_REASONING_SIGNATURE_PREFIX) or None + + +def _encrypted_reasoning_field(block: Mapping[str, object]) -> object: + match block.get("type"): + case "thinking": + return block.get("signature") + case "redacted_thinking": + return block.get("data") + case _: + return None + + +def encrypted_content_of_block(block: Mapping[str, object]) -> str | None: + return encrypted_content_from_signature(_encrypted_reasoning_field(block)) + + +def is_encrypted_reasoning_block(block: object) -> bool: + """A thinking or redacted_thinking block carrying Responses API encrypted reasoning. + + Only the Responses API that minted the content can read it back, so an Anthropic + backend has to drop such a block rather than fail signature verification on it. + """ + if not isinstance(block, Mapping): + return False + mapping: Final = cast(Mapping[str, object], block) # cast-ok: narrowed by isinstance + return _carries_encrypted_reasoning(_encrypted_reasoning_field(mapping)) + + +def strip_encrypted_reasoning_from_messages(messages: object) -> None: + """Drop the bridge-tagged reasoning blocks a routed deployment cannot decrypt from + Anthropic-shaped history. + + The whole block goes, the way #40280 drops undecryptable Responses ``input`` items: a + provider that did not mint the block rejects it signed (a foreign signature) and unsigned + (a missing signature) alike, so keeping its text as an unsigned thinking block only moves + the 400 from the router to the provider. + + Mutates the content lists in place: the router's fallback snapshot shares these + message objects, so a rebound list would replay the stripped blocks on the fallback hop. + """ + if not isinstance(messages, list): + return + for content in _anthropic_content_lists(cast(list[object], messages)): # cast-ok: untyped client json + _strip_encrypted_reasoning_from_blocks(content) + + +def _anthropic_content_lists(messages: Sequence[object]) -> Iterator[object]: + return ( + cast(list[object], content) # cast-ok: narrowed by isinstance + for message in messages + if isinstance(message, Mapping) + for content in (cast(Mapping[str, object], message).get("content"),) # cast-ok: narrowed by isinstance + if isinstance(content, list) + ) + + +def _strip_encrypted_reasoning_from_blocks(content: object) -> None: + blocks: Final = cast(list[object], content) # cast-ok: narrowed by the caller's isinstance + kept: Final = tuple(block for block in blocks if not is_encrypted_reasoning_block(block)) + blocks[:] = kept # rebind-ok: shared with fallback snapshot + + +def _reasoning_replay_group_key(indexed_block: tuple[int, Mapping[str, object]]) -> str: + index, block = indexed_block + return f"encrypted:{index}" if is_encrypted_reasoning_block(block) else "summary" + + +def _reasoning_item_from_block_group(group: tuple[Mapping[str, object], ...]) -> ChatCompletionReasoningItem | None: summary: Final[list[ChatCompletionReasoningSummaryTextBlock]] = [ # mutable-ok: API message payload ChatCompletionReasoningSummaryTextBlock(type="summary_text", text=text) - for block in thinking_blocks + for block in group if (text := _readable_thinking_text(block)) ] + encrypted_content: Final = encrypted_content_of_block(group[0]) + if encrypted_content is not None: + return ChatCompletionReasoningItem(type="reasoning", summary=summary, encrypted_content=encrypted_content) if not summary: return None return ChatCompletionReasoningItem(type="reasoning", summary=summary) +def responses_reasoning_items_from_thinking_blocks( + thinking_blocks: Iterable[Mapping[str, object]], +) -> tuple[ChatCompletionReasoningItem, ...]: + """Build Responses API `reasoning` input items from Anthropic thinking blocks. + + A block carrying encrypted reasoning replays the item it came from byte for byte; + a run of plain thinking blocks collapses into one summary-only item. No item carries + an `id`: the Responses API 404s on any id it did not mint itself and rejects an empty + one, while an item without an id is always accepted. + """ + return tuple( + item + for _, group in groupby(enumerate(thinking_blocks), key=_reasoning_replay_group_key) + if (item := _reasoning_item_from_block_group(tuple(block for _, block in group))) is not None + ) + + def _parse_content_for_reasoning( message_text: str | None, ) -> tuple[str | None, str | None]: diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 56c1d605700..ece619e3883 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -46,6 +46,7 @@ from litellm.types.utils import GenericImageParsingChunk from .common_utils import ( convert_content_list_to_str, infer_content_type_from_url_and_content, + is_encrypted_reasoning_block, is_non_content_values_set, parse_tool_call_arguments, ) @@ -2299,13 +2300,16 @@ def sanitize_messages_for_tool_calling( def _is_unsignable_thinking_block(block: object) -> bool: - """A `thinking` block that Anthropic cannot accept on input. + """A thinking block that Anthropic cannot accept on input. Anthropic verifies the thinking signature cryptographically, so a block whose signature is null, empty, or missing (e.g. from an open-source reasoning model) - is rejected with a 400 and must be dropped rather than blanked or repaired. - `redacted_thinking` blocks carry no signature and are always kept. + is rejected with a 400 and must be dropped rather than blanked or repaired, and + so is a block whose signature or data carries another provider's encrypted + reasoning. A `redacted_thinking` block Anthropic minted is always kept. """ + if is_encrypted_reasoning_block(block): + return True if not isinstance(block, dict) or block.get("type") != "thinking": return False signature: Final = block.get("signature") diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 2b57883cc13..ca520116606 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -21,6 +21,7 @@ from litellm.constants import ( ) from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_file_ids_from_messages, + is_encrypted_reasoning_block, ) from litellm.litellm_core_utils.prompt_templates.factory import ( THOUGHT_SIGNATURE_SEPARATOR, @@ -1201,6 +1202,32 @@ def strip_thinking_blocks_from_anthropic_messages(messages: list[Any]) -> list[A return out +def _without_encrypted_reasoning_blocks(message: dict) -> dict | None: # mutable-ok: Anthropic message payload shape + if not isinstance(message, Mapping): + return message + content: Final = message.get("content") + if not isinstance(content, list): + return message + kept: Final = [b for b in content if not is_encrypted_reasoning_block(b)] # mutable-ok: API message payload + if len(kept) == len(content): + return message + if not kept: + return None + return {**message, "content": kept} # mutable-ok: API message payload + + +def strip_encrypted_reasoning_blocks_from_anthropic_messages( + messages: Sequence[dict], # mutable-ok: Anthropic message payload shape +) -> list[dict]: # mutable-ok: AnthropicMessagesRequest.messages is typed list[dict] + """ + Drop thinking / redacted_thinking blocks that carry another provider's encrypted + reasoning (a turn the Responses API bridge served) before the request reaches + Anthropic, which cannot verify them. Anthropic's own signed blocks are kept. + """ + stripped: Final = (_without_encrypted_reasoning_blocks(m) for m in messages) + return [m for m in stripped if m is not None] # mutable-ok: API message payload + + def strip_thinking_blocks_from_anthropic_messages_request_dict( data: dict[str, Any], ) -> None: diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index 4bce760e943..9b3a57cc422 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -113,6 +113,7 @@ from litellm.litellm_core_utils.reasoning_effort_utils import ( from litellm.llms.anthropic.common_utils import ( is_empty_unsigned_thinking_block, normalize_anthropic_tool_use_id, + strip_encrypted_reasoning_blocks_from_anthropic_messages, ) from litellm.llms.anthropic.experimental_pass_through.context_management import ( PolyfillResult, @@ -417,7 +418,8 @@ class LiteLLMAnthropicMessagesAdapter: model: str | None = None, ) -> list: new_messages: Final[list[AllMessageValues]] = [] - for m in messages: + replayable_messages: Final = strip_encrypted_reasoning_blocks_from_anthropic_messages(messages) + for m in replayable_messages: user_message: ChatCompletionUserMessage | None = None tool_message_list: list[ChatCompletionToolMessage] = [] new_user_content_list: list[ChatCompletionTextObject | ChatCompletionImageObject] = [] diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py index 8267da157ad..9f9346fad4d 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py @@ -25,6 +25,7 @@ from ...common_utils import ( AnthropicModelInfo, optionally_handle_anthropic_oauth, strip_advisor_blocks_from_messages, + strip_encrypted_reasoning_blocks_from_anthropic_messages, ) DEFAULT_ANTHROPIC_API_VERSION: Final = "2023-06-01" @@ -613,7 +614,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): messages = strip_advisor_blocks_from_messages(messages) anthropic_messages_request: Final[AnthropicMessagesRequest] = AnthropicMessagesRequest( - messages=messages, + messages=strip_encrypted_reasoning_blocks_from_anthropic_messages(messages), max_tokens=max_tokens, model=model, **anthropic_messages_optional_request_params, diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py index 0445c23ed8c..7731c883d9f 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py @@ -19,6 +19,7 @@ from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, ) from litellm.types.llms.openai import ResponsesAPIResponse +from litellm.utils import ProviderConfigManager from ..utils import litellm_logging_obj_from_kwargs, local_model_name from .streaming_iterator import AnthropicResponsesStreamWrapper @@ -34,6 +35,15 @@ def _forwarded_kwargs(extra_kwargs: Mapping[str, object] | None) -> Mapping[str, return extra_kwargs or {} +def _provider_returns_encrypted_reasoning(model: str, custom_llm_provider: object) -> bool: + provider: Final = ( + custom_llm_provider if isinstance(custom_llm_provider, str) else litellm.get_llm_provider(model=model)[1] + ) + provider_model: Final = local_model_name(model, provider) + responses_config: Final = ProviderConfigManager.get_provider_responses_api_config(provider, provider_model) + return responses_config is not None and "include" in responses_config.get_supported_openai_params(provider_model) + + def _build_responses_kwargs( *, max_tokens: int, @@ -85,8 +95,13 @@ def _build_responses_kwargs( request_data["output_format"] = output_format anthropic_request: Final = AnthropicMessagesRequest(**request_data) - responses_kwargs: Final = _ADAPTER.translate_request(anthropic_request) forwarded_kwargs: Final = _forwarded_kwargs(extra_kwargs) + responses_kwargs: Final = _ADAPTER.translate_request( + anthropic_request, + include_encrypted_reasoning=_provider_returns_encrypted_reasoning( + model, forwarded_kwargs.get("custom_llm_provider") + ), + ) # Normalize reasoning effort based on model capabilities # (e.g. "max" → "xhigh"/"high", "minimal" → "low" if unsupported) @@ -111,7 +126,7 @@ def _build_responses_kwargs( responses_kwargs["stream"] = True # Forward litellm-specific kwargs (api_key, api_base, logging obj, etc.) - excluded: Final = {"anthropic_messages"} + excluded: Final = frozenset(("anthropic_messages",)) for key, value in forwarded_kwargs.items(): if key == "litellm_logging_obj" and value is not None: from litellm.litellm_core_utils.litellm_logging import ( @@ -132,6 +147,14 @@ def _build_responses_kwargs( if explicit_prompt_cache_key is not None: responses_kwargs["prompt_cache_key"] = explicit_prompt_cache_key + deployment_include: Final = forwarded_kwargs.get("include") + bridge_include: Final = responses_kwargs.get("include") + if isinstance(deployment_include, list) and isinstance(bridge_include, list): + responses_kwargs["include"] = [ + *bridge_include, + *(item for item in deployment_include if item not in bridge_include), + ] + return responses_kwargs diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py index 2e0a6a9df8f..f753e87fee3 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py @@ -9,13 +9,19 @@ from typing import TYPE_CHECKING, Any, Final from litellm import verbose_logger from litellm._uuid import uuid +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + encrypted_reasoning_signature, +) from litellm.llms.anthropic.experimental_pass_through.messages.utils import ( refusal_stop_details, responses_output_refusal_text, ) from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage -from .transformation import LiteLLMAnthropicToResponsesAPIAdapter +from .transformation import ( + REASONING_SUMMARY_PART_SEPARATOR, + LiteLLMAnthropicToResponsesAPIAdapter, +) if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject @@ -29,9 +35,10 @@ class AnthropicResponsesStreamWrapper: response.created -> message_start response.output_item.added -> content_block_start (if message/function_call) response.output_text.delta -> content_block_delta (text_delta) + response.reasoning_summary_part.added -> content_block_delta (thinking_delta separator) response.reasoning_summary_text.delta -> content_block_delta (thinking_delta) response.function_call_arguments.delta -> content_block_delta (input_json_delta) - response.output_item.done -> content_block_stop + response.output_item.done -> content_block_delta (signature_delta) + content_block_stop response.completed -> message_delta + message_stop """ @@ -94,6 +101,38 @@ class AnthropicResponsesStreamWrapper: ) return block_idx + @staticmethod + def _field(source: object, name: str) -> object: + return source.get(name) if isinstance(source, dict) else getattr(source, name, None) + + def _close_reasoning_item(self, item: object, item_id: str | None) -> None: + block_idx: Final = self._item_id_to_block_index.get(item_id, -1) if item_id else self._current_block_index + encrypted_content: Final = self._field(item, "encrypted_content") + signature: Final = ( + encrypted_reasoning_signature(encrypted_content) + if isinstance(encrypted_content, str) and encrypted_content + else None + ) + if block_idx < 0 and signature is None: + return + if block_idx < 0: + redacted_idx: Final = self._open_block( + item_id, + {"type": "redacted_thinking", "data": signature}, # mutable-ok: API message payload + ) + stop: Final = {"type": "content_block_stop", "index": redacted_idx} # mutable-ok: API message payload + self._chunk_queue.append(stop) + return + if signature is not None: + self._chunk_queue.append( + { # mutable-ok: API message payload + "type": "content_block_delta", + "index": block_idx, + "delta": {"type": "signature_delta", "signature": signature}, # mutable-ok: API message payload + } + ) + self._chunk_queue.append({"type": "content_block_stop", "index": block_idx}) # mutable-ok: API message payload + def _process_event(self, event: object) -> None: """Convert one Responses API event into zero or more Anthropic chunks queued for emission.""" event_type = getattr(event, "type", None) @@ -175,6 +214,26 @@ class AnthropicResponsesStreamWrapper: ) return + if event_type == "response.reasoning_summary_part.added": + part_item_id: Final = self._field(event, "item_id") + summary_index: Final = self._field(event, "summary_index") + part_block_idx: Final = ( + self._item_id_to_block_index.get(part_item_id, -1) if isinstance(part_item_id, str) else -1 + ) + if part_block_idx < 0 or not isinstance(summary_index, int) or summary_index == 0: + return + self._chunk_queue.append( + { # mutable-ok: API message payload + "type": "content_block_delta", + "index": part_block_idx, + "delta": { # mutable-ok: API message payload + "type": "thinking_delta", + "thinking": REASONING_SUMMARY_PART_SEPARATOR, + }, + } + ) + return + # ---- reasoning summary text delta ---- if event_type == "response.reasoning_summary_text.delta": item_id = getattr(event, "item_id", None) or (event.get("item_id") if isinstance(event, dict) else None) @@ -220,6 +279,9 @@ class AnthropicResponsesStreamWrapper: item_id = ( getattr(item, "id", None) or (item.get("id") if isinstance(item, dict) else None) if item else None ) + if self._field(item, "type") == "reasoning": + self._close_reasoning_item(item, item_id) + return block_idx = self._item_id_to_block_index.get(item_id, -1) if item_id else self._current_block_index if block_idx < 0: return diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py index f1daf2be42a..1fdb0318bab 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py @@ -13,7 +13,8 @@ from typing import Any, Final, cast from litellm.litellm_core_utils.prompt_templates.common_utils import ( TOOL_RESULT_IMAGE_BOUNDARY, TOOL_RESULT_IMAGE_PLACEHOLDER, - responses_reasoning_item_from_thinking_blocks, + encrypted_reasoning_signature, + responses_reasoning_items_from_thinking_blocks, with_prompt_cache_breakpoint, ) from litellm.litellm_core_utils.reasoning_effort_utils import ( @@ -33,6 +34,7 @@ from litellm.types.llms.anthropic import ( AnthropicFinishReason, AnthropicMessagesRequest, AnthropicMessagesToolChoice, + AnthropicResponseContentBlockRedactedThinking, AnthropicResponseContentBlockText, AnthropicResponseContentBlockThinking, AnthropicResponseContentBlockToolUse, @@ -43,11 +45,13 @@ from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicUsage, ) from litellm.types.llms.openai import ( - ChatCompletionThinkingBlock, ResponseAPIUsage, ResponsesAPIResponse, ) +REASONING_SUMMARY_PART_SEPARATOR: Final = "\n\n" +RESPONSES_INCLUDE_ENCRYPTED_REASONING: Final = "reasoning.encrypted_content" + class LiteLLMAnthropicToResponsesAPIAdapter: """ @@ -163,49 +167,55 @@ class LiteLLMAnthropicToResponsesAPIAdapter: return str(getattr(part, "text", None) or "") @classmethod - def _thinking_blocks_from_reasoning_item( + def _thinking_block_from_reasoning_item( cls, summary: Iterable[object], - ) -> tuple[dict[str, Any], ...]: # mutable-ok: API message payload - """Anthropic thinking blocks for one Responses reasoning item. + encrypted_content: object, + ) -> dict[str, Any] | None: # mutable-ok: API message payload + """The one Anthropic block for a Responses reasoning item. - The signature stays empty: only Anthropic can sign a thinking block, and a stand-in - value would be replayed as a real one and rejected by every backend that verifies it. + The item's encrypted reasoning rides the block's opaque field (`signature`, or + `data` when there is no summary text) so the client echoes it back and the next + turn replays the very item OpenAI produced; without it the signature stays empty, + since only Anthropic can sign a thinking block. """ - return tuple( - AnthropicResponseContentBlockThinking( - type="thinking", - thinking=text, - signature=None, - ).model_dump() - for part in summary - if (text := cls._summary_part_text(part)) + text: Final = REASONING_SUMMARY_PART_SEPARATOR.join( + part_text for part in summary if (part_text := cls._summary_part_text(part)) ) + if not isinstance(encrypted_content, str) or not encrypted_content: + if not text: + return None + return AnthropicResponseContentBlockThinking(type="thinking", thinking=text, signature=None).model_dump() + signature: Final = encrypted_reasoning_signature(encrypted_content) + if not text: + return AnthropicResponseContentBlockRedactedThinking(type="redacted_thinking", data=signature).model_dump() + return AnthropicResponseContentBlockThinking(type="thinking", thinking=text, signature=signature).model_dump() @staticmethod def _assistant_block_group_key(indexed_block: tuple[int, Mapping[str, object]]) -> str: """Group a run of consecutive thinking blocks together; keep every other block alone.""" index, block = indexed_block - return "thinking" if block.get("type") == "thinking" else f"block:{index}" + return "thinking" if block.get("type") in ("thinking", "redacted_thinking") else f"block:{index}" @classmethod - def _assistant_group_to_input_item( + def _assistant_group_to_input_items( cls, group: tuple[Mapping[str, object], ...] - ) -> dict[str, Any] | None: # mutable-ok: API message payload + ) -> tuple[dict[str, Any], ...]: # mutable-ok: API message payload first: Final = group[0] btype: Final = first.get("type") - if btype == "thinking": - blocks: Final = cast(tuple[ChatCompletionThinkingBlock, ...], group) # cast-ok: untrusted client payload - reasoning_item: Final = responses_reasoning_item_from_thinking_blocks(blocks) - return None if reasoning_item is None else dict(reasoning_item) # mutable-ok: API message payload + if btype in ("thinking", "redacted_thinking"): + replayed: Final = responses_reasoning_items_from_thinking_blocks(group) + return tuple(dict(item) for item in replayed) # mutable-ok: API message payload if btype == "tool_use": - return { # mutable-ok: API message payload - "type": "function_call", - "call_id": first.get("id", ""), - "name": first.get("name", ""), - "arguments": json.dumps(first.get("input", {})), # mutable-ok: API message payload - } - return None + return ( + { # mutable-ok: API message payload + "type": "function_call", + "call_id": first.get("id", ""), + "name": first.get("name", ""), + "arguments": json.dumps(first.get("input", {})), # mutable-ok: API message payload + }, + ) + return () def translate_messages_to_responses_input( self, @@ -362,7 +372,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: input_items.extend( item for _, group in groupby(enumerate(blocks), key=self._assistant_block_group_key) - if (item := self._assistant_group_to_input_item(tuple(block for _, block in group))) is not None + for item in self._assistant_group_to_input_items(tuple(block for _, block in group)) ) asst_parts: list[dict[str, Any]] = [ # mutable-ok: API message payload {"type": "output_text", "text": block.get("text", "")} # mutable-ok: API message payload @@ -495,10 +505,16 @@ class LiteLLMAnthropicToResponsesAPIAdapter: def translate_request( self, anthropic_request: AnthropicMessagesRequest, + include_encrypted_reasoning: bool = True, ) -> dict[str, Any]: """ Translate a full Anthropic /v1/messages request dict to litellm.responses() / litellm.aresponses() kwargs. + + ``include_encrypted_reasoning`` asks the provider for ``reasoning.encrypted_content`` + on every call, so a reasoning model's items can be replayed intact next turn even + when the client sent no ``thinking`` block; pass False for a provider whose + Responses API rejects ``include``. """ model: Final[str] = anthropic_request["model"] messages_list: Final = cast( @@ -528,6 +544,8 @@ class LiteLLMAnthropicToResponsesAPIAdapter: "model": model, "input": input_items, } + if include_encrypted_reasoning: + responses_kwargs["include"] = [RESPONSES_INCLUDE_ENCRYPTED_REASONING] # mutable-ok: API request payload if system and not developer_parts: if isinstance(system, str): @@ -634,7 +652,9 @@ class LiteLLMAnthropicToResponsesAPIAdapter: for item in response.output: if isinstance(item, ResponseReasoningItem): - content.extend(self._thinking_blocks_from_reasoning_item(item.summary)) + reasoning_block = self._thinking_block_from_reasoning_item(item.summary, item.encrypted_content) + if reasoning_block is not None: + content.append(reasoning_block) elif isinstance(item, ResponseOutputMessage): for part in item.content: @@ -684,11 +704,12 @@ class LiteLLMAnthropicToResponsesAPIAdapter: ).model_dump() ) elif item_type == "reasoning": - content.extend( - self._thinking_blocks_from_reasoning_item( - cast(Iterable[object], item.get("summary") or ()), # cast-ok: untyped provider json - ) + reasoning_block = self._thinking_block_from_reasoning_item( + cast(Iterable[object], item.get("summary") or ()), # cast-ok: untyped provider json + item.get("encrypted_content"), ) + if reasoning_block is not None: + content.append(reasoning_block) elif item_type == "function_call": try: input_data = json.loads(item.get("arguments", "{}")) diff --git a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py index d10153881d9..cdd70e6baf2 100644 --- a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py @@ -37,7 +37,7 @@ Safe to enable globally: """ import time -from collections.abc import Mapping +from collections.abc import Iterator, Mapping from typing import TYPE_CHECKING, Final, Optional, Protocol, cast import httpx @@ -48,6 +48,10 @@ from litellm.exceptions import ( ServiceUnavailableError, ) from litellm.integrations.custom_logger import CustomLogger, Span +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + encrypted_content_of_block, + strip_encrypted_reasoning_from_messages, +) from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.router_utils.cooldown_cache import CooldownCacheValue from litellm.types.llms.openai import AllMessageValues @@ -138,15 +142,48 @@ class EncryptedContentAffinityCheck(CustomLogger): # If no encoded ID, check if encrypted_content itself is wrapped encrypted_content = item.get("encrypted_content") if encrypted_content and isinstance(encrypted_content, str): - ( - model_id, - _, - ) = ResponsesAPIRequestUtils._unwrap_encrypted_content_with_model_id(encrypted_content) + model_id = EncryptedContentAffinityCheck._model_id_from_wrapped_encrypted_content(encrypted_content) if model_id: return model_id return None + @staticmethod + def _anthropic_content_blocks(messages: object) -> Iterator[Mapping[str, object]]: + if not isinstance(messages, list): + return iter(()) + return ( + cast(Mapping[str, object], block) # cast-ok: narrowed by isinstance + for message in cast(list[object], messages) # cast-ok: narrowed by isinstance + if isinstance(message, Mapping) + for content in (cast(Mapping[str, object], message).get("content"),) # cast-ok: narrowed by isinstance + if isinstance(content, list) + for block in cast(list[object], content) # cast-ok: narrowed by isinstance + if isinstance(block, Mapping) + ) + + @staticmethod + def _model_id_from_wrapped_encrypted_content(encrypted_content: str) -> str | None: + model_id, _ = ResponsesAPIRequestUtils._unwrap_encrypted_content_with_model_id(encrypted_content) + return model_id or None + + @staticmethod + def _extract_model_id_from_anthropic_messages(messages: object) -> str | None: + return next( + ( + model_id + for block in EncryptedContentAffinityCheck._anthropic_content_blocks(messages) + if (encrypted_content := encrypted_content_of_block(block)) is not None + if ( + model_id := EncryptedContentAffinityCheck._model_id_from_wrapped_encrypted_content( + encrypted_content + ) + ) + is not None + ), + None, + ) + @staticmethod def _find_deployment_by_model_id(healthy_deployments: list[dict], model_id: str) -> dict | None: for deployment in healthy_deployments: @@ -240,8 +277,9 @@ class EncryptedContentAffinityCheck(CustomLogger): parent_otel_span: Span | None = None, ) -> list[dict]: """ - If the request ``input`` contains litellm-encoded item IDs, decode the - embedded ``model_id`` and pin the request to that deployment. Raises + If the request ``input`` contains litellm-encoded item IDs, or its Anthropic + ``messages`` replay a bridge-tagged thinking block, decode the embedded + ``model_id`` and pin the request to that deployment. Raises ``RateLimitError`` / ``ServiceUnavailableError`` when the originating deployment is a member of the routed model group but currently unavailable and no encryption-boundary peer exists, rather than dispatching a doomed @@ -270,12 +308,15 @@ class EncryptedContentAffinityCheck(CustomLogger): request_kwargs["litellm_metadata"]["encrypted_content_affinity_enabled"] = True request_input: Final = request_kwargs.get("input") - model_id: Final = self._extract_model_id_from_input(request_input) + anthropic_messages: Final = messages or request_kwargs.get("messages") + model_id: Final = self._extract_model_id_from_input( + request_input + ) or self._extract_model_id_from_anthropic_messages(anthropic_messages) if not model_id: return typed_healthy_deployments verbose_router_logger.debug( - "EncryptedContentAffinityCheck: decoded model_id=%s from input item IDs", + "EncryptedContentAffinityCheck: decoded model_id=%s from the request's encrypted content markers", model_id, ) @@ -327,6 +368,7 @@ class EncryptedContentAffinityCheck(CustomLogger): model, ) ResponsesAPIRequestUtils.strip_encrypted_reasoning_from_input(request_input) + strip_encrypted_reasoning_from_messages(anthropic_messages) return typed_healthy_deployments # The origin is a member of the routed group but currently unavailable (cooled down); fail fast diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index c037f928593..7c1445d79c9 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -1,3 +1,4 @@ +import copy import functools import json import os @@ -7,14 +8,20 @@ import pytest from litellm.litellm_core_utils.prompt_templates.common_utils import ( + ENCRYPTED_REASONING_SIGNATURE_PREFIX, TOOL_RESULT_IMAGE_BOUNDARY, TOOL_RESULT_IMAGE_PLACEHOLDER, add_system_prompt_to_messages, + encrypted_content_from_signature, + encrypted_reasoning_signature, get_file_ids_from_messages, get_format_from_file_id, handle_any_messages_to_chat_completion_str_messages_conversion, hoist_images_from_tool_messages, + is_encrypted_reasoning_block, + responses_reasoning_items_from_thinking_blocks, split_concatenated_json_objects, + strip_encrypted_reasoning_from_messages, update_messages_with_model_file_ids, ) @@ -1554,3 +1561,117 @@ class TestRequestContainsImageContent: for _ in range(50): nested = {"type": "tool_result", "content": [nested]} assert request_contains_image_content([{"role": "user", "content": [nested]}]) is False + + +class TestEncryptedReasoningReplay: + """Regression for https://github.com/BerriAI/litellm/issues/40288.""" + + def test_signature_round_trips_the_encrypted_content(self): + assert encrypted_content_from_signature(encrypted_reasoning_signature("gAAAA_bytes")) == "gAAAA_bytes" + + @pytest.mark.parametrize( + "signature", [None, "", "ErcBCkgIValidAnthropicSignature", "litellm_encrypted_reasoning:", 7] + ) + def test_anything_else_is_not_encrypted_content(self, signature): + assert encrypted_content_from_signature(signature) is None + + def test_encrypted_thinking_block_replays_its_own_item(self): + items = responses_reasoning_items_from_thinking_blocks( + [{"type": "thinking", "thinking": "Plan.", "signature": encrypted_reasoning_signature("gAAAA_1")}] + ) + assert items == ( + { + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "Plan."}], + "encrypted_content": "gAAAA_1", + }, + ) + + def test_encrypted_redacted_block_replays_with_an_empty_summary(self): + items = responses_reasoning_items_from_thinking_blocks( + [{"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAAA_1")}] + ) + assert items == ({"type": "reasoning", "summary": [], "encrypted_content": "gAAAA_1"},) + + def test_plain_blocks_collapse_into_one_summary_item_around_encrypted_ones(self): + items = responses_reasoning_items_from_thinking_blocks( + [ + {"type": "thinking", "thinking": "A.", "signature": None}, + {"type": "thinking", "thinking": "B.", "signature": ""}, + {"type": "thinking", "thinking": "C.", "signature": encrypted_reasoning_signature("gAAAA_c")}, + {"type": "redacted_thinking", "data": "anthropic-minted-opaque-data"}, + {"type": "thinking", "thinking": "D."}, + ] + ) + assert items == ( + { + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "A."}, {"type": "summary_text", "text": "B."}], + }, + {"type": "reasoning", "summary": [{"type": "summary_text", "text": "C."}], "encrypted_content": "gAAAA_c"}, + {"type": "reasoning", "summary": [{"type": "summary_text", "text": "D."}]}, + ) + assert all("id" not in item for item in items) + + def test_blocks_without_text_or_encrypted_content_produce_nothing(self): + assert responses_reasoning_items_from_thinking_blocks([{"type": "thinking", "thinking": ""}]) == () + assert responses_reasoning_items_from_thinking_blocks([]) == () + + @pytest.mark.parametrize( + ("block", "expected"), + [ + ({"type": "thinking", "thinking": "x", "signature": encrypted_reasoning_signature("g")}, True), + ({"type": "redacted_thinking", "data": encrypted_reasoning_signature("g")}, True), + ({"type": "thinking", "thinking": "x", "signature": ENCRYPTED_REASONING_SIGNATURE_PREFIX}, True), + ({"type": "redacted_thinking", "data": ENCRYPTED_REASONING_SIGNATURE_PREFIX}, True), + ({"type": "thinking", "thinking": "x", "signature": "ErcBCkgIValid"}, False), + ({"type": "redacted_thinking", "data": "EmwKAhgBEgy"}, False), + ({"type": "text", "text": encrypted_reasoning_signature("g")}, False), + ("not a block", False), + ], + ) + def test_is_encrypted_reasoning_block(self, block, expected): + assert is_encrypted_reasoning_block(block) is expected + + def test_strip_drops_every_bridge_tagged_block_and_leaves_no_unsigned_thinking_behind(self): + assistant_content = [ + {"type": "thinking", "thinking": "minted by Anthropic", "signature": "ErcBCkgIValid"}, + {"type": "thinking", "thinking": "packed by the bridge", "signature": encrypted_reasoning_signature("g1")}, + {"type": "redacted_thinking", "data": encrypted_reasoning_signature("g2")}, + {"type": "thinking", "thinking": "", "signature": encrypted_reasoning_signature("g3")}, + {"type": "text", "text": "answer"}, + ] + messages = [ + {"role": "user", "content": "question"}, + {"role": "assistant", "content": assistant_content}, + {"role": "user", "content": [{"type": "text", "text": "follow-up"}]}, + ] + + strip_encrypted_reasoning_from_messages(messages) + + assert messages[1]["content"] is assistant_content + assert assistant_content == [ + {"type": "thinking", "thinking": "minted by Anthropic", "signature": "ErcBCkgIValid"}, + {"type": "text", "text": "answer"}, + ] + assert all(block["signature"] for block in assistant_content if block["type"] == "thinking") + assert messages[0] == {"role": "user", "content": "question"} + assert messages[2] == {"role": "user", "content": [{"type": "text", "text": "follow-up"}]} + + @pytest.mark.parametrize( + "messages", + [ + "not a list", + None, + [{"role": "user", "content": None}], + [{"role": "user", "content": "plain string"}], + ["not a message"], + [{"role": "assistant", "content": [{"type": "thinking", "thinking": "x", "signature": "ErcBCkgIValid"}]}], + ], + ) + def test_strip_leaves_history_without_bridge_reasoning_untouched(self, messages): + before = copy.deepcopy(messages) + + strip_encrypted_reasoning_from_messages(messages) + + assert messages == before diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index dd2d45f00c6..66d10fd1407 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -191,8 +191,16 @@ def test_bedrock_converse_assistant_with_empty_thinking_block_and_tool_calls(): {"type": "thinking", "thinking": "oss reasoning", "signature": None}, {"type": "thinking", "thinking": "oss reasoning", "signature": ""}, {"type": "thinking", "thinking": "oss reasoning"}, + {"type": "thinking", "thinking": "openai reasoning", "signature": "litellm_encrypted_reasoning:gAAAA"}, + {"type": "redacted_thinking", "data": "litellm_encrypted_reasoning:gAAAA"}, + ], + ids=[ + "null_signature", + "empty_signature", + "missing_signature", + "encrypted_reasoning_signature", + "encrypted_reasoning_redacted_data", ], - ids=["null_signature", "empty_signature", "missing_signature"], ) def test_anthropic_messages_pt_drops_unsignable_thinking_block(thinking_block): """Open-source reasoning models (DeepSeek-R1, Qwen, etc.) emit thinking blocks @@ -219,7 +227,7 @@ def test_anthropic_messages_pt_drops_unsignable_thinking_block(thinking_block): assistant = next(m for m in result if m["role"] == "assistant") content = assistant["content"] assert all( - block.get("type") != "thinking" for block in content + block.get("type") not in ("thinking", "redacted_thinking") for block in content ), f"unsignable thinking block must be dropped, got {content!r}" assert any( block.get("type") == "text" and block.get("text") == "2+2 equals 4." diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index 00b3a0633f2..f6ee1cd71c0 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -10,6 +10,7 @@ import litellm from litellm.litellm_core_utils.prompt_templates.common_utils import ( TOOL_RESULT_IMAGE_PLACEHOLDER, + encrypted_reasoning_signature, ) from litellm.litellm_core_utils.prompt_templates.factory import ( THOUGHT_SIGNATURE_SEPARATOR, @@ -423,6 +424,43 @@ def test_translate_anthropic_messages_to_openai_thinking_blocks(): assert result[1]["tool_calls"][0]["id"] == "toolu_01234" +def test_translate_anthropic_messages_to_openai_drops_bridge_encrypted_reasoning_blocks(): + """A session that moves from an OpenAI reasoning model to a chat provider replays reasoning only OpenAI can read. + + Gemini rejects the whole request when such a block reaches it as a thought_signature, so the + adapter drops those blocks and keeps the provider-signed ones. + """ + + anthropic_messages = [ + AnthropicMessagesUserMessageParam( + role="user", + content=[{"type": "text", "text": "Who drinks water?"}], + ), + AnthopicMessagesAssistantMessageParam( + role="assistant", + content=[ + {"type": "thinking", "thinking": "plan", "signature": encrypted_reasoning_signature("gAAAA_1")}, + {"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAAA_2")}, + {"type": "text", "text": "The Norwegian."}, + ], + ), + AnthopicMessagesAssistantMessageParam( + role="assistant", + content=[ + {"type": "thinking", "thinking": "native", "signature": "EqQBCkYIAxgCIkA_signed"}, + {"type": "text", "text": "Still the Norwegian."}, + ], + ), + ] + + result = LiteLLMAnthropicMessagesAdapter().translate_anthropic_messages_to_openai(messages=anthropic_messages) + + assert [m["role"] for m in result] == ["user", "assistant", "assistant"] + assert not result[1].get("thinking_blocks") + assert result[1]["content"] == "The Norwegian." + assert [b["signature"] for b in result[2]["thinking_blocks"]] == ["EqQBCkYIAxgCIkA_signed"] + + def test_translate_anthropic_messages_to_openai_sets_reasoning_content(): """Reasoning-aware chat providers read reasoning_content, so thinking text must land there. diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_messages_encrypted_reasoning.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_messages_encrypted_reasoning.py new file mode 100644 index 00000000000..c64e9d392e5 --- /dev/null +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_messages_encrypted_reasoning.py @@ -0,0 +1,50 @@ +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + encrypted_reasoning_signature, +) +from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( + AnthropicMessagesConfig, +) + + +def _transform(messages): + return AnthropicMessagesConfig().transform_anthropic_messages_request( + model="claude-sonnet-4-5", + messages=messages, + anthropic_messages_optional_request_params={"max_tokens": 1024}, + litellm_params={}, + headers={}, + ) + + +def test_reasoning_replayed_from_the_responses_bridge_never_reaches_anthropic(): + """Claude Code resumed on a Claude model echoes the thinking blocks a gpt turn produced.""" + messages = [ + {"role": "user", "content": "Solve it."}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "plan", "signature": encrypted_reasoning_signature("gAAAA_1")}, + {"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAAA_2")}, + {"type": "text", "text": "The answer."}, + ], + }, + {"role": "user", "content": "And the next one?"}, + ] + request = _transform(messages) + assert request["messages"][1]["content"] == [{"type": "text", "text": "The answer."}] + assert len(messages[1]["content"]) == 3 + + +def test_anthropic_signed_thinking_blocks_are_forwarded_untouched(): + messages = [ + {"role": "user", "content": "Solve it."}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "plan", "signature": "EqQBCkYIAxgCIkA_anthropic_signed"}, + {"type": "redacted_thinking", "data": "EmwKAhgBEgy_anthropic_minted"}, + {"type": "text", "text": "The answer."}, + ], + }, + ] + assert _transform(messages)["messages"] == messages diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_handler.py index 16e8cf0e90e..b66075f691b 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_handler.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_handler.py @@ -67,6 +67,39 @@ def test_build_responses_kwargs_prefers_explicit_prompt_cache_key_over_derived() assert responses_kwargs["prompt_cache_key"] == "explicit-key" +def test_build_responses_kwargs_asks_openai_for_encrypted_reasoning_without_thinking(): + responses_kwargs = _build_responses_kwargs( + max_tokens=1024, + messages=MESSAGES, + model="openai/gpt-5.6-luna", + extra_kwargs={"custom_llm_provider": "openai"}, + ) + assert responses_kwargs["include"] == ["reasoning.encrypted_content"] + assert "reasoning" not in responses_kwargs + + +def test_build_responses_kwargs_skips_include_for_a_responses_provider_that_rejects_it(): + responses_kwargs = _build_responses_kwargs( + max_tokens=1024, + messages=MESSAGES, + model="perplexity/sonar", + thinking={"type": "enabled", "budget_tokens": 4096}, + extra_kwargs={"custom_llm_provider": "perplexity"}, + ) + assert "include" not in responses_kwargs + assert "reasoning" in responses_kwargs + + +def test_build_responses_kwargs_keeps_the_deployment_include_next_to_encrypted_reasoning(): + responses_kwargs = _build_responses_kwargs( + max_tokens=1024, + messages=MESSAGES, + model="openai/gpt-5.6-luna", + extra_kwargs={"custom_llm_provider": "openai", "include": ["file_search_call.results"]}, + ) + assert responses_kwargs["include"] == ["reasoning.encrypted_content", "file_search_call.results"] + + def test_build_responses_kwargs_without_metadata_sets_no_prompt_cache_key(): responses_kwargs = _build_responses_kwargs( max_tokens=1024, diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py index d9df5df426a..bfe2d6b7cea 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py @@ -10,6 +10,9 @@ from types import SimpleNamespace sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../../.."))) +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + encrypted_reasoning_signature, +) from litellm.llms.anthropic.experimental_pass_through.responses_adapters.streaming_iterator import ( AnthropicResponsesStreamWrapper, ) @@ -114,7 +117,7 @@ class TestReasoningItemWithoutSummaryText: """ @staticmethod - def _gpt_turn(reasoning_summary_deltas: list) -> list: + def _gpt_turn(reasoning_summary_deltas: list, encrypted_content: str | None = None) -> list: return [ {"type": "response.created"}, {"type": "response.output_item.added", "item": {"type": "reasoning", "id": "rs_1"}}, @@ -122,7 +125,10 @@ class TestReasoningItemWithoutSummaryText: {"type": "response.reasoning_summary_text.delta", "item_id": "rs_1", "delta": delta} for delta in reasoning_summary_deltas ), - {"type": "response.output_item.done", "item": {"type": "reasoning", "id": "rs_1"}}, + { + "type": "response.output_item.done", + "item": {"type": "reasoning", "id": "rs_1", "encrypted_content": encrypted_content}, + }, {"type": "response.output_item.added", "item": {"type": "message", "id": "msg_1"}}, {"type": "response.output_text.delta", "item_id": "msg_1", "delta": "Hello"}, {"type": "response.output_item.done", "item": {"type": "message", "id": "msg_1"}}, @@ -171,6 +177,70 @@ class TestReasoningItemWithoutSummaryText: assert not [c for c in chunks if c.get("delta", {}).get("type") == "signature_delta"] +_ENCRYPTED_REASONING = "gAAAAABp_encrypted_reasoning_bytes_only_openai_can_read" + + +class TestEncryptedReasoningIsStreamedForReplay: + """Regression for https://github.com/BerriAI/litellm/issues/40288. + + The client echoes a thinking block's signature (or a redacted block's data) back on the + next turn, so the item's ``encrypted_content`` has to reach it through one of those. + """ + + def test_encrypted_content_is_streamed_as_the_signature_before_the_block_closes(self): + chunks = _drain_async( + TestReasoningItemWithoutSummaryText._gpt_turn( + reasoning_summary_deltas=["Weighing options"], encrypted_content=_ENCRYPTED_REASONING + ) + ) + + assert [(c["type"], c.get("index"), c.get("delta", {}).get("type")) for c in chunks[1:5]] == [ + ("content_block_start", 0, None), + ("content_block_delta", 0, "thinking_delta"), + ("content_block_delta", 0, "signature_delta"), + ("content_block_stop", 0, None), + ] + assert chunks[3]["delta"]["signature"] == encrypted_reasoning_signature(_ENCRYPTED_REASONING) + + def test_reasoning_without_summary_streams_a_redacted_thinking_block(self): + chunks = _drain_async( + TestReasoningItemWithoutSummaryText._gpt_turn( + reasoning_summary_deltas=[], encrypted_content=_ENCRYPTED_REASONING + ) + ) + + assert [(c["type"], c.get("index")) for c in chunks[1:]] == [ + ("content_block_start", 0), + ("content_block_stop", 0), + ("content_block_start", 1), + ("content_block_delta", 1), + ("content_block_stop", 1), + ] + assert chunks[1]["content_block"] == { + "type": "redacted_thinking", + "data": encrypted_reasoning_signature(_ENCRYPTED_REASONING), + } + + def test_summary_parts_are_separated_inside_the_one_thinking_block(self): + """Two summary parts read as two paragraphs, not as one run-on sentence.""" + events = [ + {"type": "response.created"}, + {"type": "response.output_item.added", "item": {"type": "reasoning", "id": "rs_1"}}, + {"type": "response.reasoning_summary_part.added", "item_id": "rs_1", "summary_index": 0}, + {"type": "response.reasoning_summary_text.delta", "item_id": "rs_1", "delta": "First."}, + {"type": "response.reasoning_summary_part.added", "item_id": "rs_1", "summary_index": 1}, + {"type": "response.reasoning_summary_text.delta", "item_id": "rs_1", "delta": "Second."}, + {"type": "response.output_item.done", "item": {"type": "reasoning", "id": "rs_1"}}, + ] + chunks = _process_all(events) + + thinking = "".join( + c["delta"]["thinking"] for c in chunks if c.get("delta", {}).get("type") == "thinking_delta" + ) + assert thinking == "First.\n\nSecond." + assert [c["type"] for c in chunks].count("content_block_start") == 1 + + class TestToolUseBlockClosedExactlyOnce: """Regression for https://github.com/BerriAI/litellm/issues/37273. diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py index edcc7adddb7..4ad559aa547 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py @@ -19,6 +19,7 @@ from litellm.constants import ( from litellm.litellm_core_utils.prompt_templates.common_utils import ( TOOL_RESULT_IMAGE_BOUNDARY, TOOL_RESULT_IMAGE_PLACEHOLDER, + encrypted_reasoning_signature, ) from litellm.llms.anthropic.experimental_pass_through.responses_adapters.transformation import ( LiteLLMAnthropicToResponsesAPIAdapter, @@ -566,6 +567,66 @@ class TestTranslateMessagesToResponsesInput: result = _translate_messages(messages) assert "id" not in result[0] + def test_thinking_block_with_encrypted_signature_replays_the_encrypted_content(self): + """Regression for https://github.com/BerriAI/litellm/issues/40288 (inbound fault site).""" + messages = [ + { + "role": "assistant", + "content": [ + { + "type": "thinking", + "thinking": "Private reasoning.", + "signature": encrypted_reasoning_signature("gAAAA_turn_one"), + } + ], + } + ] + result = _translate_messages(messages) + assert result == [ + { + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "Private reasoning."}], + "encrypted_content": "gAAAA_turn_one", + } + ] + + def test_redacted_thinking_with_encrypted_data_replays_the_encrypted_content(self): + messages = [ + { + "role": "assistant", + "content": [{"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAAA_turn_one")}], + } + ] + result = _translate_messages(messages) + assert result == [{"type": "reasoning", "summary": [], "encrypted_content": "gAAAA_turn_one"}] + + def test_each_encrypted_thinking_block_stays_its_own_reasoning_item(self): + """Two upstream items must not be merged into one, or the encrypted content of one is lost.""" + messages = [ + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "First.", "signature": encrypted_reasoning_signature("gAAAA_1")}, + {"type": "thinking", "thinking": "Second.", "signature": encrypted_reasoning_signature("gAAAA_2")}, + ], + } + ] + result = _translate_messages(messages) + assert [item["encrypted_content"] for item in result] == ["gAAAA_1", "gAAAA_2"] + + def test_anthropic_signed_thinking_block_replays_as_a_summary_only_item(self): + """A real Anthropic signature is opaque here, so it never masquerades as encrypted content.""" + messages = [ + { + "role": "assistant", + "content": [{"type": "thinking", "thinking": "Private reasoning.", "signature": "ErcBCkgIValid"}], + } + ] + result = _translate_messages(messages) + assert result == [ + {"type": "reasoning", "summary": [{"type": "summary_text", "text": "Private reasoning."}]} + ] + def test_consecutive_thinking_blocks_become_one_reasoning_item(self): """Summary parts of one upstream reasoning item are regrouped into that item.""" messages = [ @@ -1102,6 +1163,23 @@ class TestTranslateRequestBroaderCoverage: kwargs = _ADAPTER.translate_request(req) assert "reasoning" not in kwargs + def test_thinking_asks_for_the_encrypted_reasoning(self): + """The documented way to get reasoning that survives store=false is to ask for it.""" + req = _make_request(thinking={"type": "enabled", "budget_tokens": 12000}) + kwargs = _ADAPTER.translate_request(req) + assert kwargs["include"] == ["reasoning.encrypted_content"] + + def test_encrypted_reasoning_is_asked_for_without_a_thinking_block(self): + """A reasoning model reasons whether or not the client sent `thinking`, so the replay needs it either way.""" + kwargs = _ADAPTER.translate_request(_make_request()) + assert kwargs["include"] == ["reasoning.encrypted_content"] + + def test_encrypted_reasoning_is_not_asked_for_when_the_provider_rejects_include(self): + req = _make_request(thinking={"type": "enabled", "budget_tokens": 12000}) + kwargs = _ADAPTER.translate_request(req, include_encrypted_reasoning=False) + assert kwargs["reasoning"] == {"effort": "high"} + assert "include" not in kwargs + def test_metadata_user_id_mapped_to_user(self): req = _make_request(metadata={"user_id": "user-42"}) kwargs = _ADAPTER.translate_request(req) @@ -1246,7 +1324,9 @@ def _make_function_call_item(call_id: str, name: str, arguments: str) -> MagicMo return item -def _make_reasoning_item(summaries: List[str], item_id: str = "rs_test_1") -> MagicMock: +def _make_reasoning_item( + summaries: List[str], item_id: str = "rs_test_1", encrypted_content: str | None = None +) -> MagicMock: """Build a mock ResponseReasoningItem.""" from openai.types.responses import ResponseReasoningItem # type: ignore[import] @@ -1259,9 +1339,13 @@ def _make_reasoning_item(summaries: List[str], item_id: str = "rs_test_1") -> Ma item = MagicMock(spec=ResponseReasoningItem) item.id = item_id item.summary = summary_mocks + item.encrypted_content = encrypted_content return item +_ENCRYPTED_REASONING = "gAAAAABp_encrypted_reasoning_bytes_only_openai_can_read" + + class TestTranslateResponse: """Responses API -> AnthropicMessagesResponse conversion.""" @@ -1386,7 +1470,81 @@ class TestTranslateResponse: reasoning = _make_reasoning_item(["Part one.", "Part two."], item_id="rs_abc123") response = _make_mock_response(output=[reasoning]) result: Any = _ADAPTER.translate_response(response) - assert [block["signature"] for block in result["content"]] == [None, None] + assert [block["signature"] for block in result["content"]] == [None] + assert "rs_abc123" not in json.dumps(result["content"]) + + def test_summary_parts_join_into_one_thinking_block(self): + """One reasoning item is one block, so its signature is echoed back exactly once.""" + reasoning = _make_reasoning_item(["Part one.", "Part two."]) + response = _make_mock_response(output=[reasoning]) + result: Any = _ADAPTER.translate_response(response) + assert [block["thinking"] for block in result["content"]] == ["Part one.\n\nPart two."] + + def test_encrypted_content_rides_the_thinking_signature(self): + """Regression for https://github.com/BerriAI/litellm/issues/40288 (outbound fault site).""" + reasoning = _make_reasoning_item(["Part one."], item_id="rs_abc123", encrypted_content=_ENCRYPTED_REASONING) + response = _make_mock_response(output=[reasoning]) + result: Any = _ADAPTER.translate_response(response) + assert result["content"] == [ + { + "type": "thinking", + "thinking": "Part one.", + "signature": encrypted_reasoning_signature(_ENCRYPTED_REASONING), + } + ] + + def test_reasoning_without_summary_becomes_redacted_thinking(self): + """With summaries off the encrypted reasoning still has to reach the client to be replayed.""" + reasoning = _make_reasoning_item([], encrypted_content=_ENCRYPTED_REASONING) + response = _make_mock_response(output=[reasoning]) + result: Any = _ADAPTER.translate_response(response) + assert result["content"] == [ + {"type": "redacted_thinking", "data": encrypted_reasoning_signature(_ENCRYPTED_REASONING)} + ] + + def test_dict_reasoning_item_carries_its_encrypted_content(self): + response = _make_mock_response( + output=[ + { + "type": "reasoning", + "id": "rs_dict_1", + "encrypted_content": _ENCRYPTED_REASONING, + "summary": [{"type": "summary_text", "text": "Weighing the options."}], + } + ] + ) + result: Any = _ADAPTER.translate_response(response) + assert result["content"][0]["signature"] == encrypted_reasoning_signature(_ENCRYPTED_REASONING) + + def test_reasoning_item_round_trip_is_byte_stable(self): + """Regression for https://github.com/BerriAI/litellm/issues/40288. + + The reasoning item the next turn replays must be the one OpenAI produced, with its + encrypted reasoning intact, and identical on every later turn so the prompt cache + prefix keeps matching. + """ + reasoning = _make_reasoning_item(["Part one.", "Part two."], encrypted_content=_ENCRYPTED_REASONING) + turn: Any = _ADAPTER.translate_response(_make_mock_response(output=[reasoning])) + history = [{"role": "assistant", "content": turn["content"]}] + + replayed_items = [_translate_messages(history) for _ in range(2)] + + assert replayed_items[0] == replayed_items[1] + assert replayed_items[0] == [ + { + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "Part one.\n\nPart two."}], + "encrypted_content": _ENCRYPTED_REASONING, + } + ] + + def test_redacted_reasoning_round_trip_replays_the_encrypted_content(self): + reasoning = _make_reasoning_item([], encrypted_content=_ENCRYPTED_REASONING) + turn: Any = _ADAPTER.translate_response(_make_mock_response(output=[reasoning])) + + replayed = _translate_messages([{"role": "assistant", "content": turn["content"]}]) + + assert replayed == [{"type": "reasoning", "summary": [], "encrypted_content": _ENCRYPTED_REASONING}] def test_dict_reasoning_item_becomes_thinking_block(self): """A reasoning item arriving as a plain dict is kept, not dropped.""" @@ -1402,14 +1560,26 @@ class TestTranslateResponse: result: Any = _ADAPTER.translate_response(response) assert result["content"] == [{"type": "thinking", "thinking": "Weighing the options.", "signature": None}] - def test_thinking_blocks_are_dropped_when_replayed_to_anthropic(self): + @pytest.mark.parametrize( + ("summaries", "encrypted_content"), + [ + (["Part one."], None), + (["Part one."], _ENCRYPTED_REASONING), + ([], _ENCRYPTED_REASONING), + ], + ids=["unsigned_thinking", "encrypted_thinking", "encrypted_redacted_thinking"], + ) + def test_thinking_blocks_are_dropped_when_replayed_to_anthropic(self, summaries, encrypted_content): """Replaying this turn to an Anthropic model must not send a signature it cannot verify.""" from litellm.litellm_core_utils.prompt_templates.factory import ( _drop_unsignable_thinking_blocks, ) - response = _make_mock_response(output=[_make_reasoning_item(["Part one."], item_id="rs_abc123")]) + response = _make_mock_response( + output=[_make_reasoning_item(summaries, item_id="rs_abc123", encrypted_content=encrypted_content)] + ) result: Any = _ADAPTER.translate_response(response) + assert len(result["content"]) == 1 assert _drop_unsignable_thinking_blocks(result["content"]) == [] def test_usage_mapped_correctly(self): diff --git a/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py b/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py index ae620fdd6dc..91652c89092 100644 --- a/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py @@ -42,12 +42,15 @@ FAKE_AUTH_TOKEN = "sk-ant-aut01-fake-auth-token-for-testing-123456789" def test_is_claude_code_one_shot_subagent_request(messages, system, expected): from litellm.llms.anthropic.common_utils import is_claude_code_one_shot_subagent_request - assert is_claude_code_one_shot_subagent_request( - messages=messages, - system=system, - tools=None, - user_agent="claude-cli/2.1.263 (external, cli)", - ) is expected + assert ( + is_claude_code_one_shot_subagent_request( + messages=messages, + system=system, + tools=None, + user_agent="claude-cli/2.1.263 (external, cli)", + ) + is expected + ) class TestOptionallyHandleAnthropicOAuth: @@ -1541,6 +1544,71 @@ class TestAnthropicThinkingSignatureSelfHeal: out = strip_empty_content_blocks_from_anthropic_messages(msgs) assert [b["type"] for b in out[0]["content"]] == ["thinking"] + def test_strip_keeps_encrypted_reasoning_blocks_for_the_responses_bridge(self): + """The /v1/messages handler runs this before dispatch, so the bridge must still see the replay.""" + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + encrypted_reasoning_signature, + ) + from litellm.llms.anthropic.common_utils import ( + strip_empty_content_blocks_from_anthropic_messages, + ) + + msgs = [ + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "plan", "signature": encrypted_reasoning_signature("gAAAA_1")}, + {"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAAA_2")}, + {"type": "text", "text": "The answer."}, + ], + } + ] + assert strip_empty_content_blocks_from_anthropic_messages(msgs) == msgs + + def test_strip_encrypted_reasoning_drops_only_the_bridge_tagged_blocks(self): + """A session resumed on an Anthropic model replays reasoning only OpenAI can verify.""" + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + encrypted_reasoning_signature, + ) + from litellm.llms.anthropic.common_utils import ( + strip_encrypted_reasoning_blocks_from_anthropic_messages, + ) + + msgs = [ + {"role": "user", "content": "Solve it."}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "plan", "signature": encrypted_reasoning_signature("gAAAA_1")}, + {"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAAA_2")}, + ], + }, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "plan", "signature": encrypted_reasoning_signature("gAAAA_3")}, + {"type": "thinking", "thinking": "native", "signature": "EqQBCkYIAxgCIkA_anthropic_signed"}, + {"type": "redacted_thinking", "data": "EmwKAhgBEgy_anthropic_minted"}, + {"type": "text", "text": "The answer."}, + ], + }, + ] + out = strip_encrypted_reasoning_blocks_from_anthropic_messages(msgs) + assert [m["role"] for m in out] == ["user", "assistant"] + assert [b["type"] for b in out[1]["content"]] == ["thinking", "redacted_thinking", "text"] + assert out[1]["content"][0]["signature"] == "EqQBCkYIAxgCIkA_anthropic_signed" + assert len(msgs[1]["content"]) == 2 + assert len(msgs[2]["content"]) == 4 + + def test_strip_encrypted_reasoning_leaves_malformed_messages_for_the_provider_to_reject(self): + """A bare string in messages must reach Anthropic as a 400, not die in the stripper as a 500.""" + from litellm.llms.anthropic.common_utils import ( + strip_encrypted_reasoning_blocks_from_anthropic_messages, + ) + + msgs = ["hi", {"role": "user", "content": "hello"}] + assert strip_encrypted_reasoning_blocks_from_anthropic_messages(msgs) == msgs + def test_strip_empty_text_blocks_treats_null_text_as_empty(self): from litellm.llms.anthropic.common_utils import ( strip_empty_content_blocks_from_anthropic_messages, diff --git a/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py b/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py index 3961c8d74c2..ea8e2eacaa6 100644 --- a/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py +++ b/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py @@ -1584,6 +1584,89 @@ async def test_model_group_encrypted_content_affinity_overrides_global_deploymen router.discard() +@pytest.mark.asyncio +async def test_encrypted_content_affinity_pins_anthropic_messages_replayed_through_the_bridge(): + """ + Claude Code behind /v1/messages replays the encrypted reasoning the bridge packed + into a thinking block's signature (or a redacted block's data). The pin has to be + read from those blocks because the bridge builds the Responses `input` only after + the router has picked a deployment. + """ + check = EncryptedContentAffinityCheck() + deployments = [ + {"model_info": {"id": "openai-org-a"}, "litellm_params": {"model": "openai/gpt-5.1"}}, + {"model_info": {"id": "openai-org-b"}, "litellm_params": {"model": "openai/gpt-5.1"}}, + ] + request_kwargs = {"model": "gpt-5.1"} + + pinned = await check.async_filter_deployments( + model="gpt-5.1", + healthy_deployments=deployments, + messages=_bridge_replayed_anthropic_messages(minted_by="openai-org-b"), + request_kwargs=request_kwargs, + ) + + assert [d["model_info"]["id"] for d in pinned] == ["openai-org-b"] + assert request_kwargs["_encrypted_content_affinity_pinned"] is True + + +def _bridge_replayed_anthropic_messages(minted_by: str) -> list: + wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id("gAAAAA_turn_one", minted_by) + return [ + {"role": "user", "content": "Solve the zebra puzzle"}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "Anthropic minted this one", "signature": "ErcCCpIBCBEYAipA"}, + {"type": "redacted_thinking", "data": f"litellm_encrypted_reasoning:{wrapped}"}, + { + "type": "thinking", + "thinking": "The bridge packed this one", + "signature": f"litellm_encrypted_reasoning:{wrapped}", + }, + {"type": "text", "text": "The zebra owner lives in the green house."}, + ], + }, + {"role": "user", "content": "And who drinks water?"}, + ] + + +@pytest.mark.asyncio +async def test_encrypted_content_affinity_strips_bridge_reasoning_from_messages_routed_to_another_group(): + """ + The /v1/messages twin of the tier-change case: the routed group holds no deployment + of the org that minted the reasoning, so the bridge-tagged blocks are dropped whole + and the request dispatches to the routed pool. No unsigned thinking block may be left + behind: Anthropic and Bedrock reject a thinking block with a missing signature the + same way they reject a foreign one. + """ + originating = _make_originating_mock(None, "key-a", model_name="gpt-reasoning-tier") + mock_router = _make_router_mock_with_cooldown( + originating, cooldown_entries=[], routed_group_model_ids=["openai-org-b"] + ) + check = EncryptedContentAffinityCheck(router=mock_router) + routed_pool = [{"model_info": {"id": "openai-org-b"}, "litellm_params": {"model": "openai/gpt-5-nano"}}] + messages = _bridge_replayed_anthropic_messages(minted_by="openai-org-a") + assistant_content = messages[1]["content"] + request_kwargs = {"model": "gpt-5.1"} + + result = await check.async_filter_deployments( + model="gpt-simple-tier", + healthy_deployments=routed_pool, + messages=messages, + request_kwargs=request_kwargs, + ) + + assert result is routed_pool + assert "_encrypted_content_affinity_pinned" not in request_kwargs + assert messages[1]["content"] is assistant_content + assert assistant_content == [ + {"type": "thinking", "thinking": "Anthropic minted this one", "signature": "ErcCCpIBCBEYAipA"}, + {"type": "text", "text": "The zebra owner lives in the green house."}, + ] + assert all(block["signature"] for block in assistant_content if block["type"] == "thinking") + + class TestStripEncryptedReasoningFromInput: def test_keeps_summary_and_drops_encrypted_content_and_id(self): wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id("gAAAAA-blob", "deployment-a")