From 2bca7ff673f07812c544ce42a165dda931b8a730 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 20:27:14 -0700 Subject: [PATCH 1/6] fix(anthropic): replay OpenAI encrypted reasoning byte for byte behind /v1/messages Reasoning items the Responses API returned for a /v1/messages turn were rebuilt from their summary text on every replay, so the prompt the model saw changed between turns and the prompt cache never matched. The bridge now asks for reasoning.encrypted_content, carries it in the thinking signature (or as a redacted_thinking block when there is no summary), and replays it verbatim as the reasoning item's encrypted_content. Anthropic replay paths drop those tagged blocks so a cross-model resume never forwards OpenAI bytes to Anthropic --- .../transformation.py | 6 +- .../prompt_templates/common_utils.py | 80 +++++++-- .../prompt_templates/factory.py | 10 +- litellm/llms/anthropic/common_utils.py | 13 +- .../responses_adapters/streaming_iterator.py | 66 ++++++- .../responses_adapters/transformation.py | 84 +++++---- ...ore_utils_prompt_templates_common_utils.py | 64 +++++++ ...llm_core_utils_prompt_templates_factory.py | 12 +- ...t_responses_adapters_streaming_iterator.py | 74 +++++++- .../test_responses_adapters_transformation.py | 168 +++++++++++++++++- .../anthropic/test_anthropic_common_utils.py | 24 +++ 11 files changed, 535 insertions(+), 66 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 4fe069b0b7d..135f34afe46 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -25,7 +25,7 @@ import litellm from litellm import ModelResponse from litellm._logging import verbose_logger from litellm.litellm_core_utils.prompt_templates.common_utils import ( - responses_reasoning_item_from_thinking_blocks, + responses_reasoning_items_from_thinking_blocks, ) from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.bridges.completion_transformation import ( @@ -129,8 +129,8 @@ def _reasoning_input_items(msg: "AllMessageValues") -> list[dict[str, object]]: return stored raw_blocks: Final = msg.get("thinking_blocks") or () blocks: Final = cast("Iterable[ChatCompletionThinkingBlock]", raw_blocks) # cast-ok: untyped client json - from_thinking: Final = responses_reasoning_item_from_thinking_blocks(blocks) - return [] if from_thinking is None else [dict(from_thinking)] # mutable-ok: API message payload + replayed: Final = responses_reasoning_items_from_thinking_blocks(blocks) + return [dict(item) for item in replayed] # mutable-ok: API message payload def _build_reasoning_item( diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 00b80839dde..918cb982773 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -1823,14 +1823,11 @@ def _extract_reasoning_content(message: dict) -> tuple[str | None, str | None]: return None, message_content -def _readable_thinking_text( - block: ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock, -) -> str: +def _readable_thinking_text(block: Mapping[str, object]) -> str: """The text a chat model can read back, empty for redacted blocks and malformed ones.""" if block.get("type") != "thinking": return "" - thinking: Final = cast(ChatCompletionThinkingBlock, block).get("thinking") # cast-ok: narrowed by the type tag - return str(thinking or "") + return str(block.get("thinking") or "") def reasoning_content_from_thinking_blocks( @@ -1843,24 +1840,83 @@ def reasoning_content_from_thinking_blocks( return "\n".join(text for block in thinking_blocks if (text := _readable_thinking_text(block))) -def responses_reasoning_item_from_thinking_blocks( - thinking_blocks: Iterable[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock], -) -> ChatCompletionReasoningItem | None: - """Build a Responses API `reasoning` input item from Anthropic thinking blocks. +ENCRYPTED_REASONING_SIGNATURE_PREFIX: Final = "litellm_encrypted_reasoning:" - The item carries no `id`: the Responses API rejects an empty one and 404s on any id it - did not mint itself, while an item without an id is always accepted. + +def encrypted_reasoning_signature(encrypted_content: str) -> str: + """The opaque value a Responses API reasoning item's `encrypted_content` travels in. + + Anthropic clients echo a thinking block's `signature` and a redacted block's `data` + back verbatim, so either field can carry the encrypted reasoning across turns; the + prefix tells the two apart from a signature Anthropic minted. """ + return f"{ENCRYPTED_REASONING_SIGNATURE_PREFIX}{encrypted_content}" + + +def encrypted_content_from_signature(signature: object) -> str | None: + if not isinstance(signature, str) or not signature.startswith(ENCRYPTED_REASONING_SIGNATURE_PREFIX): + return None + return signature.removeprefix(ENCRYPTED_REASONING_SIGNATURE_PREFIX) or None + + +def _encrypted_content_of_block(block: Mapping[str, object]) -> str | None: + match block.get("type"): + case "thinking": + return encrypted_content_from_signature(block.get("signature")) + case "redacted_thinking": + return encrypted_content_from_signature(block.get("data")) + case _: + return None + + +def is_encrypted_reasoning_block(block: object) -> bool: + """A thinking or redacted_thinking block carrying Responses API encrypted reasoning. + + Only the Responses API that minted the content can read it back, so an Anthropic + backend has to drop such a block rather than fail signature verification on it. + """ + if not isinstance(block, Mapping): + return False + mapping: Final = cast(Mapping[str, object], block) # cast-ok: narrowed by isinstance + return _encrypted_content_of_block(mapping) is not None + + +def _reasoning_replay_group_key(indexed_block: tuple[int, Mapping[str, object]]) -> str: + index, block = indexed_block + return f"encrypted:{index}" if is_encrypted_reasoning_block(block) else "summary" + + +def _reasoning_item_from_block_group(group: tuple[Mapping[str, object], ...]) -> ChatCompletionReasoningItem | None: summary: Final[list[ChatCompletionReasoningSummaryTextBlock]] = [ # mutable-ok: API message payload ChatCompletionReasoningSummaryTextBlock(type="summary_text", text=text) - for block in thinking_blocks + for block in group if (text := _readable_thinking_text(block)) ] + encrypted_content: Final = _encrypted_content_of_block(group[0]) + if encrypted_content is not None: + return ChatCompletionReasoningItem(type="reasoning", summary=summary, encrypted_content=encrypted_content) if not summary: return None return ChatCompletionReasoningItem(type="reasoning", summary=summary) +def responses_reasoning_items_from_thinking_blocks( + thinking_blocks: Iterable[Mapping[str, object]], +) -> tuple[ChatCompletionReasoningItem, ...]: + """Build Responses API `reasoning` input items from Anthropic thinking blocks. + + A block carrying encrypted reasoning replays the item it came from byte for byte; + a run of plain thinking blocks collapses into one summary-only item. No item carries + an `id`: the Responses API 404s on any id it did not mint itself and rejects an empty + one, while an item without an id is always accepted. + """ + return tuple( + item + for _, group in groupby(enumerate(thinking_blocks), key=_reasoning_replay_group_key) + if (item := _reasoning_item_from_block_group(tuple(block for _, block in group))) is not None + ) + + def _parse_content_for_reasoning( message_text: str | None, ) -> tuple[str | None, str | None]: diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 56c1d605700..ece619e3883 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -46,6 +46,7 @@ from litellm.types.utils import GenericImageParsingChunk from .common_utils import ( convert_content_list_to_str, infer_content_type_from_url_and_content, + is_encrypted_reasoning_block, is_non_content_values_set, parse_tool_call_arguments, ) @@ -2299,13 +2300,16 @@ def sanitize_messages_for_tool_calling( def _is_unsignable_thinking_block(block: object) -> bool: - """A `thinking` block that Anthropic cannot accept on input. + """A thinking block that Anthropic cannot accept on input. Anthropic verifies the thinking signature cryptographically, so a block whose signature is null, empty, or missing (e.g. from an open-source reasoning model) - is rejected with a 400 and must be dropped rather than blanked or repaired. - `redacted_thinking` blocks carry no signature and are always kept. + is rejected with a 400 and must be dropped rather than blanked or repaired, and + so is a block whose signature or data carries another provider's encrypted + reasoning. A `redacted_thinking` block Anthropic minted is always kept. """ + if is_encrypted_reasoning_block(block): + return True if not isinstance(block, dict) or block.get("type") != "thinking": return False signature: Final = block.get("signature") diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 2b57883cc13..f46227239d8 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -21,6 +21,7 @@ from litellm.constants import ( ) from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_file_ids_from_messages, + is_encrypted_reasoning_block, ) from litellm.litellm_core_utils.prompt_templates.factory import ( THOUGHT_SIGNATURE_SEPARATOR, @@ -1235,8 +1236,10 @@ def strip_empty_content_blocks_from_anthropic_messages( on the unified ``/v1/messages`` path. ``/v1/chat/completions`` already handles this in ``anthropic_messages_pt``; this helper provides the equivalent guarantee for the native Anthropic Messages path. - ``redacted_thinking`` blocks are never touched: they carry opaque - ``data`` instead of thinking text. + A thinking or ``redacted_thinking`` block whose signature or data carries + another provider's encrypted reasoning (a turn served by the Responses API + bridge) is dropped too, since Anthropic cannot verify it; every other + ``redacted_thinking`` block is left alone. Messages whose content is a list and becomes empty after stripping are omitted, matching :func:`strip_thinking_blocks_from_anthropic_messages`. @@ -1249,7 +1252,11 @@ def strip_empty_content_blocks_from_anthropic_messages( out.append(m) continue content = m["content"] - filtered = [b for b in content if not _is_empty_text_block(b) and not is_empty_thinking_block(b)] + filtered = [ # mutable-ok: rebuilt message content list + b + for b in content + if not _is_empty_text_block(b) and not is_empty_thinking_block(b) and not is_encrypted_reasoning_block(b) + ] if len(filtered) == len(content): out.append(m) elif filtered: diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py index 2e0a6a9df8f..f753e87fee3 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py @@ -9,13 +9,19 @@ from typing import TYPE_CHECKING, Any, Final from litellm import verbose_logger from litellm._uuid import uuid +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + encrypted_reasoning_signature, +) from litellm.llms.anthropic.experimental_pass_through.messages.utils import ( refusal_stop_details, responses_output_refusal_text, ) from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage -from .transformation import LiteLLMAnthropicToResponsesAPIAdapter +from .transformation import ( + REASONING_SUMMARY_PART_SEPARATOR, + LiteLLMAnthropicToResponsesAPIAdapter, +) if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject @@ -29,9 +35,10 @@ class AnthropicResponsesStreamWrapper: response.created -> message_start response.output_item.added -> content_block_start (if message/function_call) response.output_text.delta -> content_block_delta (text_delta) + response.reasoning_summary_part.added -> content_block_delta (thinking_delta separator) response.reasoning_summary_text.delta -> content_block_delta (thinking_delta) response.function_call_arguments.delta -> content_block_delta (input_json_delta) - response.output_item.done -> content_block_stop + response.output_item.done -> content_block_delta (signature_delta) + content_block_stop response.completed -> message_delta + message_stop """ @@ -94,6 +101,38 @@ class AnthropicResponsesStreamWrapper: ) return block_idx + @staticmethod + def _field(source: object, name: str) -> object: + return source.get(name) if isinstance(source, dict) else getattr(source, name, None) + + def _close_reasoning_item(self, item: object, item_id: str | None) -> None: + block_idx: Final = self._item_id_to_block_index.get(item_id, -1) if item_id else self._current_block_index + encrypted_content: Final = self._field(item, "encrypted_content") + signature: Final = ( + encrypted_reasoning_signature(encrypted_content) + if isinstance(encrypted_content, str) and encrypted_content + else None + ) + if block_idx < 0 and signature is None: + return + if block_idx < 0: + redacted_idx: Final = self._open_block( + item_id, + {"type": "redacted_thinking", "data": signature}, # mutable-ok: API message payload + ) + stop: Final = {"type": "content_block_stop", "index": redacted_idx} # mutable-ok: API message payload + self._chunk_queue.append(stop) + return + if signature is not None: + self._chunk_queue.append( + { # mutable-ok: API message payload + "type": "content_block_delta", + "index": block_idx, + "delta": {"type": "signature_delta", "signature": signature}, # mutable-ok: API message payload + } + ) + self._chunk_queue.append({"type": "content_block_stop", "index": block_idx}) # mutable-ok: API message payload + def _process_event(self, event: object) -> None: """Convert one Responses API event into zero or more Anthropic chunks queued for emission.""" event_type = getattr(event, "type", None) @@ -175,6 +214,26 @@ class AnthropicResponsesStreamWrapper: ) return + if event_type == "response.reasoning_summary_part.added": + part_item_id: Final = self._field(event, "item_id") + summary_index: Final = self._field(event, "summary_index") + part_block_idx: Final = ( + self._item_id_to_block_index.get(part_item_id, -1) if isinstance(part_item_id, str) else -1 + ) + if part_block_idx < 0 or not isinstance(summary_index, int) or summary_index == 0: + return + self._chunk_queue.append( + { # mutable-ok: API message payload + "type": "content_block_delta", + "index": part_block_idx, + "delta": { # mutable-ok: API message payload + "type": "thinking_delta", + "thinking": REASONING_SUMMARY_PART_SEPARATOR, + }, + } + ) + return + # ---- reasoning summary text delta ---- if event_type == "response.reasoning_summary_text.delta": item_id = getattr(event, "item_id", None) or (event.get("item_id") if isinstance(event, dict) else None) @@ -220,6 +279,9 @@ class AnthropicResponsesStreamWrapper: item_id = ( getattr(item, "id", None) or (item.get("id") if isinstance(item, dict) else None) if item else None ) + if self._field(item, "type") == "reasoning": + self._close_reasoning_item(item, item_id) + return block_idx = self._item_id_to_block_index.get(item_id, -1) if item_id else self._current_block_index if block_idx < 0: return diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py index f1daf2be42a..07419db443c 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py @@ -13,7 +13,8 @@ from typing import Any, Final, cast from litellm.litellm_core_utils.prompt_templates.common_utils import ( TOOL_RESULT_IMAGE_BOUNDARY, TOOL_RESULT_IMAGE_PLACEHOLDER, - responses_reasoning_item_from_thinking_blocks, + encrypted_reasoning_signature, + responses_reasoning_items_from_thinking_blocks, with_prompt_cache_breakpoint, ) from litellm.litellm_core_utils.reasoning_effort_utils import ( @@ -33,6 +34,7 @@ from litellm.types.llms.anthropic import ( AnthropicFinishReason, AnthropicMessagesRequest, AnthropicMessagesToolChoice, + AnthropicResponseContentBlockRedactedThinking, AnthropicResponseContentBlockText, AnthropicResponseContentBlockThinking, AnthropicResponseContentBlockToolUse, @@ -43,11 +45,13 @@ from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicUsage, ) from litellm.types.llms.openai import ( - ChatCompletionThinkingBlock, ResponseAPIUsage, ResponsesAPIResponse, ) +REASONING_SUMMARY_PART_SEPARATOR: Final = "\n\n" +RESPONSES_INCLUDE_ENCRYPTED_REASONING: Final = "reasoning.encrypted_content" + class LiteLLMAnthropicToResponsesAPIAdapter: """ @@ -163,49 +167,55 @@ class LiteLLMAnthropicToResponsesAPIAdapter: return str(getattr(part, "text", None) or "") @classmethod - def _thinking_blocks_from_reasoning_item( + def _thinking_block_from_reasoning_item( cls, summary: Iterable[object], - ) -> tuple[dict[str, Any], ...]: # mutable-ok: API message payload - """Anthropic thinking blocks for one Responses reasoning item. + encrypted_content: object, + ) -> dict[str, Any] | None: # mutable-ok: API message payload + """The one Anthropic block for a Responses reasoning item. - The signature stays empty: only Anthropic can sign a thinking block, and a stand-in - value would be replayed as a real one and rejected by every backend that verifies it. + The item's encrypted reasoning rides the block's opaque field (`signature`, or + `data` when there is no summary text) so the client echoes it back and the next + turn replays the very item OpenAI produced; without it the signature stays empty, + since only Anthropic can sign a thinking block. """ - return tuple( - AnthropicResponseContentBlockThinking( - type="thinking", - thinking=text, - signature=None, - ).model_dump() - for part in summary - if (text := cls._summary_part_text(part)) + text: Final = REASONING_SUMMARY_PART_SEPARATOR.join( + part_text for part in summary if (part_text := cls._summary_part_text(part)) ) + if not isinstance(encrypted_content, str) or not encrypted_content: + if not text: + return None + return AnthropicResponseContentBlockThinking(type="thinking", thinking=text, signature=None).model_dump() + signature: Final = encrypted_reasoning_signature(encrypted_content) + if not text: + return AnthropicResponseContentBlockRedactedThinking(type="redacted_thinking", data=signature).model_dump() + return AnthropicResponseContentBlockThinking(type="thinking", thinking=text, signature=signature).model_dump() @staticmethod def _assistant_block_group_key(indexed_block: tuple[int, Mapping[str, object]]) -> str: """Group a run of consecutive thinking blocks together; keep every other block alone.""" index, block = indexed_block - return "thinking" if block.get("type") == "thinking" else f"block:{index}" + return "thinking" if block.get("type") in ("thinking", "redacted_thinking") else f"block:{index}" @classmethod - def _assistant_group_to_input_item( + def _assistant_group_to_input_items( cls, group: tuple[Mapping[str, object], ...] - ) -> dict[str, Any] | None: # mutable-ok: API message payload + ) -> tuple[dict[str, Any], ...]: # mutable-ok: API message payload first: Final = group[0] btype: Final = first.get("type") - if btype == "thinking": - blocks: Final = cast(tuple[ChatCompletionThinkingBlock, ...], group) # cast-ok: untrusted client payload - reasoning_item: Final = responses_reasoning_item_from_thinking_blocks(blocks) - return None if reasoning_item is None else dict(reasoning_item) # mutable-ok: API message payload + if btype in ("thinking", "redacted_thinking"): + replayed: Final = responses_reasoning_items_from_thinking_blocks(group) + return tuple(dict(item) for item in replayed) # mutable-ok: API message payload if btype == "tool_use": - return { # mutable-ok: API message payload - "type": "function_call", - "call_id": first.get("id", ""), - "name": first.get("name", ""), - "arguments": json.dumps(first.get("input", {})), # mutable-ok: API message payload - } - return None + return ( + { # mutable-ok: API message payload + "type": "function_call", + "call_id": first.get("id", ""), + "name": first.get("name", ""), + "arguments": json.dumps(first.get("input", {})), # mutable-ok: API message payload + }, + ) + return () def translate_messages_to_responses_input( self, @@ -362,7 +372,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: input_items.extend( item for _, group in groupby(enumerate(blocks), key=self._assistant_block_group_key) - if (item := self._assistant_group_to_input_item(tuple(block for _, block in group))) is not None + for item in self._assistant_group_to_input_items(tuple(block for _, block in group)) ) asst_parts: list[dict[str, Any]] = [ # mutable-ok: API message payload {"type": "output_text", "text": block.get("text", "")} # mutable-ok: API message payload @@ -572,6 +582,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: ) if reasoning: responses_kwargs["reasoning"] = reasoning + responses_kwargs["include"] = [RESPONSES_INCLUDE_ENCRYPTED_REASONING] # mutable-ok: json list # output_format / output_config.format -> text format # output_format: {"type": "json_schema", "schema": {...}} @@ -634,7 +645,9 @@ class LiteLLMAnthropicToResponsesAPIAdapter: for item in response.output: if isinstance(item, ResponseReasoningItem): - content.extend(self._thinking_blocks_from_reasoning_item(item.summary)) + reasoning_block = self._thinking_block_from_reasoning_item(item.summary, item.encrypted_content) + if reasoning_block is not None: + content.append(reasoning_block) elif isinstance(item, ResponseOutputMessage): for part in item.content: @@ -684,11 +697,12 @@ class LiteLLMAnthropicToResponsesAPIAdapter: ).model_dump() ) elif item_type == "reasoning": - content.extend( - self._thinking_blocks_from_reasoning_item( - cast(Iterable[object], item.get("summary") or ()), # cast-ok: untyped provider json - ) + reasoning_block = self._thinking_block_from_reasoning_item( + cast(Iterable[object], item.get("summary") or ()), # cast-ok: untyped provider json + item.get("encrypted_content"), ) + if reasoning_block is not None: + content.append(reasoning_block) elif item_type == "function_call": try: input_data = json.loads(item.get("arguments", "{}")) diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index c037f928593..9df124576a0 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -10,10 +10,14 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( TOOL_RESULT_IMAGE_BOUNDARY, TOOL_RESULT_IMAGE_PLACEHOLDER, add_system_prompt_to_messages, + encrypted_content_from_signature, + encrypted_reasoning_signature, get_file_ids_from_messages, get_format_from_file_id, handle_any_messages_to_chat_completion_str_messages_conversion, hoist_images_from_tool_messages, + is_encrypted_reasoning_block, + responses_reasoning_items_from_thinking_blocks, split_concatenated_json_objects, update_messages_with_model_file_ids, ) @@ -1554,3 +1558,63 @@ class TestRequestContainsImageContent: for _ in range(50): nested = {"type": "tool_result", "content": [nested]} assert request_contains_image_content([{"role": "user", "content": [nested]}]) is False + + +class TestEncryptedReasoningReplay: + """Regression for https://github.com/BerriAI/litellm/issues/40288.""" + + def test_signature_round_trips_the_encrypted_content(self): + assert encrypted_content_from_signature(encrypted_reasoning_signature("gAAAA_bytes")) == "gAAAA_bytes" + + @pytest.mark.parametrize("signature", [None, "", "ErcBCkgIValidAnthropicSignature", "litellm_encrypted_reasoning:", 7]) + def test_anything_else_is_not_encrypted_content(self, signature): + assert encrypted_content_from_signature(signature) is None + + def test_encrypted_thinking_block_replays_its_own_item(self): + items = responses_reasoning_items_from_thinking_blocks( + [{"type": "thinking", "thinking": "Plan.", "signature": encrypted_reasoning_signature("gAAAA_1")}] + ) + assert items == ( + {"type": "reasoning", "summary": [{"type": "summary_text", "text": "Plan."}], "encrypted_content": "gAAAA_1"}, + ) + + def test_encrypted_redacted_block_replays_with_an_empty_summary(self): + items = responses_reasoning_items_from_thinking_blocks( + [{"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAAA_1")}] + ) + assert items == ({"type": "reasoning", "summary": [], "encrypted_content": "gAAAA_1"},) + + def test_plain_blocks_collapse_into_one_summary_item_around_encrypted_ones(self): + items = responses_reasoning_items_from_thinking_blocks( + [ + {"type": "thinking", "thinking": "A.", "signature": None}, + {"type": "thinking", "thinking": "B.", "signature": ""}, + {"type": "thinking", "thinking": "C.", "signature": encrypted_reasoning_signature("gAAAA_c")}, + {"type": "redacted_thinking", "data": "anthropic-minted-opaque-data"}, + {"type": "thinking", "thinking": "D."}, + ] + ) + assert items == ( + {"type": "reasoning", "summary": [{"type": "summary_text", "text": "A."}, {"type": "summary_text", "text": "B."}]}, + {"type": "reasoning", "summary": [{"type": "summary_text", "text": "C."}], "encrypted_content": "gAAAA_c"}, + {"type": "reasoning", "summary": [{"type": "summary_text", "text": "D."}]}, + ) + assert all("id" not in item for item in items) + + def test_blocks_without_text_or_encrypted_content_produce_nothing(self): + assert responses_reasoning_items_from_thinking_blocks([{"type": "thinking", "thinking": ""}]) == () + assert responses_reasoning_items_from_thinking_blocks([]) == () + + @pytest.mark.parametrize( + ("block", "expected"), + [ + ({"type": "thinking", "thinking": "x", "signature": encrypted_reasoning_signature("g")}, True), + ({"type": "redacted_thinking", "data": encrypted_reasoning_signature("g")}, True), + ({"type": "thinking", "thinking": "x", "signature": "ErcBCkgIValid"}, False), + ({"type": "redacted_thinking", "data": "EmwKAhgBEgy"}, False), + ({"type": "text", "text": encrypted_reasoning_signature("g")}, False), + ("not a block", False), + ], + ) + def test_is_encrypted_reasoning_block(self, block, expected): + assert is_encrypted_reasoning_block(block) is expected diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index dd2d45f00c6..66d10fd1407 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -191,8 +191,16 @@ def test_bedrock_converse_assistant_with_empty_thinking_block_and_tool_calls(): {"type": "thinking", "thinking": "oss reasoning", "signature": None}, {"type": "thinking", "thinking": "oss reasoning", "signature": ""}, {"type": "thinking", "thinking": "oss reasoning"}, + {"type": "thinking", "thinking": "openai reasoning", "signature": "litellm_encrypted_reasoning:gAAAA"}, + {"type": "redacted_thinking", "data": "litellm_encrypted_reasoning:gAAAA"}, + ], + ids=[ + "null_signature", + "empty_signature", + "missing_signature", + "encrypted_reasoning_signature", + "encrypted_reasoning_redacted_data", ], - ids=["null_signature", "empty_signature", "missing_signature"], ) def test_anthropic_messages_pt_drops_unsignable_thinking_block(thinking_block): """Open-source reasoning models (DeepSeek-R1, Qwen, etc.) emit thinking blocks @@ -219,7 +227,7 @@ def test_anthropic_messages_pt_drops_unsignable_thinking_block(thinking_block): assistant = next(m for m in result if m["role"] == "assistant") content = assistant["content"] assert all( - block.get("type") != "thinking" for block in content + block.get("type") not in ("thinking", "redacted_thinking") for block in content ), f"unsignable thinking block must be dropped, got {content!r}" assert any( block.get("type") == "text" and block.get("text") == "2+2 equals 4." diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py index d9df5df426a..bfe2d6b7cea 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py @@ -10,6 +10,9 @@ from types import SimpleNamespace sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../../.."))) +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + encrypted_reasoning_signature, +) from litellm.llms.anthropic.experimental_pass_through.responses_adapters.streaming_iterator import ( AnthropicResponsesStreamWrapper, ) @@ -114,7 +117,7 @@ class TestReasoningItemWithoutSummaryText: """ @staticmethod - def _gpt_turn(reasoning_summary_deltas: list) -> list: + def _gpt_turn(reasoning_summary_deltas: list, encrypted_content: str | None = None) -> list: return [ {"type": "response.created"}, {"type": "response.output_item.added", "item": {"type": "reasoning", "id": "rs_1"}}, @@ -122,7 +125,10 @@ class TestReasoningItemWithoutSummaryText: {"type": "response.reasoning_summary_text.delta", "item_id": "rs_1", "delta": delta} for delta in reasoning_summary_deltas ), - {"type": "response.output_item.done", "item": {"type": "reasoning", "id": "rs_1"}}, + { + "type": "response.output_item.done", + "item": {"type": "reasoning", "id": "rs_1", "encrypted_content": encrypted_content}, + }, {"type": "response.output_item.added", "item": {"type": "message", "id": "msg_1"}}, {"type": "response.output_text.delta", "item_id": "msg_1", "delta": "Hello"}, {"type": "response.output_item.done", "item": {"type": "message", "id": "msg_1"}}, @@ -171,6 +177,70 @@ class TestReasoningItemWithoutSummaryText: assert not [c for c in chunks if c.get("delta", {}).get("type") == "signature_delta"] +_ENCRYPTED_REASONING = "gAAAAABp_encrypted_reasoning_bytes_only_openai_can_read" + + +class TestEncryptedReasoningIsStreamedForReplay: + """Regression for https://github.com/BerriAI/litellm/issues/40288. + + The client echoes a thinking block's signature (or a redacted block's data) back on the + next turn, so the item's ``encrypted_content`` has to reach it through one of those. + """ + + def test_encrypted_content_is_streamed_as_the_signature_before_the_block_closes(self): + chunks = _drain_async( + TestReasoningItemWithoutSummaryText._gpt_turn( + reasoning_summary_deltas=["Weighing options"], encrypted_content=_ENCRYPTED_REASONING + ) + ) + + assert [(c["type"], c.get("index"), c.get("delta", {}).get("type")) for c in chunks[1:5]] == [ + ("content_block_start", 0, None), + ("content_block_delta", 0, "thinking_delta"), + ("content_block_delta", 0, "signature_delta"), + ("content_block_stop", 0, None), + ] + assert chunks[3]["delta"]["signature"] == encrypted_reasoning_signature(_ENCRYPTED_REASONING) + + def test_reasoning_without_summary_streams_a_redacted_thinking_block(self): + chunks = _drain_async( + TestReasoningItemWithoutSummaryText._gpt_turn( + reasoning_summary_deltas=[], encrypted_content=_ENCRYPTED_REASONING + ) + ) + + assert [(c["type"], c.get("index")) for c in chunks[1:]] == [ + ("content_block_start", 0), + ("content_block_stop", 0), + ("content_block_start", 1), + ("content_block_delta", 1), + ("content_block_stop", 1), + ] + assert chunks[1]["content_block"] == { + "type": "redacted_thinking", + "data": encrypted_reasoning_signature(_ENCRYPTED_REASONING), + } + + def test_summary_parts_are_separated_inside_the_one_thinking_block(self): + """Two summary parts read as two paragraphs, not as one run-on sentence.""" + events = [ + {"type": "response.created"}, + {"type": "response.output_item.added", "item": {"type": "reasoning", "id": "rs_1"}}, + {"type": "response.reasoning_summary_part.added", "item_id": "rs_1", "summary_index": 0}, + {"type": "response.reasoning_summary_text.delta", "item_id": "rs_1", "delta": "First."}, + {"type": "response.reasoning_summary_part.added", "item_id": "rs_1", "summary_index": 1}, + {"type": "response.reasoning_summary_text.delta", "item_id": "rs_1", "delta": "Second."}, + {"type": "response.output_item.done", "item": {"type": "reasoning", "id": "rs_1"}}, + ] + chunks = _process_all(events) + + thinking = "".join( + c["delta"]["thinking"] for c in chunks if c.get("delta", {}).get("type") == "thinking_delta" + ) + assert thinking == "First.\n\nSecond." + assert [c["type"] for c in chunks].count("content_block_start") == 1 + + class TestToolUseBlockClosedExactlyOnce: """Regression for https://github.com/BerriAI/litellm/issues/37273. diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py index 9f8414afa38..79eaf6b60f2 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py @@ -19,6 +19,7 @@ from litellm.constants import ( from litellm.litellm_core_utils.prompt_templates.common_utils import ( TOOL_RESULT_IMAGE_BOUNDARY, TOOL_RESULT_IMAGE_PLACEHOLDER, + encrypted_reasoning_signature, ) from litellm.llms.anthropic.experimental_pass_through.responses_adapters.transformation import ( LiteLLMAnthropicToResponsesAPIAdapter, @@ -566,6 +567,66 @@ class TestTranslateMessagesToResponsesInput: result = _translate_messages(messages) assert "id" not in result[0] + def test_thinking_block_with_encrypted_signature_replays_the_encrypted_content(self): + """Regression for https://github.com/BerriAI/litellm/issues/40288 (inbound fault site).""" + messages = [ + { + "role": "assistant", + "content": [ + { + "type": "thinking", + "thinking": "Private reasoning.", + "signature": encrypted_reasoning_signature("gAAAA_turn_one"), + } + ], + } + ] + result = _translate_messages(messages) + assert result == [ + { + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "Private reasoning."}], + "encrypted_content": "gAAAA_turn_one", + } + ] + + def test_redacted_thinking_with_encrypted_data_replays_the_encrypted_content(self): + messages = [ + { + "role": "assistant", + "content": [{"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAAA_turn_one")}], + } + ] + result = _translate_messages(messages) + assert result == [{"type": "reasoning", "summary": [], "encrypted_content": "gAAAA_turn_one"}] + + def test_each_encrypted_thinking_block_stays_its_own_reasoning_item(self): + """Two upstream items must not be merged into one, or the encrypted content of one is lost.""" + messages = [ + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "First.", "signature": encrypted_reasoning_signature("gAAAA_1")}, + {"type": "thinking", "thinking": "Second.", "signature": encrypted_reasoning_signature("gAAAA_2")}, + ], + } + ] + result = _translate_messages(messages) + assert [item["encrypted_content"] for item in result] == ["gAAAA_1", "gAAAA_2"] + + def test_anthropic_signed_thinking_block_replays_as_a_summary_only_item(self): + """A real Anthropic signature is opaque here, so it never masquerades as encrypted content.""" + messages = [ + { + "role": "assistant", + "content": [{"type": "thinking", "thinking": "Private reasoning.", "signature": "ErcBCkgIValid"}], + } + ] + result = _translate_messages(messages) + assert result == [ + {"type": "reasoning", "summary": [{"type": "summary_text", "text": "Private reasoning."}]} + ] + def test_consecutive_thinking_blocks_become_one_reasoning_item(self): """Summary parts of one upstream reasoning item are regrouped into that item.""" messages = [ @@ -1101,6 +1162,13 @@ class TestTranslateRequestBroaderCoverage: req = _make_request(thinking={"type": "disabled"}) kwargs = _ADAPTER.translate_request(req) assert "reasoning" not in kwargs + assert "include" not in kwargs + + def test_thinking_asks_for_the_encrypted_reasoning(self): + """The documented way to get reasoning that survives store=false is to ask for it.""" + req = _make_request(thinking={"type": "enabled", "budget_tokens": 12000}) + kwargs = _ADAPTER.translate_request(req) + assert kwargs["include"] == ["reasoning.encrypted_content"] def test_metadata_user_id_mapped_to_user(self): req = _make_request(metadata={"user_id": "user-42"}) @@ -1229,7 +1297,9 @@ def _make_function_call_item(call_id: str, name: str, arguments: str) -> MagicMo return item -def _make_reasoning_item(summaries: List[str], item_id: str = "rs_test_1") -> MagicMock: +def _make_reasoning_item( + summaries: List[str], item_id: str = "rs_test_1", encrypted_content: str | None = None +) -> MagicMock: """Build a mock ResponseReasoningItem.""" from openai.types.responses import ResponseReasoningItem # type: ignore[import] @@ -1242,9 +1312,13 @@ def _make_reasoning_item(summaries: List[str], item_id: str = "rs_test_1") -> Ma item = MagicMock(spec=ResponseReasoningItem) item.id = item_id item.summary = summary_mocks + item.encrypted_content = encrypted_content return item +_ENCRYPTED_REASONING = "gAAAAABp_encrypted_reasoning_bytes_only_openai_can_read" + + class TestTranslateResponse: """Responses API -> AnthropicMessagesResponse conversion.""" @@ -1369,7 +1443,81 @@ class TestTranslateResponse: reasoning = _make_reasoning_item(["Part one.", "Part two."], item_id="rs_abc123") response = _make_mock_response(output=[reasoning]) result: Any = _ADAPTER.translate_response(response) - assert [block["signature"] for block in result["content"]] == [None, None] + assert [block["signature"] for block in result["content"]] == [None] + assert "rs_abc123" not in json.dumps(result["content"]) + + def test_summary_parts_join_into_one_thinking_block(self): + """One reasoning item is one block, so its signature is echoed back exactly once.""" + reasoning = _make_reasoning_item(["Part one.", "Part two."]) + response = _make_mock_response(output=[reasoning]) + result: Any = _ADAPTER.translate_response(response) + assert [block["thinking"] for block in result["content"]] == ["Part one.\n\nPart two."] + + def test_encrypted_content_rides_the_thinking_signature(self): + """Regression for https://github.com/BerriAI/litellm/issues/40288 (outbound fault site).""" + reasoning = _make_reasoning_item(["Part one."], item_id="rs_abc123", encrypted_content=_ENCRYPTED_REASONING) + response = _make_mock_response(output=[reasoning]) + result: Any = _ADAPTER.translate_response(response) + assert result["content"] == [ + { + "type": "thinking", + "thinking": "Part one.", + "signature": encrypted_reasoning_signature(_ENCRYPTED_REASONING), + } + ] + + def test_reasoning_without_summary_becomes_redacted_thinking(self): + """With summaries off the encrypted reasoning still has to reach the client to be replayed.""" + reasoning = _make_reasoning_item([], encrypted_content=_ENCRYPTED_REASONING) + response = _make_mock_response(output=[reasoning]) + result: Any = _ADAPTER.translate_response(response) + assert result["content"] == [ + {"type": "redacted_thinking", "data": encrypted_reasoning_signature(_ENCRYPTED_REASONING)} + ] + + def test_dict_reasoning_item_carries_its_encrypted_content(self): + response = _make_mock_response( + output=[ + { + "type": "reasoning", + "id": "rs_dict_1", + "encrypted_content": _ENCRYPTED_REASONING, + "summary": [{"type": "summary_text", "text": "Weighing the options."}], + } + ] + ) + result: Any = _ADAPTER.translate_response(response) + assert result["content"][0]["signature"] == encrypted_reasoning_signature(_ENCRYPTED_REASONING) + + def test_reasoning_item_round_trip_is_byte_stable(self): + """Regression for https://github.com/BerriAI/litellm/issues/40288. + + The reasoning item the next turn replays must be the one OpenAI produced, with its + encrypted reasoning intact, and identical on every later turn so the prompt cache + prefix keeps matching. + """ + reasoning = _make_reasoning_item(["Part one.", "Part two."], encrypted_content=_ENCRYPTED_REASONING) + turn: Any = _ADAPTER.translate_response(_make_mock_response(output=[reasoning])) + history = [{"role": "assistant", "content": turn["content"]}] + + replayed_items = [_translate_messages(history) for _ in range(2)] + + assert replayed_items[0] == replayed_items[1] + assert replayed_items[0] == [ + { + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "Part one.\n\nPart two."}], + "encrypted_content": _ENCRYPTED_REASONING, + } + ] + + def test_redacted_reasoning_round_trip_replays_the_encrypted_content(self): + reasoning = _make_reasoning_item([], encrypted_content=_ENCRYPTED_REASONING) + turn: Any = _ADAPTER.translate_response(_make_mock_response(output=[reasoning])) + + replayed = _translate_messages([{"role": "assistant", "content": turn["content"]}]) + + assert replayed == [{"type": "reasoning", "summary": [], "encrypted_content": _ENCRYPTED_REASONING}] def test_dict_reasoning_item_becomes_thinking_block(self): """A reasoning item arriving as a plain dict is kept, not dropped.""" @@ -1385,14 +1533,26 @@ class TestTranslateResponse: result: Any = _ADAPTER.translate_response(response) assert result["content"] == [{"type": "thinking", "thinking": "Weighing the options.", "signature": None}] - def test_thinking_blocks_are_dropped_when_replayed_to_anthropic(self): + @pytest.mark.parametrize( + ("summaries", "encrypted_content"), + [ + (["Part one."], None), + (["Part one."], _ENCRYPTED_REASONING), + ([], _ENCRYPTED_REASONING), + ], + ids=["unsigned_thinking", "encrypted_thinking", "encrypted_redacted_thinking"], + ) + def test_thinking_blocks_are_dropped_when_replayed_to_anthropic(self, summaries, encrypted_content): """Replaying this turn to an Anthropic model must not send a signature it cannot verify.""" from litellm.litellm_core_utils.prompt_templates.factory import ( _drop_unsignable_thinking_blocks, ) - response = _make_mock_response(output=[_make_reasoning_item(["Part one."], item_id="rs_abc123")]) + response = _make_mock_response( + output=[_make_reasoning_item(summaries, item_id="rs_abc123", encrypted_content=encrypted_content)] + ) result: Any = _ADAPTER.translate_response(response) + assert len(result["content"]) == 1 assert _drop_unsignable_thinking_blocks(result["content"]) == [] def test_usage_mapped_correctly(self): diff --git a/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py b/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py index ae620fdd6dc..336cbf57e29 100644 --- a/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py @@ -1541,6 +1541,30 @@ class TestAnthropicThinkingSignatureSelfHeal: out = strip_empty_content_blocks_from_anthropic_messages(msgs) assert [b["type"] for b in out[0]["content"]] == ["thinking"] + def test_strip_drops_encrypted_reasoning_blocks_from_the_responses_bridge(self): + """A session resumed on an Anthropic model replays reasoning only OpenAI can verify.""" + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + encrypted_reasoning_signature, + ) + from litellm.llms.anthropic.common_utils import ( + strip_empty_content_blocks_from_anthropic_messages, + ) + + msgs = [ + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "plan", "signature": encrypted_reasoning_signature("gAAAA_1")}, + {"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAAA_2")}, + {"type": "redacted_thinking", "data": "EmwKAhgBEgy_anthropic_minted"}, + {"type": "text", "text": "The answer."}, + ], + } + ] + out = strip_empty_content_blocks_from_anthropic_messages(msgs) + assert [b["type"] for b in out[0]["content"]] == ["redacted_thinking", "text"] + assert len(msgs[0]["content"]) == 4 + def test_strip_empty_text_blocks_treats_null_text_as_empty(self): from litellm.llms.anthropic.common_utils import ( strip_empty_content_blocks_from_anthropic_messages, From 751431dc3f782b1aa93908533dbc9d9e0062b400 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 12:41:38 -0700 Subject: [PATCH 2/6] fix(anthropic): strip bridge reasoning in the native messages transform, not the empty-block pass --- litellm/llms/anthropic/common_utils.py | 36 ++++++++--- .../messages/transformation.py | 3 +- ..._anthropic_messages_encrypted_reasoning.py | 50 ++++++++++++++++ .../anthropic/test_anthropic_common_utils.py | 59 +++++++++++++++---- 4 files changed, 126 insertions(+), 22 deletions(-) create mode 100644 tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_messages_encrypted_reasoning.py diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index f46227239d8..cb24f4153f9 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -1202,6 +1202,30 @@ def strip_thinking_blocks_from_anthropic_messages(messages: list[Any]) -> list[A return out +def _without_encrypted_reasoning_blocks(message: dict) -> dict | None: # mutable-ok: Anthropic message payload shape + content: Final = message.get("content") + if not isinstance(content, list): + return message + kept: Final = [b for b in content if not is_encrypted_reasoning_block(b)] # mutable-ok: API message payload + if len(kept) == len(content): + return message + if not kept: + return None + return {**message, "content": kept} # mutable-ok: API message payload + + +def strip_encrypted_reasoning_blocks_from_anthropic_messages( + messages: Sequence[dict], # mutable-ok: Anthropic message payload shape +) -> list[dict]: # mutable-ok: AnthropicMessagesRequest.messages is typed list[dict] + """ + Drop thinking / redacted_thinking blocks that carry another provider's encrypted + reasoning (a turn the Responses API bridge served) before the request reaches + Anthropic, which cannot verify them. Anthropic's own signed blocks are kept. + """ + stripped: Final = (_without_encrypted_reasoning_blocks(m) for m in messages) + return [m for m in stripped if m is not None] # mutable-ok: API message payload + + def strip_thinking_blocks_from_anthropic_messages_request_dict( data: dict[str, Any], ) -> None: @@ -1236,10 +1260,8 @@ def strip_empty_content_blocks_from_anthropic_messages( on the unified ``/v1/messages`` path. ``/v1/chat/completions`` already handles this in ``anthropic_messages_pt``; this helper provides the equivalent guarantee for the native Anthropic Messages path. - A thinking or ``redacted_thinking`` block whose signature or data carries - another provider's encrypted reasoning (a turn served by the Responses API - bridge) is dropped too, since Anthropic cannot verify it; every other - ``redacted_thinking`` block is left alone. + ``redacted_thinking`` blocks are never touched: they carry opaque + ``data`` instead of thinking text. Messages whose content is a list and becomes empty after stripping are omitted, matching :func:`strip_thinking_blocks_from_anthropic_messages`. @@ -1252,11 +1274,7 @@ def strip_empty_content_blocks_from_anthropic_messages( out.append(m) continue content = m["content"] - filtered = [ # mutable-ok: rebuilt message content list - b - for b in content - if not _is_empty_text_block(b) and not is_empty_thinking_block(b) and not is_encrypted_reasoning_block(b) - ] + filtered = [b for b in content if not _is_empty_text_block(b) and not is_empty_thinking_block(b)] if len(filtered) == len(content): out.append(m) elif filtered: diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py index 8267da157ad..9f9346fad4d 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py @@ -25,6 +25,7 @@ from ...common_utils import ( AnthropicModelInfo, optionally_handle_anthropic_oauth, strip_advisor_blocks_from_messages, + strip_encrypted_reasoning_blocks_from_anthropic_messages, ) DEFAULT_ANTHROPIC_API_VERSION: Final = "2023-06-01" @@ -613,7 +614,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): messages = strip_advisor_blocks_from_messages(messages) anthropic_messages_request: Final[AnthropicMessagesRequest] = AnthropicMessagesRequest( - messages=messages, + messages=strip_encrypted_reasoning_blocks_from_anthropic_messages(messages), max_tokens=max_tokens, model=model, **anthropic_messages_optional_request_params, diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_messages_encrypted_reasoning.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_messages_encrypted_reasoning.py new file mode 100644 index 00000000000..c64e9d392e5 --- /dev/null +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_messages_encrypted_reasoning.py @@ -0,0 +1,50 @@ +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + encrypted_reasoning_signature, +) +from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( + AnthropicMessagesConfig, +) + + +def _transform(messages): + return AnthropicMessagesConfig().transform_anthropic_messages_request( + model="claude-sonnet-4-5", + messages=messages, + anthropic_messages_optional_request_params={"max_tokens": 1024}, + litellm_params={}, + headers={}, + ) + + +def test_reasoning_replayed_from_the_responses_bridge_never_reaches_anthropic(): + """Claude Code resumed on a Claude model echoes the thinking blocks a gpt turn produced.""" + messages = [ + {"role": "user", "content": "Solve it."}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "plan", "signature": encrypted_reasoning_signature("gAAAA_1")}, + {"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAAA_2")}, + {"type": "text", "text": "The answer."}, + ], + }, + {"role": "user", "content": "And the next one?"}, + ] + request = _transform(messages) + assert request["messages"][1]["content"] == [{"type": "text", "text": "The answer."}] + assert len(messages[1]["content"]) == 3 + + +def test_anthropic_signed_thinking_blocks_are_forwarded_untouched(): + messages = [ + {"role": "user", "content": "Solve it."}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "plan", "signature": "EqQBCkYIAxgCIkA_anthropic_signed"}, + {"type": "redacted_thinking", "data": "EmwKAhgBEgy_anthropic_minted"}, + {"type": "text", "text": "The answer."}, + ], + }, + ] + assert _transform(messages)["messages"] == messages diff --git a/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py b/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py index 336cbf57e29..97d22274b1c 100644 --- a/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py @@ -42,12 +42,15 @@ FAKE_AUTH_TOKEN = "sk-ant-aut01-fake-auth-token-for-testing-123456789" def test_is_claude_code_one_shot_subagent_request(messages, system, expected): from litellm.llms.anthropic.common_utils import is_claude_code_one_shot_subagent_request - assert is_claude_code_one_shot_subagent_request( - messages=messages, - system=system, - tools=None, - user_agent="claude-cli/2.1.263 (external, cli)", - ) is expected + assert ( + is_claude_code_one_shot_subagent_request( + messages=messages, + system=system, + tools=None, + user_agent="claude-cli/2.1.263 (external, cli)", + ) + is expected + ) class TestOptionallyHandleAnthropicOAuth: @@ -1541,8 +1544,8 @@ class TestAnthropicThinkingSignatureSelfHeal: out = strip_empty_content_blocks_from_anthropic_messages(msgs) assert [b["type"] for b in out[0]["content"]] == ["thinking"] - def test_strip_drops_encrypted_reasoning_blocks_from_the_responses_bridge(self): - """A session resumed on an Anthropic model replays reasoning only OpenAI can verify.""" + def test_strip_keeps_encrypted_reasoning_blocks_for_the_responses_bridge(self): + """The /v1/messages handler runs this before dispatch, so the bridge must still see the replay.""" from litellm.litellm_core_utils.prompt_templates.common_utils import ( encrypted_reasoning_signature, ) @@ -1556,14 +1559,46 @@ class TestAnthropicThinkingSignatureSelfHeal: "content": [ {"type": "thinking", "thinking": "plan", "signature": encrypted_reasoning_signature("gAAAA_1")}, {"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAAA_2")}, - {"type": "redacted_thinking", "data": "EmwKAhgBEgy_anthropic_minted"}, {"type": "text", "text": "The answer."}, ], } ] - out = strip_empty_content_blocks_from_anthropic_messages(msgs) - assert [b["type"] for b in out[0]["content"]] == ["redacted_thinking", "text"] - assert len(msgs[0]["content"]) == 4 + assert strip_empty_content_blocks_from_anthropic_messages(msgs) == msgs + + def test_strip_encrypted_reasoning_drops_only_the_bridge_tagged_blocks(self): + """A session resumed on an Anthropic model replays reasoning only OpenAI can verify.""" + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + encrypted_reasoning_signature, + ) + from litellm.llms.anthropic.common_utils import ( + strip_encrypted_reasoning_blocks_from_anthropic_messages, + ) + + msgs = [ + {"role": "user", "content": "Solve it."}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "plan", "signature": encrypted_reasoning_signature("gAAAA_1")}, + {"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAAA_2")}, + ], + }, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "plan", "signature": encrypted_reasoning_signature("gAAAA_3")}, + {"type": "thinking", "thinking": "native", "signature": "EqQBCkYIAxgCIkA_anthropic_signed"}, + {"type": "redacted_thinking", "data": "EmwKAhgBEgy_anthropic_minted"}, + {"type": "text", "text": "The answer."}, + ], + }, + ] + out = strip_encrypted_reasoning_blocks_from_anthropic_messages(msgs) + assert [m["role"] for m in out] == ["user", "assistant"] + assert [b["type"] for b in out[1]["content"]] == ["thinking", "redacted_thinking", "text"] + assert out[1]["content"][0]["signature"] == "EqQBCkYIAxgCIkA_anthropic_signed" + assert len(msgs[1]["content"]) == 2 + assert len(msgs[2]["content"]) == 4 def test_strip_empty_text_blocks_treats_null_text_as_empty(self): from litellm.llms.anthropic.common_utils import ( From f49ebc93eaf108d2312779346485ce6808c26184 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 14:11:43 -0700 Subject: [PATCH 3/6] fix(anthropic): request encrypted reasoning only where the Responses provider returns it The bridge now asks for reasoning.encrypted_content whenever the provider's Responses config lists include, independent of the client's thinking block, and leaves it out for providers such as Perplexity that reject the param. Bridge-tagged blocks are stripped on the chat adapter path too, so a mid session model switch to Gemini or Bedrock no longer forwards them as real signatures, a bare prefix counts as bridge-tagged, and non-mapping messages pass through the strip untouched. --- .../prompt_templates/common_utils.py | 18 ++++++--- litellm/llms/anthropic/common_utils.py | 2 + .../adapters/transformation.py | 4 +- .../responses_adapters/handler.py | 17 ++++++++- .../responses_adapters/transformation.py | 9 ++++- ...ore_utils_prompt_templates_common_utils.py | 3 ++ ...al_pass_through_adapters_transformation.py | 38 +++++++++++++++++++ .../test_responses_adapters_handler.py | 23 +++++++++++ .../test_responses_adapters_transformation.py | 12 +++++- .../anthropic/test_anthropic_common_utils.py | 9 +++++ 10 files changed, 126 insertions(+), 9 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 918cb982773..75a3aefa98b 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -1853,22 +1853,30 @@ def encrypted_reasoning_signature(encrypted_content: str) -> str: return f"{ENCRYPTED_REASONING_SIGNATURE_PREFIX}{encrypted_content}" +def _carries_encrypted_reasoning(signature: object) -> bool: + return isinstance(signature, str) and signature.startswith(ENCRYPTED_REASONING_SIGNATURE_PREFIX) + + def encrypted_content_from_signature(signature: object) -> str | None: - if not isinstance(signature, str) or not signature.startswith(ENCRYPTED_REASONING_SIGNATURE_PREFIX): + if not isinstance(signature, str) or not _carries_encrypted_reasoning(signature): return None return signature.removeprefix(ENCRYPTED_REASONING_SIGNATURE_PREFIX) or None -def _encrypted_content_of_block(block: Mapping[str, object]) -> str | None: +def _encrypted_reasoning_field(block: Mapping[str, object]) -> object: match block.get("type"): case "thinking": - return encrypted_content_from_signature(block.get("signature")) + return block.get("signature") case "redacted_thinking": - return encrypted_content_from_signature(block.get("data")) + return block.get("data") case _: return None +def _encrypted_content_of_block(block: Mapping[str, object]) -> str | None: + return encrypted_content_from_signature(_encrypted_reasoning_field(block)) + + def is_encrypted_reasoning_block(block: object) -> bool: """A thinking or redacted_thinking block carrying Responses API encrypted reasoning. @@ -1878,7 +1886,7 @@ def is_encrypted_reasoning_block(block: object) -> bool: if not isinstance(block, Mapping): return False mapping: Final = cast(Mapping[str, object], block) # cast-ok: narrowed by isinstance - return _encrypted_content_of_block(mapping) is not None + return _carries_encrypted_reasoning(_encrypted_reasoning_field(mapping)) def _reasoning_replay_group_key(indexed_block: tuple[int, Mapping[str, object]]) -> str: diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index cb24f4153f9..ca520116606 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -1203,6 +1203,8 @@ def strip_thinking_blocks_from_anthropic_messages(messages: list[Any]) -> list[A def _without_encrypted_reasoning_blocks(message: dict) -> dict | None: # mutable-ok: Anthropic message payload shape + if not isinstance(message, Mapping): + return message content: Final = message.get("content") if not isinstance(content, list): return message diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index db890662132..aaffc84b872 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -113,6 +113,7 @@ from litellm.litellm_core_utils.reasoning_effort_utils import ( from litellm.llms.anthropic.common_utils import ( is_empty_unsigned_thinking_block, normalize_anthropic_tool_use_id, + strip_encrypted_reasoning_blocks_from_anthropic_messages, ) from litellm.llms.anthropic.experimental_pass_through.context_management import ( PolyfillResult, @@ -417,7 +418,8 @@ class LiteLLMAnthropicMessagesAdapter: model: str | None = None, ) -> list: new_messages: Final[list[AllMessageValues]] = [] - for m in messages: + replayable_messages: Final = strip_encrypted_reasoning_blocks_from_anthropic_messages(messages) + for m in replayable_messages: user_message: ChatCompletionUserMessage | None = None tool_message_list: list[ChatCompletionToolMessage] = [] new_user_content_list: list[ChatCompletionTextObject | ChatCompletionImageObject] = [] diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py index 0445c23ed8c..24825cc87e0 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py @@ -19,6 +19,7 @@ from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, ) from litellm.types.llms.openai import ResponsesAPIResponse +from litellm.utils import ProviderConfigManager from ..utils import litellm_logging_obj_from_kwargs, local_model_name from .streaming_iterator import AnthropicResponsesStreamWrapper @@ -34,6 +35,15 @@ def _forwarded_kwargs(extra_kwargs: Mapping[str, object] | None) -> Mapping[str, return extra_kwargs or {} +def _provider_returns_encrypted_reasoning(model: str, custom_llm_provider: object) -> bool: + provider: Final = ( + custom_llm_provider if isinstance(custom_llm_provider, str) else litellm.get_llm_provider(model=model)[1] + ) + provider_model: Final = local_model_name(model, provider) + responses_config: Final = ProviderConfigManager.get_provider_responses_api_config(provider, provider_model) + return responses_config is not None and "include" in responses_config.get_supported_openai_params(provider_model) + + def _build_responses_kwargs( *, max_tokens: int, @@ -85,8 +95,13 @@ def _build_responses_kwargs( request_data["output_format"] = output_format anthropic_request: Final = AnthropicMessagesRequest(**request_data) - responses_kwargs: Final = _ADAPTER.translate_request(anthropic_request) forwarded_kwargs: Final = _forwarded_kwargs(extra_kwargs) + responses_kwargs: Final = _ADAPTER.translate_request( + anthropic_request, + include_encrypted_reasoning=_provider_returns_encrypted_reasoning( + model, forwarded_kwargs.get("custom_llm_provider") + ), + ) # Normalize reasoning effort based on model capabilities # (e.g. "max" → "xhigh"/"high", "minimal" → "low" if unsupported) diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py index 07419db443c..1fdb0318bab 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py @@ -505,10 +505,16 @@ class LiteLLMAnthropicToResponsesAPIAdapter: def translate_request( self, anthropic_request: AnthropicMessagesRequest, + include_encrypted_reasoning: bool = True, ) -> dict[str, Any]: """ Translate a full Anthropic /v1/messages request dict to litellm.responses() / litellm.aresponses() kwargs. + + ``include_encrypted_reasoning`` asks the provider for ``reasoning.encrypted_content`` + on every call, so a reasoning model's items can be replayed intact next turn even + when the client sent no ``thinking`` block; pass False for a provider whose + Responses API rejects ``include``. """ model: Final[str] = anthropic_request["model"] messages_list: Final = cast( @@ -538,6 +544,8 @@ class LiteLLMAnthropicToResponsesAPIAdapter: "model": model, "input": input_items, } + if include_encrypted_reasoning: + responses_kwargs["include"] = [RESPONSES_INCLUDE_ENCRYPTED_REASONING] # mutable-ok: API request payload if system and not developer_parts: if isinstance(system, str): @@ -582,7 +590,6 @@ class LiteLLMAnthropicToResponsesAPIAdapter: ) if reasoning: responses_kwargs["reasoning"] = reasoning - responses_kwargs["include"] = [RESPONSES_INCLUDE_ENCRYPTED_REASONING] # mutable-ok: json list # output_format / output_config.format -> text format # output_format: {"type": "json_schema", "schema": {...}} diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index 9df124576a0..ecb07d8c937 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -7,6 +7,7 @@ import pytest from litellm.litellm_core_utils.prompt_templates.common_utils import ( + ENCRYPTED_REASONING_SIGNATURE_PREFIX, TOOL_RESULT_IMAGE_BOUNDARY, TOOL_RESULT_IMAGE_PLACEHOLDER, add_system_prompt_to_messages, @@ -1610,6 +1611,8 @@ class TestEncryptedReasoningReplay: [ ({"type": "thinking", "thinking": "x", "signature": encrypted_reasoning_signature("g")}, True), ({"type": "redacted_thinking", "data": encrypted_reasoning_signature("g")}, True), + ({"type": "thinking", "thinking": "x", "signature": ENCRYPTED_REASONING_SIGNATURE_PREFIX}, True), + ({"type": "redacted_thinking", "data": ENCRYPTED_REASONING_SIGNATURE_PREFIX}, True), ({"type": "thinking", "thinking": "x", "signature": "ErcBCkgIValid"}, False), ({"type": "redacted_thinking", "data": "EmwKAhgBEgy"}, False), ({"type": "text", "text": encrypted_reasoning_signature("g")}, False), diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index 30465ca25ba..279ed7a8b33 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -9,6 +9,7 @@ import litellm from litellm.litellm_core_utils.prompt_templates.common_utils import ( TOOL_RESULT_IMAGE_PLACEHOLDER, + encrypted_reasoning_signature, ) from litellm.litellm_core_utils.prompt_templates.factory import ( THOUGHT_SIGNATURE_SEPARATOR, @@ -407,6 +408,43 @@ def test_translate_anthropic_messages_to_openai_thinking_blocks(): assert result[1]["tool_calls"][0]["id"] == "toolu_01234" +def test_translate_anthropic_messages_to_openai_drops_bridge_encrypted_reasoning_blocks(): + """A session that moves from an OpenAI reasoning model to a chat provider replays reasoning only OpenAI can read. + + Gemini rejects the whole request when such a block reaches it as a thought_signature, so the + adapter drops those blocks and keeps the provider-signed ones. + """ + + anthropic_messages = [ + AnthropicMessagesUserMessageParam( + role="user", + content=[{"type": "text", "text": "Who drinks water?"}], + ), + AnthopicMessagesAssistantMessageParam( + role="assistant", + content=[ + {"type": "thinking", "thinking": "plan", "signature": encrypted_reasoning_signature("gAAAA_1")}, + {"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAAA_2")}, + {"type": "text", "text": "The Norwegian."}, + ], + ), + AnthopicMessagesAssistantMessageParam( + role="assistant", + content=[ + {"type": "thinking", "thinking": "native", "signature": "EqQBCkYIAxgCIkA_signed"}, + {"type": "text", "text": "Still the Norwegian."}, + ], + ), + ] + + result = LiteLLMAnthropicMessagesAdapter().translate_anthropic_messages_to_openai(messages=anthropic_messages) + + assert [m["role"] for m in result] == ["user", "assistant", "assistant"] + assert not result[1].get("thinking_blocks") + assert result[1]["content"] == "The Norwegian." + assert [b["signature"] for b in result[2]["thinking_blocks"]] == ["EqQBCkYIAxgCIkA_signed"] + + def test_translate_anthropic_messages_to_openai_sets_reasoning_content(): """Reasoning-aware chat providers read reasoning_content, so thinking text must land there. diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_handler.py index 3383813245a..2cbfaa17d23 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_handler.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_handler.py @@ -54,6 +54,29 @@ def test_build_responses_kwargs_prefers_explicit_prompt_cache_key_over_derived() assert responses_kwargs["prompt_cache_key"] == "explicit-key" +def test_build_responses_kwargs_asks_openai_for_encrypted_reasoning_without_thinking(): + responses_kwargs = _build_responses_kwargs( + max_tokens=1024, + messages=MESSAGES, + model="openai/gpt-5.6-luna", + extra_kwargs={"custom_llm_provider": "openai"}, + ) + assert responses_kwargs["include"] == ["reasoning.encrypted_content"] + assert "reasoning" not in responses_kwargs + + +def test_build_responses_kwargs_skips_include_for_a_responses_provider_that_rejects_it(): + responses_kwargs = _build_responses_kwargs( + max_tokens=1024, + messages=MESSAGES, + model="perplexity/sonar", + thinking={"type": "enabled", "budget_tokens": 4096}, + extra_kwargs={"custom_llm_provider": "perplexity"}, + ) + assert "include" not in responses_kwargs + assert "reasoning" in responses_kwargs + + def test_build_responses_kwargs_without_metadata_sets_no_prompt_cache_key(): responses_kwargs = _build_responses_kwargs( max_tokens=1024, diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py index 79eaf6b60f2..9b7b15ace61 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py @@ -1162,7 +1162,6 @@ class TestTranslateRequestBroaderCoverage: req = _make_request(thinking={"type": "disabled"}) kwargs = _ADAPTER.translate_request(req) assert "reasoning" not in kwargs - assert "include" not in kwargs def test_thinking_asks_for_the_encrypted_reasoning(self): """The documented way to get reasoning that survives store=false is to ask for it.""" @@ -1170,6 +1169,17 @@ class TestTranslateRequestBroaderCoverage: kwargs = _ADAPTER.translate_request(req) assert kwargs["include"] == ["reasoning.encrypted_content"] + def test_encrypted_reasoning_is_asked_for_without_a_thinking_block(self): + """A reasoning model reasons whether or not the client sent `thinking`, so the replay needs it either way.""" + kwargs = _ADAPTER.translate_request(_make_request()) + assert kwargs["include"] == ["reasoning.encrypted_content"] + + def test_encrypted_reasoning_is_not_asked_for_when_the_provider_rejects_include(self): + req = _make_request(thinking={"type": "enabled", "budget_tokens": 12000}) + kwargs = _ADAPTER.translate_request(req, include_encrypted_reasoning=False) + assert kwargs["reasoning"] == {"effort": "high"} + assert "include" not in kwargs + def test_metadata_user_id_mapped_to_user(self): req = _make_request(metadata={"user_id": "user-42"}) kwargs = _ADAPTER.translate_request(req) diff --git a/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py b/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py index 97d22274b1c..91652c89092 100644 --- a/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/test_litellm/llms/anthropic/test_anthropic_common_utils.py @@ -1600,6 +1600,15 @@ class TestAnthropicThinkingSignatureSelfHeal: assert len(msgs[1]["content"]) == 2 assert len(msgs[2]["content"]) == 4 + def test_strip_encrypted_reasoning_leaves_malformed_messages_for_the_provider_to_reject(self): + """A bare string in messages must reach Anthropic as a 400, not die in the stripper as a 500.""" + from litellm.llms.anthropic.common_utils import ( + strip_encrypted_reasoning_blocks_from_anthropic_messages, + ) + + msgs = ["hi", {"role": "user", "content": "hello"}] + assert strip_encrypted_reasoning_blocks_from_anthropic_messages(msgs) == msgs + def test_strip_empty_text_blocks_treats_null_text_as_empty(self): from litellm.llms.anthropic.common_utils import ( strip_empty_content_blocks_from_anthropic_messages, From e27a018a6ce1ecf0fdbfa071da8e50d5b9460d96 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 16:29:52 -0700 Subject: [PATCH 4/6] fix(router): pin bridge-replayed encrypted reasoning to the deployment that minted it The encrypted_content_affinity check only read the pin from the Responses input, which /v1/messages builds after the router has picked a deployment, so a model group spread across OpenAI orgs sent follow-up turns to the wrong org and got invalid_encrypted_content back. The check now also decodes the pin from bridge-tagged thinking and redacted_thinking blocks in the Anthropic messages. The bridge also keeps a deployment's own include list next to reasoning.encrypted_content instead of replacing it. --- .../prompt_templates/common_utils.py | 4 +- .../responses_adapters/handler.py | 10 +++- .../encrypted_content_affinity_check.py | 50 ++++++++++++++++--- .../test_responses_adapters_handler.py | 10 ++++ .../test_encrypted_content_affinity_check.py | 40 +++++++++++++++ 5 files changed, 104 insertions(+), 10 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 75a3aefa98b..ae9b7f6bc4b 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -1873,7 +1873,7 @@ def _encrypted_reasoning_field(block: Mapping[str, object]) -> object: return None -def _encrypted_content_of_block(block: Mapping[str, object]) -> str | None: +def encrypted_content_of_block(block: Mapping[str, object]) -> str | None: return encrypted_content_from_signature(_encrypted_reasoning_field(block)) @@ -1900,7 +1900,7 @@ def _reasoning_item_from_block_group(group: tuple[Mapping[str, object], ...]) -> for block in group if (text := _readable_thinking_text(block)) ] - encrypted_content: Final = _encrypted_content_of_block(group[0]) + encrypted_content: Final = encrypted_content_of_block(group[0]) if encrypted_content is not None: return ChatCompletionReasoningItem(type="reasoning", summary=summary, encrypted_content=encrypted_content) if not summary: diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py index 24825cc87e0..7731c883d9f 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py @@ -126,7 +126,7 @@ def _build_responses_kwargs( responses_kwargs["stream"] = True # Forward litellm-specific kwargs (api_key, api_base, logging obj, etc.) - excluded: Final = {"anthropic_messages"} + excluded: Final = frozenset(("anthropic_messages",)) for key, value in forwarded_kwargs.items(): if key == "litellm_logging_obj" and value is not None: from litellm.litellm_core_utils.litellm_logging import ( @@ -147,6 +147,14 @@ def _build_responses_kwargs( if explicit_prompt_cache_key is not None: responses_kwargs["prompt_cache_key"] = explicit_prompt_cache_key + deployment_include: Final = forwarded_kwargs.get("include") + bridge_include: Final = responses_kwargs.get("include") + if isinstance(deployment_include, list) and isinstance(bridge_include, list): + responses_kwargs["include"] = [ + *bridge_include, + *(item for item in deployment_include if item not in bridge_include), + ] + return responses_kwargs diff --git a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py index b623e31ce06..ba94bd86a56 100644 --- a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py @@ -37,6 +37,7 @@ Safe to enable globally: """ import time +from collections.abc import Iterator, Mapping from typing import TYPE_CHECKING, Final, Optional, Protocol, cast import httpx @@ -48,6 +49,7 @@ from litellm.exceptions import ( ServiceUnavailableError, ) from litellm.integrations.custom_logger import CustomLogger, Span +from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_content_of_block from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.router_utils.cooldown_cache import CooldownCacheValue from litellm.types.llms.openai import AllMessageValues @@ -138,15 +140,48 @@ class EncryptedContentAffinityCheck(CustomLogger): # If no encoded ID, check if encrypted_content itself is wrapped encrypted_content = item.get("encrypted_content") if encrypted_content and isinstance(encrypted_content, str): - ( - model_id, - _, - ) = ResponsesAPIRequestUtils._unwrap_encrypted_content_with_model_id(encrypted_content) + model_id = EncryptedContentAffinityCheck._model_id_from_wrapped_encrypted_content(encrypted_content) if model_id: return model_id return None + @staticmethod + def _anthropic_content_blocks(messages: object) -> Iterator[Mapping[str, object]]: + if not isinstance(messages, list): + return iter(()) + return ( + cast(Mapping[str, object], block) # cast-ok: narrowed by isinstance + for message in cast(list[object], messages) # cast-ok: narrowed by isinstance + if isinstance(message, Mapping) + for content in (cast(Mapping[str, object], message).get("content"),) # cast-ok: narrowed by isinstance + if isinstance(content, list) + for block in cast(list[object], content) # cast-ok: narrowed by isinstance + if isinstance(block, Mapping) + ) + + @staticmethod + def _model_id_from_wrapped_encrypted_content(encrypted_content: str) -> str | None: + model_id, _ = ResponsesAPIRequestUtils._unwrap_encrypted_content_with_model_id(encrypted_content) + return model_id or None + + @staticmethod + def _extract_model_id_from_anthropic_messages(messages: object) -> str | None: + return next( + ( + model_id + for block in EncryptedContentAffinityCheck._anthropic_content_blocks(messages) + if (encrypted_content := encrypted_content_of_block(block)) is not None + if ( + model_id := EncryptedContentAffinityCheck._model_id_from_wrapped_encrypted_content( + encrypted_content + ) + ) + is not None + ), + None, + ) + @staticmethod def _find_deployment_by_model_id(healthy_deployments: list[dict], model_id: str) -> dict | None: for deployment in healthy_deployments: @@ -248,13 +283,14 @@ class EncryptedContentAffinityCheck(CustomLogger): if "litellm_metadata" in request_kwargs: request_kwargs["litellm_metadata"]["encrypted_content_affinity_enabled"] = True - request_input: Final = request_kwargs.get("input") - model_id: Final = self._extract_model_id_from_input(request_input) + model_id: Final = self._extract_model_id_from_input( + request_kwargs.get("input") + ) or self._extract_model_id_from_anthropic_messages(request_kwargs.get("messages")) if not model_id: return typed_healthy_deployments verbose_router_logger.debug( - "EncryptedContentAffinityCheck: decoded model_id=%s from input item IDs", + "EncryptedContentAffinityCheck: decoded model_id=%s from the request's encrypted content markers", model_id, ) diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_handler.py index 2cbfaa17d23..2aa84f05a86 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_handler.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_handler.py @@ -77,6 +77,16 @@ def test_build_responses_kwargs_skips_include_for_a_responses_provider_that_reje assert "reasoning" in responses_kwargs +def test_build_responses_kwargs_keeps_the_deployment_include_next_to_encrypted_reasoning(): + responses_kwargs = _build_responses_kwargs( + max_tokens=1024, + messages=MESSAGES, + model="openai/gpt-5.6-luna", + extra_kwargs={"custom_llm_provider": "openai", "include": ["file_search_call.results"]}, + ) + assert responses_kwargs["include"] == ["reasoning.encrypted_content", "file_search_call.results"] + + def test_build_responses_kwargs_without_metadata_sets_no_prompt_cache_key(): responses_kwargs = _build_responses_kwargs( max_tokens=1024, diff --git a/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py b/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py index dac991a41c4..e858f7eb0a8 100644 --- a/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py +++ b/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py @@ -1656,3 +1656,43 @@ async def test_model_group_encrypted_content_affinity_overrides_global_deploymen assert request_kwargs.get("_encrypted_content_affinity_pinned") is True finally: router.discard() + + +@pytest.mark.asyncio +async def test_encrypted_content_affinity_pins_anthropic_messages_replayed_through_the_bridge(): + """ + Claude Code behind /v1/messages replays the encrypted reasoning the bridge packed + into a thinking block's signature (or a redacted block's data). The pin has to be + read from those blocks because the bridge builds the Responses `input` only after + the router has picked a deployment. + """ + check = EncryptedContentAffinityCheck() + deployments = [ + {"model_info": {"id": "openai-org-a"}, "litellm_params": {"model": "openai/gpt-5.1"}}, + {"model_info": {"id": "openai-org-b"}, "litellm_params": {"model": "openai/gpt-5.1"}}, + ] + wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id("gAAAAA_turn_one", "openai-org-b") + request_kwargs = { + "messages": [ + {"role": "user", "content": "Solve the zebra puzzle"}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "Anthropic minted this one", "signature": "ErcCCpIBCBEYAipA"}, + {"type": "redacted_thinking", "data": f"litellm_encrypted_reasoning:{wrapped}"}, + {"type": "text", "text": "The zebra owner lives in the green house."}, + ], + }, + {"role": "user", "content": "And who drinks water?"}, + ], + } + + pinned = await check.async_filter_deployments( + model="gpt-5.1", + healthy_deployments=deployments, + messages=None, + request_kwargs=request_kwargs, + ) + + assert [d["model_info"]["id"] for d in pinned] == ["openai-org-b"] + assert request_kwargs["_encrypted_content_affinity_pinned"] is True From 4716b46c243b90d833ea08de8bebcc9a20c963d2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 18:31:08 -0700 Subject: [PATCH 5/6] fix(router): read the affinity pin from the messages argument and strip bridge reasoning on a cross-group route The encrypted_content_affinity check only read the Anthropic history from request_kwargs["messages"], so a caller that passes it through the callback's messages argument alone skipped the pin. Read the argument first and fall back to the kwargs. When the minting deployment is not a candidate of the routed group, the base already strips the Responses input's encrypted reasoning; do the same for the bridge-tagged thinking blocks in Anthropic messages so the routed deployment gets the readable thinking text instead of ciphertext it cannot decrypt. --- .../prompt_templates/common_utils.py | 43 ++++++++++- .../encrypted_content_affinity_check.py | 14 +++- ...ore_utils_prompt_templates_common_utils.py | 60 ++++++++++++++- .../test_encrypted_content_affinity_check.py | 73 +++++++++++++++---- 4 files changed, 166 insertions(+), 24 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index ae9b7f6bc4b..7ca2a8aad6a 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -6,7 +6,7 @@ import io import json import mimetypes import re -from collections.abc import Iterable, Mapping, Sequence +from collections.abc import Iterable, Iterator, Mapping, Sequence from itertools import groupby from os import PathLike from pathlib import Path @@ -1889,6 +1889,47 @@ def is_encrypted_reasoning_block(block: object) -> bool: return _carries_encrypted_reasoning(_encrypted_reasoning_field(mapping)) +def strip_encrypted_reasoning_from_messages(messages: object) -> None: + """Drop the encrypted reasoning a routed deployment cannot decrypt from Anthropic-shaped + history, keeping the readable thinking text. + + Mutates the content lists in place: the router's fallback snapshot shares these + message objects, so a rebound list would replay the stripped blocks on the fallback hop. + """ + if not isinstance(messages, list): + return + for content in _anthropic_content_lists(cast(list[object], messages)): # cast-ok: untyped client json + _strip_encrypted_reasoning_from_blocks(content) + + +def _anthropic_content_lists(messages: Sequence[object]) -> Iterator[object]: + return ( + cast(list[object], content) # cast-ok: narrowed by isinstance + for message in messages + if isinstance(message, Mapping) + for content in (cast(Mapping[str, object], message).get("content"),) # cast-ok: narrowed by isinstance + if isinstance(content, list) + ) + + +def _strip_encrypted_reasoning_from_blocks(content: object) -> None: + blocks: Final = cast(list[object], content) # cast-ok: narrowed by the caller's isinstance + stripped: Final = tuple(_without_encrypted_reasoning_block(block) for block in blocks) + blocks[:] = (block for block in stripped if block is not None) # rebind-ok: list shared with fallback snapshot + + +def _without_encrypted_reasoning_block(block: object) -> object | None: + if not is_encrypted_reasoning_block(block): + return block + mapping: Final = cast(Mapping[str, object], block) # cast-ok: narrowed by is_encrypted_reasoning_block + if mapping.get("type") != "thinking" or not mapping.get("thinking"): + return None + kept: Final[dict[str, object]] = { # mutable-ok: thinking block rebuilt without the undecryptable signature + key: value for key, value in mapping.items() if key != "signature" + } + return kept + + def _reasoning_replay_group_key(indexed_block: tuple[int, Mapping[str, object]]) -> str: index, block = indexed_block return f"encrypted:{index}" if is_encrypted_reasoning_block(block) else "summary" diff --git a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py index b2edb13a4bf..cdd70e6baf2 100644 --- a/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py +++ b/litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py @@ -48,7 +48,10 @@ from litellm.exceptions import ( ServiceUnavailableError, ) from litellm.integrations.custom_logger import CustomLogger, Span -from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_content_of_block +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + encrypted_content_of_block, + strip_encrypted_reasoning_from_messages, +) from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.router_utils.cooldown_cache import CooldownCacheValue from litellm.types.llms.openai import AllMessageValues @@ -274,8 +277,9 @@ class EncryptedContentAffinityCheck(CustomLogger): parent_otel_span: Span | None = None, ) -> list[dict]: """ - If the request ``input`` contains litellm-encoded item IDs, decode the - embedded ``model_id`` and pin the request to that deployment. Raises + If the request ``input`` contains litellm-encoded item IDs, or its Anthropic + ``messages`` replay a bridge-tagged thinking block, decode the embedded + ``model_id`` and pin the request to that deployment. Raises ``RateLimitError`` / ``ServiceUnavailableError`` when the originating deployment is a member of the routed model group but currently unavailable and no encryption-boundary peer exists, rather than dispatching a doomed @@ -304,9 +308,10 @@ class EncryptedContentAffinityCheck(CustomLogger): request_kwargs["litellm_metadata"]["encrypted_content_affinity_enabled"] = True request_input: Final = request_kwargs.get("input") + anthropic_messages: Final = messages or request_kwargs.get("messages") model_id: Final = self._extract_model_id_from_input( request_input - ) or self._extract_model_id_from_anthropic_messages(request_kwargs.get("messages")) + ) or self._extract_model_id_from_anthropic_messages(anthropic_messages) if not model_id: return typed_healthy_deployments @@ -363,6 +368,7 @@ class EncryptedContentAffinityCheck(CustomLogger): model, ) ResponsesAPIRequestUtils.strip_encrypted_reasoning_from_input(request_input) + strip_encrypted_reasoning_from_messages(anthropic_messages) return typed_healthy_deployments # The origin is a member of the routed group but currently unavailable (cooled down); fail fast diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index ecb07d8c937..71b71d55fa7 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -1,3 +1,4 @@ +import copy import functools import json import os @@ -20,6 +21,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( is_encrypted_reasoning_block, responses_reasoning_items_from_thinking_blocks, split_concatenated_json_objects, + strip_encrypted_reasoning_from_messages, update_messages_with_model_file_ids, ) @@ -1567,7 +1569,9 @@ class TestEncryptedReasoningReplay: def test_signature_round_trips_the_encrypted_content(self): assert encrypted_content_from_signature(encrypted_reasoning_signature("gAAAA_bytes")) == "gAAAA_bytes" - @pytest.mark.parametrize("signature", [None, "", "ErcBCkgIValidAnthropicSignature", "litellm_encrypted_reasoning:", 7]) + @pytest.mark.parametrize( + "signature", [None, "", "ErcBCkgIValidAnthropicSignature", "litellm_encrypted_reasoning:", 7] + ) def test_anything_else_is_not_encrypted_content(self, signature): assert encrypted_content_from_signature(signature) is None @@ -1576,7 +1580,11 @@ class TestEncryptedReasoningReplay: [{"type": "thinking", "thinking": "Plan.", "signature": encrypted_reasoning_signature("gAAAA_1")}] ) assert items == ( - {"type": "reasoning", "summary": [{"type": "summary_text", "text": "Plan."}], "encrypted_content": "gAAAA_1"}, + { + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "Plan."}], + "encrypted_content": "gAAAA_1", + }, ) def test_encrypted_redacted_block_replays_with_an_empty_summary(self): @@ -1596,7 +1604,10 @@ class TestEncryptedReasoningReplay: ] ) assert items == ( - {"type": "reasoning", "summary": [{"type": "summary_text", "text": "A."}, {"type": "summary_text", "text": "B."}]}, + { + "type": "reasoning", + "summary": [{"type": "summary_text", "text": "A."}, {"type": "summary_text", "text": "B."}], + }, {"type": "reasoning", "summary": [{"type": "summary_text", "text": "C."}], "encrypted_content": "gAAAA_c"}, {"type": "reasoning", "summary": [{"type": "summary_text", "text": "D."}]}, ) @@ -1621,3 +1632,46 @@ class TestEncryptedReasoningReplay: ) def test_is_encrypted_reasoning_block(self, block, expected): assert is_encrypted_reasoning_block(block) is expected + + def test_strip_keeps_the_readable_thinking_and_drops_the_undecryptable_bytes(self): + assistant_content = [ + {"type": "thinking", "thinking": "minted by Anthropic", "signature": "ErcBCkgIValid"}, + {"type": "thinking", "thinking": "packed by the bridge", "signature": encrypted_reasoning_signature("g1")}, + {"type": "redacted_thinking", "data": encrypted_reasoning_signature("g2")}, + {"type": "thinking", "thinking": "", "signature": encrypted_reasoning_signature("g3")}, + {"type": "text", "text": "answer"}, + ] + messages = [ + {"role": "user", "content": "question"}, + {"role": "assistant", "content": assistant_content}, + {"role": "user", "content": [{"type": "text", "text": "follow-up"}]}, + ] + + strip_encrypted_reasoning_from_messages(messages) + + assert messages[1]["content"] is assistant_content + assert assistant_content == [ + {"type": "thinking", "thinking": "minted by Anthropic", "signature": "ErcBCkgIValid"}, + {"type": "thinking", "thinking": "packed by the bridge"}, + {"type": "text", "text": "answer"}, + ] + assert messages[0] == {"role": "user", "content": "question"} + assert messages[2] == {"role": "user", "content": [{"type": "text", "text": "follow-up"}]} + + @pytest.mark.parametrize( + "messages", + [ + "not a list", + None, + [{"role": "user", "content": None}], + [{"role": "user", "content": "plain string"}], + ["not a message"], + [{"role": "assistant", "content": [{"type": "thinking", "thinking": "x", "signature": "ErcBCkgIValid"}]}], + ], + ) + def test_strip_leaves_history_without_bridge_reasoning_untouched(self, messages): + before = copy.deepcopy(messages) + + strip_encrypted_reasoning_from_messages(messages) + + assert messages == before diff --git a/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py b/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py index 99a650436b8..57c02e43fc1 100644 --- a/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py +++ b/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py @@ -1597,26 +1597,12 @@ async def test_encrypted_content_affinity_pins_anthropic_messages_replayed_throu {"model_info": {"id": "openai-org-a"}, "litellm_params": {"model": "openai/gpt-5.1"}}, {"model_info": {"id": "openai-org-b"}, "litellm_params": {"model": "openai/gpt-5.1"}}, ] - wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id("gAAAAA_turn_one", "openai-org-b") - request_kwargs = { - "messages": [ - {"role": "user", "content": "Solve the zebra puzzle"}, - { - "role": "assistant", - "content": [ - {"type": "thinking", "thinking": "Anthropic minted this one", "signature": "ErcCCpIBCBEYAipA"}, - {"type": "redacted_thinking", "data": f"litellm_encrypted_reasoning:{wrapped}"}, - {"type": "text", "text": "The zebra owner lives in the green house."}, - ], - }, - {"role": "user", "content": "And who drinks water?"}, - ], - } + request_kwargs = {"model": "gpt-5.1"} pinned = await check.async_filter_deployments( model="gpt-5.1", healthy_deployments=deployments, - messages=None, + messages=_bridge_replayed_anthropic_messages(minted_by="openai-org-b"), request_kwargs=request_kwargs, ) @@ -1624,6 +1610,61 @@ async def test_encrypted_content_affinity_pins_anthropic_messages_replayed_throu assert request_kwargs["_encrypted_content_affinity_pinned"] is True +def _bridge_replayed_anthropic_messages(minted_by: str) -> list: + wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id("gAAAAA_turn_one", minted_by) + return [ + {"role": "user", "content": "Solve the zebra puzzle"}, + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "Anthropic minted this one", "signature": "ErcCCpIBCBEYAipA"}, + {"type": "redacted_thinking", "data": f"litellm_encrypted_reasoning:{wrapped}"}, + { + "type": "thinking", + "thinking": "The bridge packed this one", + "signature": f"litellm_encrypted_reasoning:{wrapped}", + }, + {"type": "text", "text": "The zebra owner lives in the green house."}, + ], + }, + {"role": "user", "content": "And who drinks water?"}, + ] + + +@pytest.mark.asyncio +async def test_encrypted_content_affinity_strips_bridge_reasoning_from_messages_routed_to_another_group(): + """ + The /v1/messages twin of the tier-change case: the routed group holds no deployment + of the org that minted the reasoning, so the bridge-tagged blocks are stripped down + to their readable thinking text and the request dispatches to the routed pool. + """ + originating = _make_originating_mock(None, "key-a", model_name="gpt-reasoning-tier") + mock_router = _make_router_mock_with_cooldown( + originating, cooldown_entries=[], routed_group_model_ids=["openai-org-b"] + ) + check = EncryptedContentAffinityCheck(router=mock_router) + routed_pool = [{"model_info": {"id": "openai-org-b"}, "litellm_params": {"model": "openai/gpt-5-nano"}}] + messages = _bridge_replayed_anthropic_messages(minted_by="openai-org-a") + assistant_content = messages[1]["content"] + request_kwargs = {"model": "gpt-5.1"} + + result = await check.async_filter_deployments( + model="gpt-simple-tier", + healthy_deployments=routed_pool, + messages=messages, + request_kwargs=request_kwargs, + ) + + assert result is routed_pool + assert "_encrypted_content_affinity_pinned" not in request_kwargs + assert messages[1]["content"] is assistant_content + assert assistant_content == [ + {"type": "thinking", "thinking": "Anthropic minted this one", "signature": "ErcCCpIBCBEYAipA"}, + {"type": "thinking", "thinking": "The bridge packed this one"}, + {"type": "text", "text": "The zebra owner lives in the green house."}, + ] + + class TestStripEncryptedReasoningFromInput: def test_keeps_summary_and_drops_encrypted_content_and_id(self): wrapped = ResponsesAPIRequestUtils._wrap_encrypted_content_with_model_id("gAAAAA-blob", "deployment-a") From 5a1be5642606eb56945e0dfd91ca02647a592a52 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 9 Sep 2026 19:27:54 -0700 Subject: [PATCH 6/6] fix(router): drop bridge-tagged reasoning blocks whole when the minting deployment is not in the routed group The cross-group branch of EncryptedContentAffinityCheck removed only the signature from Anthropic-shaped thinking blocks, which left unsigned thinking blocks that Anthropic and Bedrock reject (thinking.signature: Field required). Drop the whole block, the way #40280 drops undecryptable Responses input items, so the routed request carries the conversation text with no reasoning item for those turns --- .../prompt_templates/common_utils.py | 25 +++++++------------ ...ore_utils_prompt_templates_common_utils.py | 4 +-- .../test_encrypted_content_affinity_check.py | 8 +++--- 3 files changed, 16 insertions(+), 21 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 7ca2a8aad6a..d70530534da 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -1890,8 +1890,13 @@ def is_encrypted_reasoning_block(block: object) -> bool: def strip_encrypted_reasoning_from_messages(messages: object) -> None: - """Drop the encrypted reasoning a routed deployment cannot decrypt from Anthropic-shaped - history, keeping the readable thinking text. + """Drop the bridge-tagged reasoning blocks a routed deployment cannot decrypt from + Anthropic-shaped history. + + The whole block goes, the way #40280 drops undecryptable Responses ``input`` items: a + provider that did not mint the block rejects it signed (a foreign signature) and unsigned + (a missing signature) alike, so keeping its text as an unsigned thinking block only moves + the 400 from the router to the provider. Mutates the content lists in place: the router's fallback snapshot shares these message objects, so a rebound list would replay the stripped blocks on the fallback hop. @@ -1914,20 +1919,8 @@ def _anthropic_content_lists(messages: Sequence[object]) -> Iterator[object]: def _strip_encrypted_reasoning_from_blocks(content: object) -> None: blocks: Final = cast(list[object], content) # cast-ok: narrowed by the caller's isinstance - stripped: Final = tuple(_without_encrypted_reasoning_block(block) for block in blocks) - blocks[:] = (block for block in stripped if block is not None) # rebind-ok: list shared with fallback snapshot - - -def _without_encrypted_reasoning_block(block: object) -> object | None: - if not is_encrypted_reasoning_block(block): - return block - mapping: Final = cast(Mapping[str, object], block) # cast-ok: narrowed by is_encrypted_reasoning_block - if mapping.get("type") != "thinking" or not mapping.get("thinking"): - return None - kept: Final[dict[str, object]] = { # mutable-ok: thinking block rebuilt without the undecryptable signature - key: value for key, value in mapping.items() if key != "signature" - } - return kept + kept: Final = tuple(block for block in blocks if not is_encrypted_reasoning_block(block)) + blocks[:] = kept # rebind-ok: shared with fallback snapshot def _reasoning_replay_group_key(indexed_block: tuple[int, Mapping[str, object]]) -> str: diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index 71b71d55fa7..7c1445d79c9 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -1633,7 +1633,7 @@ class TestEncryptedReasoningReplay: def test_is_encrypted_reasoning_block(self, block, expected): assert is_encrypted_reasoning_block(block) is expected - def test_strip_keeps_the_readable_thinking_and_drops_the_undecryptable_bytes(self): + def test_strip_drops_every_bridge_tagged_block_and_leaves_no_unsigned_thinking_behind(self): assistant_content = [ {"type": "thinking", "thinking": "minted by Anthropic", "signature": "ErcBCkgIValid"}, {"type": "thinking", "thinking": "packed by the bridge", "signature": encrypted_reasoning_signature("g1")}, @@ -1652,9 +1652,9 @@ class TestEncryptedReasoningReplay: assert messages[1]["content"] is assistant_content assert assistant_content == [ {"type": "thinking", "thinking": "minted by Anthropic", "signature": "ErcBCkgIValid"}, - {"type": "thinking", "thinking": "packed by the bridge"}, {"type": "text", "text": "answer"}, ] + assert all(block["signature"] for block in assistant_content if block["type"] == "thinking") assert messages[0] == {"role": "user", "content": "question"} assert messages[2] == {"role": "user", "content": [{"type": "text", "text": "follow-up"}]} diff --git a/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py b/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py index 57c02e43fc1..ea8e2eacaa6 100644 --- a/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py +++ b/tests/test_litellm/router_utils/pre_call_checks/test_encrypted_content_affinity_check.py @@ -1635,8 +1635,10 @@ def _bridge_replayed_anthropic_messages(minted_by: str) -> list: async def test_encrypted_content_affinity_strips_bridge_reasoning_from_messages_routed_to_another_group(): """ The /v1/messages twin of the tier-change case: the routed group holds no deployment - of the org that minted the reasoning, so the bridge-tagged blocks are stripped down - to their readable thinking text and the request dispatches to the routed pool. + of the org that minted the reasoning, so the bridge-tagged blocks are dropped whole + and the request dispatches to the routed pool. No unsigned thinking block may be left + behind: Anthropic and Bedrock reject a thinking block with a missing signature the + same way they reject a foreign one. """ originating = _make_originating_mock(None, "key-a", model_name="gpt-reasoning-tier") mock_router = _make_router_mock_with_cooldown( @@ -1660,9 +1662,9 @@ async def test_encrypted_content_affinity_strips_bridge_reasoning_from_messages_ assert messages[1]["content"] is assistant_content assert assistant_content == [ {"type": "thinking", "thinking": "Anthropic minted this one", "signature": "ErcCCpIBCBEYAipA"}, - {"type": "thinking", "thinking": "The bridge packed this one"}, {"type": "text", "text": "The zebra owner lives in the green house."}, ] + assert all(block["signature"] for block in assistant_content if block["type"] == "thinking") class TestStripEncryptedReasoningFromInput: