fix(anthropic): keep Responses reasoning on tool turns and retry once when a backend cannot decrypt it

Backport of the second #49076 commit to lts/1.108.x. Cherry-picked from 10c953d855.

Adapted to the line: chat_completion_ends_in_responses_api has no deployment_model_info
parameter here, so only tools, reasoning_effort, and api_base are threaded through, and
the moved tests import the adapters from pass_through instead of messages.
This commit is contained in:
mateo-berri 2026-11-06 13:17:32 -08:00
parent 1e513a4314
commit 6fef70ac72
8 changed files with 443 additions and 85 deletions

View file

@ -448,6 +448,7 @@ def _as_chat_reasoning_items(
def _with_stream_block_index(
block: Mapping[str, object],
index: int,
summary_streamed: bool,
) -> ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock:
if block.get("type") == "redacted_thinking":
data: Final = block.get("data")
@ -460,21 +461,33 @@ def _with_stream_block_index(
signature: Final = block.get("signature")
return ChatCompletionThinkingBlock(
type="thinking",
thinking=thinking if isinstance(thinking, str) else "",
thinking=thinking if isinstance(thinking, str) and not summary_streamed else "",
signature=signature if isinstance(signature, str) else None,
index=index,
)
def _stream_output_index(chunk: Mapping[str, object]) -> int | None:
output_index: Final = chunk.get("output_index")
return output_index if isinstance(output_index, int) and not isinstance(output_index, bool) else None
def _reasoning_done_thinking_blocks(
item: Mapping[str, object],
output_index: object,
output_index: int | None,
summary_streamed: bool,
) -> tuple[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock, ...]:
index: Final = output_index if isinstance(output_index, int) and not isinstance(output_index, bool) else 0
return tuple(_with_stream_block_index(block, index) for block in thinking_blocks_from_reasoning_items((item,)))
index: Final = output_index if output_index is not None else 0
return tuple(
_with_stream_block_index(block, index, summary_streamed)
for block in thinking_blocks_from_reasoning_items((item,))
)
def _reasoning_done_stream_chunk(chunk: Mapping[str, object]) -> ModelResponseStream | None:
def _reasoning_done_stream_chunk(
chunk: Mapping[str, object],
summary_streamed_outputs: frozenset[int | None],
) -> ModelResponseStream | None:
event_type: Final = chunk.get("type")
event_name: Final = (
event_type.value
@ -488,7 +501,8 @@ def _reasoning_done_stream_chunk(chunk: Mapping[str, object]) -> ModelResponseSt
item: Final = as_str_mapping(chunk.get("item"))
if item is None or item.get("type") != "reasoning":
return None
signed_blocks: Final = _reasoning_done_thinking_blocks(item, chunk.get("output_index"))
output_index: Final = _stream_output_index(chunk)
signed_blocks: Final = _reasoning_done_thinking_blocks(item, output_index, output_index in summary_streamed_outputs)
if not signed_blocks:
return None
return ModelResponseStream(
@ -556,7 +570,6 @@ def _fold_choices_into_one(
*chain.from_iterable(_message_reasoning_items(m) for m in messages),
*(() if trailing_reasoning_item is None else (trailing_reasoning_item,)),
)
encrypted_thinking_blocks: Final = thinking_blocks_from_reasoning_items(reasoning_items)
return Choices(
message=Message(
role=messages[0].role if messages else "assistant",
@ -565,7 +578,6 @@ def _fold_choices_into_one(
tool_calls=list(tool_calls) or None,
reasoning_content=" ".join(value for value in reasoning_contents if value) or None,
reasoning_items=_as_chat_reasoning_items(reasoning_items),
thinking_blocks=list(encrypted_thinking_blocks) or None,
provider_specific_fields=_tool_search_provider_fields(tool_search_items),
),
finish_reason="tool_calls" if tool_calls else "stop",
@ -1371,12 +1383,10 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
for summary_block in reasoning_item["summary"]
if summary_block.get("text")
)
encrypted_thinking_blocks: Final = thinking_blocks_from_reasoning_items(reasoning_items)
message: Final = Message(
content=None if tool_search_items else "",
reasoning_content=reasoning_content if reasoning_content else None,
reasoning_items=_as_chat_reasoning_items(reasoning_items),
thinking_blocks=list(encrypted_thinking_blocks) or None,
provider_specific_fields=_tool_search_provider_fields(tool_search_items),
)
return Choices(message=message, finish_reason=finish_reason, index=0)
@ -1941,6 +1951,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
self._tool_call_index_map: dict[int, int] = {} # mutable-ok: per-stream accumulator state
self._structured_output: Final = structured_output
self._first_message_item_done: bool = False
self._summary_streamed_outputs: frozenset[int | None] = frozenset()
def _handle_string_chunk(
self, str_line: Union[str, "BaseModel"]
@ -2255,7 +2266,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
"""
verbose_logger.debug("Chat provider: transform_streaming_response called with chunk: %s", chunk)
self._remember_served_service_tier(chunk)
reasoning_chunk: Final = _reasoning_done_stream_chunk(chunk)
reasoning_chunk: Final = _reasoning_done_stream_chunk(chunk, self._summary_streamed_outputs)
translated: Final = (
self._empty_content_chunk()
if self._is_text_delta_after_structured_answer(chunk)
@ -2267,6 +2278,8 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
)
if self._structured_output and _is_message_item_done(chunk):
self._first_message_item_done = True
if _responses_event_type(chunk) == "response.reasoning_summary_text.delta" and chunk.get("delta"):
self._summary_streamed_outputs = self._summary_streamed_outputs | {_stream_output_index(chunk)}
return self._with_served_service_tier(self._with_stream_scoped_id(translated))
def _is_text_delta_after_structured_answer(self, chunk: Mapping[str, object]) -> bool:

View file

@ -2462,6 +2462,14 @@ def anthropic_content_lists(messages: Sequence[object]) -> Iterator[object]:
)
def _content_carries_encrypted_reasoning(content: object) -> bool:
return any(is_encrypted_reasoning_block(block) for block in _object_sequence(content))
def messages_carry_encrypted_reasoning(messages: Sequence[object]) -> bool:
return any(_content_carries_encrypted_reasoning(content) for content in anthropic_content_lists(messages))
def _strip_encrypted_reasoning_from_blocks(
content: object,
*,

View file

@ -1,5 +1,6 @@
from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence
from collections.abc import AsyncIterator, Callable, Coroutine, Iterator, Mapping, Sequence
from dataclasses import dataclass
from functools import partial
from typing import (
TYPE_CHECKING,
Final,
@ -13,12 +14,15 @@ from typing_extensions import TypedDict
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.asyncify import run_async_function
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
from litellm.litellm_core_utils.prompt_templates.common_utils import messages_carry_encrypted_reasoning
from litellm.litellm_core_utils.tool_fields import (
ROUTES_WITHOUT_MCP_SERVERS,
drops_mcp_servers,
tools_without_mcp_servers,
)
from litellm.llms.anthropic.common_utils import strip_encrypted_reasoning_blocks_from_anthropic_messages
from litellm.llms.anthropic.pass_through.adapters.transformation import (
AnthropicAdapter,
)
@ -51,6 +55,7 @@ from litellm.types.utils import ModelResponse
from litellm.utils import get_model_info
if TYPE_CHECKING:
from litellm.chat_completions.dispatch import ChatResult
from litellm.router import Router
from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth
@ -68,22 +73,68 @@ class _CompletionKwargs(TypedDict, total=False, extra_items=object):
custom_llm_provider: str
_ANTHROPIC_MESSAGES: Final = TypeAdapter(list[dict[str, object]])
_PrepareReplay: TypeAlias = "Callable[[_AnthropicMessages], _CompletionKwargs]"
@dataclass(frozen=True, slots=True)
class _ChatTargetReasoning:
effort: str
summary: str | None = None
def _messages_without_rejected_reasoning(
messages: _AnthropicMessages,
error: litellm.BadRequestError,
) -> _AnthropicMessages | None:
error_message: Final = str(error)
rejected: Final = "invalid_encrypted_content" in error_message or "could not be verified" in error_message
if not rejected or not messages_carry_encrypted_reasoning(messages):
return None
verbose_logger.warning("Backend rejected replayed encrypted reasoning, retrying once without it: %s", error)
return _ANTHROPIC_MESSAGES.validate_python(strip_encrypted_reasoning_blocks_from_anthropic_messages(messages))
async def _acompletion_without_rejected_reasoning(
completion_kwargs: _CompletionKwargs,
messages: _AnthropicMessages,
prepare_replay: _PrepareReplay,
) -> tuple[_CompletionKwargs, "ChatResult"]:
try:
return completion_kwargs, await litellm.acompletion(**completion_kwargs)
except litellm.BadRequestError as error:
replayed_messages: Final = _messages_without_rejected_reasoning(messages, error)
if replayed_messages is None:
raise
replay_kwargs: Final = prepare_replay(replayed_messages)
return replay_kwargs, await litellm.acompletion(**replay_kwargs)
def _completion_without_rejected_reasoning(
completion_kwargs: _CompletionKwargs,
messages: _AnthropicMessages,
prepare_replay: _PrepareReplay,
) -> tuple[_CompletionKwargs, "ChatResult | Coroutine[object, object, ChatResult]"]:
try:
return completion_kwargs, litellm.completion(**completion_kwargs)
except litellm.BadRequestError as error:
replayed_messages: Final = _messages_without_rejected_reasoning(messages, error)
if replayed_messages is None:
raise
replay_kwargs: Final = prepare_replay(replayed_messages)
return replay_kwargs, litellm.completion(**replay_kwargs)
def _is_compaction_block(block: object) -> bool:
mapping: Final = as_str_mapping(block)
return mapping is not None and mapping.get("type") == "compaction"
def _messages_have_compaction_block(messages: _AnthropicMessages) -> bool:
"""Return True when any message carries a ``compaction`` content block."""
for msg in messages:
content = msg.get("content")
if not isinstance(content, list):
continue
for block in content:
if isinstance(block, dict) and block.get("type") == "compaction":
return True
return False
return any(
any(map(_is_compaction_block, content)) for msg in messages if isinstance(content := msg.get("content"), list)
)
def _proxy_router_fallback() -> "Router | None":
@ -646,12 +697,14 @@ class LiteLLMMessagesToCompletionTransformationHandler:
if "output_config" in extra_kwargs:
request_data["output_config"] = extra_kwargs["output_config"]
api_base: Final = provider_kwargs.get("api_base")
(
openai_request,
tool_name_mapping,
) = ANTHROPIC_ADAPTER.translate_completion_input_params_with_tool_mapping(
request_data,
custom_llm_provider=target_provider,
api_base=api_base if isinstance(api_base, str) else None,
)
if openai_request is None:
@ -779,12 +832,9 @@ class LiteLLMMessagesToCompletionTransformationHandler:
parent_trace_id=sub_call_trace_id,
)
(
completion_kwargs,
tool_name_mapping,
) = LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs(
prepare: Final = partial(
LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs,
max_tokens=max_tokens,
messages=effective_messages,
model=model,
metadata=metadata,
stop_sequences=stop_sequences,
@ -799,8 +849,10 @@ class LiteLLMMessagesToCompletionTransformationHandler:
output_format=output_format,
extra_kwargs=kwargs,
)
completion_response: Final = await litellm.acompletion(**completion_kwargs)
initial_kwargs, tool_name_mapping = prepare(messages=effective_messages)
completion_kwargs, completion_response = await _acompletion_without_rejected_reasoning(
initial_kwargs, effective_messages, lambda replayed_messages: prepare(messages=replayed_messages)[0]
)
logging_obj: Final = litellm_logging_obj_from_kwargs(kwargs)
if stream:
@ -924,12 +976,9 @@ class LiteLLMMessagesToCompletionTransformationHandler:
effective_messages: Final = polyfill_result.messages if polyfill_result is not None else messages
effective_system: Final = polyfill_result.system if polyfill_result is not None else system
(
completion_kwargs,
tool_name_mapping,
) = LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs(
prepare: Final = partial(
LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs,
max_tokens=max_tokens,
messages=effective_messages,
model=model,
metadata=metadata,
stop_sequences=stop_sequences,
@ -944,8 +993,10 @@ class LiteLLMMessagesToCompletionTransformationHandler:
output_format=output_format,
extra_kwargs=kwargs,
)
completion_response: Final = litellm.completion(**completion_kwargs)
initial_kwargs, tool_name_mapping = prepare(messages=effective_messages)
completion_kwargs, completion_response = _completion_without_rejected_reasoning(
initial_kwargs, effective_messages, lambda replayed_messages: prepare(messages=replayed_messages)[0]
)
logging_obj: Final = litellm_logging_obj_from_kwargs(kwargs)
if stream:

View file

@ -24,9 +24,7 @@ from typing_extensions import assert_never
from litellm._logging import verbose_logger
from litellm._uuid import uuid
from litellm.exceptions import ContextWindowExceededError, MidStreamFallbackError
from litellm.litellm_core_utils.core_helpers import attribute_or_none
from litellm.litellm_core_utils.hidden_params import set_hidden_params
from litellm.litellm_core_utils.prompt_templates.common_utils import thinking_blocks_from_reasoning_items
from litellm.llms.anthropic.pass_through.safeguards import StreamedSafeguardResults
from litellm.llms.anthropic.pass_through.utils import anthropic_sse_frame
from litellm.llms.base_llm.chat.transformation import BaseLLMException
@ -68,10 +66,6 @@ def _optional_attr(obj: object, name: str) -> object:
return getattr(obj, name, None)
def _delta_has_encrypted_reasoning(delta: object) -> bool:
return bool(thinking_blocks_from_reasoning_items(attribute_or_none(delta, "reasoning_items")))
def _optional_attr_sequence(obj: object, name: str) -> Sequence[object]:
value: Final = getattr(obj, name, None)
return value if value else ()
@ -1312,8 +1306,6 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
isinstance(b, dict) and not is_empty_unsigned_thinking_block(b) for b in thinking_blocks
):
return False
if _delta_has_encrypted_reasoning(delta):
return False
return True
def _should_start_new_content_block(self, chunk: "ModelResponseStream") -> bool:

View file

@ -2,12 +2,14 @@ import copy
import hashlib
import json
from collections.abc import AsyncIterator, Iterator, Mapping, Sequence
from dataclasses import dataclass
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast
from pydantic import JsonValue, TypeAdapter
import litellm
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_items
from litellm.litellm_core_utils.hidden_params import get_hidden_params
from litellm.llms.anthropic.pass_through.utils import (
is_reasoning_auto_summary_enabled,
@ -220,10 +222,30 @@ def target_supports_mid_conversation_system(model: str | None, custom_llm_provid
return supports_mid_conversation_system(model=model, custom_llm_provider=custom_llm_provider)
def _keeps_encrypted_reasoning_for_responses(model: str | None, custom_llm_provider: str | None) -> bool:
@dataclass(frozen=True, slots=True)
class ChatRouteInputs:
tools: Sequence[Mapping[str, object]] | None = None
reasoning_effort: str | Mapping[str, object] | None = None
api_base: str | None = None
_NO_ROUTE_INPUTS: Final = ChatRouteInputs()
def _keeps_encrypted_reasoning_for_responses(
model: str | None,
custom_llm_provider: str | None,
route: ChatRouteInputs,
) -> bool:
from litellm.responses.bridge_check import chat_completion_ends_in_responses_api
return chat_completion_ends_in_responses_api(model, custom_llm_provider)
return chat_completion_ends_in_responses_api(
model,
custom_llm_provider,
tools=route.tools,
reasoning_effort=route.reasoning_effort,
api_base=route.api_base,
)
def _anthropic_thinking_blocks(source: object) -> tuple[Mapping[str, object], ...]:
@ -233,6 +255,10 @@ def _anthropic_thinking_blocks(source: object) -> tuple[Mapping[str, object], ..
)
def _streamed_thinking_blocks(delta: object) -> tuple[Mapping[str, object], ...]:
return tuple(block for item in attribute_items(delta, "thinking_blocks") if (block := as_str_mapping(item)))
def _chat_tool_param(function_chunk: ChatCompletionToolParamFunctionChunk, tool: object) -> ChatCompletionToolParam:
eager_input_streaming: Final = eager_input_streaming_flag(tool)
if eager_input_streaming is None:
@ -260,7 +286,7 @@ class AnthropicAdapter:
return result
def translate_completion_input_params_with_tool_mapping(
self, kwargs, *, custom_llm_provider: str | None = None
self, kwargs, *, custom_llm_provider: str | None = None, api_base: str | None = None
) -> tuple[ChatCompletionRequest | None, dict[str, str]]:
"""
Translate Anthropic request params to OpenAI format, returning tool name mapping.
@ -294,6 +320,7 @@ class AnthropicAdapter:
) = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(
anthropic_message_request=request_body,
custom_llm_provider=custom_llm_provider,
api_base=api_base,
)
return translated_body, tool_name_mapping
@ -479,11 +506,12 @@ class LiteLLMAnthropicMessagesAdapter:
*,
custom_llm_provider: str | None = None,
preserve_midturn_system: bool = False,
route: ChatRouteInputs = _NO_ROUTE_INPUTS,
) -> list[AllMessageValues]:
new_messages: Final[list[AllMessageValues]] = []
replayable_messages: Final = (
messages
if _keeps_encrypted_reasoning_for_responses(model, custom_llm_provider)
if _keeps_encrypted_reasoning_for_responses(model, custom_llm_provider, route)
else strip_encrypted_reasoning_blocks_from_anthropic_messages(messages)
)
leading_count: Final = next(
@ -1239,6 +1267,7 @@ class LiteLLMAnthropicMessagesAdapter:
anthropic_message_request: AnthropicMessagesRequest,
*,
custom_llm_provider: str | None = None,
api_base: str | None = None,
preserve_midturn_system: bool = False,
) -> tuple[ChatCompletionRequest, dict[str, str]]:
"""
@ -1250,26 +1279,15 @@ class LiteLLMAnthropicMessagesAdapter:
for tools that exceeded OpenAI's 64-char limit
"""
# Debug: Processing Anthropic message request
new_messages: list[AllMessageValues] = []
tool_name_mapping: dict[str, str] = {}
## CONVERT ANTHROPIC MESSAGES TO OPENAI
messages_list: Final[list[AllAnthropicPassThroughMessageValues]] = cast(
list[AllAnthropicPassThroughMessageValues],
anthropic_message_request["messages"],
)
new_messages = self.translate_anthropic_messages_to_openai(
messages=messages_list,
model=anthropic_message_request.get("model"),
custom_llm_provider=custom_llm_provider,
preserve_midturn_system=preserve_midturn_system,
)
## ADD SYSTEM MESSAGE TO MESSAGES
self._add_system_message_to_messages(new_messages, anthropic_message_request)
new_kwargs: Final[ChatCompletionRequest] = {
"model": anthropic_message_request["model"],
"messages": new_messages,
"messages": [],
}
## CONVERT METADATA (user_id + litellm metadata)
self._translate_metadata_to_openai(
@ -1293,6 +1311,21 @@ class LiteLLMAnthropicMessagesAdapter:
new_kwargs=new_kwargs,
custom_llm_provider=custom_llm_provider,
)
## CONVERT ANTHROPIC MESSAGES TO OPENAI, after tools and thinking pick the route
new_messages: Final = self.translate_anthropic_messages_to_openai(
messages=messages_list,
model=anthropic_message_request.get("model"),
custom_llm_provider=custom_llm_provider,
preserve_midturn_system=preserve_midturn_system,
route=ChatRouteInputs(
tools=new_kwargs.get("tools"),
reasoning_effort=new_kwargs.get("reasoning_effort"),
api_base=api_base,
),
)
## ADD SYSTEM MESSAGE TO MESSAGES
self._add_system_message_to_messages(new_messages, anthropic_message_request)
new_kwargs["messages"] = new_messages
## CONVERT STOP_SEQUENCES
self._translate_stop_sequences_to_openai(
anthropic_message_request=anthropic_message_request,
@ -1678,7 +1711,7 @@ class LiteLLMAnthropicMessagesAdapter:
choice.delta
) is not None:
return "text", TextBlock(type="text", text="")
elif isinstance(choice, StreamingChoices) and (thinking_blocks := _anthropic_thinking_blocks(choice.delta)):
elif isinstance(choice, StreamingChoices) and (thinking_blocks := _streamed_thinking_blocks(choice.delta)):
thinking_block = thinking_blocks[0]
if thinking_block["type"] == "redacted_thinking":
redacted_data = thinking_block.get("data")
@ -1728,7 +1761,7 @@ class LiteLLMAnthropicMessagesAdapter:
for tool in choice.delta.tool_calls:
if tool.function is not None and tool.function.arguments is not None:
partial_json = (partial_json or "") + tool.function.arguments
elif isinstance(choice, StreamingChoices) and (thinking_blocks := _anthropic_thinking_blocks(choice.delta)):
elif isinstance(choice, StreamingChoices) and (thinking_blocks := _streamed_thinking_blocks(choice.delta)):
for thinking_block in thinking_blocks:
if thinking_block["type"] == "thinking":
thinking = thinking_block.get("thinking") or ""

View file

@ -191,12 +191,19 @@ def responses_api_bridge_check(
return model_info, model
def chat_completion_ends_in_responses_api(model: str | None, custom_llm_provider: str | None) -> bool:
def chat_completion_ends_in_responses_api(
model: str | None,
custom_llm_provider: str | None,
*,
tools: Sequence[Mapping[str, object]] | None = None,
reasoning_effort: str | Mapping[str, object] | None = None,
api_base: str | None = None,
) -> bool:
"""Whether chat completion for this model is served by a Responses API.
Encrypted reasoning can be replayed only there. A normal chat provider cannot
verify the block, so the messages adapter strips it. ``chatgpt`` and a
``responses/`` model keep it.
verify the block, so the messages adapter strips it. ``chatgpt``, a ``responses/``
model, and a request whose tools or reasoning effort the bridge sends to Responses keep it.
"""
if model is None and custom_llm_provider is None:
return False
@ -207,5 +214,11 @@ def chat_completion_ends_in_responses_api(model: str | None, custom_llm_provider
return custom_llm_provider == LlmProviders.CHATGPT.value or (
isinstance(model, str) and (model == "chatgpt" or model.startswith("chatgpt/"))
)
model_info, _resolved = responses_api_bridge_check(model=resolved_model, custom_llm_provider=provider)
model_info, _resolved = responses_api_bridge_check(
model=resolved_model,
custom_llm_provider=provider,
tools=tools,
reasoning_effort=reasoning_effort,
api_base=api_base,
)
return model_info.get("mode") == "responses"

View file

@ -3253,7 +3253,7 @@ def test_reasoning_output_item_done_emits_signed_thinking_blocks_the_stream_keep
)
from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
from litellm.llms.anthropic.messages.adapters.transformation import LiteLLMAnthropicMessagesAdapter
from litellm.llms.anthropic.pass_through.adapters.transformation import LiteLLMAnthropicMessagesAdapter
encrypted: Final = "gAAAA_round_trip"
done: Final = {
@ -3318,6 +3318,74 @@ def test_reasoning_output_item_done_with_summary_signs_the_thinking_text():
assert block["index"] == 2
def test_streamed_reasoning_summary_reaches_messages_clients_once_with_its_signature():
from litellm.completion_extras.litellm_responses_transformation.transformation import (
OpenAiResponsesToChatCompletionStreamIterator,
)
from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature
from litellm.llms.anthropic.pass_through.adapters.streaming_iterator import AnthropicStreamWrapper
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
encrypted: Final = "gAAAA_streamed_summary"
events: Final = (
{"type": "response.reasoning_summary_part.added", "output_index": 0, "summary_index": 0},
{"type": "response.reasoning_summary_text.delta", "output_index": 0, "delta": "Oslo is "},
{"type": "response.reasoning_summary_text.delta", "output_index": 0, "delta": "colder."},
{
"type": "response.output_item.done",
"output_index": 0,
"item": {
"type": "reasoning",
"id": "rs_streamed",
"encrypted_content": encrypted,
"summary": [{"type": "summary_text", "text": "Oslo is colder."}],
},
},
{"type": "response.output_text.delta", "output_index": 1, "delta": "Oslo."},
)
parser: Final = OpenAiResponsesToChatCompletionStreamIterator(streaming_response=None, sync_stream=True)
chunks: Final = (
*(parser.chunk_parser(event) for event in events),
ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(), finish_reason="stop")]),
)
sse_events: Final = list(AnthropicStreamWrapper(completion_stream=iter(chunks), model="chatgpt/gpt-5.5"))
deltas: Final = [event["delta"] for event in sse_events if event.get("type") == "content_block_delta"]
assert "".join(d.get("thinking", "") for d in deltas if d["type"] == "thinking_delta") == "Oslo is colder."
assert [d["signature"] for d in deltas if d["type"] == "signature_delta"] == [
encrypted_reasoning_signature(encrypted)
]
def test_reasoning_item_repeated_at_stream_end_reaches_messages_clients_once():
from litellm.completion_extras.litellm_responses_transformation.transformation import (
OpenAiResponsesToChatCompletionStreamIterator,
)
from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature
from litellm.llms.anthropic.pass_through.adapters.streaming_iterator import AnthropicStreamWrapper
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
encrypted: Final = "gAAAA_completed_repeat"
reasoning_item: Final = {"type": "reasoning", "id": "rs_repeat", "summary": [], "encrypted_content": encrypted}
events: Final = (
{"type": "response.output_item.done", "output_index": 0, "item": reasoning_item},
{"type": "response.output_text.delta", "output_index": 1, "delta": "Oslo."},
)
parser: Final = OpenAiResponsesToChatCompletionStreamIterator(streaming_response=None, sync_stream=True)
chunks: Final = (
*(parser.chunk_parser(event) for event in events),
ModelResponseStream(
choices=[StreamingChoices(index=0, delta=Delta(reasoning_items=[reasoning_item]), finish_reason=None)]
),
ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(), finish_reason="stop")]),
)
sse_events: Final = list(AnthropicStreamWrapper(completion_stream=iter(chunks), model="azure/gpt-5.4-nano"))
started: Final = [event["content_block"] for event in sse_events if event.get("type") == "content_block_start"]
assert [block["type"] for block in started] == ["redacted_thinking", "text"]
assert started[0]["data"] == encrypted_reasoning_signature(encrypted)
def test_reasoning_output_item_done_without_encrypted_content_stays_empty():
from litellm.completion_extras.litellm_responses_transformation.transformation import (
OpenAiResponsesToChatCompletionStreamIterator,
@ -3338,10 +3406,13 @@ def test_reasoning_output_item_done_without_encrypted_content_stays_empty():
assert getattr(result.choices[0].delta, "thinking_blocks", None) in (None, [])
def test_transform_response_copies_encrypted_reasoning_onto_thinking_blocks():
def test_transform_response_hands_encrypted_reasoning_to_messages_clients_only():
from unittest.mock import Mock
from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature
from litellm.llms.anthropic.pass_through.adapters.transformation import (
LiteLLMAnthropicMessagesAdapter,
)
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.utils import ModelResponse, Usage
@ -3394,10 +3465,16 @@ def test_transform_response_copies_encrypted_reasoning_onto_thinking_blocks():
)
message: Final = result.choices[0].message
assert message.reasoning_items[0]["encrypted_content"] == encrypted
assert message.thinking_blocks[0]["type"] == "redacted_thinking"
assert message.thinking_blocks[0]["data"] == encrypted_reasoning_signature(encrypted)
assert "thinking_blocks" not in result.model_dump()["choices"][0]["message"]
assert message.tool_calls[0].function.name == "get_weather"
anthropic_response: Final = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic(
response=result, tool_name_mapping=None
)
content: Final = anthropic_response["content"]
assert [block["type"] for block in content] == ["redacted_thinking", "tool_use"]
assert content[0]["data"] == encrypted_reasoning_signature(encrypted)
def test_streaming_function_call_tool_id_for_degenerate_call_id():
"""In streaming, Bedrock Mantle's function_call event carries a unique ``id``

View file

@ -4,7 +4,9 @@ import json
from types import MappingProxyType
from typing import Any, Final, cast
import httpx
import pytest
import respx
import litellm
@ -31,6 +33,7 @@ from litellm.llms.anthropic.pass_through.messages.mid_conversation_system import
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
from litellm.types.llms.anthropic import (
AnthopicMessagesAssistantMessageParam,
AnthropicMessagesRequest,
AnthropicMessagesUserMessageParam,
)
from litellm.types.llms.openai import ChatCompletionAssistantToolCall
@ -647,26 +650,50 @@ def test_translate_openai_reasoning_items_are_not_duplicated_when_thinking_block
assert [block["type"] for block in result] == ["redacted_thinking", "text"]
def test_streaming_chunk_with_only_reasoning_items_opens_a_signed_thinking_block():
encrypted: Final = "gAAAA_delta"
choices: Final = [
StreamingChoices(
finish_reason=None,
index=0,
delta=Delta(
reasoning_items=[
{"type": "reasoning", "id": "rs_1", "encrypted_content": encrypted, "summary": []}
]
def _replayed_reasoning_request(tools: list[dict[str, object]]) -> AnthropicMessagesRequest:
return AnthropicMessagesRequest(
model="azure/gpt-5.4-nano",
max_tokens=64,
messages=[
AnthropicMessagesUserMessageParam(role="user", content="Which city is colder?"),
AnthopicMessagesAssistantMessageParam(
role="assistant",
content=[
{"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAAA_azure_tools")},
{"type": "tool_use", "id": "call_1", "name": "get_weather", "input": {"city": "Oslo"}},
],
),
)
]
block_type, start = LiteLLMAnthropicMessagesAdapter().translate_streaming_openai_chunk_to_anthropic_content_block(
choices=choices
AnthropicMessagesUserMessageParam(
role="user",
content=[{"type": "tool_result", "tool_use_id": "call_1", "content": "-7 C"}],
),
],
tools=tools,
)
assert block_type == "redacted_thinking"
assert start["data"] == encrypted_reasoning_signature(encrypted)
@pytest.mark.parametrize(
("tools", "kept"),
[
pytest.param(
[{"name": "get_weather", "input_schema": {"type": "object", "properties": {}}}],
True,
id="function_tools_bridge_to_responses",
),
pytest.param([], False, id="no_tools_stay_on_chat"),
],
)
def test_encrypted_reasoning_is_kept_when_the_request_tools_route_it_to_responses(
tools: list[dict[str, object]], kept: bool
) -> None:
openai_request, _mapping = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(
_replayed_reasoning_request(tools),
custom_llm_provider="azure",
api_base="https://keep-check.openai.azure.com",
)
assistant: Final = openai_request["messages"][1]
assert bool(assistant.get("thinking_blocks")) is kept
def test_translate_anthropic_messages_to_openai_sets_reasoning_content():
@ -5395,3 +5422,147 @@ def test_eager_input_streaming_tool_reaches_bedrock_converse_as_beta():
assert data["additionalModelRequestFields"]["anthropic_beta"] == ["fine-grained-tool-streaming-2025-05-14"]
assert data["toolConfig"]["tools"][0]["toolSpec"]["inputSchema"]["json"] == EAGER_INPUT_SCHEMA
@pytest.fixture
def _chat_bridge_httpx(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
litellm.in_memory_llm_clients_cache.flush_cache()
_RETRY_API_BASE: Final = "https://encrypted-retry.openai.azure.com"
_RETRY_URL: Final = r".*encrypted-retry\.openai\.azure\.com/openai/.*responses.*"
_RETRY_TOOLS: Final = [{"name": "get_weather", "input_schema": {"type": "object", "properties": {}}}]
_RETRY_REJECTION: Final = httpx.Response(
400,
json={
"error": {
"message": "The encrypted content gAAA_foreign could not be verified.",
"type": "invalid_request_error",
"param": None,
"code": "invalid_encrypted_content",
}
},
)
_UNRELATED_REJECTION: Final = httpx.Response(
400, json={"error": {"message": "max_output_tokens is too small", "type": "invalid_request_error"}}
)
_RETRY_REPLY: Final = {
"id": "resp_retry",
"object": "response",
"created_at": 1700000000,
"status": "completed",
"model": "gpt-5.4-nano",
"output": [
{
"type": "message",
"id": "msg_retry",
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": "Oslo", "annotations": []}],
}
],
"usage": {"input_tokens": 10, "output_tokens": 2, "total_tokens": 12},
"parallel_tool_calls": True,
"tool_choice": "auto",
"tools": [],
}
def _history_with_foreign_reasoning() -> list[dict[str, object]]:
return [
{"role": "user", "content": "Which city is colder?"},
{
"role": "assistant",
"content": [
{"type": "redacted_thinking", "data": encrypted_reasoning_signature("gAAA_foreign")},
{"type": "tool_use", "id": "call_1", "name": "get_weather", "input": {"city": "Oslo"}},
],
},
{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "call_1", "content": "-7 C"}]},
]
def _sent_reasoning_items(request: httpx.Request) -> list[object]:
sent: Final = json.loads(request.read())
return [item for item in sent["input"] if item.get("type") == "reasoning"]
@pytest.mark.asyncio
@pytest.mark.usefixtures("_chat_bridge_httpx")
async def test_async_messages_retry_once_without_reasoning_the_backend_cannot_decrypt(
respx_mock: respx.MockRouter,
) -> None:
route: Final = respx_mock.post(url__regex=_RETRY_URL).mock(
side_effect=[_RETRY_REJECTION, httpx.Response(200, json=_RETRY_REPLY)]
)
response: Final = await litellm.anthropic.messages.acreate(
model="azure/gpt-5.4-nano",
api_base=_RETRY_API_BASE,
api_key="fake-key",
api_version="2025-04-01-preview",
max_tokens=64,
tools=_RETRY_TOOLS,
messages=_history_with_foreign_reasoning(),
)
assert route.call_count == 2
assert len(_sent_reasoning_items(route.calls[0].request)) == 1
assert _sent_reasoning_items(route.calls[1].request) == []
assert [block["text"] for block in response["content"] if block["type"] == "text"] == ["Oslo"]
@pytest.mark.usefixtures("_chat_bridge_httpx")
def test_sync_messages_retry_once_without_reasoning_the_backend_cannot_decrypt(
respx_mock: respx.MockRouter,
) -> None:
route: Final = respx_mock.post(url__regex=_RETRY_URL).mock(
side_effect=[_RETRY_REJECTION, httpx.Response(200, json=_RETRY_REPLY)]
)
response: Final = litellm.anthropic.messages.create(
model="azure/gpt-5.4-nano",
api_base=_RETRY_API_BASE,
api_key="fake-key",
api_version="2025-04-01-preview",
max_tokens=64,
tools=_RETRY_TOOLS,
messages=_history_with_foreign_reasoning(),
)
assert route.call_count == 2
assert _sent_reasoning_items(route.calls[1].request) == []
assert [block["text"] for block in response["content"] if block["type"] == "text"] == ["Oslo"]
@pytest.mark.asyncio
@pytest.mark.usefixtures("_chat_bridge_httpx")
@pytest.mark.parametrize(
("messages", "rejection"),
[
pytest.param(
[{"role": "user", "content": "Which city is colder?"}],
_RETRY_REJECTION,
id="no_encrypted_reasoning_to_drop",
),
pytest.param(_history_with_foreign_reasoning(), _UNRELATED_REJECTION, id="unrelated_rejection"),
],
)
async def test_messages_do_not_retry_other_rejections(
respx_mock: respx.MockRouter, messages: list[dict[str, object]], rejection: httpx.Response
) -> None:
route: Final = respx_mock.post(url__regex=_RETRY_URL).mock(return_value=rejection)
with pytest.raises(litellm.BadRequestError):
await litellm.anthropic.messages.acreate(
model="azure/gpt-5.4-nano",
api_base=_RETRY_API_BASE,
api_key="fake-key",
api_version="2025-04-01-preview",
max_tokens=64,
tools=_RETRY_TOOLS,
messages=messages,
)
assert route.call_count == 1