diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 9902fe1d8fc..562719f962e 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -39,7 +39,7 @@ from litellm.litellm_core_utils.allowed_tools_tool_choice import ( parse_chat_allowed_tools_tool_choice, responses_allowed_tools_tool_choice, ) -from litellm.litellm_core_utils.core_helpers import drops_unsupported_params +from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params from litellm.litellm_core_utils.hidden_params import get_hidden_params, get_or_create_hidden_params from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import ( prompt_cache_breakpoint_with_readable_mode, @@ -48,6 +48,7 @@ from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils i ) from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( responses_reasoning_items_from_thinking_blocks, + thinking_blocks_from_reasoning_items, ) from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.bridges.completion_transformation import ( @@ -68,6 +69,8 @@ from litellm.responses.utils import ( from litellm.types.llms.openai import ( ChatCompletionAnnotation, ChatCompletionReasoningItem, + ChatCompletionRedactedThinkingBlock, + ChatCompletionThinkingBlock, ChatCompletionToolCallChunk, ChatCompletionToolCallFunctionChunk, ChatCompletionToolParamFunctionChunk, @@ -76,7 +79,7 @@ from litellm.types.llms.openai import ( ResponsesAPIResponse, ResponsesAPIStreamEvents, ) -from litellm.types.utils import GenericStreamingChunk, ModelResponseStream +from litellm.types.utils import Delta, GenericStreamingChunk, ModelResponseStream, StreamingChoices if TYPE_CHECKING: from openai.types.responses import ResponseInputImageParam, ResponseOutputItem @@ -481,6 +484,63 @@ def _as_chat_reasoning_items( return cast(list[ChatCompletionReasoningItem], list(reasoning_items)) +def _with_stream_block_index( + block: ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock, + index: int, +) -> ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock: + if block["type"] == "redacted_thinking": + data: Final = block.get("data") + return ChatCompletionRedactedThinkingBlock( + type="redacted_thinking", + data=data if isinstance(data, str) else "", + index=index, + ) + thinking: Final = block.get("thinking") + signature: Final = block.get("signature") + return ChatCompletionThinkingBlock( + type="thinking", + thinking=thinking if isinstance(thinking, str) else "", + signature=signature if isinstance(signature, str) else None, + index=index, + ) + + +def _reasoning_done_thinking_blocks( + item: Mapping[str, object], + output_index: object, +) -> tuple[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock, ...]: + index: Final = output_index if isinstance(output_index, int) and not isinstance(output_index, bool) else 0 + return tuple(_with_stream_block_index(block, index) for block in thinking_blocks_from_reasoning_items((item,))) + + +def _reasoning_done_stream_chunk(chunk: Mapping[str, object]) -> ModelResponseStream | None: + event_type: Final = chunk.get("type") + event_name: Final = ( + event_type.value + if isinstance(event_type, ResponsesAPIStreamEvents) + else event_type + if isinstance(event_type, str) + else None + ) + if event_name != ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE: + return None + item: Final = as_str_mapping(chunk.get("item")) + if item is None or item.get("type") != "reasoning": + return None + signed_blocks: Final = _reasoning_done_thinking_blocks(item, chunk.get("output_index")) + if not signed_blocks: + return None + return ModelResponseStream( + choices=[ + StreamingChoices( + index=0, + delta=Delta(thinking_blocks=list(signed_blocks)), + finish_reason=None, + ) + ] + ) + + def _shift_annotation(annotation: ChatCompletionAnnotation, offset: int) -> ChatCompletionAnnotation: shifted: Final = { key: value + offset if key in ("start_index", "end_index") and isinstance(value, int) else value @@ -535,6 +595,7 @@ def _fold_choices_into_one( *chain.from_iterable(_message_reasoning_items(m) for m in messages), *trailing_reasoning_items, ) + encrypted_thinking_blocks: Final = thinking_blocks_from_reasoning_items(reasoning_items) return Choices( message=Message( role=messages[0].role if messages else "assistant", @@ -543,6 +604,7 @@ def _fold_choices_into_one( tool_calls=list(tool_calls) or None, reasoning_content=" ".join(value for value in reasoning_contents if value) or None, reasoning_items=_as_chat_reasoning_items(reasoning_items), + thinking_blocks=list(encrypted_thinking_blocks) or None, provider_specific_fields=_tool_search_provider_fields(tool_search_items), ), finish_reason="tool_calls" if tool_calls else "stop", @@ -1436,10 +1498,12 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): for summary_block in reasoning_item["summary"] if summary_block.get("text") ) + encrypted_thinking_blocks: Final = thinking_blocks_from_reasoning_items(reasoning_items) message: Final = Message( content=None if tool_search_items else "", reasoning_content=reasoning_content if reasoning_content else None, reasoning_items=_as_chat_reasoning_items(reasoning_items), + thinking_blocks=list(encrypted_thinking_blocks) or None, provider_specific_fields=_tool_search_provider_fields(tool_search_items), ) return Choices(message=message, finish_reason=finish_reason, index=0) @@ -2248,7 +2312,6 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): ) ] ) - elif event_type == "response.output_text.delta": # Content part added to output content_part = parsed_chunk.get("delta", None) @@ -2365,9 +2428,12 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): """ verbose_logger.debug("Chat provider: transform_streaming_response called with chunk: %s", chunk) self._remember_served_service_tier(chunk) + reasoning_chunk: Final = _reasoning_done_stream_chunk(chunk) translated: Final = ( self._empty_content_chunk() if self._is_text_delta_after_structured_answer(chunk) + else reasoning_chunk + if reasoning_chunk is not None else OpenAiResponsesToChatCompletionStreamIterator.translate_responses_chunk_to_openai_stream( chunk, tool_call_index_map=self._tool_call_index_map, diff --git a/litellm/litellm_core_utils/prompt_templates/reasoning_utils.py b/litellm/litellm_core_utils/prompt_templates/reasoning_utils.py index 817b1fa5915..afffc8d7146 100644 --- a/litellm/litellm_core_utils/prompt_templates/reasoning_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/reasoning_utils.py @@ -5,6 +5,7 @@ from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence from itertools import groupby from typing import Final, cast # noqa: TID251 # Existing casts retained unchanged in this pure move +from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.types.llms.openai import ( ChatCompletionReasoningItem, ChatCompletionReasoningSummaryTextBlock, @@ -94,6 +95,87 @@ def encrypted_content_of_block(block: Mapping[str, object]) -> str | None: return encrypted_content_from_signature(_encrypted_reasoning_field(block)) +_REASONING_SUMMARY_SEPARATOR: Final = "\n\n" + + +def _object_sequence(value: object) -> tuple[object, ...]: + if not isinstance(value, Sequence) or isinstance(value, (str, bytes)): + return () + return tuple(value) + + +def _iter_mappings(value: object) -> tuple[Mapping[str, object], ...]: + return tuple(mapping for item in _object_sequence(value) if (mapping := as_str_mapping(item)) is not None) + + +def _reasoning_summary_part_text(part: object) -> str: + mapping: Final = as_str_mapping(part) + if mapping is not None: + text: Final = mapping.get("text") + return text if isinstance(text, str) else "" + text_attr: Final = getattr(part, "text", None) + return text_attr if isinstance(text_attr, str) else "" + + +def _reasoning_item_summary_text(item: Mapping[str, object]) -> str: + parts: Final = tuple(_reasoning_summary_part_text(part) for part in _object_sequence(item.get("summary"))) + return _REASONING_SUMMARY_SEPARATOR.join(part for part in parts if part) + + +def thinking_block_from_reasoning_item( + item: object, +) -> ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock | None: + """The chat thinking block that carries one Responses `reasoning` item back to an Anthropic client. + + `encrypted_content` rides the same `litellm_encrypted_reasoning:` signature the Responses + adapter uses. A summary becomes thinking text. Bytes with no summary become `redacted_thinking`, + because an empty thinking string is not something the client can replay. + """ + mapping: Final = as_str_mapping(item) + if mapping is None or mapping.get("type") not in (None, "reasoning"): + return None + encrypted_content: Final = mapping.get("encrypted_content") + if not isinstance(encrypted_content, str) or not encrypted_content: + return None + signature: Final = encrypted_reasoning_signature(encrypted_content) + text: Final = _reasoning_item_summary_text(mapping) + if not text: + return ChatCompletionRedactedThinkingBlock(type="redacted_thinking", data=signature) + return ChatCompletionThinkingBlock(type="thinking", thinking=text, signature=signature) + + +def thinking_blocks_from_reasoning_items( + items: object, +) -> tuple[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock, ...]: + return tuple( + block for item in _object_sequence(items) if (block := thinking_block_from_reasoning_item(item)) is not None + ) + + +def _block_encrypted_content(block: object) -> str | None: + mapping: Final = as_str_mapping(block) + if mapping is None: + return None + return encrypted_content_of_block(mapping) + + +def thinking_blocks_with_encrypted_reasoning( + thinking_blocks: object, + reasoning_items: object, +) -> tuple[Mapping[str, object], ...]: + """Thinking blocks already on the message, plus any encrypted reasoning item they do not already carry.""" + direct: Final = _iter_mappings(thinking_blocks) + represented: Final = frozenset( + content for block in direct if (content := _block_encrypted_content(block)) is not None + ) + extra: Final = tuple( + block + for block in thinking_blocks_from_reasoning_items(_iter_mappings(reasoning_items)) + if _block_encrypted_content(block) not in represented + ) + return (*extra, *direct) + + def is_encrypted_reasoning_block(block: object) -> bool: """A thinking or redacted_thinking block carrying Responses API encrypted reasoning. diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index aa6ce6ea99e..1713cc5a11c 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -1129,6 +1129,7 @@ class CustomStreamWrapper: and model_response.choices[0].delta.role is not None ) or (getattr(model_response.choices[0].delta, "reasoning_items", None) is not None) + or getattr(model_response.choices[0].delta, "thinking_blocks", None) ): return True else: diff --git a/litellm/llms/anthropic/messages/adapters/streaming_iterator.py b/litellm/llms/anthropic/messages/adapters/streaming_iterator.py index 87be996ebbf..75a83d2da10 100644 --- a/litellm/llms/anthropic/messages/adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/messages/adapters/streaming_iterator.py @@ -27,8 +27,9 @@ import litellm from litellm._logging import verbose_logger from litellm._uuid import uuid from litellm.exceptions import ContextWindowExceededError, MidStreamFallbackError -from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_items +from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_items, attribute_or_none from litellm.litellm_core_utils.hidden_params import set_hidden_params +from litellm.litellm_core_utils.prompt_templates.reasoning_utils import thinking_blocks_from_reasoning_items from litellm.llms.anthropic.messages.safeguards import StreamedSafeguardResults from litellm.llms.anthropic.messages.utils import anthropic_sse_frame from litellm.llms.base_llm.chat.transformation import BaseLLMException @@ -92,6 +93,10 @@ def _optional_attr(obj: object, name: str) -> object: return getattr(obj, name, None) +def _delta_has_encrypted_reasoning(delta: object) -> bool: + return bool(thinking_blocks_from_reasoning_items(attribute_or_none(delta, "reasoning_items"))) + + def _optional_attr_sequence(obj: object, name: str) -> Sequence[object]: value: Final = getattr(obj, name, None) return value if value else () @@ -1518,6 +1523,8 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): for b in attribute_items(delta, "thinking_blocks") ): return False + if _delta_has_encrypted_reasoning(delta): + return False return True def _record_anthropic_stop_fields(self, delta: object) -> None: diff --git a/litellm/llms/anthropic/messages/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py index 5379dcd6d68..bc30bcdd0a0 100644 --- a/litellm/llms/anthropic/messages/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -133,6 +133,7 @@ from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils i ) from litellm.litellm_core_utils.prompt_templates.reasoning_utils import ( reasoning_content_from_thinking_blocks, + thinking_blocks_with_encrypted_reasoning, ) from litellm.litellm_core_utils.prompt_templates.tool_call_json_repair import ( parse_tool_call_arguments, @@ -237,6 +238,19 @@ def target_supports_mid_conversation_system(model: str | None, custom_llm_provid return supports_mid_conversation_system(model=model, custom_llm_provider=custom_llm_provider) +def _keeps_encrypted_reasoning_for_responses(model: str | None, custom_llm_provider: str | None) -> bool: + from litellm.responses.bridge_check import chat_completion_ends_in_responses_api + + return chat_completion_ends_in_responses_api(model, custom_llm_provider) + + +def _anthropic_thinking_blocks(source: object) -> tuple[Mapping[str, object], ...]: + return thinking_blocks_with_encrypted_reasoning( + getattr(source, "thinking_blocks", None), + getattr(source, "reasoning_items", None), + ) + + def _chat_tool_param(function_chunk: ChatCompletionToolParamFunctionChunk, tool: object) -> ChatCompletionToolParam: eager_input_streaming: Final = eager_input_streaming_flag(tool) if eager_input_streaming is None: @@ -505,7 +519,11 @@ class LiteLLMAnthropicMessagesAdapter: preserve_midturn_system: bool = False, ) -> list[AllMessageValues]: new_messages: Final[list[AllMessageValues]] = [] - replayable_messages: Final = strip_encrypted_reasoning_blocks_from_anthropic_messages(messages) + replayable_messages: Final = ( + messages + if _keeps_encrypted_reasoning_for_responses(model, custom_llm_provider) + else strip_encrypted_reasoning_blocks_from_anthropic_messages(messages) + ) leading_count: Final = next( (i for i, m in enumerate(replayable_messages) if not is_system_role_message(m)), len(replayable_messages), @@ -1443,9 +1461,9 @@ class LiteLLMAnthropicMessagesAdapter: for choice in choices ): new_content.extend(dict(_COMPACTION_BLOCK.validate_python(block)) for block in compaction_blocks) - # Handle thinking blocks first - if hasattr(choice.message, "thinking_blocks") and choice.message.thinking_blocks: - for thinking_block in choice.message.thinking_blocks: + signed_thinking_blocks = _anthropic_thinking_blocks(choice.message) + if signed_thinking_blocks: + for thinking_block in signed_thinking_blocks: if thinking_block.get("type") == "thinking": if is_empty_unsigned_thinking_block(thinking_block): continue @@ -1756,24 +1774,24 @@ class LiteLLMAnthropicMessagesAdapter: choice.delta ) is not None: return "text", TextBlock(type="text", text="") - elif isinstance(choice, StreamingChoices) and hasattr(choice.delta, "thinking_blocks"): - thinking_blocks = choice.delta.thinking_blocks or [] - if len(thinking_blocks) > 0: - thinking_block = thinking_blocks[0] - if thinking_block["type"] == "redacted_thinking": - return "redacted_thinking", ChatCompletionRedactedThinkingBlock( - type="redacted_thinking", data=thinking_block.get("data") or "" - ) - if thinking_block["type"] == "thinking": - thinking = thinking_block.get("thinking") or "" - signature = thinking_block.get("signature") or "" + elif isinstance(choice, StreamingChoices) and (thinking_blocks := _anthropic_thinking_blocks(choice.delta)): + thinking_block = thinking_blocks[0] + if thinking_block["type"] == "redacted_thinking": + redacted_data = thinking_block.get("data") + return "redacted_thinking", ChatCompletionRedactedThinkingBlock( + type="redacted_thinking", + data=redacted_data if isinstance(redacted_data, str) else "", + ) + if thinking_block["type"] == "thinking": + thinking = thinking_block.get("thinking") or "" + signature = thinking_block.get("signature") or "" - assert isinstance(thinking, str) - assert isinstance(signature, str) + assert isinstance(thinking, str) + assert isinstance(signature, str) - return "thinking", ChatCompletionThinkingBlock( - type="thinking", thinking=thinking if signature else "", signature=signature - ) + return "thinking", ChatCompletionThinkingBlock( + type="thinking", thinking=thinking if signature else "", signature=signature + ) # OpenAI-compatible reasoning backends (e.g. vLLM/SGLang reasoning # parsers) populate ``reasoning_content`` without ``thinking_blocks``. # ``Delta`` deletes the ``thinking_blocks`` attribute when unset, so the @@ -1806,19 +1824,17 @@ class LiteLLMAnthropicMessagesAdapter: for tool in choice.delta.tool_calls: if tool.function is not None and tool.function.arguments is not None: partial_json = (partial_json or "") + tool.function.arguments - elif isinstance(choice, StreamingChoices) and hasattr(choice.delta, "thinking_blocks"): - thinking_blocks = choice.delta.thinking_blocks or [] - if len(thinking_blocks) > 0: - for thinking_block in thinking_blocks: - if thinking_block["type"] == "thinking": - thinking = thinking_block.get("thinking") or "" - signature = thinking_block.get("signature") or "" + elif isinstance(choice, StreamingChoices) and (thinking_blocks := _anthropic_thinking_blocks(choice.delta)): + for thinking_block in thinking_blocks: + if thinking_block["type"] == "thinking": + thinking = thinking_block.get("thinking") or "" + signature = thinking_block.get("signature") or "" - assert isinstance(thinking, str) - assert isinstance(signature, str) + assert isinstance(thinking, str) + assert isinstance(signature, str) - reasoning_content += thinking - reasoning_signature += signature + reasoning_content += thinking + reasoning_signature += signature # Handle reasoning_content when thinking_blocks is not present # This handles providers like OpenRouter that return reasoning_content elif isinstance(choice, StreamingChoices) and hasattr(choice.delta, "reasoning_content"): diff --git a/litellm/responses/bridge_check.py b/litellm/responses/bridge_check.py index 826de8cfdc9..68009abcbe6 100644 --- a/litellm/responses/bridge_check.py +++ b/litellm/responses/bridge_check.py @@ -4,7 +4,10 @@ from urllib.parse import urlsplit import litellm from litellm._logging import verbose_logger -from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.litellm_core_utils.get_llm_provider_logic import ( + get_llm_provider, + resolve_model_and_provider_for_metadata, +) from litellm.llms.azure_ai.chat.transformation import AzureAIGPT5Config from litellm.llms.azure_ai.common_utils import ( azure_ai_supports_native_responses, @@ -291,3 +294,23 @@ def responses_api_bridge_check( model = model.replace("responses/", "") return model_info, model + + +def chat_completion_ends_in_responses_api(model: str | None, custom_llm_provider: str | None) -> bool: + """Whether chat completion for this model is served by a Responses API. + + Encrypted reasoning can be replayed only there. A normal chat provider cannot + verify the block, so the messages adapter strips it. ``chatgpt`` and a + ``responses/`` model keep it. + """ + if model is None and custom_llm_provider is None: + return False + try: + resolved_model, provider = resolve_model_and_provider_for_metadata(model or "", custom_llm_provider) + except Exception as error: # noqa: BLE001 # unknown model names must not fail message translation + verbose_logger.debug("responses replay check failed for %s: %s", model, error) + return custom_llm_provider == LlmProviders.CHATGPT.value or ( + isinstance(model, str) and (model == "chatgpt" or model.startswith("chatgpt/")) + ) + model_info, _resolved = responses_api_bridge_check(model=resolved_model, custom_llm_provider=provider) + return model_info.get("mode") == "responses" diff --git a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index 19aefb430c7..eca1bdd58df 100644 --- a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -3837,6 +3837,165 @@ def test_reasoning_items_streaming_emitted_on_response_completed(): assert ri["summary"][0]["text"] == summary_text +def test_reasoning_output_item_done_emits_signed_thinking_blocks_the_stream_keeps(): + """A chatgpt /v1/messages stream drops the reasoning item unless this delta carries thinking_blocks. + + response.output_item.done is the only event that has encrypted_content. The chat delta has to + expose it as a signed thinking block, and that block alone has to count as stream content. + """ + from unittest.mock import MagicMock + + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + OpenAiResponsesToChatCompletionStreamIterator, + ) + from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature + from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper + from litellm.llms.anthropic.messages.adapters.transformation import LiteLLMAnthropicMessagesAdapter + + encrypted: Final = "gAAAA_round_trip" + done: Final = { + "type": "response.output_item.done", + "output_index": 0, + "item": { + "type": "reasoning", + "id": "rs_done", + "encrypted_content": encrypted, + "summary": [], + }, + } + result: Final = OpenAiResponsesToChatCompletionStreamIterator( + streaming_response=None, sync_stream=True + ).chunk_parser(done) + blocks: Final = result.choices[0].delta.thinking_blocks + assert blocks is not None + assert blocks[0]["type"] == "redacted_thinking" + assert blocks[0]["data"] == encrypted_reasoning_signature(encrypted) + assert blocks[0]["index"] == 0 + assert getattr(result.choices[0].delta, "reasoning_content", None) in (None, "") + + wrapper: Final = CustomStreamWrapper( + completion_stream=None, + model="chatgpt/gpt-6.1-sol", + logging_obj=MagicMock(), + custom_llm_provider="chatgpt", + ) + assert wrapper.is_chunk_non_empty({}, result, {}) is True + + block_type, start = LiteLLMAnthropicMessagesAdapter().translate_streaming_openai_chunk_to_anthropic_content_block( + choices=result.choices + ) + assert block_type == "redacted_thinking" + assert start["data"] == encrypted_reasoning_signature(encrypted) + + +def test_reasoning_output_item_done_with_summary_signs_the_thinking_text(): + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + OpenAiResponsesToChatCompletionStreamIterator, + ) + from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature + + encrypted: Final = "gAAAA_with_summary" + done: Final = { + "type": "response.output_item.done", + "output_index": 2, + "item": { + "type": "reasoning", + "id": "rs_sum", + "encrypted_content": encrypted, + "summary": [{"type": "summary_text", "text": "Oslo is colder."}], + }, + } + result: Final = OpenAiResponsesToChatCompletionStreamIterator( + streaming_response=None, sync_stream=True + ).chunk_parser(done) + block: Final = result.choices[0].delta.thinking_blocks[0] + assert block["type"] == "thinking" + assert block["thinking"] == "Oslo is colder." + assert block["signature"] == encrypted_reasoning_signature(encrypted) + assert block["index"] == 2 + + +def test_reasoning_output_item_done_without_encrypted_content_stays_empty(): + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + OpenAiResponsesToChatCompletionStreamIterator, + ) + + done: Final = { + "type": "response.output_item.done", + "output_index": 0, + "item": { + "type": "reasoning", + "id": "rs_plain", + "summary": [{"type": "summary_text", "text": "already streamed"}], + }, + } + result: Final = OpenAiResponsesToChatCompletionStreamIterator( + streaming_response=None, sync_stream=True + ).chunk_parser(done) + assert getattr(result.choices[0].delta, "thinking_blocks", None) in (None, []) + + +def test_transform_response_copies_encrypted_reasoning_onto_thinking_blocks(): + from unittest.mock import Mock + + from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature + from litellm.types.llms.openai import ResponsesAPIResponse + from litellm.types.utils import ModelResponse, Usage + + encrypted: Final = "gAAAA_non_stream" + reasoning: Final = ResponseReasoningItem( + id="rs_tool", + summary=[], + type="reasoning", + content=None, + encrypted_content=encrypted, + status=None, + ) + function_call: Final = ResponseFunctionToolCall( + id="fc_1", + type="function_call", + status="completed", + arguments='{"city":"Oslo"}', + call_id="call_1", + name="get_weather", + ) + raw: Final = ResponsesAPIResponse.model_construct( + id="resp_tool", + created_at=1, + error=None, + incomplete_details=None, + model="gpt-6.1-sol", + output=[reasoning, function_call], + status="completed", + usage=None, + ) + model_response: Final = ModelResponse( + id="chatcmpl-tool", + created=1, + model=None, + object="chat.completion", + system_fingerprint=None, + choices=[], + usage=Usage(completion_tokens=0, prompt_tokens=0, total_tokens=0), + ) + result: Final = LiteLLMResponsesTransformationHandler().transform_response( + model="gpt-6.1-sol", + raw_response=raw, + model_response=model_response, + logging_obj=Mock(), + request_data={"model": "gpt-6.1-sol"}, + messages=[{"role": "user", "content": "Which city?"}], + optional_params={}, + litellm_params={}, + encoding=Mock(), + ) + message: Final = result.choices[0].message + assert message.reasoning_items[0]["encrypted_content"] == encrypted + assert message.thinking_blocks[0]["type"] == "redacted_thinking" + assert message.thinking_blocks[0]["data"] == encrypted_reasoning_signature(encrypted) + assert message.tool_calls[0].function.name == "get_weather" + + def test_streaming_function_call_tool_id_for_degenerate_call_id(): """In streaming, Bedrock Mantle's function_call event carries a unique ``id`` (``fc_...``) and a non-unique, index-based ``call_id`` (``call_0``). For that diff --git a/tests/unit/litellm_core_utils/test_streaming_handler.py b/tests/unit/litellm_core_utils/test_streaming_handler.py index cf8de9c40f5..f2f3ea5864a 100644 --- a/tests/unit/litellm_core_utils/test_streaming_handler.py +++ b/tests/unit/litellm_core_utils/test_streaming_handler.py @@ -276,6 +276,37 @@ def test_is_chunk_non_empty(initialized_custom_stream_wrapper: CustomStreamWrapp ) +def test_is_chunk_non_empty_keeps_thinking_blocks_without_other_content( + initialized_custom_stream_wrapper: CustomStreamWrapper, +): + """Encrypted reasoning arrives as thinking_blocks and nothing else, so that delta has to be yielded.""" + chunk: Final = ModelResponseStream( + id="chunk-thinking", + object="chat.completion.chunk", + created=1, + model="chatgpt/gpt-6.1-sol", + choices=[ + StreamingChoices( + index=0, + finish_reason=None, + delta=Delta( + thinking_blocks=[ + {"type": "redacted_thinking", "data": "litellm_encrypted_reasoning:gAAAA"} + ] + ), + ) + ], + ) + assert ( + initialized_custom_stream_wrapper.is_chunk_non_empty( + completion_obj={}, + model_response=chunk, + response_obj={}, + ) + is True + ) + + def test_is_chunk_non_empty_with_annotations( initialized_custom_stream_wrapper: CustomStreamWrapper, ): diff --git a/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py b/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py index fb4f19a862c..98fa4caf012 100644 --- a/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py @@ -563,6 +563,166 @@ def test_translate_anthropic_messages_to_openai_drops_bridge_encrypted_reasoning assert [b["signature"] for b in result[2]["thinking_blocks"]] == ["EqQBCkYIAxgCIkA_signed"] +def test_chatgpt_messages_keep_encrypted_reasoning_and_replay_it(): + """chatgpt /v1/messages ends in the Responses API, so the signed block has to survive into the next request.""" + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, + ) + + summary_encrypted: Final = "gAAAA_summary" + bytes_encrypted: Final = "gAAAA_bytes" + messages: Final = [ + AnthropicMessagesUserMessageParam(role="user", content="Which city is colder?"), + AnthopicMessagesAssistantMessageParam( + role="assistant", + content=[ + { + "type": "thinking", + "thinking": "Oslo is colder.", + "signature": encrypted_reasoning_signature(summary_encrypted), + }, + {"type": "redacted_thinking", "data": encrypted_reasoning_signature(bytes_encrypted)}, + {"type": "tool_use", "id": "call_1", "name": "get_weather", "input": {"city": "Oslo"}}, + ], + ), + AnthropicMessagesUserMessageParam( + role="user", + content=[{"type": "tool_result", "tool_use_id": "call_1", "content": "-7 C"}], + ), + ] + + kept: Final = LiteLLMAnthropicMessagesAdapter().translate_anthropic_messages_to_openai( + messages=messages, + model="chatgpt/gpt-6.1-sol", + custom_llm_provider="chatgpt", + ) + assistant: Final = kept[1] + assert [block["type"] for block in assistant["thinking_blocks"]] == ["thinking", "redacted_thinking"] + assert assistant["thinking_blocks"][0]["signature"] == encrypted_reasoning_signature(summary_encrypted) + assert assistant["thinking_blocks"][1]["data"] == encrypted_reasoning_signature(bytes_encrypted) + + responses_input, _instructions = LiteLLMResponsesTransformationHandler().convert_chat_completion_messages_to_responses_api( + kept + ) + reasoning: Final = tuple(item for item in responses_input if item.get("type") == "reasoning") + assert [item["encrypted_content"] for item in reasoning] == [summary_encrypted, bytes_encrypted] + types: Final = [item.get("type") for item in responses_input] + assert types.index("reasoning") < types.index("function_call") + + stripped: Final = LiteLLMAnthropicMessagesAdapter().translate_anthropic_messages_to_openai( + messages=messages, + model="gpt-4o", + custom_llm_provider="openai", + ) + assert not stripped[1].get("thinking_blocks") + + prefixed: Final = LiteLLMAnthropicMessagesAdapter().translate_anthropic_messages_to_openai( + messages=messages, + model="responses/gpt-5-mini", + custom_llm_provider="openai", + ) + assert prefixed[1]["thinking_blocks"][1]["data"] == encrypted_reasoning_signature(bytes_encrypted) + + +def test_translate_openai_reasoning_items_become_signed_thinking_blocks(): + encrypted: Final = "gAAAA_outbound" + openai_choices: Final = [ + Choices( + message=Message( + role="assistant", + content="Oslo.", + reasoning_items=[ + { + "type": "reasoning", + "id": "rs_1", + "encrypted_content": encrypted, + "summary": [{"type": "summary_text", "text": "Oslo is colder."}], + } + ], + tool_calls=[ + ChatCompletionAssistantToolCall( + id="call_1", + type="function", + function=Function(name="get_weather", arguments='{"city": "Oslo"}'), + ) + ], + ) + ) + ] + + result: Final = LiteLLMAnthropicMessagesAdapter()._translate_openai_content_to_anthropic(choices=openai_choices) + + assert result[0]["type"] == "thinking" + assert result[0]["thinking"] == "Oslo is colder." + assert result[0]["signature"] == encrypted_reasoning_signature(encrypted) + assert result[1]["type"] == "text" + assert result[2]["type"] == "tool_use" + + +def test_translate_openai_reasoning_items_without_summary_become_redacted_thinking(): + encrypted: Final = "gAAAA_bytes_only" + openai_choices: Final = [ + Choices( + message=Message( + role="assistant", + content=None, + reasoning_items=[ + {"type": "reasoning", "id": "rs_1", "encrypted_content": encrypted, "summary": []} + ], + ) + ) + ] + + result: Final = LiteLLMAnthropicMessagesAdapter()._translate_openai_content_to_anthropic(choices=openai_choices) + + assert result == [ + {"type": "redacted_thinking", "data": encrypted_reasoning_signature(encrypted)}, + ] + + +def test_translate_openai_reasoning_items_are_not_duplicated_when_thinking_blocks_already_carry_them(): + encrypted: Final = "gAAAA_once" + signature: Final = encrypted_reasoning_signature(encrypted) + openai_choices: Final = [ + Choices( + message=Message( + role="assistant", + content="Oslo.", + thinking_blocks=[{"type": "redacted_thinking", "data": signature}], + reasoning_items=[ + {"type": "reasoning", "id": "rs_1", "encrypted_content": encrypted, "summary": []} + ], + ) + ) + ] + + result: Final = LiteLLMAnthropicMessagesAdapter()._translate_openai_content_to_anthropic(choices=openai_choices) + + assert [block["type"] for block in result] == ["redacted_thinking", "text"] + + +def test_streaming_chunk_with_only_reasoning_items_opens_a_signed_thinking_block(): + encrypted: Final = "gAAAA_delta" + choices: Final = [ + StreamingChoices( + finish_reason=None, + index=0, + delta=Delta( + reasoning_items=[ + {"type": "reasoning", "id": "rs_1", "encrypted_content": encrypted, "summary": []} + ] + ), + ) + ] + + block_type, start = LiteLLMAnthropicMessagesAdapter().translate_streaming_openai_chunk_to_anthropic_content_block( + choices=choices + ) + + assert block_type == "redacted_thinking" + assert start["data"] == encrypted_reasoning_signature(encrypted) + + def test_translate_anthropic_messages_to_openai_sets_reasoning_content(): """Reasoning-aware chat providers read reasoning_content, so thinking text must land there.