diff --git a/litellm/llms/anthropic/messages/adapters/handler.py b/litellm/llms/anthropic/messages/adapters/handler.py index d13fd4f47ff..6b05db094e6 100644 --- a/litellm/llms/anthropic/messages/adapters/handler.py +++ b/litellm/llms/anthropic/messages/adapters/handler.py @@ -720,8 +720,54 @@ class LiteLLMMessagesToCompletionTransformationHandler: if chat_reasoning is not None and chat_reasoning.summary is not None: completion_kwargs["reasoning_summary"] = chat_reasoning.summary + if LiteLLMMessagesToCompletionTransformationHandler._uses_responses_bridge(completion_kwargs): + from litellm.llms.anthropic.messages.adapters.transformation import LiteLLMAnthropicMessagesAdapter + from litellm.types.llms.anthropic import AllAnthropicPassThroughMessageValues, AnthropicMessagesRequest + + adapter: Final = LiteLLMAnthropicMessagesAdapter() + replay_messages: Final = adapter.translate_anthropic_messages_to_openai( + cast(list[AllAnthropicPassThroughMessageValues], messages), + model=model, + custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None, + preserve_encrypted_reasoning=True, + ) + replay_request: Final = AnthropicMessagesRequest(model=model, messages=messages, system=system) + adapter._add_system_message_to_messages(replay_messages, replay_request) + completion_kwargs["messages"] = replay_messages + return completion_kwargs, tool_name_mapping + @staticmethod + def _uses_responses_bridge(completion_kwargs: _CompletionKwargs) -> bool: + from litellm.responses.bridge_check import responses_api_bridge_check, str_keyed_mapping + + model: Final = completion_kwargs.get("model") + if not isinstance(model, str): + return False + provider: Final = completion_kwargs.get("custom_llm_provider") + api_base: Final = completion_kwargs.get("api_base") + try: + local_model, resolved_provider, _, resolved_api_base = litellm.utils.get_llm_provider( + model=model, + custom_llm_provider=provider if isinstance(provider, str) else None, + api_base=api_base if isinstance(api_base, str) else None, + ) + except (ValueError, litellm.BadRequestError): + return False + tools: Final = completion_kwargs.get("tools") + web_search: Final = completion_kwargs.get("web_search_options") + effort: Final = completion_kwargs.get("reasoning_effort") + model_info, _ = responses_api_bridge_check( + model=local_model, + custom_llm_provider=resolved_provider, + tools=cast("list[dict[str, object]]", tools) if isinstance(tools, list) else None, + web_search_options=cast(OpenAIWebSearchOptions, web_search) if isinstance(web_search, dict) else None, + reasoning_effort=effort if isinstance(effort, str) else str_keyed_mapping(effort), + api_base=resolved_api_base, + deployment_model_info=str_keyed_mapping(completion_kwargs.get("model_info")), + ) + return model_info.get("mode") == "responses" + @staticmethod async def async_anthropic_messages_handler( max_tokens: int, diff --git a/litellm/llms/anthropic/messages/adapters/streaming_iterator.py b/litellm/llms/anthropic/messages/adapters/streaming_iterator.py index 61d23c2de0d..cd72c60f4c9 100644 --- a/litellm/llms/anthropic/messages/adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/messages/adapters/streaming_iterator.py @@ -29,6 +29,7 @@ from litellm._uuid import uuid from litellm.exceptions import ContextWindowExceededError, MidStreamFallbackError from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_items from litellm.litellm_core_utils.hidden_params import set_hidden_params +from litellm.llms.anthropic.messages.responses_adapters.transformation import LiteLLMAnthropicToResponsesAPIAdapter from litellm.llms.anthropic.messages.safeguards import StreamedSafeguardResults from litellm.llms.anthropic.messages.utils import anthropic_sse_frame from litellm.llms.base_llm.chat.transformation import BaseLLMException @@ -494,9 +495,10 @@ class _CombinedChunkSplitter: ) def _expand(self, chunk: "ModelResponseStream") -> "tuple[ModelResponseStream, ...]": + normalized: Final = self._with_reasoning_items(chunk) split_chunks: Final = tuple( split_chunk - for combined_chunk in self._split(chunk) + for combined_chunk in self._split(normalized) for split_chunk in self._split_by_payload_kind(combined_chunk) # comprehension-ok: flattens nested splits ) block_chunks: Final = tuple(itertools.chain.from_iterable(map(self._split_thinking_blocks, split_chunks))) @@ -540,6 +542,26 @@ class _CombinedChunkSplitter: def _restates_unsigned_thinking(self, text: str) -> bool: return len(text) == self._unsigned_thinking_length and text == self._unsigned_thinking.getvalue() + @staticmethod + def _with_reasoning_items(chunk: "ModelResponseStream") -> "ModelResponseStream": + if not chunk.choices: + return chunk + choice: Final = chunk.choices[0] + blocks: Final = tuple( + block + for item in (choice.delta.reasoning_items or ()) + if ( + block := LiteLLMAnthropicToResponsesAPIAdapter._thinking_block_from_reasoning_item( + (), item.get("encrypted_content") + ) + ) + is not None + ) + if not blocks: + return chunk + delta: Final = choice.delta.model_copy(update={"thinking_blocks": list(blocks)}) + return chunk.model_copy(update={"choices": [choice.model_copy(update={"delta": delta})]}) + def __iter__(self) -> "Iterator[ModelResponseStream]": return self diff --git a/litellm/llms/anthropic/messages/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py index 4b1b5a8a11a..70a7dd4fdfe 100644 --- a/litellm/llms/anthropic/messages/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -155,6 +155,7 @@ from litellm.llms.anthropic.messages.native.utils import ( refusal_reply_text, refusal_stop_details, ) +from litellm.llms.anthropic.messages.responses_adapters.transformation import LiteLLMAnthropicToResponsesAPIAdapter from litellm.types.llms.anthropic import ( ANTHROPIC_HOSTED_TOOLS, AllAnthropicPassThroughMessageValues, @@ -495,9 +496,14 @@ class LiteLLMAnthropicMessagesAdapter: *, custom_llm_provider: str | None = None, preserve_midturn_system: bool = False, + preserve_encrypted_reasoning: bool = False, ) -> list[AllMessageValues]: new_messages: Final[list[AllMessageValues]] = [] - replayable_messages: Final = strip_encrypted_reasoning_blocks_from_anthropic_messages(messages) + replayable_messages: Final = ( + messages + if preserve_encrypted_reasoning + else strip_encrypted_reasoning_blocks_from_anthropic_messages(messages) + ) leading_count: Final = next( (i for i, m in enumerate(replayable_messages) if not is_system_role_message(m)), len(replayable_messages), @@ -1435,8 +1441,19 @@ class LiteLLMAnthropicMessagesAdapter: for choice in choices ): new_content.extend(dict(_COMPACTION_BLOCK.validate_python(block)) for block in compaction_blocks) - # Handle thinking blocks first - if hasattr(choice.message, "thinking_blocks") and choice.message.thinking_blocks: + reasoning_blocks: Final = tuple( + block + for item in (choice.message.reasoning_items or ()) + if ( + block := LiteLLMAnthropicToResponsesAPIAdapter._thinking_block_from_reasoning_item( + item.get("summary", ()), item.get("encrypted_content") + ) + ) + is not None + ) + if reasoning_blocks: + new_content.extend(reasoning_blocks) + elif hasattr(choice.message, "thinking_blocks") and choice.message.thinking_blocks: for thinking_block in choice.message.thinking_blocks: if thinking_block.get("type") == "thinking": if is_empty_unsigned_thinking_block(thinking_block): diff --git a/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py b/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py index cc366dcde4c..b3b49436802 100644 --- a/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py @@ -11,9 +11,9 @@ import pytest import respx import litellm - - - +from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, +) from litellm.litellm_core_utils.prompt_templates.common_utils import ( TOOL_RESULT_IMAGE_PLACEHOLDER, encrypted_reasoning_signature, @@ -22,6 +22,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( THOUGHT_SIGNATURE_SEPARATOR, _bedrock_converse_messages_pt, ) +from litellm.llms.anthropic.messages.adapters.handler import LiteLLMMessagesToCompletionTransformationHandler from litellm.llms.anthropic.messages.adapters.transformation import ( OPENAI_MAX_TOOL_NAME_LENGTH, AnthropicAdapter, @@ -37,7 +38,7 @@ from litellm.types.llms.anthropic import ( AnthopicMessagesAssistantMessageParam, AnthropicMessagesUserMessageParam, ) -from litellm.types.llms.openai import ChatCompletionAssistantToolCall +from litellm.types.llms.openai import ChatCompletionAssistantToolCall, ChatCompletionReasoningSummaryTextBlock from litellm.types.utils import ( ChatCompletionDeltaToolCall, Choices, @@ -52,6 +53,52 @@ from litellm.types.utils import ( from litellm.utils import remove_additional_properties, remove_strict_from_schema +@pytest.mark.parametrize("model", ["chatgpt/gpt-5", "openai/responses/gpt-5"]) +@pytest.mark.parametrize("summary", [[], [{"type": "summary_text", "text": "Check the weather"}]]) +def test_responses_reasoning_round_trips_through_anthropic_tool_turn( + model: str, summary: list[ChatCompletionReasoningSummaryTextBlock] +) -> None: + response: Final = ModelResponse( + model=model, + choices=[ + { + "index": 0, + "finish_reason": "tool_calls", + "message": { + "role": "assistant", + "content": None, + "reasoning_items": [{"type": "reasoning", "summary": summary, "encrypted_content": "opaque-test"}], + "tool_calls": [ + {"id": "call_weather", "type": "function", "function": {"name": "weather", "arguments": "{}"}} + ], + }, + } + ], + ) + result: Final = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic(response) + assert result["content"][0] == ( + {"type": "thinking", "thinking": "Check the weather", "signature": encrypted_reasoning_signature("opaque-test")} + if summary + else {"type": "redacted_thinking", "data": encrypted_reasoning_signature("opaque-test")} + ) + prepared, _ = LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs( + model=model, + max_tokens=100, + system="Keep the system instruction", + messages=[ + {"role": "user", "content": "Weather?"}, + {"role": "assistant", "content": result["content"]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "call_weather", "content": "sunny"}]}, + ], + ) + items, instructions = LiteLLMResponsesTransformationHandler().convert_chat_completion_messages_to_responses_api( + prepared["messages"] + ) + assert instructions == "Keep the system instruction" + assert [item["type"] for item in items] == ["message", "reasoning", "function_call", "function_call_output"] + assert items[1]["encrypted_content"] == "opaque-test" + + def test_translate_openai_response_to_anthropic_empty_choices() -> None: response: Final = ModelResponse( id="chatcmpl-empty", diff --git a/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_combined_chunk.py b/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_combined_chunk.py index d4911427e73..9cd0fc67d21 100644 --- a/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_combined_chunk.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_streaming_iterator_combined_chunk.py @@ -22,6 +22,7 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo convert_to_streaming_response, convert_to_streaming_response_async, ) +from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature from litellm.llms.anthropic.messages.adapters.streaming_iterator import ( AnthropicStreamWrapper, _CombinedChunkSplitter, @@ -39,6 +40,27 @@ from litellm.types.utils import ( ) +async def _stream_with_encrypted_reasoning() -> AsyncIterator[ModelResponseStream]: + yield ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(reasoning_content="Visible summary"))]) + yield ModelResponseStream( + choices=[ + StreamingChoices( + index=0, + delta=Delta(reasoning_items=[{"type": "reasoning", "summary": [], "encrypted_content": "opaque-test"}]), + finish_reason="stop", + ) + ] + ) + + +def test_stream_replays_encrypted_reasoning_without_repeating_summary() -> None: + content: Final = _anthropic_sdk_final_content(_stream_with_encrypted_reasoning()) + assert content == [ + {"type": "thinking", "thinking": "Visible summary", "signature": ""}, + {"type": "redacted_thinking", "data": encrypted_reasoning_signature("opaque-test")}, + ] + + def _build_fake_stream(content: str, finish_reason: str = "stop") -> MockResponseIterator: """Mimic a Vertex Gemma `:predict` fake stream: one collapsed chunk.""" model_response = ModelResponse() @@ -346,9 +368,7 @@ def _anthropic_sdk_final_content( lambda request: httpx.Response(200, content=sse.encode(), headers={"content-type": "text/event-stream"}) ) client: Final = anthropic.Anthropic(api_key="transport-only", http_client=httpx.Client(transport=transport)) - with client.messages.stream( - model="claude", max_tokens=16, messages=[{"role": "user", "content": "hi"}] - ) as stream: + with client.messages.stream(model="claude", max_tokens=16, messages=[{"role": "user", "content": "hi"}]) as stream: return [block.model_dump(exclude_none=True) for block in stream.get_final_message().content] @@ -377,7 +397,12 @@ def test_fake_stream_gives_each_thinking_block_its_own_content_block(thinking_bl def _native_thinking_chunk(thinking: str, index: int, signature: str | None = None) -> ModelResponseStream: - block: Final = {"type": "thinking", "thinking": thinking, "index": index, **({"signature": signature} if signature else {})} + block: Final = { + "type": "thinking", + "thinking": thinking, + "index": index, + **({"signature": signature} if signature else {}), + } return ModelResponseStream( choices=[StreamingChoices(index=0, delta=Delta(reasoning_content=thinking or None, thinking_blocks=[block]))] )