From ce35f27bfaecc0b513b0cd5921b095d80b406a83 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 7 Nov 2026 04:08:08 -0800 Subject: [PATCH] fix(anthropic): keep every non-streamed reasoning item for Messages clients and read a deployment's reasoning_summary in the keep check Backport of #49076 to lts/1.108.x. Ported from cadfe1d1798 (cursor/chatgpt-encrypted-reasoning-messages-1723) onto the line's pass_through/adapters layout. --- .../handler.py | 18 +++-- .../transformation.py | 40 +++++++---- .../pass_through/adapters/handler.py | 2 + .../pass_through/adapters/transformation.py | 12 +++- litellm/responses/bridge_check.py | 2 + ...responses_transformation_transformation.py | 66 +++++++++++++++++++ ...al_pass_through_adapters_transformation.py | 28 ++++++++ 7 files changed, 144 insertions(+), 24 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/handler.py b/litellm/completion_extras/litellm_responses_transformation/handler.py index eef9cc6c4c9..e824d9bcfc1 100644 --- a/litellm/completion_extras/litellm_responses_transformation/handler.py +++ b/litellm/completion_extras/litellm_responses_transformation/handler.py @@ -13,9 +13,7 @@ from typing import ( from typing_extensions import TypedDict -from litellm.litellm_core_utils.core_helpers import attribute_or_none from litellm.types.llms.openai import ResponsesAPIResponse -from litellm.types.utils import CallTypes if TYPE_CHECKING: from litellm import CustomStreamWrapper, LiteLLMLoggingObj, ModelResponse @@ -38,10 +36,6 @@ def _restore_routing_prefix(model: str, custom_llm_provider: str) -> str: return f"{custom_llm_provider}/{model}" -def _serves_anthropic_messages(logging_obj: "LiteLLMLoggingObj") -> bool: - return attribute_or_none(logging_obj, "call_type") == CallTypes.anthropic_messages.value - - class ResponsesToCompletionBridgeHandler: def __init__(self): from .transformation import LiteLLMResponsesTransformationHandler @@ -188,7 +182,9 @@ class ResponsesToCompletionBridgeHandler: model_response: Final = validated_kwargs["model_response"] logging_obj: Final = validated_kwargs["logging_obj"] custom_llm_provider: Final = validated_kwargs["custom_llm_provider"] - serves_anthropic_messages: Final = _serves_anthropic_messages(logging_obj) + from .transformation import serves_anthropic_messages + + streams_reasoning_signatures: Final = serves_anthropic_messages(logging_obj) if kwargs.get("stream") is True and "stream" not in optional_params: optional_params = {**optional_params, "stream": True} @@ -267,7 +263,7 @@ class ResponsesToCompletionBridgeHandler: structured_output=is_structured_output_request( cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict ), - streams_reasoning_signatures=serves_anthropic_messages, + streams_reasoning_signatures=streams_reasoning_signatures, ) streamwrapper: Final = CustomStreamWrapper( completion_stream=completion_stream, @@ -290,7 +286,9 @@ class ResponsesToCompletionBridgeHandler: model_response: Final = validated_kwargs["model_response"] logging_obj: Final = validated_kwargs["logging_obj"] custom_llm_provider: Final = validated_kwargs["custom_llm_provider"] - serves_anthropic_messages: Final = _serves_anthropic_messages(logging_obj) + from .transformation import serves_anthropic_messages + + streams_reasoning_signatures: Final = serves_anthropic_messages(logging_obj) if kwargs.get("stream") is True and "stream" not in optional_params: optional_params = {**optional_params, "stream": True} @@ -372,7 +370,7 @@ class ResponsesToCompletionBridgeHandler: structured_output=is_structured_output_request( cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict ), - streams_reasoning_signatures=serves_anthropic_messages, + streams_reasoning_signatures=streams_reasoning_signatures, ) streamwrapper: Final = CustomStreamWrapper( completion_stream=completion_stream, diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 33ac222b06b..cf8d845bf26 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -26,7 +26,7 @@ import litellm from litellm import ModelResponse from litellm._logging import verbose_logger from litellm.integrations.anthropic_cache_control_hook import supports_openai_prompt_cache_breakpoint -from litellm.litellm_core_utils.core_helpers import as_str_mapping +from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none from litellm.litellm_core_utils.hidden_params import get_hidden_params, get_or_create_hidden_params from litellm.litellm_core_utils.prompt_templates.common_utils import ( responses_reasoning_items_from_thinking_blocks, @@ -58,7 +58,7 @@ from litellm.types.llms.openai import ( ResponsesAPIResponse, ResponsesAPIStreamEvents, ) -from litellm.types.utils import Delta, GenericStreamingChunk, ModelResponseStream, StreamingChoices +from litellm.types.utils import CallTypes, Delta, GenericStreamingChunk, ModelResponseStream, StreamingChoices if TYPE_CHECKING: from openai.types.responses import ResponseInputImageParam, ResponseOutputItem @@ -227,6 +227,10 @@ class _BuiltReasoningItem(TypedDict): summary: Sequence[_ReasoningSummaryText] +def serves_anthropic_messages(logging_obj: object) -> bool: + return attribute_or_none(logging_obj, "call_type") == CallTypes.anthropic_messages.value + + def _get_reasoning_items( msg: "AllMessageValues", ) -> list[ChatCompletionReasoningItem]: @@ -549,7 +553,7 @@ def _fold_choices_into_one( choices: Sequence["Choices"], tool_calls: Sequence[Mapping[str, object]], trailing_reasoning_content: str | None, - trailing_reasoning_item: _BuiltReasoningItem | None, + trailing_reasoning_items: Sequence[_BuiltReasoningItem], tool_search_items: Sequence[Mapping[str, object]], structured_output: bool = False, ) -> "Choices": @@ -568,7 +572,7 @@ def _fold_choices_into_one( reasoning_contents: Final = (*(_message_reasoning_content(m) for m in messages), trailing_reasoning_content) reasoning_items: Final = ( *chain.from_iterable(_message_reasoning_items(m) for m in messages), - *(() if trailing_reasoning_item is None else (trailing_reasoning_item,)), + *trailing_reasoning_items, ) return Choices( message=Message( @@ -1222,6 +1226,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): output_items: Sequence[object], handle_raw_dict_callback: Callable[..., tuple["Choices | None", int]] | None = None, structured_output: bool = False, + keeps_every_reasoning_item: bool = False, ) -> list["Choices"]: """ Convert Responses API output items to chat completion choices. @@ -1231,6 +1236,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): handle_raw_dict_callback: Optional callback for handling raw dict items structured_output: The chat request carried a json response_format, so only the first message item becomes content + keeps_every_reasoning_item: Keep each reasoning item before an answer, not only the last Returns: List of Choices objects @@ -1253,7 +1259,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): choices: Final[list[Choices]] = [] index = 0 reasoning_content: str | None = None - pending_reasoning_item: _BuiltReasoningItem | None = None + pending_reasoning_items: tuple[_BuiltReasoningItem, ...] = () # Collect all tool calls to put them in a single choice # (Chat Completions API expects all tool calls in one message) @@ -1262,12 +1268,19 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): for item in output_items: if isinstance(item, ResponseReasoningItem): - pending_reasoning_item = _build_reasoning_item( - item_id=item.id, - encrypted_content=getattr(item, "encrypted_content", None), - summary_raw=item.summary, + pending_reasoning_items = ( + *(pending_reasoning_items if keeps_every_reasoning_item else ()), + _build_reasoning_item( + item_id=item.id, + encrypted_content=getattr(item, "encrypted_content", None), + summary_raw=item.summary, + ), + ) + reasoning_content = " ".join( + s["text"] + for s in chain.from_iterable(built["summary"] for built in pending_reasoning_items) + if s.get("text") ) - reasoning_content = " ".join(s["text"] for s in pending_reasoning_item["summary"] if s.get("text")) elif isinstance(item, ResponseOutputMessage): for content in item.content: @@ -1284,7 +1297,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): annotations=annotations, reasoning_items=cast( list[ChatCompletionReasoningItem] | None, - ([pending_reasoning_item] if pending_reasoning_item is not None else None), + (list(pending_reasoning_items) or None), ), ) @@ -1297,7 +1310,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): ) reasoning_content = None # flush - pending_reasoning_item = None # flush + pending_reasoning_items = () # flush index += 1 elif isinstance(item, ResponseFunctionToolCall): @@ -1362,7 +1375,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): choices=choices, tool_calls=accumulated_tool_calls, trailing_reasoning_content=reasoning_content, - trailing_reasoning_item=pending_reasoning_item, + trailing_reasoning_items=pending_reasoning_items, tool_search_items=tool_search_items, structured_output=structured_output, ) @@ -1500,6 +1513,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): structured_output=is_structured_output_request( cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict ), + keeps_every_reasoning_item=serves_anthropic_messages(logging_obj), ) response_is_incomplete: Final = raw_response.status == "incomplete" or ( diff --git a/litellm/llms/anthropic/pass_through/adapters/handler.py b/litellm/llms/anthropic/pass_through/adapters/handler.py index 9cd08aca1d6..4f21c7dbe5e 100644 --- a/litellm/llms/anthropic/pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/pass_through/adapters/handler.py @@ -17,6 +17,7 @@ from litellm._logging import verbose_logger from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.core_helpers import as_str_mapping, iter_items from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject +from litellm.litellm_core_utils.model_param_helper import peek_reasoning_summary_aliases from litellm.litellm_core_utils.prompt_templates.common_utils import ( is_encrypted_reasoning_block, thinking_blocks_with_encrypted_reasoning, @@ -760,6 +761,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: request_data, custom_llm_provider=target_provider, api_base=api_base if isinstance(api_base, str) else None, + reasoning_summary=peek_reasoning_summary_aliases(provider_kwargs), ) if openai_request is None: diff --git a/litellm/llms/anthropic/pass_through/adapters/transformation.py b/litellm/llms/anthropic/pass_through/adapters/transformation.py index ee553749c74..17e50498d46 100644 --- a/litellm/llms/anthropic/pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/pass_through/adapters/transformation.py @@ -228,6 +228,7 @@ class ChatRouteInputs: reasoning_effort: str | Mapping[str, object] | None = None api_base: str | None = None thinking: Mapping[str, object] | None = None + reasoning_summary: object | None = None _NO_ROUTE_INPUTS: Final = ChatRouteInputs() @@ -271,6 +272,7 @@ def _keeps_encrypted_reasoning_for_responses( custom_llm_provider, tools=route.tools, reasoning_effort=route.reasoning_effort, + reasoning_summary=route.reasoning_summary, api_base=route.api_base, ) @@ -313,7 +315,12 @@ class AnthropicAdapter: return result def translate_completion_input_params_with_tool_mapping( - self, kwargs, *, custom_llm_provider: str | None = None, api_base: str | None = None + self, + kwargs, + *, + custom_llm_provider: str | None = None, + api_base: str | None = None, + reasoning_summary: object | None = None, ) -> tuple[ChatCompletionRequest | None, dict[str, str]]: """ Translate Anthropic request params to OpenAI format, returning tool name mapping. @@ -348,6 +355,7 @@ class AnthropicAdapter: anthropic_message_request=request_body, custom_llm_provider=custom_llm_provider, api_base=api_base, + reasoning_summary=reasoning_summary, ) return translated_body, tool_name_mapping @@ -1295,6 +1303,7 @@ class LiteLLMAnthropicMessagesAdapter: *, custom_llm_provider: str | None = None, api_base: str | None = None, + reasoning_summary: object | None = None, preserve_midturn_system: bool = False, ) -> tuple[ChatCompletionRequest, dict[str, str]]: """ @@ -1349,6 +1358,7 @@ class LiteLLMAnthropicMessagesAdapter: reasoning_effort=new_kwargs.get("reasoning_effort"), api_base=api_base, thinking=anthropic_message_request.get("thinking"), + reasoning_summary=reasoning_summary, ), ) ## ADD SYSTEM MESSAGE TO MESSAGES diff --git a/litellm/responses/bridge_check.py b/litellm/responses/bridge_check.py index d8a9440bc57..6b80132a077 100644 --- a/litellm/responses/bridge_check.py +++ b/litellm/responses/bridge_check.py @@ -197,6 +197,7 @@ def chat_completion_ends_in_responses_api( *, tools: Sequence[Mapping[str, object]] | None = None, reasoning_effort: str | Mapping[str, object] | None = None, + reasoning_summary: object | None = None, api_base: str | None = None, ) -> bool: """Whether chat completion for this model is served by a Responses API. @@ -219,6 +220,7 @@ def chat_completion_ends_in_responses_api( custom_llm_provider=provider, tools=tools, reasoning_effort=reasoning_effort, + reasoning_summary=reasoning_summary, api_base=api_base, ) return model_info.get("mode") == "responses" diff --git a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index df9d048fb45..cb4cbfc1c57 100644 --- a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -3570,6 +3570,72 @@ def test_transform_response_hands_encrypted_reasoning_to_messages_clients_only() assert content[0]["data"] == encrypted_reasoning_signature(encrypted) +@pytest.mark.parametrize( + ("call_type", "kept_ids"), + [ + pytest.param("anthropic_messages", ["rs_first", "rs_second"], id="messages_client_keeps_every_item"), + pytest.param("acompletion", ["rs_second"], id="chat_client_keeps_the_last_item"), + ], +) +def test_non_streamed_tool_turn_hands_messages_clients_every_reasoning_item( + call_type: str, kept_ids: list[str] +) -> None: + from types import SimpleNamespace + from unittest.mock import Mock + + from litellm.types.llms.openai import ResponsesAPIResponse + from litellm.types.utils import ModelResponse, Usage + + def reasoning(item_id: str) -> ResponseReasoningItem: + return ResponseReasoningItem( + id=item_id, summary=[], type="reasoning", content=None, encrypted_content=f"gAAAA_{item_id}", status=None + ) + + def call(index: int, city: str) -> ResponseFunctionToolCall: + return ResponseFunctionToolCall( + id=f"fc_{index}", + type="function_call", + status="completed", + arguments=json.dumps({"city": city}), + call_id=f"call_{index}", + name="get_weather", + ) + + raw: Final = ResponsesAPIResponse.model_construct( + id="resp_two_reasoning", + created_at=1, + error=None, + incomplete_details=None, + model="gpt-5.4-nano", + output=[reasoning("rs_first"), call(1, "Oslo"), reasoning("rs_second"), call(2, "Lyon")], + status="completed", + usage=None, + ) + result: Final = LiteLLMResponsesTransformationHandler().transform_response( + model="gpt-5.4-nano", + raw_response=raw, + model_response=ModelResponse( + id="chatcmpl-two-reasoning", + created=1, + model=None, + object="chat.completion", + system_fingerprint=None, + choices=[], + usage=Usage(completion_tokens=0, prompt_tokens=0, total_tokens=0), + ), + logging_obj=SimpleNamespace(call_type=call_type), + request_data={"model": "gpt-5.4-nano"}, + messages=[{"role": "user", "content": "Oslo or Lyon?"}], + optional_params={}, + litellm_params={"custom_llm_provider": "azure"}, + encoding=Mock(), + ) + + message: Final = result.choices[0].message + assert [item["id"] for item in message.reasoning_items] == kept_ids + assert [tool_call.function.name for tool_call in message.tool_calls] == ["get_weather", "get_weather"] + + def test_streaming_function_call_tool_id_for_degenerate_call_id(): """In streaming, Bedrock Mantle's function_call event carries a unique ``id`` (``fc_...``) and a non-unique, index-based ``call_id`` (``call_0``). For that diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index 595f9bb9837..1223185a05c 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -698,6 +698,34 @@ def test_encrypted_reasoning_is_kept_when_the_request_tools_route_it_to_response assert bool(assistant.get("thinking_blocks")) is kept +@pytest.mark.parametrize( + ("deployment_params", "kept"), + [ + pytest.param({"reasoning_summary": "auto"}, True, id="deployment_summary_bridges_to_responses"), + pytest.param({"extra_body": {"reasoningSummary": "auto"}}, True, id="extra_body_summary_bridges_to_responses"), + pytest.param({}, False, id="no_summary_stays_on_chat"), + ], +) +def test_encrypted_reasoning_is_kept_when_the_deployment_reasoning_summary_routes_it_to_responses( + deployment_params: dict[str, object], kept: bool +) -> None: + from litellm.llms.anthropic.pass_through.adapters.handler import ( + LiteLLMMessagesToCompletionTransformationHandler, + ) + + request: Final = _replayed_reasoning_request([]) + completion_kwargs, _mapping = LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs( + max_tokens=64, + messages=cast(list[dict[str, object]], request["messages"]), + model="azure/gpt-5.1", + thinking={"type": "enabled", "budget_tokens": 2048}, + extra_kwargs={"api_base": "https://keep-check.openai.azure.com", **deployment_params}, + ) + + assistant: Final = cast(list[Mapping[str, object]], completion_kwargs["messages"])[1] + assert bool(assistant.get("thinking_blocks")) is kept + + def test_translate_anthropic_messages_to_openai_sets_reasoning_content(): """Reasoning-aware chat providers read reasoning_content, so thinking text must land there.