fix(anthropic): keep every non-streamed reasoning item for Messages clients and read a deployment's reasoning_summary in the keep check

Backport of #49076 to lts/1.108.x.
Ported from cadfe1d179 (cursor/chatgpt-encrypted-reasoning-messages-1723) onto the line's pass_through/adapters layout.
This commit is contained in:
mateo-berri 2026-11-07 04:08:08 -08:00
parent d6625245b4
commit ce35f27bfa
7 changed files with 144 additions and 24 deletions

View file

@ -13,9 +13,7 @@ from typing import (
from typing_extensions import TypedDict
from litellm.litellm_core_utils.core_helpers import attribute_or_none
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.utils import CallTypes
if TYPE_CHECKING:
from litellm import CustomStreamWrapper, LiteLLMLoggingObj, ModelResponse
@ -38,10 +36,6 @@ def _restore_routing_prefix(model: str, custom_llm_provider: str) -> str:
return f"{custom_llm_provider}/{model}"
def _serves_anthropic_messages(logging_obj: "LiteLLMLoggingObj") -> bool:
return attribute_or_none(logging_obj, "call_type") == CallTypes.anthropic_messages.value
class ResponsesToCompletionBridgeHandler:
def __init__(self):
from .transformation import LiteLLMResponsesTransformationHandler
@ -188,7 +182,9 @@ class ResponsesToCompletionBridgeHandler:
model_response: Final = validated_kwargs["model_response"]
logging_obj: Final = validated_kwargs["logging_obj"]
custom_llm_provider: Final = validated_kwargs["custom_llm_provider"]
serves_anthropic_messages: Final = _serves_anthropic_messages(logging_obj)
from .transformation import serves_anthropic_messages
streams_reasoning_signatures: Final = serves_anthropic_messages(logging_obj)
if kwargs.get("stream") is True and "stream" not in optional_params:
optional_params = {**optional_params, "stream": True}
@ -267,7 +263,7 @@ class ResponsesToCompletionBridgeHandler:
structured_output=is_structured_output_request(
cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict
),
streams_reasoning_signatures=serves_anthropic_messages,
streams_reasoning_signatures=streams_reasoning_signatures,
)
streamwrapper: Final = CustomStreamWrapper(
completion_stream=completion_stream,
@ -290,7 +286,9 @@ class ResponsesToCompletionBridgeHandler:
model_response: Final = validated_kwargs["model_response"]
logging_obj: Final = validated_kwargs["logging_obj"]
custom_llm_provider: Final = validated_kwargs["custom_llm_provider"]
serves_anthropic_messages: Final = _serves_anthropic_messages(logging_obj)
from .transformation import serves_anthropic_messages
streams_reasoning_signatures: Final = serves_anthropic_messages(logging_obj)
if kwargs.get("stream") is True and "stream" not in optional_params:
optional_params = {**optional_params, "stream": True}
@ -372,7 +370,7 @@ class ResponsesToCompletionBridgeHandler:
structured_output=is_structured_output_request(
cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict
),
streams_reasoning_signatures=serves_anthropic_messages,
streams_reasoning_signatures=streams_reasoning_signatures,
)
streamwrapper: Final = CustomStreamWrapper(
completion_stream=completion_stream,

View file

@ -26,7 +26,7 @@ import litellm
from litellm import ModelResponse
from litellm._logging import verbose_logger
from litellm.integrations.anthropic_cache_control_hook import supports_openai_prompt_cache_breakpoint
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none
from litellm.litellm_core_utils.hidden_params import get_hidden_params, get_or_create_hidden_params
from litellm.litellm_core_utils.prompt_templates.common_utils import (
responses_reasoning_items_from_thinking_blocks,
@ -58,7 +58,7 @@ from litellm.types.llms.openai import (
ResponsesAPIResponse,
ResponsesAPIStreamEvents,
)
from litellm.types.utils import Delta, GenericStreamingChunk, ModelResponseStream, StreamingChoices
from litellm.types.utils import CallTypes, Delta, GenericStreamingChunk, ModelResponseStream, StreamingChoices
if TYPE_CHECKING:
from openai.types.responses import ResponseInputImageParam, ResponseOutputItem
@ -227,6 +227,10 @@ class _BuiltReasoningItem(TypedDict):
summary: Sequence[_ReasoningSummaryText]
def serves_anthropic_messages(logging_obj: object) -> bool:
return attribute_or_none(logging_obj, "call_type") == CallTypes.anthropic_messages.value
def _get_reasoning_items(
msg: "AllMessageValues",
) -> list[ChatCompletionReasoningItem]:
@ -549,7 +553,7 @@ def _fold_choices_into_one(
choices: Sequence["Choices"],
tool_calls: Sequence[Mapping[str, object]],
trailing_reasoning_content: str | None,
trailing_reasoning_item: _BuiltReasoningItem | None,
trailing_reasoning_items: Sequence[_BuiltReasoningItem],
tool_search_items: Sequence[Mapping[str, object]],
structured_output: bool = False,
) -> "Choices":
@ -568,7 +572,7 @@ def _fold_choices_into_one(
reasoning_contents: Final = (*(_message_reasoning_content(m) for m in messages), trailing_reasoning_content)
reasoning_items: Final = (
*chain.from_iterable(_message_reasoning_items(m) for m in messages),
*(() if trailing_reasoning_item is None else (trailing_reasoning_item,)),
*trailing_reasoning_items,
)
return Choices(
message=Message(
@ -1222,6 +1226,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
output_items: Sequence[object],
handle_raw_dict_callback: Callable[..., tuple["Choices | None", int]] | None = None,
structured_output: bool = False,
keeps_every_reasoning_item: bool = False,
) -> list["Choices"]:
"""
Convert Responses API output items to chat completion choices.
@ -1231,6 +1236,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
handle_raw_dict_callback: Optional callback for handling raw dict items
structured_output: The chat request carried a json response_format, so only the
first message item becomes content
keeps_every_reasoning_item: Keep each reasoning item before an answer, not only the last
Returns:
List of Choices objects
@ -1253,7 +1259,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
choices: Final[list[Choices]] = []
index = 0
reasoning_content: str | None = None
pending_reasoning_item: _BuiltReasoningItem | None = None
pending_reasoning_items: tuple[_BuiltReasoningItem, ...] = ()
# Collect all tool calls to put them in a single choice
# (Chat Completions API expects all tool calls in one message)
@ -1262,12 +1268,19 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
for item in output_items:
if isinstance(item, ResponseReasoningItem):
pending_reasoning_item = _build_reasoning_item(
item_id=item.id,
encrypted_content=getattr(item, "encrypted_content", None),
summary_raw=item.summary,
pending_reasoning_items = (
*(pending_reasoning_items if keeps_every_reasoning_item else ()),
_build_reasoning_item(
item_id=item.id,
encrypted_content=getattr(item, "encrypted_content", None),
summary_raw=item.summary,
),
)
reasoning_content = " ".join(
s["text"]
for s in chain.from_iterable(built["summary"] for built in pending_reasoning_items)
if s.get("text")
)
reasoning_content = " ".join(s["text"] for s in pending_reasoning_item["summary"] if s.get("text"))
elif isinstance(item, ResponseOutputMessage):
for content in item.content:
@ -1284,7 +1297,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
annotations=annotations,
reasoning_items=cast(
list[ChatCompletionReasoningItem] | None,
([pending_reasoning_item] if pending_reasoning_item is not None else None),
(list(pending_reasoning_items) or None),
),
)
@ -1297,7 +1310,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
)
reasoning_content = None # flush
pending_reasoning_item = None # flush
pending_reasoning_items = () # flush
index += 1
elif isinstance(item, ResponseFunctionToolCall):
@ -1362,7 +1375,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
choices=choices,
tool_calls=accumulated_tool_calls,
trailing_reasoning_content=reasoning_content,
trailing_reasoning_item=pending_reasoning_item,
trailing_reasoning_items=pending_reasoning_items,
tool_search_items=tool_search_items,
structured_output=structured_output,
)
@ -1500,6 +1513,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
structured_output=is_structured_output_request(
cast("Mapping[str, object]", optional_params) # cast-ok: optional_params is an untyped dict
),
keeps_every_reasoning_item=serves_anthropic_messages(logging_obj),
)
response_is_incomplete: Final = raw_response.status == "incomplete" or (

View file

@ -17,6 +17,7 @@ from litellm._logging import verbose_logger
from litellm.litellm_core_utils.asyncify import run_async_function
from litellm.litellm_core_utils.core_helpers import as_str_mapping, iter_items
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
from litellm.litellm_core_utils.model_param_helper import peek_reasoning_summary_aliases
from litellm.litellm_core_utils.prompt_templates.common_utils import (
is_encrypted_reasoning_block,
thinking_blocks_with_encrypted_reasoning,
@ -760,6 +761,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
request_data,
custom_llm_provider=target_provider,
api_base=api_base if isinstance(api_base, str) else None,
reasoning_summary=peek_reasoning_summary_aliases(provider_kwargs),
)
if openai_request is None:

View file

@ -228,6 +228,7 @@ class ChatRouteInputs:
reasoning_effort: str | Mapping[str, object] | None = None
api_base: str | None = None
thinking: Mapping[str, object] | None = None
reasoning_summary: object | None = None
_NO_ROUTE_INPUTS: Final = ChatRouteInputs()
@ -271,6 +272,7 @@ def _keeps_encrypted_reasoning_for_responses(
custom_llm_provider,
tools=route.tools,
reasoning_effort=route.reasoning_effort,
reasoning_summary=route.reasoning_summary,
api_base=route.api_base,
)
@ -313,7 +315,12 @@ class AnthropicAdapter:
return result
def translate_completion_input_params_with_tool_mapping(
self, kwargs, *, custom_llm_provider: str | None = None, api_base: str | None = None
self,
kwargs,
*,
custom_llm_provider: str | None = None,
api_base: str | None = None,
reasoning_summary: object | None = None,
) -> tuple[ChatCompletionRequest | None, dict[str, str]]:
"""
Translate Anthropic request params to OpenAI format, returning tool name mapping.
@ -348,6 +355,7 @@ class AnthropicAdapter:
anthropic_message_request=request_body,
custom_llm_provider=custom_llm_provider,
api_base=api_base,
reasoning_summary=reasoning_summary,
)
return translated_body, tool_name_mapping
@ -1295,6 +1303,7 @@ class LiteLLMAnthropicMessagesAdapter:
*,
custom_llm_provider: str | None = None,
api_base: str | None = None,
reasoning_summary: object | None = None,
preserve_midturn_system: bool = False,
) -> tuple[ChatCompletionRequest, dict[str, str]]:
"""
@ -1349,6 +1358,7 @@ class LiteLLMAnthropicMessagesAdapter:
reasoning_effort=new_kwargs.get("reasoning_effort"),
api_base=api_base,
thinking=anthropic_message_request.get("thinking"),
reasoning_summary=reasoning_summary,
),
)
## ADD SYSTEM MESSAGE TO MESSAGES

View file

@ -197,6 +197,7 @@ def chat_completion_ends_in_responses_api(
*,
tools: Sequence[Mapping[str, object]] | None = None,
reasoning_effort: str | Mapping[str, object] | None = None,
reasoning_summary: object | None = None,
api_base: str | None = None,
) -> bool:
"""Whether chat completion for this model is served by a Responses API.
@ -219,6 +220,7 @@ def chat_completion_ends_in_responses_api(
custom_llm_provider=provider,
tools=tools,
reasoning_effort=reasoning_effort,
reasoning_summary=reasoning_summary,
api_base=api_base,
)
return model_info.get("mode") == "responses"

View file

@ -3570,6 +3570,72 @@ def test_transform_response_hands_encrypted_reasoning_to_messages_clients_only()
assert content[0]["data"] == encrypted_reasoning_signature(encrypted)
@pytest.mark.parametrize(
("call_type", "kept_ids"),
[
pytest.param("anthropic_messages", ["rs_first", "rs_second"], id="messages_client_keeps_every_item"),
pytest.param("acompletion", ["rs_second"], id="chat_client_keeps_the_last_item"),
],
)
def test_non_streamed_tool_turn_hands_messages_clients_every_reasoning_item(
call_type: str, kept_ids: list[str]
) -> None:
from types import SimpleNamespace
from unittest.mock import Mock
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.utils import ModelResponse, Usage
def reasoning(item_id: str) -> ResponseReasoningItem:
return ResponseReasoningItem(
id=item_id, summary=[], type="reasoning", content=None, encrypted_content=f"gAAAA_{item_id}", status=None
)
def call(index: int, city: str) -> ResponseFunctionToolCall:
return ResponseFunctionToolCall(
id=f"fc_{index}",
type="function_call",
status="completed",
arguments=json.dumps({"city": city}),
call_id=f"call_{index}",
name="get_weather",
)
raw: Final = ResponsesAPIResponse.model_construct(
id="resp_two_reasoning",
created_at=1,
error=None,
incomplete_details=None,
model="gpt-5.4-nano",
output=[reasoning("rs_first"), call(1, "Oslo"), reasoning("rs_second"), call(2, "Lyon")],
status="completed",
usage=None,
)
result: Final = LiteLLMResponsesTransformationHandler().transform_response(
model="gpt-5.4-nano",
raw_response=raw,
model_response=ModelResponse(
id="chatcmpl-two-reasoning",
created=1,
model=None,
object="chat.completion",
system_fingerprint=None,
choices=[],
usage=Usage(completion_tokens=0, prompt_tokens=0, total_tokens=0),
),
logging_obj=SimpleNamespace(call_type=call_type),
request_data={"model": "gpt-5.4-nano"},
messages=[{"role": "user", "content": "Oslo or Lyon?"}],
optional_params={},
litellm_params={"custom_llm_provider": "azure"},
encoding=Mock(),
)
message: Final = result.choices[0].message
assert [item["id"] for item in message.reasoning_items] == kept_ids
assert [tool_call.function.name for tool_call in message.tool_calls] == ["get_weather", "get_weather"]
def test_streaming_function_call_tool_id_for_degenerate_call_id():
"""In streaming, Bedrock Mantle's function_call event carries a unique ``id``
(``fc_...``) and a non-unique, index-based ``call_id`` (``call_0``). For that

View file

@ -698,6 +698,34 @@ def test_encrypted_reasoning_is_kept_when_the_request_tools_route_it_to_response
assert bool(assistant.get("thinking_blocks")) is kept
@pytest.mark.parametrize(
("deployment_params", "kept"),
[
pytest.param({"reasoning_summary": "auto"}, True, id="deployment_summary_bridges_to_responses"),
pytest.param({"extra_body": {"reasoningSummary": "auto"}}, True, id="extra_body_summary_bridges_to_responses"),
pytest.param({}, False, id="no_summary_stays_on_chat"),
],
)
def test_encrypted_reasoning_is_kept_when_the_deployment_reasoning_summary_routes_it_to_responses(
deployment_params: dict[str, object], kept: bool
) -> None:
from litellm.llms.anthropic.pass_through.adapters.handler import (
LiteLLMMessagesToCompletionTransformationHandler,
)
request: Final = _replayed_reasoning_request([])
completion_kwargs, _mapping = LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs(
max_tokens=64,
messages=cast(list[dict[str, object]], request["messages"]),
model="azure/gpt-5.1",
thinking={"type": "enabled", "budget_tokens": 2048},
extra_kwargs={"api_base": "https://keep-check.openai.azure.com", **deployment_params},
)
assistant: Final = cast(list[Mapping[str, object]], completion_kwargs["messages"])[1]
assert bool(assistant.get("thinking_blocks")) is kept
def test_translate_anthropic_messages_to_openai_sets_reasoning_content():
"""Reasoning-aware chat providers read reasoning_content, so thinking text must land there.