From 37ae1ac75cd3c8a68e8a99fe314224f5d08e1bbe Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Wed, 28 Oct 2026 09:03:47 -0700 Subject: [PATCH 1/2] fix(responses): announce the reasoning summary part so SDK stream readers can place reasoning deltas When /v1/responses streams a chat-completions model that thinks, the bridge announced the reasoning item with a null summary, never sent response.reasoning_summary_part.added, and the proxy dropped the defaulted summary_index from every reasoning_summary_text.delta. The OpenAI JS SDK accumulator indexes output.summary[summary_index] on the first delta and threw. The reasoning item now opens with an empty summary list, a summary part is announced before the first delta, every delta names its summary index explicitly, and the reasoning item in response.completed carries the same summary_text part the stream built. Refs #45552, #40519. Carves the reasoning summary lifecycle out of #36375. --- .../streaming_iterator.py | 30 +++++++- .../transformation.py | 42 ++++++----- litellm/types/responses/main.py | 7 ++ .../test_litellm_completion_responses.py | 11 +-- .../test_streaming_iterator_transformation.py | 69 +++++++++++++++++-- 5 files changed, 132 insertions(+), 27 deletions(-) diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index 219944de521..b241107d43d 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -48,6 +48,7 @@ from litellm.types.llms.openai import ( ResponseIncompleteEvent, ResponseInProgressEvent, ResponseInputParam, + ResponsePartAddedEvent, ResponsesAPIOptionalRequestParams, ResponsesAPIResponse, ResponsesAPIStreamEvents, @@ -842,6 +843,28 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): chunk_dict["usage"] = hidden_params["usage"] return chunk_dict + def create_reasoning_summary_part_added_event(self, reasoning_item_id: str) -> ResponsePartAddedEvent: + """ + Create response.reasoning_summary_part.added event, which opens the summary part + that the following response.reasoning_summary_text.delta events append to. + + Example: + { + "type": "response.reasoning_summary_part.added", + "item_id": "rs_0c5dae30e53172980069708ba2f59c8197b71ca9820edad07c", + "output_index": 0, + "summary_index": 0, + "part": {"type": "summary_text", "text": ""} + } + """ + return ResponsePartAddedEvent( + type=ResponsesAPIStreamEvents.RESPONSE_PART_ADDED, + item_id=reasoning_item_id, + output_index=self._reasoning_index(), + summary_index=0, + part={"type": "summary_text", "text": ""}, + ) + def create_reasoning_summary_text_done_event( self, reasoning_item_id: str, @@ -1129,12 +1152,14 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): "id": self._cached_reasoning_item_id, "type": "reasoning", "status": "in_progress", - "summary": None, + "summary": [], } ), ) event.__dict__["sequence_number"] = self._sequence_number - self._pending_response_events.append(event) + self._pending_response_events.extend( + [event, self.create_reasoning_summary_part_added_event(self._cached_reasoning_item_id)] + ) return # Tool-first @@ -1357,6 +1382,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): type=ResponsesAPIStreamEvents.REASONING_SUMMARY_TEXT_DELTA, item_id=self._cached_reasoning_item_id, output_index=self._reasoning_index(), + summary_index=0, delta=reasoning_content, ) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index bc92bc9cb3a..f1241fd8784 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -113,6 +113,7 @@ from litellm.types.responses.main import ( OutputImageGenerationCall, OutputText, OutputToolSearchCall, + ReasoningSummaryText, ) from litellm.types.utils import ( ChatCompletionAnnotation, @@ -3437,23 +3438,30 @@ class LiteLLMCompletionResponsesConfig: if reasoning_content or encrypted_content: # Only check the first choice for reasoning content return [ - GenericResponseOutputItem( - type="reasoning", - id=mint_reasoning_item_id(), - status=LiteLLMCompletionResponsesConfig._map_chat_completion_finish_reason_to_responses_status( - choice.finish_reason - ), - role="assistant", - content=[ - OutputText( - type="output_text", - text=text, - annotations=[], - ) - for text in (reasoning_content,) - if text - ], - encrypted_content=encrypted_content, + GenericResponseOutputItem.model_validate( + { + "type": "reasoning", + "id": mint_reasoning_item_id(), + "status": LiteLLMCompletionResponsesConfig._map_chat_completion_finish_reason_to_responses_status( + choice.finish_reason + ), + "role": "assistant", + "content": [ + OutputText( + type="output_text", + text=text, + annotations=[], + ) + for text in (reasoning_content,) + if text + ], + "summary": [ + ReasoningSummaryText(type="summary_text", text=text) + for text in (reasoning_content,) + if text + ], + "encrypted_content": encrypted_content, + } ) ] return [] diff --git a/litellm/types/responses/main.py b/litellm/types/responses/main.py index 72dc63a581f..ae4a817ad53 100644 --- a/litellm/types/responses/main.py +++ b/litellm/types/responses/main.py @@ -158,6 +158,13 @@ class GenericResponseOutputItem(BaseLiteLLMOpenAIResponseObject): phase: Phase = None +class ReasoningSummaryText(BaseLiteLLMOpenAIResponseObject): + model_config = ConfigDict(frozen=True) + + type: Literal["summary_text"] + text: str + + class DeleteResponseResult(BaseLiteLLMOpenAIResponseObject): """ Result of a delete response request diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 3fd4be6f722..8a94d475918 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -4834,9 +4834,9 @@ class TestEnsureOutputItemContentPartAdded: assert completed_event.type == ResponsesAPIStreamEvents.RESPONSE_COMPLETED assert completed_event.response.status == "completed" - def test_reasoning_item_does_not_emit_content_part_added(self): - """Reasoning items should not get a content_part.added event.""" - from litellm.types.llms.openai import OutputItemAddedEvent + def test_reasoning_item_opens_a_summary_part_instead_of_a_content_part(self): + """Reasoning items announce their summary part, never a content_part.added event.""" + from litellm.types.llms.openai import OutputItemAddedEvent, ResponsePartAddedEvent iterator = self._make_iterator() chunk = self._make_reasoning_chunk() @@ -4844,8 +4844,11 @@ class TestEnsureOutputItemContentPartAdded: iterator._ensure_output_item_for_chunk(chunk) events = iterator._pending_response_events - assert len(events) == 1 + assert len(events) == 2 assert isinstance(events[0], OutputItemAddedEvent) + assert isinstance(events[1], ResponsePartAddedEvent) + assert events[1].item_id == events[0].item.id + assert events[1].output_index == events[0].output_index assert iterator.sent_content_part_added_event is False def test_only_emits_once(self): diff --git a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py index ed90acbd2fa..8c5b620f51c 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py +++ b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py @@ -11,11 +11,13 @@ spend tracking stores, so a follow-up previous_response_id still finds the conve """ import json -from collections.abc import Callable +from collections.abc import Callable, Mapping, Sequence from typing import Final from unittest.mock import AsyncMock, MagicMock import pytest +from openai.types.responses.response_reasoning_item import Summary as OpenAIReasoningSummaryPart +from pydantic import TypeAdapter from litellm.responses.litellm_completion_transformation.streaming_iterator import ( LiteLLMCompletionStreamingIterator, @@ -1239,6 +1241,67 @@ async def test_reasoning_item_closes_before_message_item_opens(stream_chunks: St assert _event_text(events, ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA) == "Hello!" +def _split_reasoning_then_text_chunks() -> tuple[ModelResponseStream, ...]: + return (_reasoning_chunk("let me "), _reasoning_chunk("think"), _chunk("Hello!", finish_reason="stop")) + + +def _streamed_reasoning_text(chunks: Sequence[ModelResponseStream]) -> str: + return "".join(getattr(chunk.choices[0].delta, "reasoning_content", None) or "" for chunk in chunks) + + +def _proxy_sse_payload(event: BaseLiteLLMOpenAIResponseObject) -> Mapping[str, object]: + """The same dump litellm.proxy.proxy_server._serialize_streaming_chunk puts on the SSE wire.""" + return json.loads(event.model_dump_json(exclude_none=True, exclude_unset=True)) + + +def _reasoning_summary_as_the_openai_sdk_accumulates_it( + payloads: Sequence[Mapping[str, object]], +) -> Sequence[Mapping[str, str]]: + """Replays openai-node ResponseAccumulator's lookups, so a key or index missing on the wire raises as the SDK throws.""" + items: Final = {} + for payload in payloads: + match payload["type"]: + case "response.output_item.added": + items[payload["output_index"]] = payload["item"] + case "response.reasoning_summary_part.added": + items[payload["output_index"]]["summary"].append(payload["part"]) + case "response.reasoning_summary_text.delta": + items[payload["output_index"]]["summary"][payload["summary_index"]]["text"] += payload["delta"] + reasoning_item: Final = next(item for item in items.values() if item["type"] == "reasoning") + return reasoning_item["summary"] + + +def _reasoning_item(items: Sequence[Mapping[str, object]]) -> Mapping[str, object]: + return next(item for item in items if item["type"] == "reasoning") + + +@pytest.mark.parametrize( + "stream_chunks", + [_split_reasoning_then_text_chunks, _reasoning_and_text_in_one_chunk, _signed_thinking_and_text_in_one_chunk], + ids=[ + "split_reasoning_then_text_chunks", + "reasoning_and_text_in_one_chunk", + "signed_thinking_and_text_in_one_chunk", + ], +) +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +async def test_sdk_stream_readers_can_place_every_reasoning_summary_delta(stream_chunks: StreamChunks, sync_mode: bool): + chunks: Final = stream_chunks() + iterator: Final = _build_iterator(chunks) + + payloads: Final = [_proxy_sse_payload(event) for event in await _collect_events(iterator, sync_mode)] + + accumulated: Final = _reasoning_summary_as_the_openai_sdk_accumulates_it(payloads) + done_items: Final = [payload["item"] for payload in payloads if payload["type"] == "response.output_item.done"] + completed_payload: Final = next(payload for payload in payloads if payload["type"] == "response.completed") + completed_summary: Final = _reasoning_item(completed_payload["response"]["output"])["summary"] + assert "".join(part["text"] for part in accumulated) == _streamed_reasoning_text(chunks) + assert _reasoning_item(done_items)["summary"] == accumulated + assert "".join(part["text"] for part in completed_summary) == _streamed_reasoning_text(chunks) + TypeAdapter(list[OpenAIReasoningSummaryPart]).validate_python([*accumulated, *completed_summary]) + + @pytest.mark.parametrize("sync_mode", [True, False]) @pytest.mark.asyncio async def test_tool_then_reasoning_then_text_gives_message_its_own_output_index(sync_mode: bool): @@ -1421,9 +1484,7 @@ async def test_anthropic_tool_search_call_streams_as_tool_search_call_item(sync_ model="claude-sonnet-5-5", object="chat.completion.chunk", choices=[ - StreamingChoices( - index=0, delta=delta, finish_reason="tool_calls" if index == len(deltas) - 1 else None - ) + StreamingChoices(index=0, delta=delta, finish_reason="tool_calls" if index == len(deltas) - 1 else None) ], ) for index, delta in enumerate(deltas) From 964f8aa309a3dcd26a8e8e554577010294897846 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Wed, 28 Oct 2026 10:33:58 -0700 Subject: [PATCH 2/2] fix(responses): open the reasoning summary part only once reasoning text streams A reasoning item with no summary text, such as signature-only thinking, streamed one empty summary part and closed with it while response.completed carried an empty summary list. The part is now announced right before the first reasoning delta, the done events are skipped when there is no text, and the item closes with the same empty summary the final response carries, matching OpenAI --- .../streaming_iterator.py | 45 ++++++++++--------- .../test_litellm_completion_responses.py | 11 ++--- .../test_streaming_iterator_transformation.py | 14 +++++- 3 files changed, 39 insertions(+), 31 deletions(-) diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index b241107d43d..83e7bfc5e60 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -177,6 +177,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self._cached_reasoning_item_id: str | None = None self._sent_reasoning_summary_text_done_event: bool = False self._sent_reasoning_summary_part_done_event: bool = False + self._sent_reasoning_summary_part_added_event: bool = False self._reasoning_summary_text: str = "" # -- GENERIC RESPONSE-EVENTS PENDING QUEUE as required by fix -- self._pending_response_events: list[BaseLiteLLMOpenAIResponseObject] = [] @@ -1035,12 +1036,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): **{ "id": reasoning_item_id, "type": "reasoning", - "summary": [ - { - "type": "summary_text", - "text": reasoning_content, - } - ], + "summary": [{"type": "summary_text", "text": reasoning_content}] if reasoning_content else [], } ), ) @@ -1157,9 +1153,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): ), ) event.__dict__["sequence_number"] = self._sequence_number - self._pending_response_events.extend( - [event, self.create_reasoning_summary_part_added_event(self._cached_reasoning_item_id)] - ) + self._pending_response_events.append(event) return # Tool-first @@ -1186,25 +1180,27 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): ) reasoning_item_id: Final = self._cached_reasoning_item_id self._queue_compaction_item_done() - self._sequence_number += 1 - text_done_event: Final = self.create_reasoning_summary_text_done_event( - reasoning_item_id=reasoning_item_id, - reasoning_content=reasoning_content, - sequence_number=self._sequence_number, - ) - self._sequence_number += 1 - part_done_event: Final = self.create_reasoning_summary_part_done_event( - reasoning_item_id=reasoning_item_id, - reasoning_content=reasoning_content, - sequence_number=self._sequence_number, - ) + if reasoning_content: + self._sequence_number += 1 + text_done_event: Final = self.create_reasoning_summary_text_done_event( + reasoning_item_id=reasoning_item_id, + reasoning_content=reasoning_content, + sequence_number=self._sequence_number, + ) + self._sequence_number += 1 + part_done_event: Final = self.create_reasoning_summary_part_done_event( + reasoning_item_id=reasoning_item_id, + reasoning_content=reasoning_content, + sequence_number=self._sequence_number, + ) + self._pending_response_events.extend([text_done_event, part_done_event]) self._sequence_number += 1 reasoning_output_item_done_event: Final = self.create_reasoning_output_item_done_event( reasoning_item_id=reasoning_item_id, reasoning_content=reasoning_content, sequence_number=self._sequence_number, ) - self._pending_response_events.extend([text_done_event, part_done_event, reasoning_output_item_done_event]) + self._pending_response_events.append(reasoning_output_item_done_event) self._reasoning_done_emitted = True self._reasoning_active = False @@ -1377,6 +1373,11 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): if self._cached_reasoning_item_id is None: self._cached_reasoning_item_id = mint_reasoning_item_id() + if not self._sent_reasoning_summary_part_added_event: + self._sent_reasoning_summary_part_added_event = True + self._pending_response_events.append( + self.create_reasoning_summary_part_added_event(self._cached_reasoning_item_id) + ) return ReasoningSummaryTextDeltaEvent( type=ResponsesAPIStreamEvents.REASONING_SUMMARY_TEXT_DELTA, diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 8a94d475918..3fd4be6f722 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -4834,9 +4834,9 @@ class TestEnsureOutputItemContentPartAdded: assert completed_event.type == ResponsesAPIStreamEvents.RESPONSE_COMPLETED assert completed_event.response.status == "completed" - def test_reasoning_item_opens_a_summary_part_instead_of_a_content_part(self): - """Reasoning items announce their summary part, never a content_part.added event.""" - from litellm.types.llms.openai import OutputItemAddedEvent, ResponsePartAddedEvent + def test_reasoning_item_does_not_emit_content_part_added(self): + """Reasoning items should not get a content_part.added event.""" + from litellm.types.llms.openai import OutputItemAddedEvent iterator = self._make_iterator() chunk = self._make_reasoning_chunk() @@ -4844,11 +4844,8 @@ class TestEnsureOutputItemContentPartAdded: iterator._ensure_output_item_for_chunk(chunk) events = iterator._pending_response_events - assert len(events) == 2 + assert len(events) == 1 assert isinstance(events[0], OutputItemAddedEvent) - assert isinstance(events[1], ResponsePartAddedEvent) - assert events[1].item_id == events[0].item.id - assert events[1].output_index == events[0].output_index assert iterator.sent_content_part_added_event is False def test_only_emits_once(self): diff --git a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py index 8c5b620f51c..65bcafef683 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py +++ b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py @@ -1245,6 +1245,10 @@ def _split_reasoning_then_text_chunks() -> tuple[ModelResponseStream, ...]: return (_reasoning_chunk("let me "), _reasoning_chunk("think"), _chunk("Hello!", finish_reason="stop")) +def _signature_only_thinking_then_text_chunks() -> tuple[ModelResponseStream, ...]: + return (_signature_only_thinking_chunk("sig_only"), _chunk("4", finish_reason="stop")) + + def _streamed_reasoning_text(chunks: Sequence[ModelResponseStream]) -> str: return "".join(getattr(chunk.choices[0].delta, "reasoning_content", None) or "" for chunk in chunks) @@ -1277,11 +1281,17 @@ def _reasoning_item(items: Sequence[Mapping[str, object]]) -> Mapping[str, objec @pytest.mark.parametrize( "stream_chunks", - [_split_reasoning_then_text_chunks, _reasoning_and_text_in_one_chunk, _signed_thinking_and_text_in_one_chunk], + [ + _split_reasoning_then_text_chunks, + _reasoning_and_text_in_one_chunk, + _signed_thinking_and_text_in_one_chunk, + _signature_only_thinking_then_text_chunks, + ], ids=[ "split_reasoning_then_text_chunks", "reasoning_and_text_in_one_chunk", "signed_thinking_and_text_in_one_chunk", + "signature_only_thinking_then_text_chunks", ], ) @pytest.mark.parametrize("sync_mode", [True, False]) @@ -1298,7 +1308,7 @@ async def test_sdk_stream_readers_can_place_every_reasoning_summary_delta(stream completed_summary: Final = _reasoning_item(completed_payload["response"]["output"])["summary"] assert "".join(part["text"] for part in accumulated) == _streamed_reasoning_text(chunks) assert _reasoning_item(done_items)["summary"] == accumulated - assert "".join(part["text"] for part in completed_summary) == _streamed_reasoning_text(chunks) + assert completed_summary == accumulated TypeAdapter(list[OpenAIReasoningSummaryPart]).validate_python([*accumulated, *completed_summary])