From 37ae1ac75cd3c8a68e8a99fe314224f5d08e1bbe Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Wed, 28 Oct 2026 09:03:47 -0700 Subject: [PATCH] fix(responses): announce the reasoning summary part so SDK stream readers can place reasoning deltas When /v1/responses streams a chat-completions model that thinks, the bridge announced the reasoning item with a null summary, never sent response.reasoning_summary_part.added, and the proxy dropped the defaulted summary_index from every reasoning_summary_text.delta. The OpenAI JS SDK accumulator indexes output.summary[summary_index] on the first delta and threw. The reasoning item now opens with an empty summary list, a summary part is announced before the first delta, every delta names its summary index explicitly, and the reasoning item in response.completed carries the same summary_text part the stream built. Refs #45552, #40519. Carves the reasoning summary lifecycle out of #36375. --- .../streaming_iterator.py | 30 +++++++- .../transformation.py | 42 ++++++----- litellm/types/responses/main.py | 7 ++ .../test_litellm_completion_responses.py | 11 +-- .../test_streaming_iterator_transformation.py | 69 +++++++++++++++++-- 5 files changed, 132 insertions(+), 27 deletions(-) diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index 219944de521..b241107d43d 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -48,6 +48,7 @@ from litellm.types.llms.openai import ( ResponseIncompleteEvent, ResponseInProgressEvent, ResponseInputParam, + ResponsePartAddedEvent, ResponsesAPIOptionalRequestParams, ResponsesAPIResponse, ResponsesAPIStreamEvents, @@ -842,6 +843,28 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): chunk_dict["usage"] = hidden_params["usage"] return chunk_dict + def create_reasoning_summary_part_added_event(self, reasoning_item_id: str) -> ResponsePartAddedEvent: + """ + Create response.reasoning_summary_part.added event, which opens the summary part + that the following response.reasoning_summary_text.delta events append to. + + Example: + { + "type": "response.reasoning_summary_part.added", + "item_id": "rs_0c5dae30e53172980069708ba2f59c8197b71ca9820edad07c", + "output_index": 0, + "summary_index": 0, + "part": {"type": "summary_text", "text": ""} + } + """ + return ResponsePartAddedEvent( + type=ResponsesAPIStreamEvents.RESPONSE_PART_ADDED, + item_id=reasoning_item_id, + output_index=self._reasoning_index(), + summary_index=0, + part={"type": "summary_text", "text": ""}, + ) + def create_reasoning_summary_text_done_event( self, reasoning_item_id: str, @@ -1129,12 +1152,14 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): "id": self._cached_reasoning_item_id, "type": "reasoning", "status": "in_progress", - "summary": None, + "summary": [], } ), ) event.__dict__["sequence_number"] = self._sequence_number - self._pending_response_events.append(event) + self._pending_response_events.extend( + [event, self.create_reasoning_summary_part_added_event(self._cached_reasoning_item_id)] + ) return # Tool-first @@ -1357,6 +1382,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): type=ResponsesAPIStreamEvents.REASONING_SUMMARY_TEXT_DELTA, item_id=self._cached_reasoning_item_id, output_index=self._reasoning_index(), + summary_index=0, delta=reasoning_content, ) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index bc92bc9cb3a..f1241fd8784 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -113,6 +113,7 @@ from litellm.types.responses.main import ( OutputImageGenerationCall, OutputText, OutputToolSearchCall, + ReasoningSummaryText, ) from litellm.types.utils import ( ChatCompletionAnnotation, @@ -3437,23 +3438,30 @@ class LiteLLMCompletionResponsesConfig: if reasoning_content or encrypted_content: # Only check the first choice for reasoning content return [ - GenericResponseOutputItem( - type="reasoning", - id=mint_reasoning_item_id(), - status=LiteLLMCompletionResponsesConfig._map_chat_completion_finish_reason_to_responses_status( - choice.finish_reason - ), - role="assistant", - content=[ - OutputText( - type="output_text", - text=text, - annotations=[], - ) - for text in (reasoning_content,) - if text - ], - encrypted_content=encrypted_content, + GenericResponseOutputItem.model_validate( + { + "type": "reasoning", + "id": mint_reasoning_item_id(), + "status": LiteLLMCompletionResponsesConfig._map_chat_completion_finish_reason_to_responses_status( + choice.finish_reason + ), + "role": "assistant", + "content": [ + OutputText( + type="output_text", + text=text, + annotations=[], + ) + for text in (reasoning_content,) + if text + ], + "summary": [ + ReasoningSummaryText(type="summary_text", text=text) + for text in (reasoning_content,) + if text + ], + "encrypted_content": encrypted_content, + } ) ] return [] diff --git a/litellm/types/responses/main.py b/litellm/types/responses/main.py index 72dc63a581f..ae4a817ad53 100644 --- a/litellm/types/responses/main.py +++ b/litellm/types/responses/main.py @@ -158,6 +158,13 @@ class GenericResponseOutputItem(BaseLiteLLMOpenAIResponseObject): phase: Phase = None +class ReasoningSummaryText(BaseLiteLLMOpenAIResponseObject): + model_config = ConfigDict(frozen=True) + + type: Literal["summary_text"] + text: str + + class DeleteResponseResult(BaseLiteLLMOpenAIResponseObject): """ Result of a delete response request diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 3fd4be6f722..8a94d475918 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -4834,9 +4834,9 @@ class TestEnsureOutputItemContentPartAdded: assert completed_event.type == ResponsesAPIStreamEvents.RESPONSE_COMPLETED assert completed_event.response.status == "completed" - def test_reasoning_item_does_not_emit_content_part_added(self): - """Reasoning items should not get a content_part.added event.""" - from litellm.types.llms.openai import OutputItemAddedEvent + def test_reasoning_item_opens_a_summary_part_instead_of_a_content_part(self): + """Reasoning items announce their summary part, never a content_part.added event.""" + from litellm.types.llms.openai import OutputItemAddedEvent, ResponsePartAddedEvent iterator = self._make_iterator() chunk = self._make_reasoning_chunk() @@ -4844,8 +4844,11 @@ class TestEnsureOutputItemContentPartAdded: iterator._ensure_output_item_for_chunk(chunk) events = iterator._pending_response_events - assert len(events) == 1 + assert len(events) == 2 assert isinstance(events[0], OutputItemAddedEvent) + assert isinstance(events[1], ResponsePartAddedEvent) + assert events[1].item_id == events[0].item.id + assert events[1].output_index == events[0].output_index assert iterator.sent_content_part_added_event is False def test_only_emits_once(self): diff --git a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py index ed90acbd2fa..8c5b620f51c 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py +++ b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py @@ -11,11 +11,13 @@ spend tracking stores, so a follow-up previous_response_id still finds the conve """ import json -from collections.abc import Callable +from collections.abc import Callable, Mapping, Sequence from typing import Final from unittest.mock import AsyncMock, MagicMock import pytest +from openai.types.responses.response_reasoning_item import Summary as OpenAIReasoningSummaryPart +from pydantic import TypeAdapter from litellm.responses.litellm_completion_transformation.streaming_iterator import ( LiteLLMCompletionStreamingIterator, @@ -1239,6 +1241,67 @@ async def test_reasoning_item_closes_before_message_item_opens(stream_chunks: St assert _event_text(events, ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA) == "Hello!" +def _split_reasoning_then_text_chunks() -> tuple[ModelResponseStream, ...]: + return (_reasoning_chunk("let me "), _reasoning_chunk("think"), _chunk("Hello!", finish_reason="stop")) + + +def _streamed_reasoning_text(chunks: Sequence[ModelResponseStream]) -> str: + return "".join(getattr(chunk.choices[0].delta, "reasoning_content", None) or "" for chunk in chunks) + + +def _proxy_sse_payload(event: BaseLiteLLMOpenAIResponseObject) -> Mapping[str, object]: + """The same dump litellm.proxy.proxy_server._serialize_streaming_chunk puts on the SSE wire.""" + return json.loads(event.model_dump_json(exclude_none=True, exclude_unset=True)) + + +def _reasoning_summary_as_the_openai_sdk_accumulates_it( + payloads: Sequence[Mapping[str, object]], +) -> Sequence[Mapping[str, str]]: + """Replays openai-node ResponseAccumulator's lookups, so a key or index missing on the wire raises as the SDK throws.""" + items: Final = {} + for payload in payloads: + match payload["type"]: + case "response.output_item.added": + items[payload["output_index"]] = payload["item"] + case "response.reasoning_summary_part.added": + items[payload["output_index"]]["summary"].append(payload["part"]) + case "response.reasoning_summary_text.delta": + items[payload["output_index"]]["summary"][payload["summary_index"]]["text"] += payload["delta"] + reasoning_item: Final = next(item for item in items.values() if item["type"] == "reasoning") + return reasoning_item["summary"] + + +def _reasoning_item(items: Sequence[Mapping[str, object]]) -> Mapping[str, object]: + return next(item for item in items if item["type"] == "reasoning") + + +@pytest.mark.parametrize( + "stream_chunks", + [_split_reasoning_then_text_chunks, _reasoning_and_text_in_one_chunk, _signed_thinking_and_text_in_one_chunk], + ids=[ + "split_reasoning_then_text_chunks", + "reasoning_and_text_in_one_chunk", + "signed_thinking_and_text_in_one_chunk", + ], +) +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +async def test_sdk_stream_readers_can_place_every_reasoning_summary_delta(stream_chunks: StreamChunks, sync_mode: bool): + chunks: Final = stream_chunks() + iterator: Final = _build_iterator(chunks) + + payloads: Final = [_proxy_sse_payload(event) for event in await _collect_events(iterator, sync_mode)] + + accumulated: Final = _reasoning_summary_as_the_openai_sdk_accumulates_it(payloads) + done_items: Final = [payload["item"] for payload in payloads if payload["type"] == "response.output_item.done"] + completed_payload: Final = next(payload for payload in payloads if payload["type"] == "response.completed") + completed_summary: Final = _reasoning_item(completed_payload["response"]["output"])["summary"] + assert "".join(part["text"] for part in accumulated) == _streamed_reasoning_text(chunks) + assert _reasoning_item(done_items)["summary"] == accumulated + assert "".join(part["text"] for part in completed_summary) == _streamed_reasoning_text(chunks) + TypeAdapter(list[OpenAIReasoningSummaryPart]).validate_python([*accumulated, *completed_summary]) + + @pytest.mark.parametrize("sync_mode", [True, False]) @pytest.mark.asyncio async def test_tool_then_reasoning_then_text_gives_message_its_own_output_index(sync_mode: bool): @@ -1421,9 +1484,7 @@ async def test_anthropic_tool_search_call_streams_as_tool_search_call_item(sync_ model="claude-sonnet-5-5", object="chat.completion.chunk", choices=[ - StreamingChoices( - index=0, delta=delta, finish_reason="tool_calls" if index == len(deltas) - 1 else None - ) + StreamingChoices(index=0, delta=delta, finish_reason="tool_calls" if index == len(deltas) - 1 else None) ], ) for index, delta in enumerate(deltas)