diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index 982158040c4..a5705194bbb 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -48,6 +48,7 @@ from litellm.types.llms.openai import ( ResponseIncompleteEvent, ResponseInProgressEvent, ResponseInputParam, + ResponsePartAddedEvent, ResponsesAPIOptionalRequestParams, ResponsesAPIResponse, ResponsesAPIStreamEvents, @@ -177,6 +178,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self._cached_reasoning_item_id: str | None = None self._sent_reasoning_summary_text_done_event: bool = False self._sent_reasoning_summary_part_done_event: bool = False + self._sent_reasoning_summary_part_added_event: bool = False self._reasoning_summary_text: str = "" # -- GENERIC RESPONSE-EVENTS PENDING QUEUE as required by fix -- self._pending_response_events: list[BaseLiteLLMOpenAIResponseObject] = [] @@ -852,6 +854,28 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): chunk_dict["usage"] = hidden_params["usage"] return chunk_dict + def create_reasoning_summary_part_added_event(self, reasoning_item_id: str) -> ResponsePartAddedEvent: + """ + Create response.reasoning_summary_part.added event, which opens the summary part + that the following response.reasoning_summary_text.delta events append to. + + Example: + { + "type": "response.reasoning_summary_part.added", + "item_id": "rs_0c5dae30e53172980069708ba2f59c8197b71ca9820edad07c", + "output_index": 0, + "summary_index": 0, + "part": {"type": "summary_text", "text": ""} + } + """ + return ResponsePartAddedEvent( + type=ResponsesAPIStreamEvents.RESPONSE_PART_ADDED, + item_id=reasoning_item_id, + output_index=self._reasoning_index(), + summary_index=0, + part={"type": "summary_text", "text": ""}, + ) + def create_reasoning_summary_text_done_event( self, reasoning_item_id: str, @@ -1022,12 +1046,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): **{ "id": reasoning_item_id, "type": "reasoning", - "summary": [ - { - "type": "summary_text", - "text": reasoning_content, - } - ], + "summary": [{"type": "summary_text", "text": reasoning_content}] if reasoning_content else [], } ), ) @@ -1139,7 +1158,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): "id": self._cached_reasoning_item_id, "type": "reasoning", "status": "in_progress", - "summary": None, + "summary": [], } ), ) @@ -1171,25 +1190,27 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): ) reasoning_item_id: Final = self._cached_reasoning_item_id self._queue_compaction_item_done() - self._sequence_number += 1 - text_done_event: Final = self.create_reasoning_summary_text_done_event( - reasoning_item_id=reasoning_item_id, - reasoning_content=reasoning_content, - sequence_number=self._sequence_number, - ) - self._sequence_number += 1 - part_done_event: Final = self.create_reasoning_summary_part_done_event( - reasoning_item_id=reasoning_item_id, - reasoning_content=reasoning_content, - sequence_number=self._sequence_number, - ) + if reasoning_content: + self._sequence_number += 1 + text_done_event: Final = self.create_reasoning_summary_text_done_event( + reasoning_item_id=reasoning_item_id, + reasoning_content=reasoning_content, + sequence_number=self._sequence_number, + ) + self._sequence_number += 1 + part_done_event: Final = self.create_reasoning_summary_part_done_event( + reasoning_item_id=reasoning_item_id, + reasoning_content=reasoning_content, + sequence_number=self._sequence_number, + ) + self._pending_response_events.extend([text_done_event, part_done_event]) self._sequence_number += 1 reasoning_output_item_done_event: Final = self.create_reasoning_output_item_done_event( reasoning_item_id=reasoning_item_id, reasoning_content=reasoning_content, sequence_number=self._sequence_number, ) - self._pending_response_events.extend([text_done_event, part_done_event, reasoning_output_item_done_event]) + self._pending_response_events.append(reasoning_output_item_done_event) self._reasoning_done_emitted = True self._reasoning_active = False @@ -1362,11 +1383,17 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): if self._cached_reasoning_item_id is None: self._cached_reasoning_item_id = mint_reasoning_item_id() + if not self._sent_reasoning_summary_part_added_event: + self._sent_reasoning_summary_part_added_event = True + self._pending_response_events.append( + self.create_reasoning_summary_part_added_event(self._cached_reasoning_item_id) + ) return ReasoningSummaryTextDeltaEvent( type=ResponsesAPIStreamEvents.REASONING_SUMMARY_TEXT_DELTA, item_id=self._cached_reasoning_item_id, output_index=self._reasoning_index(), + summary_index=0, delta=reasoning_content, ) diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index a342d704a9d..4afc79119c0 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -113,6 +113,7 @@ from litellm.types.responses.main import ( OutputImageGenerationCall, OutputText, OutputToolSearchCall, + ReasoningSummaryText, ) from litellm.types.utils import ( ChatCompletionAnnotation, @@ -3427,23 +3428,30 @@ class LiteLLMCompletionResponsesConfig: if reasoning_content or encrypted_content: # Only check the first choice for reasoning content return [ - GenericResponseOutputItem( - type="reasoning", - id=mint_reasoning_item_id(), - status=LiteLLMCompletionResponsesConfig._map_chat_completion_finish_reason_to_responses_status( - choice.finish_reason - ), - role="assistant", - content=[ - OutputText( - type="output_text", - text=text, - annotations=[], - ) - for text in (reasoning_content,) - if text - ], - encrypted_content=encrypted_content, + GenericResponseOutputItem.model_validate( + { + "type": "reasoning", + "id": mint_reasoning_item_id(), + "status": LiteLLMCompletionResponsesConfig._map_chat_completion_finish_reason_to_responses_status( + choice.finish_reason + ), + "role": "assistant", + "content": [ + OutputText( + type="output_text", + text=text, + annotations=[], + ) + for text in (reasoning_content,) + if text + ], + "summary": [ + ReasoningSummaryText(type="summary_text", text=text) + for text in (reasoning_content,) + if text + ], + "encrypted_content": encrypted_content, + } ) ] return [] diff --git a/litellm/types/responses/main.py b/litellm/types/responses/main.py index 72dc63a581f..ae4a817ad53 100644 --- a/litellm/types/responses/main.py +++ b/litellm/types/responses/main.py @@ -158,6 +158,13 @@ class GenericResponseOutputItem(BaseLiteLLMOpenAIResponseObject): phase: Phase = None +class ReasoningSummaryText(BaseLiteLLMOpenAIResponseObject): + model_config = ConfigDict(frozen=True) + + type: Literal["summary_text"] + text: str + + class DeleteResponseResult(BaseLiteLLMOpenAIResponseObject): """ Result of a delete response request diff --git a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py index d9a0033b179..0eef8d0b5f8 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py +++ b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py @@ -12,11 +12,13 @@ spend tracking stores, so a follow-up previous_response_id still finds the conve import itertools import json -from collections.abc import Callable +from collections.abc import Callable, Mapping, Sequence from typing import Final from unittest.mock import AsyncMock, MagicMock import pytest +from openai.types.responses.response_reasoning_item import Summary as OpenAIReasoningSummaryPart +from pydantic import TypeAdapter from litellm.responses.litellm_completion_transformation.streaming_iterator import ( LiteLLMCompletionStreamingIterator, @@ -1241,6 +1243,77 @@ async def test_reasoning_item_closes_before_message_item_opens(stream_chunks: St assert _event_text(events, ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA) == "Hello!" +def _split_reasoning_then_text_chunks() -> tuple[ModelResponseStream, ...]: + return (_reasoning_chunk("let me "), _reasoning_chunk("think"), _chunk("Hello!", finish_reason="stop")) + + +def _signature_only_thinking_then_text_chunks() -> tuple[ModelResponseStream, ...]: + return (_signature_only_thinking_chunk("sig_only"), _chunk("4", finish_reason="stop")) + + +def _streamed_reasoning_text(chunks: Sequence[ModelResponseStream]) -> str: + return "".join(getattr(chunk.choices[0].delta, "reasoning_content", None) or "" for chunk in chunks) + + +def _proxy_sse_payload(event: BaseLiteLLMOpenAIResponseObject) -> Mapping[str, object]: + """The same dump litellm.proxy.proxy_server._serialize_streaming_chunk puts on the SSE wire.""" + return json.loads(event.model_dump_json(exclude_none=True, exclude_unset=True)) + + +def _reasoning_summary_as_the_openai_sdk_accumulates_it( + payloads: Sequence[Mapping[str, object]], +) -> Sequence[Mapping[str, str]]: + """Replays openai-node ResponseAccumulator's lookups, so a key or index missing on the wire raises as the SDK throws.""" + items: Final = {} + for payload in payloads: + match payload["type"]: + case "response.output_item.added": + items[payload["output_index"]] = payload["item"] + case "response.reasoning_summary_part.added": + items[payload["output_index"]]["summary"].append(payload["part"]) + case "response.reasoning_summary_text.delta": + items[payload["output_index"]]["summary"][payload["summary_index"]]["text"] += payload["delta"] + reasoning_item: Final = next(item for item in items.values() if item["type"] == "reasoning") + return reasoning_item["summary"] + + +def _reasoning_item(items: Sequence[Mapping[str, object]]) -> Mapping[str, object]: + return next(item for item in items if item["type"] == "reasoning") + + +@pytest.mark.parametrize( + "stream_chunks", + [ + _split_reasoning_then_text_chunks, + _reasoning_and_text_in_one_chunk, + _signed_thinking_and_text_in_one_chunk, + _signature_only_thinking_then_text_chunks, + ], + ids=[ + "split_reasoning_then_text_chunks", + "reasoning_and_text_in_one_chunk", + "signed_thinking_and_text_in_one_chunk", + "signature_only_thinking_then_text_chunks", + ], +) +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +async def test_sdk_stream_readers_can_place_every_reasoning_summary_delta(stream_chunks: StreamChunks, sync_mode: bool): + chunks: Final = stream_chunks() + iterator: Final = _build_iterator(chunks) + + payloads: Final = [_proxy_sse_payload(event) for event in await _collect_events(iterator, sync_mode)] + + accumulated: Final = _reasoning_summary_as_the_openai_sdk_accumulates_it(payloads) + done_items: Final = [payload["item"] for payload in payloads if payload["type"] == "response.output_item.done"] + completed_payload: Final = next(payload for payload in payloads if payload["type"] == "response.completed") + completed_summary: Final = _reasoning_item(completed_payload["response"]["output"])["summary"] + assert "".join(part["text"] for part in accumulated) == _streamed_reasoning_text(chunks) + assert _reasoning_item(done_items)["summary"] == accumulated + assert completed_summary == accumulated + TypeAdapter(list[OpenAIReasoningSummaryPart]).validate_python([*accumulated, *completed_summary]) + + @pytest.mark.parametrize("sync_mode", [True, False]) @pytest.mark.asyncio async def test_tool_then_reasoning_then_text_gives_message_its_own_output_index(sync_mode: bool): @@ -1523,9 +1596,7 @@ async def test_anthropic_tool_search_call_streams_as_tool_search_call_item(sync_ model="claude-sonnet-5-5", object="chat.completion.chunk", choices=[ - StreamingChoices( - index=0, delta=delta, finish_reason="tool_calls" if index == len(deltas) - 1 else None - ) + StreamingChoices(index=0, delta=delta, finish_reason="tool_calls" if index == len(deltas) - 1 else None) ], ) for index, delta in enumerate(deltas)