From dafe96c05a5153706670893ca5afbb47e85ae700 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 12 Nov 2026 04:59:45 +0000 Subject: [PATCH] fix(responses): preserve mixed reasoning and answer deltas on lts Partial backport of #46754 to lts/1.108.x. Source merge commit a8113b0dd721804c752a6f1952367fb642205224. Preserve the reasoning/answer split only, excluding the unrelated Anthropic compaction changes. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../streaming_iterator.py | 31 ++++++++++++++-- .../test_streaming_iterator_transformation.py | 36 ++++++++++++++++++- 2 files changed, 63 insertions(+), 4 deletions(-) diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index 39a7f06a85a..678ad0ee029 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -91,6 +91,11 @@ def _delta_has_signed_thinking_block(delta: object) -> bool: return any(isinstance(b, dict) and (b.get("signature") or b.get("data")) for b in blocks) +def _delta_carries_reasoning_and_answer(delta: ChatCompletionDelta) -> bool: + has_reasoning: Final = bool(getattr(delta, "reasoning_content", None)) or _delta_has_signed_thinking_block(delta) + return has_reasoning and bool(delta.content or delta.tool_calls) + + def _delta_carries_output(delta: ChatCompletionDelta) -> bool: fields: Final = ( delta.content, @@ -553,6 +558,26 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self._buffered_chunk = None return buffered + def _reasoning_part_first(self, chunk: ModelResponseStream) -> ModelResponseStream: + if not chunk.choices or not _delta_carries_reasoning_and_answer(chunk.choices[0].delta): + return chunk + first_choice: Final = chunk.choices[0] + delta: Final = first_choice.delta + answer_delta: Final = delta.model_copy(update={"reasoning_content": None, "thinking_blocks": None}) + self._buffered_chunk = chunk.model_copy( + update={"choices": [first_choice.model_copy(update={"delta": answer_delta}), *chunk.choices[1:]]} + ) + reasoning_delta: Final = ChatCompletionDelta( + reasoning_content=getattr(delta, "reasoning_content", None), + thinking_blocks=delta.thinking_blocks if hasattr(delta, "thinking_blocks") else None, + ) + return chunk.model_copy( + update={ + "provider_specific_fields": None, + "choices": [first_choice.model_copy(update={"delta": reasoning_delta, "finish_reason": None})], + } + ) + def _with_encoded_response_id(self, response: ResponsesAPIResponse) -> ResponsesAPIResponse: return ResponsesAPIRequestUtils.update_responses_api_response_id_with_model_id( responses_api_response=response, @@ -1052,7 +1077,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): raise StopAsyncIteration chunk = await self.litellm_custom_stream_wrapper.__anext__() if chunk is not None: - chunk = cast(ModelResponseStream, chunk) + chunk = self._reasoning_part_first(cast(ModelResponseStream, chunk)) for src in ( getattr(chunk, "provider_specific_fields", None), getattr( @@ -1151,11 +1176,11 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): try: buffered_chunk = self._take_buffered_chunk() if buffered_chunk is not None: - chunk = buffered_chunk + chunk = self._reasoning_part_first(buffered_chunk) elif self._upstream_exhausted: raise StopIteration else: - chunk = self.litellm_custom_stream_wrapper.__next__() + chunk = self._reasoning_part_first(self.litellm_custom_stream_wrapper.__next__()) for src in ( getattr(chunk, "provider_specific_fields", None), getattr( diff --git a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py index 3fd580792b8..b278f9b8b02 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py +++ b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py @@ -16,6 +16,7 @@ from typing import Final from unittest.mock import AsyncMock, MagicMock import pytest +from openai.types.responses import ResponseOutputMessage from litellm.responses.litellm_completion_transformation.streaming_iterator import ( LiteLLMCompletionStreamingIterator, @@ -23,9 +24,12 @@ from litellm.responses.litellm_completion_transformation.streaming_iterator impo from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.openai import ( BaseLiteLLMOpenAIResponseObject, + OutputTextDeltaEvent, + ReasoningSummaryTextDeltaEvent, + ResponseCompletedEvent, ResponsesAPIStreamEvents, ) -from litellm.types.responses.main import build_web_search_call +from litellm.types.responses.main import GenericResponseOutputItem, build_web_search_call from litellm.types.utils import ( Delta, ModelResponse, @@ -1016,6 +1020,36 @@ async def _collect_events( return [event async for event in iterator] +@pytest.mark.asyncio +@pytest.mark.parametrize("sync_mode", (True, False)) +@pytest.mark.parametrize("prefix", ("", "First ")) +async def test_mixed_reasoning_chunk_keeps_the_first_answer_delta(sync_mode: bool, prefix: str) -> None: + mixed: Final = ModelResponseStream( + id=CHAT_COMPLETION_ID, + created=1748575031, + model="claude-haiku-4-5", + choices=[StreamingChoices(index=0, delta=Delta(reasoning_content="multiply", content="1"))], + ) + chunks: Final = ( + *((_reasoning_chunk(prefix),) if prefix else ()), + mixed, + _chunk("9 × 29 = **551**", finish_reason="stop"), + ) + events: Final = await _collect_events(_build_iterator(chunks), sync_mode) + text: Final = "".join(event.delta for event in events if isinstance(event, OutputTextDeltaEvent)) + reasoning: Final = "".join(event.delta for event in events if isinstance(event, ReasoningSummaryTextDeltaEvent)) + completed: Final = next(event for event in events if isinstance(event, ResponseCompletedEvent)) + message_texts: Final = ( + "".join(part.text for part in item.content if part.type == "output_text") + for item in completed.response.output + if isinstance(item, (ResponseOutputMessage, GenericResponseOutputItem)) and item.type == "message" + ) + completed_text: Final = "".join(message_texts) + + assert text == completed_text == "19 × 29 = **551**" + assert reasoning == f"{prefix}multiply" + + def _is_message_item(event: BaseLiteLLMOpenAIResponseObject) -> bool: return getattr(getattr(event, "item", None), "type", None) == "message"