From 92fe745a7eb34fc92db18bb1f8ca5b6d8610166a Mon Sep 17 00:00:00 2001 From: liracle Date: Sat, 29 Aug 2026 16:10:36 +0800 Subject: [PATCH] fix(responses): treat annotations as content when deferring the message item An annotation-only chunk carries no content, reasoning or tool calls, so _ensure_output_item_for_chunk deferred the message item past it. Annotation events are buffered until the item they reference is announced, so the stream ended at response.completed with the annotation stranded in the buffer. Annotations now count as content (as does an already buffered annotation), which keeps the ordering guarantee -- annotation.added still follows its item's output_item.added -- without starving annotation-only streams. --- .../streaming_iterator.py | 28 +++++++--- .../test_streaming_event_order.py | 55 +++++++++++++++++++ 2 files changed, 75 insertions(+), 8 deletions(-) diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index 25e83b1cdc7..b49ef284752 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -915,17 +915,29 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): return delta: Final = chunk.choices[0].delta - # Role-only chunks (no content / reasoning_content / tool_calls) - # must not decide the item type: defer message output_item.added - # until the first chunk carrying actual content. Reasoning - # providers commonly start with a role-only chunk; announcing the - # message item there breaks the native event order for reasoning - # streams. Pure text streams are unaffected (the first content - # chunk still takes the default branch, one chunk later). + # Role-only chunks (no content / reasoning_content / tool_calls / + # annotations) must not decide the item type: defer message + # output_item.added until the first chunk carrying actual content. + # Reasoning providers commonly start with a role-only chunk; + # announcing the message item there breaks the native event order + # for reasoning streams. Pure text streams are unaffected (the first + # content chunk still takes the default branch, one chunk later). + # + # Annotations count as content: an annotation event references the + # message item, and it is buffered until that item is announced, so + # deferring here would strand it -- an annotation-only stream would + # end with the item never announced and the buffer never drained. _chunk_has_reasoning: Final = hasattr(delta, "reasoning_content") and delta.reasoning_content _chunk_has_tool_calls: Final = hasattr(delta, "tool_calls") and delta.tool_calls _chunk_has_content: Final = hasattr(delta, "content") and delta.content - if not (_chunk_has_reasoning or _chunk_has_tool_calls or _chunk_has_content): + _chunk_has_annotations: Final = hasattr(delta, "annotations") and delta.annotations + if not ( + _chunk_has_reasoning + or _chunk_has_tool_calls + or _chunk_has_content + or _chunk_has_annotations + or getattr(self, "_pending_annotation_events", None) + ): return self._sequence_number += 1 diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_event_order.py b/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_event_order.py index 036f8bdb504..22095d500fb 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_event_order.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_event_order.py @@ -21,6 +21,7 @@ sequences that violate the OpenAI Responses streaming contract: which strict consumers reject. """ +from typing import Final from unittest.mock import MagicMock import pytest @@ -316,3 +317,57 @@ async def test_annotation_events_wait_for_message_item_announcement(): assert len(annotation_events) == 1 message_added = events[message_added_idx] assert annotation_events[0].item_id == message_added.item.id + + +URL_CITATION_ANNOTATION: Final = { + "type": "url_citation", + "url_citation": { + "start_index": 0, + "end_index": 6, + "url": "https://example.com/citation", + "title": "Example", + }, +} + + +@pytest.mark.asyncio +async def test_annotation_only_stream_still_delivers_the_annotation(): + """ + Regression: an annotation-only chunk carries no content, so deferring the + message item past it strands the annotation -- annotation events are + buffered until their item is announced, and the stream would reach + response.completed with the buffer never drained. + """ + events = await _collect_events( + [ + _chunk(annotations=[URL_CITATION_ANNOTATION]), + _chunk(finish_reason="stop"), + ] + ) + types = [_event_type(event) for event in events] + annotation_added = str(ResponsesAPIStreamEvents.OUTPUT_TEXT_ANNOTATION_ADDED) + assert annotation_added in types, f"annotation-only stream dropped the annotation: types={types}" + assert types.index(annotation_added) > types.index(str(ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED)), ( + f"annotation.added must follow the message output_item.added: types={types}" + ) + + +@pytest.mark.asyncio +async def test_annotation_on_the_final_chunk_is_not_dropped(): + """ + The per-chunk emit path only runs while chunks are still arriving, so an + annotation queued from the final chunk has to be drained during + finalization or it never reaches the consumer. + """ + events = await _collect_events( + [ + _chunk(content="Answer with a citation"), + _chunk(annotations=[URL_CITATION_ANNOTATION], finish_reason="stop"), + ] + ) + types = [_event_type(event) for event in events] + annotation_added = str(ResponsesAPIStreamEvents.OUTPUT_TEXT_ANNOTATION_ADDED) + assert annotation_added in types, f"annotation from the final chunk was dropped: types={types}" + assert types.index(annotation_added) < types.index(str(ResponsesAPIStreamEvents.OUTPUT_TEXT_DONE)), ( + f"annotation.added must precede output_text.done: types={types}" + )