mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(responses): treat annotations as content when deferring the message item
An annotation-only chunk carries no content, reasoning or tool calls, so _ensure_output_item_for_chunk deferred the message item past it. Annotation events are buffered until the item they reference is announced, so the stream ended at response.completed with the annotation stranded in the buffer. Annotations now count as content (as does an already buffered annotation), which keeps the ordering guarantee -- annotation.added still follows its item's output_item.added -- without starving annotation-only streams.
This commit is contained in:
parent
f5ec303e41
commit
92fe745a7e
2 changed files with 75 additions and 8 deletions
|
|
@ -915,17 +915,29 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
|
|||
return
|
||||
delta: Final = chunk.choices[0].delta
|
||||
|
||||
# Role-only chunks (no content / reasoning_content / tool_calls)
|
||||
# must not decide the item type: defer message output_item.added
|
||||
# until the first chunk carrying actual content. Reasoning
|
||||
# providers commonly start with a role-only chunk; announcing the
|
||||
# message item there breaks the native event order for reasoning
|
||||
# streams. Pure text streams are unaffected (the first content
|
||||
# chunk still takes the default branch, one chunk later).
|
||||
# Role-only chunks (no content / reasoning_content / tool_calls /
|
||||
# annotations) must not decide the item type: defer message
|
||||
# output_item.added until the first chunk carrying actual content.
|
||||
# Reasoning providers commonly start with a role-only chunk;
|
||||
# announcing the message item there breaks the native event order
|
||||
# for reasoning streams. Pure text streams are unaffected (the first
|
||||
# content chunk still takes the default branch, one chunk later).
|
||||
#
|
||||
# Annotations count as content: an annotation event references the
|
||||
# message item, and it is buffered until that item is announced, so
|
||||
# deferring here would strand it -- an annotation-only stream would
|
||||
# end with the item never announced and the buffer never drained.
|
||||
_chunk_has_reasoning: Final = hasattr(delta, "reasoning_content") and delta.reasoning_content
|
||||
_chunk_has_tool_calls: Final = hasattr(delta, "tool_calls") and delta.tool_calls
|
||||
_chunk_has_content: Final = hasattr(delta, "content") and delta.content
|
||||
if not (_chunk_has_reasoning or _chunk_has_tool_calls or _chunk_has_content):
|
||||
_chunk_has_annotations: Final = hasattr(delta, "annotations") and delta.annotations
|
||||
if not (
|
||||
_chunk_has_reasoning
|
||||
or _chunk_has_tool_calls
|
||||
or _chunk_has_content
|
||||
or _chunk_has_annotations
|
||||
or getattr(self, "_pending_annotation_events", None)
|
||||
):
|
||||
return
|
||||
|
||||
self._sequence_number += 1
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ sequences that violate the OpenAI Responses streaming contract:
|
|||
which strict consumers reject.
|
||||
"""
|
||||
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
|
@ -316,3 +317,57 @@ async def test_annotation_events_wait_for_message_item_announcement():
|
|||
assert len(annotation_events) == 1
|
||||
message_added = events[message_added_idx]
|
||||
assert annotation_events[0].item_id == message_added.item.id
|
||||
|
||||
|
||||
URL_CITATION_ANNOTATION: Final = {
|
||||
"type": "url_citation",
|
||||
"url_citation": {
|
||||
"start_index": 0,
|
||||
"end_index": 6,
|
||||
"url": "https://example.com/citation",
|
||||
"title": "Example",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_annotation_only_stream_still_delivers_the_annotation():
|
||||
"""
|
||||
Regression: an annotation-only chunk carries no content, so deferring the
|
||||
message item past it strands the annotation -- annotation events are
|
||||
buffered until their item is announced, and the stream would reach
|
||||
response.completed with the buffer never drained.
|
||||
"""
|
||||
events = await _collect_events(
|
||||
[
|
||||
_chunk(annotations=[URL_CITATION_ANNOTATION]),
|
||||
_chunk(finish_reason="stop"),
|
||||
]
|
||||
)
|
||||
types = [_event_type(event) for event in events]
|
||||
annotation_added = str(ResponsesAPIStreamEvents.OUTPUT_TEXT_ANNOTATION_ADDED)
|
||||
assert annotation_added in types, f"annotation-only stream dropped the annotation: types={types}"
|
||||
assert types.index(annotation_added) > types.index(str(ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED)), (
|
||||
f"annotation.added must follow the message output_item.added: types={types}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_annotation_on_the_final_chunk_is_not_dropped():
|
||||
"""
|
||||
The per-chunk emit path only runs while chunks are still arriving, so an
|
||||
annotation queued from the final chunk has to be drained during
|
||||
finalization or it never reaches the consumer.
|
||||
"""
|
||||
events = await _collect_events(
|
||||
[
|
||||
_chunk(content="Answer with a citation"),
|
||||
_chunk(annotations=[URL_CITATION_ANNOTATION], finish_reason="stop"),
|
||||
]
|
||||
)
|
||||
types = [_event_type(event) for event in events]
|
||||
annotation_added = str(ResponsesAPIStreamEvents.OUTPUT_TEXT_ANNOTATION_ADDED)
|
||||
assert annotation_added in types, f"annotation from the final chunk was dropped: types={types}"
|
||||
assert types.index(annotation_added) < types.index(str(ResponsesAPIStreamEvents.OUTPUT_TEXT_DONE)), (
|
||||
f"annotation.added must precede output_text.done: types={types}"
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue