From 35718878e44d7ef67eb4023c7a83c59506a1a00e Mon Sep 17 00:00:00 2001 From: JingHao-Leon <102573344+JingHao-Leon@users.noreply.github.com> Date: Sun, 11 Oct 2026 00:48:46 +0800 Subject: [PATCH] fix(responses bridge): stop dropping reasoning items and url_citation annotations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bridging a Responses API reply back to Chat Completions lost data on two independent paths, and both losses were invisible to streaming clients. 1. Reasoning items (stream=False) — a reply can contain several `reasoning` items before the next message or tool call. `_convert_response_output_to_choices` held them in a single `pending_reasoning_item` slot, so each new item overwrote the previous one: `[reasoning rs_1, reasoning rs_2, function_call]` came back with `reasoning_items=[rs_2]`, while the same reply streamed returned both. It now accumulates in a tuple and flushes every item, into the assistant `Message` at the next message and into the merged trailing `tool_calls` choice. `reasoning_content` is unchanged (summary text of the most recent item), and single-item replies behave exactly as before. 2. url_citation annotations (stream=True) — the Responses API delivers annotations as `response.output_text.annotation.added` events. The stream bridge had no branch for that event, so a streamed reply never carried the citation that the same reply with `stream=False` reports on `message.annotations`. It now converts the annotation with the existing `_convert_annotations_to_chat_format` helper and emits it on one delta. Deliberate scope: the annotation is emitted exactly once, on the `annotation.added` event. `output_item.done` / `response.completed` repeat the annotations, and re-emitting there would duplicate citations for clients that accumulate `delta.annotations` across chunks — which is how native Chat Completions consumers collect them. Fixes #43620 Fixes #43817 --- .../transformation.py | 58 +++++--- ...responses_transformation_transformation.py | 133 +++++++++++++++++- 2 files changed, 173 insertions(+), 18 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 3b68b58a6d9..4f31e40e4a5 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -865,7 +865,10 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): choices: Final[list[Choices]] = [] index = 0 reasoning_content: str | None = None - pending_reasoning_item: _BuiltReasoningItem | None = None + # Tuple accumulator: the Responses API can emit several reasoning items + # before the next message / tool call, and each flush below consumes + # every item seen since the previous flush. + pending_reasoning_items: tuple[_BuiltReasoningItem, ...] = () # Collect all tool calls to put them in a single choice # (Chat Completions API expects all tool calls in one message) @@ -874,12 +877,13 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): for item in output_items: if isinstance(item, ResponseReasoningItem): - pending_reasoning_item = _build_reasoning_item( + built_reasoning_item = _build_reasoning_item( item_id=item.id, encrypted_content=getattr(item, "encrypted_content", None), summary_raw=item.summary, ) - reasoning_content = " ".join(s["text"] for s in pending_reasoning_item["summary"] if s.get("text")) + pending_reasoning_items = (*pending_reasoning_items, built_reasoning_item) + reasoning_content = " ".join(s["text"] for s in built_reasoning_item["summary"] if s.get("text")) elif isinstance(item, ResponseOutputMessage): for content in item.content: @@ -894,10 +898,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): content=response_text if response_text else "", reasoning_content=reasoning_content, annotations=annotations, - reasoning_items=cast( - list[ChatCompletionReasoningItem] | None, - ([pending_reasoning_item] if pending_reasoning_item is not None else None), - ), + reasoning_items=_as_chat_reasoning_items(pending_reasoning_items), ) choices.append( @@ -909,7 +910,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): ) reasoning_content = None # flush - pending_reasoning_item = None # flush + pending_reasoning_items = () # flush index += 1 elif isinstance(item, ResponseFunctionToolCall): @@ -966,10 +967,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): " ".join(value for value in (last_reasoning_content, reasoning_content) if value) or None ) merged_reasoning_items: Final = _as_chat_reasoning_items( - ( - *(last_reasoning_items or ()), - *(() if pending_reasoning_item is None else (pending_reasoning_item,)), - ) + (*(last_reasoning_items or ()), *pending_reasoning_items) ) merged_message: Final = Message( role=last_choice.message.role, @@ -989,14 +987,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): content=None, tool_calls=accumulated_tool_calls, reasoning_content=reasoning_content, - reasoning_items=cast( - list[ChatCompletionReasoningItem] | None, - ([pending_reasoning_item] if pending_reasoning_item is not None else None), - ), + reasoning_items=_as_chat_reasoning_items(pending_reasoning_items), ) choices.append(Choices(message=msg, finish_reason="tool_calls", index=index)) reasoning_content = None - pending_reasoning_item = None + pending_reasoning_items = () return choices @@ -1788,6 +1783,35 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator): ) else: raise ValueError(f"Chat provider: Invalid text delta {parsed_chunk}") + elif event_type == ResponsesAPIStreamEvents.OUTPUT_TEXT_ANNOTATION_ADDED: + # A url_citation / file_citation arrived for the in-flight message. + # Emit it on the delta exactly once in Chat Completions format — + # this is the only event that carries it, so later + # output_item.done / response.completed events must not repeat it + # or accumulating clients would see duplicates (issue #43817). + raw_annotation: Final = parsed_chunk.get("annotation", None) + annotation: Final = LiteLLMResponsesTransformationHandler._convert_annotations_to_chat_format( + [raw_annotation] if raw_annotation is not None else None + ) + if annotation: + return ModelResponseStream( + choices=[ + StreamingChoices( + index=0, + delta=Delta(annotations=annotation), + finish_reason=None, + ) + ] + ) + return ModelResponseStream( + choices=[ + StreamingChoices( + index=0, + delta=Delta(), + finish_reason=None, + ) + ] + ) elif event_type == "response.reasoning_summary_text.delta": content_part = parsed_chunk.get("delta", None) if content_part: diff --git a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index 00653a5d127..f82caac97c7 100644 --- a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -5043,6 +5043,14 @@ def test_transform_request_keeps_the_preceding_part_breakpoint_over_the_dropped_ def test_convert_chat_completion_messages_to_responses_api_keeps_prompt_cache_breakpoint_on_unknown_block(): """The hook marks the last block of its target message, so a message ending in a block the bridge cannot map reaches the stringify path and has to keep the marker there.""" + + +def test_convert_response_output_keeps_every_reasoning_item_before_a_message(): + """Regression for issue #43620 (stream=False): the Responses API can emit several + reasoning items before the assistant message, and the bridge used to overwrite + its pending reasoning item per item, so only the last one reached + message.reasoning_items. Every item must survive the flush.""" + from openai.types.responses import ResponseOutputMessage, ResponseOutputText, ResponseReasoningItem from litellm.completion_extras.litellm_responses_transformation.transformation import ( LiteLLMResponsesTransformationHandler, ) @@ -5138,4 +5146,127 @@ def test_transform_request_drop_params_in_litellm_params_gates_the_prompt_cache_ litellm_logging_obj=Mock(), ) - assert "prompt_cache_breakpoint" not in result["input"][0]["content"][0] \ No newline at end of file + assert "prompt_cache_breakpoint" not in result["input"][0]["content"][0] + + + rs_1 = ResponseReasoningItem(type="reasoning", id="rs_1", summary=[], encrypted_content="enc1") + rs_2 = ResponseReasoningItem(type="reasoning", id="rs_2", summary=[], encrypted_content="enc2") + message = ResponseOutputMessage( + type="message", + id="msg_1", + role="assistant", + status="completed", + content=[ResponseOutputText(type="output_text", text="hello", annotations=[])], + ) + + choices = LiteLLMResponsesTransformationHandler._convert_response_output_to_choices([rs_1, rs_2, message]) + + assert len(choices) == 1 + reasoning_items = choices[0].message.reasoning_items + assert reasoning_items is not None + assert [item["id"] for item in reasoning_items] == ["rs_1", "rs_2"] + + +def test_convert_response_output_keeps_every_reasoning_item_before_a_tool_call(): + """Regression for issue #43620 (stream=False): [reasoning rs_1, reasoning rs_2, + function_call] must merge both reasoning items into the trailing tool_calls + choice instead of keeping only rs_2.""" + from openai.types.responses import ResponseFunctionToolCall, ResponseReasoningItem + + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, + ) + + rs_1 = ResponseReasoningItem(type="reasoning", id="rs_1", summary=[], encrypted_content="enc1") + rs_2 = ResponseReasoningItem(type="reasoning", id="rs_2", summary=[], encrypted_content="enc2") + function_call = ResponseFunctionToolCall( + type="function_call", + id="fc_1", + call_id="call_1", + name="get_weather", + arguments="{}", + status="completed", + ) + + choices = LiteLLMResponsesTransformationHandler._convert_response_output_to_choices([rs_1, rs_2, function_call]) + + assert len(choices) == 1 + choice = choices[0] + assert choice.finish_reason == "tool_calls" + reasoning_items = choice.message.reasoning_items + assert reasoning_items is not None + assert [item["id"] for item in reasoning_items] == ["rs_1", "rs_2"] + assert choice.message.tool_calls[0].id == "fc_1" + + +def test_stream_translator_emits_url_citation_annotations_exactly_once(): + """Regression for issue #43817: the Responses API delivers annotations as + response.output_text.annotation.added events, and the Chat Completions + stream bridge dropped them, so a streamed reply never carried the + url_citation that the same reply with stream=False reports on + message.annotations. The annotation must ride exactly one delta — the + terminal response.completed event repeats it in the response payload and + re-emitting it there would duplicate it for accumulating clients.""" + from litellm.completion_extras.litellm_responses_transformation.transformation import ( + OpenAiResponsesToChatCompletionStreamIterator, + ) + + citation = { + "type": "url_citation", + "url": "https://a.example", + "title": "A", + "start_index": 0, + "end_index": 5, + } + reply = { + "id": "resp_1", + "object": "response", + "created_at": 0, + "status": "completed", + "model": "gpt-5-mini", + "output": [ + { + "type": "message", + "id": "msg_1", + "role": "assistant", + "status": "completed", + "content": [ + { + "type": "output_text", + "text": "Sunny in Paris.", + "annotations": [citation], + } + ], + } + ], + } + iterator = OpenAiResponsesToChatCompletionStreamIterator(streaming_response=None, sync_stream=True) + events = [ + {"type": "response.created", "response": {**reply, "status": "in_progress", "output": []}}, + { + "type": "response.output_text.delta", + "output_index": 0, + "item_id": "msg_1", + "content_index": 0, + "delta": "Sunny in Paris.", + }, + { + "type": "response.output_text.annotation.added", + "output_index": 0, + "item_id": "msg_1", + "content_index": 0, + "annotation_index": 0, + "annotation": citation, + }, + {"type": "response.completed", "response": reply}, + ] + + chunks = [iterator.chunk_parser(event) for event in events] + + carried = [ + annotation + for chunk in chunks + for choice in chunk.choices + for annotation in (getattr(choice.delta, "annotations", None) or []) + ] + assert carried == [citation]