mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-15 04:19:08 +00:00
fix(responses bridge): stop dropping reasoning items and url_citation annotations
Bridging a Responses API reply back to Chat Completions lost data on two independent paths, and both losses were invisible to streaming clients. 1. Reasoning items (stream=False) — a reply can contain several `reasoning` items before the next message or tool call. `_convert_response_output_to_choices` held them in a single `pending_reasoning_item` slot, so each new item overwrote the previous one: `[reasoning rs_1, reasoning rs_2, function_call]` came back with `reasoning_items=[rs_2]`, while the same reply streamed returned both. It now accumulates in a tuple and flushes every item, into the assistant `Message` at the next message and into the merged trailing `tool_calls` choice. `reasoning_content` is unchanged (summary text of the most recent item), and single-item replies behave exactly as before. 2. url_citation annotations (stream=True) — the Responses API delivers annotations as `response.output_text.annotation.added` events. The stream bridge had no branch for that event, so a streamed reply never carried the citation that the same reply with `stream=False` reports on `message.annotations`. It now converts the annotation with the existing `_convert_annotations_to_chat_format` helper and emits it on one delta. Deliberate scope: the annotation is emitted exactly once, on the `annotation.added` event. `output_item.done` / `response.completed` repeat the annotations, and re-emitting there would duplicate citations for clients that accumulate `delta.annotations` across chunks — which is how native Chat Completions consumers collect them. Fixes #43620 Fixes #43817
This commit is contained in:
JingHao-Leon
parent
541e3ed530
commit
35718878e4
2 changed files with 173 additions and 18 deletions
|
|
@ -865,7 +865,10 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
choices: Final[list[Choices]] = []
|
||||
index = 0
|
||||
reasoning_content: str | None = None
|
||||
pending_reasoning_item: _BuiltReasoningItem | None = None
|
||||
# Tuple accumulator: the Responses API can emit several reasoning items
|
||||
# before the next message / tool call, and each flush below consumes
|
||||
# every item seen since the previous flush.
|
||||
pending_reasoning_items: tuple[_BuiltReasoningItem, ...] = ()
|
||||
|
||||
# Collect all tool calls to put them in a single choice
|
||||
# (Chat Completions API expects all tool calls in one message)
|
||||
|
|
@ -874,12 +877,13 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
|
||||
for item in output_items:
|
||||
if isinstance(item, ResponseReasoningItem):
|
||||
pending_reasoning_item = _build_reasoning_item(
|
||||
built_reasoning_item = _build_reasoning_item(
|
||||
item_id=item.id,
|
||||
encrypted_content=getattr(item, "encrypted_content", None),
|
||||
summary_raw=item.summary,
|
||||
)
|
||||
reasoning_content = " ".join(s["text"] for s in pending_reasoning_item["summary"] if s.get("text"))
|
||||
pending_reasoning_items = (*pending_reasoning_items, built_reasoning_item)
|
||||
reasoning_content = " ".join(s["text"] for s in built_reasoning_item["summary"] if s.get("text"))
|
||||
|
||||
elif isinstance(item, ResponseOutputMessage):
|
||||
for content in item.content:
|
||||
|
|
@ -894,10 +898,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
content=response_text if response_text else "",
|
||||
reasoning_content=reasoning_content,
|
||||
annotations=annotations,
|
||||
reasoning_items=cast(
|
||||
list[ChatCompletionReasoningItem] | None,
|
||||
([pending_reasoning_item] if pending_reasoning_item is not None else None),
|
||||
),
|
||||
reasoning_items=_as_chat_reasoning_items(pending_reasoning_items),
|
||||
)
|
||||
|
||||
choices.append(
|
||||
|
|
@ -909,7 +910,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
)
|
||||
|
||||
reasoning_content = None # flush
|
||||
pending_reasoning_item = None # flush
|
||||
pending_reasoning_items = () # flush
|
||||
index += 1
|
||||
|
||||
elif isinstance(item, ResponseFunctionToolCall):
|
||||
|
|
@ -966,10 +967,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
" ".join(value for value in (last_reasoning_content, reasoning_content) if value) or None
|
||||
)
|
||||
merged_reasoning_items: Final = _as_chat_reasoning_items(
|
||||
(
|
||||
*(last_reasoning_items or ()),
|
||||
*(() if pending_reasoning_item is None else (pending_reasoning_item,)),
|
||||
)
|
||||
(*(last_reasoning_items or ()), *pending_reasoning_items)
|
||||
)
|
||||
merged_message: Final = Message(
|
||||
role=last_choice.message.role,
|
||||
|
|
@ -989,14 +987,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
content=None,
|
||||
tool_calls=accumulated_tool_calls,
|
||||
reasoning_content=reasoning_content,
|
||||
reasoning_items=cast(
|
||||
list[ChatCompletionReasoningItem] | None,
|
||||
([pending_reasoning_item] if pending_reasoning_item is not None else None),
|
||||
),
|
||||
reasoning_items=_as_chat_reasoning_items(pending_reasoning_items),
|
||||
)
|
||||
choices.append(Choices(message=msg, finish_reason="tool_calls", index=index))
|
||||
reasoning_content = None
|
||||
pending_reasoning_item = None
|
||||
pending_reasoning_items = ()
|
||||
|
||||
return choices
|
||||
|
||||
|
|
@ -1788,6 +1783,35 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
|
|||
)
|
||||
else:
|
||||
raise ValueError(f"Chat provider: Invalid text delta {parsed_chunk}")
|
||||
elif event_type == ResponsesAPIStreamEvents.OUTPUT_TEXT_ANNOTATION_ADDED:
|
||||
# A url_citation / file_citation arrived for the in-flight message.
|
||||
# Emit it on the delta exactly once in Chat Completions format —
|
||||
# this is the only event that carries it, so later
|
||||
# output_item.done / response.completed events must not repeat it
|
||||
# or accumulating clients would see duplicates (issue #43817).
|
||||
raw_annotation: Final = parsed_chunk.get("annotation", None)
|
||||
annotation: Final = LiteLLMResponsesTransformationHandler._convert_annotations_to_chat_format(
|
||||
[raw_annotation] if raw_annotation is not None else None
|
||||
)
|
||||
if annotation:
|
||||
return ModelResponseStream(
|
||||
choices=[
|
||||
StreamingChoices(
|
||||
index=0,
|
||||
delta=Delta(annotations=annotation),
|
||||
finish_reason=None,
|
||||
)
|
||||
]
|
||||
)
|
||||
return ModelResponseStream(
|
||||
choices=[
|
||||
StreamingChoices(
|
||||
index=0,
|
||||
delta=Delta(),
|
||||
finish_reason=None,
|
||||
)
|
||||
]
|
||||
)
|
||||
elif event_type == "response.reasoning_summary_text.delta":
|
||||
content_part = parsed_chunk.get("delta", None)
|
||||
if content_part:
|
||||
|
|
|
|||
|
|
@ -5043,6 +5043,14 @@ def test_transform_request_keeps_the_preceding_part_breakpoint_over_the_dropped_
|
|||
def test_convert_chat_completion_messages_to_responses_api_keeps_prompt_cache_breakpoint_on_unknown_block():
|
||||
"""The hook marks the last block of its target message, so a message ending in a block the bridge
|
||||
cannot map reaches the stringify path and has to keep the marker there."""
|
||||
|
||||
|
||||
def test_convert_response_output_keeps_every_reasoning_item_before_a_message():
|
||||
"""Regression for issue #43620 (stream=False): the Responses API can emit several
|
||||
reasoning items before the assistant message, and the bridge used to overwrite
|
||||
its pending reasoning item per item, so only the last one reached
|
||||
message.reasoning_items. Every item must survive the flush."""
|
||||
from openai.types.responses import ResponseOutputMessage, ResponseOutputText, ResponseReasoningItem
|
||||
from litellm.completion_extras.litellm_responses_transformation.transformation import (
|
||||
LiteLLMResponsesTransformationHandler,
|
||||
)
|
||||
|
|
@ -5138,4 +5146,127 @@ def test_transform_request_drop_params_in_litellm_params_gates_the_prompt_cache_
|
|||
litellm_logging_obj=Mock(),
|
||||
)
|
||||
|
||||
assert "prompt_cache_breakpoint" not in result["input"][0]["content"][0]
|
||||
assert "prompt_cache_breakpoint" not in result["input"][0]["content"][0]
|
||||
|
||||
|
||||
rs_1 = ResponseReasoningItem(type="reasoning", id="rs_1", summary=[], encrypted_content="enc1")
|
||||
rs_2 = ResponseReasoningItem(type="reasoning", id="rs_2", summary=[], encrypted_content="enc2")
|
||||
message = ResponseOutputMessage(
|
||||
type="message",
|
||||
id="msg_1",
|
||||
role="assistant",
|
||||
status="completed",
|
||||
content=[ResponseOutputText(type="output_text", text="hello", annotations=[])],
|
||||
)
|
||||
|
||||
choices = LiteLLMResponsesTransformationHandler._convert_response_output_to_choices([rs_1, rs_2, message])
|
||||
|
||||
assert len(choices) == 1
|
||||
reasoning_items = choices[0].message.reasoning_items
|
||||
assert reasoning_items is not None
|
||||
assert [item["id"] for item in reasoning_items] == ["rs_1", "rs_2"]
|
||||
|
||||
|
||||
def test_convert_response_output_keeps_every_reasoning_item_before_a_tool_call():
|
||||
"""Regression for issue #43620 (stream=False): [reasoning rs_1, reasoning rs_2,
|
||||
function_call] must merge both reasoning items into the trailing tool_calls
|
||||
choice instead of keeping only rs_2."""
|
||||
from openai.types.responses import ResponseFunctionToolCall, ResponseReasoningItem
|
||||
|
||||
from litellm.completion_extras.litellm_responses_transformation.transformation import (
|
||||
LiteLLMResponsesTransformationHandler,
|
||||
)
|
||||
|
||||
rs_1 = ResponseReasoningItem(type="reasoning", id="rs_1", summary=[], encrypted_content="enc1")
|
||||
rs_2 = ResponseReasoningItem(type="reasoning", id="rs_2", summary=[], encrypted_content="enc2")
|
||||
function_call = ResponseFunctionToolCall(
|
||||
type="function_call",
|
||||
id="fc_1",
|
||||
call_id="call_1",
|
||||
name="get_weather",
|
||||
arguments="{}",
|
||||
status="completed",
|
||||
)
|
||||
|
||||
choices = LiteLLMResponsesTransformationHandler._convert_response_output_to_choices([rs_1, rs_2, function_call])
|
||||
|
||||
assert len(choices) == 1
|
||||
choice = choices[0]
|
||||
assert choice.finish_reason == "tool_calls"
|
||||
reasoning_items = choice.message.reasoning_items
|
||||
assert reasoning_items is not None
|
||||
assert [item["id"] for item in reasoning_items] == ["rs_1", "rs_2"]
|
||||
assert choice.message.tool_calls[0].id == "fc_1"
|
||||
|
||||
|
||||
def test_stream_translator_emits_url_citation_annotations_exactly_once():
|
||||
"""Regression for issue #43817: the Responses API delivers annotations as
|
||||
response.output_text.annotation.added events, and the Chat Completions
|
||||
stream bridge dropped them, so a streamed reply never carried the
|
||||
url_citation that the same reply with stream=False reports on
|
||||
message.annotations. The annotation must ride exactly one delta — the
|
||||
terminal response.completed event repeats it in the response payload and
|
||||
re-emitting it there would duplicate it for accumulating clients."""
|
||||
from litellm.completion_extras.litellm_responses_transformation.transformation import (
|
||||
OpenAiResponsesToChatCompletionStreamIterator,
|
||||
)
|
||||
|
||||
citation = {
|
||||
"type": "url_citation",
|
||||
"url": "https://a.example",
|
||||
"title": "A",
|
||||
"start_index": 0,
|
||||
"end_index": 5,
|
||||
}
|
||||
reply = {
|
||||
"id": "resp_1",
|
||||
"object": "response",
|
||||
"created_at": 0,
|
||||
"status": "completed",
|
||||
"model": "gpt-5-mini",
|
||||
"output": [
|
||||
{
|
||||
"type": "message",
|
||||
"id": "msg_1",
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [
|
||||
{
|
||||
"type": "output_text",
|
||||
"text": "Sunny in Paris.",
|
||||
"annotations": [citation],
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
iterator = OpenAiResponsesToChatCompletionStreamIterator(streaming_response=None, sync_stream=True)
|
||||
events = [
|
||||
{"type": "response.created", "response": {**reply, "status": "in_progress", "output": []}},
|
||||
{
|
||||
"type": "response.output_text.delta",
|
||||
"output_index": 0,
|
||||
"item_id": "msg_1",
|
||||
"content_index": 0,
|
||||
"delta": "Sunny in Paris.",
|
||||
},
|
||||
{
|
||||
"type": "response.output_text.annotation.added",
|
||||
"output_index": 0,
|
||||
"item_id": "msg_1",
|
||||
"content_index": 0,
|
||||
"annotation_index": 0,
|
||||
"annotation": citation,
|
||||
},
|
||||
{"type": "response.completed", "response": reply},
|
||||
]
|
||||
|
||||
chunks = [iterator.chunk_parser(event) for event in events]
|
||||
|
||||
carried = [
|
||||
annotation
|
||||
for chunk in chunks
|
||||
for choice in chunk.choices
|
||||
for annotation in (getattr(choice.delta, "annotations", None) or [])
|
||||
]
|
||||
assert carried == [citation]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue