fix(responses): fold bridged Responses output into one chat choice (#46099)

* fix(responses): fold bridged Responses output into one chat choice

A Chat Completions request bridged to /responses got one choice per
Responses message item, so a reply shaped [message, message,
function_call] put the tool call on choices[1] and clients reading
only choices[0] never saw it. Fold every message text, annotation,
reasoning item and tool call into choices[0]

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(responses): drop mutable annotations and the extra cast in the choice fold

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(responses): remove a cast-ok marker that suppressed nothing

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(responses): join folded message texts with a newline

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* revert(responses): join folded message texts with no separator so stream and non-stream match

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com>
(cherry picked from commit 4b7b8efd4c)

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -5,6 +5,7 @@ Handler for transforming /chat/completions api requests to litellm.responses req
 import json
 import os
 from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence
+from itertools import accumulate, chain
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, TypeVar, Union, cast
 
@@ -69,7 +70,7 @@ if TYPE_CHECKING:
         ChatCompletionToolReferenceObject,
         OpenAIMessageContentListBlock,
     )
-    from litellm.types.utils import Choices
+    from litellm.types.utils import Choices, Message
 
 
 _CHAT_COMPLETION_FIELDS: Final = frozenset((*ModelResponse.model_fields, "usage"))
@@ -194,6 +195,67 @@ def _as_chat_reasoning_items(
     return cast(list[ChatCompletionReasoningItem], list(reasoning_items))
 
 
+def _shift_annotation(annotation: ChatCompletionAnnotation, offset: int) -> ChatCompletionAnnotation:
+    shifted: Final = {
+        key: value + offset if key in ("start_index", "end_index") and isinstance(value, int) else value
+        for key, value in annotation.items()
+    }
+    return cast(ChatCompletionAnnotation, shifted)
+
+
+def _message_annotations(message: "Message") -> Sequence[ChatCompletionAnnotation]:
+    annotations: Sequence[ChatCompletionAnnotation] | None = getattr(message, "annotations", None)
+    return annotations or ()
+
+
+def _message_reasoning_content(message: "Message") -> str | None:
+    reasoning_content: str | None = getattr(message, "reasoning_content", None)
+    return reasoning_content
+
+
+def _message_reasoning_items(message: "Message") -> Sequence[ChatCompletionReasoningItem]:
+    reasoning_items: Sequence[ChatCompletionReasoningItem] | None = getattr(message, "reasoning_items", None)
+    return reasoning_items or ()
+
+
+def _shifted_annotations(messages: Sequence["Message"], offsets: Sequence[int]) -> Iterator[ChatCompletionAnnotation]:
+    for message, offset in zip(messages, offsets):
+        for annotation in _message_annotations(message):
+            yield _shift_annotation(annotation, offset)
+
+
+def _fold_choices_into_one(
+    choices: Sequence["Choices"],
+    tool_calls: Sequence[Mapping[str, object]],
+    trailing_reasoning_content: str | None,
+    trailing_reasoning_item: _BuiltReasoningItem | None,
+) -> "Choices":
+    """Clients that only read choices[0] must still see every text and tool call."""
+    from litellm.types.utils import Choices, Message
+
+    messages: Final = tuple(choice.message for choice in choices)
+    texts: Final = tuple(str(message.content or "") for message in messages)
+    offsets: Final = (0, *accumulate(len(text) for text in texts))
+    annotations: Final = tuple(_shifted_annotations(messages, offsets))
+    reasoning_contents: Final = (*(_message_reasoning_content(m) for m in messages), trailing_reasoning_content)
+    reasoning_items: Final = (
+        *chain.from_iterable(_message_reasoning_items(m) for m in messages),
+        *(() if trailing_reasoning_item is None else (trailing_reasoning_item,)),
+    )
+    return Choices(
+        message=Message(
+            role=messages[0].role if messages else "assistant",
+            content="".join(texts) if messages else None,
+            annotations=list(annotations) or None,
+            tool_calls=list(tool_calls) or None,
+            reasoning_content=" ".join(value for value in reasoning_contents if value) or None,
+            reasoning_items=_as_chat_reasoning_items(reasoning_items),
+        ),
+        finish_reason="tool_calls" if tool_calls else "stop",
+        index=0,
+    )
+
+
 _ToolChoiceT = TypeVar("_ToolChoiceT")
 
 
@@ -784,47 +846,16 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             else:
                 pass  # don't fail request if item in list is not supported
 
-        if accumulated_tool_calls and choices:
-            last_choice: Final = choices[-1]
-            last_reasoning_content: Final = getattr(last_choice.message, "reasoning_content", None)
-            last_reasoning_items: Final = getattr(last_choice.message, "reasoning_items", None)
-            merged_reasoning_content: Final = (
-                " ".join(value for value in (last_reasoning_content, reasoning_content) if value) or None
-            )
-            merged_reasoning_items: Final = _as_chat_reasoning_items(
-                (
-                    *(last_reasoning_items or ()),
-                    *(() if pending_reasoning_item is None else (pending_reasoning_item,)),
-                )
-            )
-            merged_message: Final = Message(
-                role=last_choice.message.role,
-                content=last_choice.message.content,
-                annotations=getattr(last_choice.message, "annotations", None),
+        if not choices and not accumulated_tool_calls:
+            return choices
+        return [
+            _fold_choices_into_one(
+                choices=choices,
                 tool_calls=accumulated_tool_calls,
-                reasoning_content=merged_reasoning_content,
-                reasoning_items=merged_reasoning_items,
+                trailing_reasoning_content=reasoning_content,
+                trailing_reasoning_item=pending_reasoning_item,
             )
-            return [
-                *choices[:-1],
-                Choices(message=merged_message, finish_reason="tool_calls", index=last_choice.index),
-            ]
-
-        if accumulated_tool_calls:
-            msg = Message(
-                content=None,
-                tool_calls=accumulated_tool_calls,
-                reasoning_content=reasoning_content,
-                reasoning_items=cast(
-                    list[ChatCompletionReasoningItem] | None,
-                    ([pending_reasoning_item] if pending_reasoning_item is not None else None),
-                ),
-            )
-            choices.append(Choices(message=msg, finish_reason="tool_calls", index=index))
-            reasoning_content = None
-            pending_reasoning_item = None
-
-        return choices
+        ]
 
     @staticmethod
     def _build_empty_incomplete_choice(

tests/integration/_support/client.py

@@ -23,6 +23,11 @@ def object_value(value: JsonValue) -> dict[str, JsonValue]:
     return JSON_OBJECT.validate_python(value)
 
 
+def list_value(value: JsonValue) -> Sequence[JsonValue]:
+    assert isinstance(value, list), f"Expected a list, received {type(value).__name__}"
+    return value
+
+
 def string_value(value: JsonValue) -> str:
     assert isinstance(value, str), f"Expected a string, received {type(value).__name__}"
     return value

tests/integration/providers/test_responses_bridge_incomplete.py

@@ -3,8 +3,9 @@ import uuid
 from typing import Final
 
 import pytest
-from integration._support.client import Gateway
+from integration._support.client import Gateway, list_value, object_value
 from integration._support.wire import Reply, Request, wire_server
+from pydantic import JsonValue
 
 
 @pytest.mark.covers("other.provider_wire.responses_bridge.max_output_tokens_incomplete_maps_to_length")
@@ -572,3 +573,256 @@ def test_chat_over_responses_deployment_merges_function_call_followed_by_message
             }
         ], response.text
         assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/responses")]
+
+
+_WEATHER_TOOL: Final[dict[str, JsonValue]] = {
+    "type": "function",
+    "function": {
+        "name": "get_weather",
+        "description": "Get the weather for a city.",
+        "parameters": {
+            "type": "object",
+            "properties": {"city": {"type": "string"}},
+            "required": ["city"],
+        },
+    },
+}
+
+
+def _message_item(item_id: str, text: str) -> dict[str, JsonValue]:
+    return {
+        "type": "message",
+        "id": item_id,
+        "status": "completed",
+        "role": "assistant",
+        "content": [{"type": "output_text", "text": text, "annotations": []}],
+    }
+
+
+_WEATHER_CALL: Final[dict[str, JsonValue]] = {
+    "type": "function_call",
+    "id": "fc_1",
+    "call_id": "call_1",
+    "name": "get_weather",
+    "arguments": '{"city":"Paris"}',
+    "status": "completed",
+}
+
+_TWO_MESSAGES_THEN_CALL: Final = (
+    _message_item("msg_first", "First message."),
+    _message_item("msg_second", "Second message."),
+    _WEATHER_CALL,
+)
+
+
+def test_chat_over_responses_deployment_folds_two_messages_and_function_call_into_one_choice(
+    gateway: Gateway,
+) -> None:
+    identity: Final = "responses-bridge-two-messages-" + uuid.uuid4().hex
+
+    def respond(request: Request) -> Reply:
+        if request.method == "GET" and request.target == "/v1/models":
+            return Reply(body=b'{"object":"list","data":[]}')
+        assert request.method == "POST" and request.target == "/responses", request.target
+        return Reply(
+            body=json.dumps(
+                {
+                    "id": "resp_two_messages",
+                    "object": "response",
+                    "created_at": 1789788253,
+                    "status": "completed",
+                    "model": "gpt-6-sol",
+                    "output": list(_TWO_MESSAGES_THEN_CALL),
+                    "usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15},
+                }
+            ).encode()
+        )
+
+    with wire_server(respond) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(
+            model="openai/responses/gpt-6-sol", api_base=wire.url, api_key="synthetic-openai-key"
+        )
+        response: Final = gateway.request(
+            "POST",
+            "/v1/chat/completions",
+            {
+                "model": model,
+                "messages": [{"role": "user", "content": f"What is the weather in Paris? {identity}"}],
+                "tools": [_WEATHER_TOOL],
+                "cache": {"no-cache": True},
+            },
+        )
+        assert response.status_code == 200, response.text
+        body: Final = response.json()
+        assert body["choices"] == [
+            {
+                "finish_reason": "tool_calls",
+                "index": 0,
+                "message": {
+                    "role": "assistant",
+                    "content": "First message.Second message.",
+                    "tool_calls": [
+                        {
+                            "id": "fc_1",
+                            "type": "function",
+                            "function": {"name": "get_weather", "arguments": '{"city":"Paris"}'},
+                            "index": 0,
+                        }
+                    ],
+                },
+            }
+        ], response.text
+        assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/responses")]
+
+
+def _responses_frame(event: str, payload: dict[str, JsonValue]) -> bytes:
+    return f"event: {event}\ndata: {json.dumps(payload, separators=(',', ':'))}\n\n".encode()
+
+
+def _message_item_frames(output_index: int, item: dict[str, JsonValue]) -> tuple[bytes, ...]:
+    item_id: Final = item["id"]
+    text: Final = object_value(list_value(item["content"])[0])["text"]
+    return (
+        _responses_frame(
+            "response.output_item.added",
+            {
+                "type": "response.output_item.added",
+                "output_index": output_index,
+                "item": {**item, "status": "in_progress", "content": []},
+            },
+        ),
+        _responses_frame(
+            "response.output_text.delta",
+            {
+                "type": "response.output_text.delta",
+                "item_id": item_id,
+                "output_index": output_index,
+                "content_index": 0,
+                "delta": text,
+            },
+        ),
+        _responses_frame(
+            "response.output_item.done",
+            {"type": "response.output_item.done", "output_index": output_index, "item": item},
+        ),
+    )
+
+
+def _function_call_frames(output_index: int, item: dict[str, JsonValue]) -> tuple[bytes, ...]:
+    return (
+        _responses_frame(
+            "response.output_item.added",
+            {
+                "type": "response.output_item.added",
+                "output_index": output_index,
+                "item": {**item, "status": "in_progress", "arguments": ""},
+            },
+        ),
+        _responses_frame(
+            "response.function_call_arguments.delta",
+            {
+                "type": "response.function_call_arguments.delta",
+                "item_id": item["id"],
+                "output_index": output_index,
+                "delta": item["arguments"],
+            },
+        ),
+        _responses_frame(
+            "response.function_call_arguments.done",
+            {
+                "type": "response.function_call_arguments.done",
+                "item_id": item["id"],
+                "output_index": output_index,
+                "arguments": item["arguments"],
+            },
+        ),
+        _responses_frame(
+            "response.output_item.done",
+            {"type": "response.output_item.done", "output_index": output_index, "item": item},
+        ),
+    )
+
+
+def _sse_payloads(lines: list[str]) -> list[dict[str, JsonValue]]:
+    return [json.loads(line[6:]) for line in lines if line.startswith("data: ") and line != "data: [DONE]"]
+
+
+def test_chat_stream_over_responses_deployment_folds_two_messages_and_function_call_into_one_choice(
+    gateway: Gateway,
+) -> None:
+    identity: Final = "responses-bridge-two-messages-stream-" + uuid.uuid4().hex
+    response_payload: Final[dict[str, JsonValue]] = {
+        "id": "resp_two_messages_stream",
+        "object": "response",
+        "created_at": 1789788253,
+        "model": "gpt-6-sol",
+        "status": "in_progress",
+        "output": [],
+    }
+
+    def respond(request: Request) -> Reply:
+        if request.method == "GET" and request.target == "/v1/models":
+            return Reply(body=b'{"object":"list","data":[]}')
+        assert request.method == "POST" and request.target == "/responses", request.target
+        assert json.loads(request.body)["stream"] is True, request.body
+        return Reply(
+            content_type="text/event-stream",
+            chunks=(
+                _responses_frame("response.created", {"type": "response.created", "response": response_payload}),
+                *_message_item_frames(0, _TWO_MESSAGES_THEN_CALL[0]),
+                *_message_item_frames(1, _TWO_MESSAGES_THEN_CALL[1]),
+                *_function_call_frames(2, _TWO_MESSAGES_THEN_CALL[2]),
+                _responses_frame(
+                    "response.completed",
+                    {
+                        "type": "response.completed",
+                        "response": {
+                            **response_payload,
+                            "status": "completed",
+                            "output": list(_TWO_MESSAGES_THEN_CALL),
+                            "usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15},
+                        },
+                    },
+                ),
+            ),
+        )
+
+    with wire_server(respond) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(
+            model="openai/responses/gpt-6-sol", api_base=wire.url, api_key="synthetic-openai-key"
+        )
+        response: Final = gateway.request(
+            "POST",
+            "/v1/chat/completions",
+            {
+                "model": model,
+                "messages": [{"role": "user", "content": f"What is the weather in Paris? {identity}"}],
+                "tools": [_WEATHER_TOOL],
+                "stream": True,
+                "cache": {"no-cache": True},
+            },
+        )
+        assert response.status_code == 200, response.text
+        chunks: Final = _sse_payloads(list(response.iter_lines()))
+        choices: Final = [object_value(choice) for chunk in chunks for choice in list_value(chunk["choices"])]
+        assert [choice["index"] for choice in choices] == [0] * len(choices), chunks
+        deltas: Final = [object_value(choice["delta"]) for choice in choices]
+        assert (
+            "".join(str(delta["content"]) for delta in deltas if delta.get("content"))
+            == "First message.Second message."
+        )
+        tool_calls: Final = [
+            object_value(call)
+            for delta in deltas
+            if delta.get("tool_calls")
+            for call in list_value(delta["tool_calls"])
+        ]
+        assert [
+            object_value(call["function"])["name"] for call in tool_calls if object_value(call["function"]).get("name")
+        ] == ["get_weather"], chunks
+        assert (
+            "".join(str(object_value(call["function"]).get("arguments") or "") for call in tool_calls)
+            == '{"city":"Paris"}'
+        ), chunks
+        assert [choice["finish_reason"] for choice in choices if choice.get("finish_reason")] == ["tool_calls"], chunks
+        assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/responses")]

tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py

@@ -4539,3 +4539,61 @@ def test_every_bridged_chunk_after_response_created_carries_the_served_service_t
     relayed = [iterator.chunk_parser(event).model_dump().get("service_tier") for event in events]
 
     assert relayed == ["default"] * len(events), relayed
+
+
+def test_convert_response_output_folds_two_messages_and_function_call_into_one_choice() -> None:
+    first: Final = ResponseOutputMessage.model_validate(
+        {
+            "id": "msg_first",
+            "type": "message",
+            "role": "assistant",
+            "status": "completed",
+            "content": [{"type": "output_text", "text": "First message.", "annotations": []}],
+        }
+    )
+    second_annotation: Final = {
+        "type": "url_citation",
+        "start_index": 0,
+        "end_index": 6,
+        "title": "Second",
+        "url": "https://example.com/second",
+    }
+    second: Final = ResponseOutputMessage.model_validate(
+        {
+            "id": "msg_second",
+            "type": "message",
+            "role": "assistant",
+            "status": "completed",
+            "content": [{"type": "output_text", "text": "Second message.", "annotations": [second_annotation]}],
+        }
+    )
+    function_call: Final = ResponseFunctionToolCall(
+        id="fc_1",
+        type="function_call",
+        status="completed",
+        arguments='{"city":"Paris"}',
+        call_id="call_1",
+        name="get_weather",
+    )
+
+    choices: Final = LiteLLMResponsesTransformationHandler._convert_response_output_to_choices(  # pyright: ignore[reportPrivateUsage]  # exercises the converter directly
+        (first, second, function_call)
+    )
+
+    assert len(choices) == 1
+    assert choices[0].index == 0
+    assert choices[0].finish_reason == "tool_calls"
+    assert choices[0].message.content == "First message.Second message."
+    assert choices[0].message.annotations == [
+        {**second_annotation, "start_index": len("First message."), "end_index": len("First message.") + 6}
+    ]
+    tool_calls: Final = choices[0].message.tool_calls
+    assert tool_calls is not None
+    assert [call.model_dump(exclude_none=True) for call in tool_calls] == [
+        {
+            "id": "fc_1",
+            "type": "function",
+            "function": {"name": "get_weather", "arguments": '{"city":"Paris"}'},
+            "index": 0,
+        }
+    ]