Merge pull request #46949 from BerriAI/litellm_backport_responses_bridge_stable_1_103_x

fix(responses): backport #46099 and #46109 to stable/1.103.x for v1.103.5

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -5,6 +5,7 @@ Handler for transforming /chat/completions api requests to litellm.responses req
 import json
 import os
 from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence
+from itertools import accumulate, chain
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, Union, cast, get_args
 
@@ -70,7 +71,7 @@ if TYPE_CHECKING:
         ChatCompletionToolReferenceObject,
         OpenAIMessageContentListBlock,
     )
-    from litellm.types.utils import Choices
+    from litellm.types.utils import Choices, Message
 
 
 _CHAT_COMPLETION_FIELDS: Final = frozenset((*ModelResponse.model_fields, "usage"))
@@ -197,6 +198,67 @@ def _as_chat_reasoning_items(
     return cast(list[ChatCompletionReasoningItem], list(reasoning_items))
 
 
+def _shift_annotation(annotation: ChatCompletionAnnotation, offset: int) -> ChatCompletionAnnotation:
+    shifted: Final = {
+        key: value + offset if key in ("start_index", "end_index") and isinstance(value, int) else value
+        for key, value in annotation.items()
+    }
+    return cast(ChatCompletionAnnotation, shifted)
+
+
+def _message_annotations(message: "Message") -> Sequence[ChatCompletionAnnotation]:
+    annotations: Sequence[ChatCompletionAnnotation] | None = getattr(message, "annotations", None)
+    return annotations or ()
+
+
+def _message_reasoning_content(message: "Message") -> str | None:
+    reasoning_content: str | None = getattr(message, "reasoning_content", None)
+    return reasoning_content
+
+
+def _message_reasoning_items(message: "Message") -> Sequence[ChatCompletionReasoningItem]:
+    reasoning_items: Sequence[ChatCompletionReasoningItem] | None = getattr(message, "reasoning_items", None)
+    return reasoning_items or ()
+
+
+def _shifted_annotations(messages: Sequence["Message"], offsets: Sequence[int]) -> Iterator[ChatCompletionAnnotation]:
+    for message, offset in zip(messages, offsets):
+        for annotation in _message_annotations(message):
+            yield _shift_annotation(annotation, offset)
+
+
+def _fold_choices_into_one(
+    choices: Sequence["Choices"],
+    tool_calls: Sequence[Mapping[str, object]],
+    trailing_reasoning_content: str | None,
+    trailing_reasoning_item: _BuiltReasoningItem | None,
+) -> "Choices":
+    """Clients that only read choices[0] must still see every text and tool call."""
+    from litellm.types.utils import Choices, Message
+
+    messages: Final = tuple(choice.message for choice in choices)
+    texts: Final = tuple(str(message.content or "") for message in messages)
+    offsets: Final = (0, *accumulate(len(text) for text in texts))
+    annotations: Final = tuple(_shifted_annotations(messages, offsets))
+    reasoning_contents: Final = (*(_message_reasoning_content(m) for m in messages), trailing_reasoning_content)
+    reasoning_items: Final = (
+        *chain.from_iterable(_message_reasoning_items(m) for m in messages),
+        *(() if trailing_reasoning_item is None else (trailing_reasoning_item,)),
+    )
+    return Choices(
+        message=Message(
+            role=messages[0].role if messages else "assistant",
+            content="".join(texts) if messages else None,
+            annotations=list(annotations) or None,
+            tool_calls=list(tool_calls) or None,
+            reasoning_content=" ".join(value for value in reasoning_contents if value) or None,
+            reasoning_items=_as_chat_reasoning_items(reasoning_items),
+        ),
+        finish_reason="tool_calls" if tool_calls else "stop",
+        index=0,
+    )
+
+
 def _map_incomplete_reason_to_finish_reason(incomplete_reason: str | None) -> Literal["length", "content_filter"]:
     if incomplete_reason == "content_filter":
         return "content_filter"
@@ -782,47 +844,16 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             else:
                 pass  # don't fail request if item in list is not supported
 
-        if accumulated_tool_calls and choices:
-            last_choice: Final = choices[-1]
-            last_reasoning_content: Final = getattr(last_choice.message, "reasoning_content", None)
-            last_reasoning_items: Final = getattr(last_choice.message, "reasoning_items", None)
-            merged_reasoning_content: Final = (
-                " ".join(value for value in (last_reasoning_content, reasoning_content) if value) or None
-            )
-            merged_reasoning_items: Final = _as_chat_reasoning_items(
-                (
-                    *(last_reasoning_items or ()),
-                    *(() if pending_reasoning_item is None else (pending_reasoning_item,)),
-                )
-            )
-            merged_message: Final = Message(
-                role=last_choice.message.role,
-                content=last_choice.message.content,
-                annotations=getattr(last_choice.message, "annotations", None),
+        if not choices and not accumulated_tool_calls:
+            return choices
+        return [
+            _fold_choices_into_one(
+                choices=choices,
                 tool_calls=accumulated_tool_calls,
-                reasoning_content=merged_reasoning_content,
-                reasoning_items=merged_reasoning_items,
+                trailing_reasoning_content=reasoning_content,
+                trailing_reasoning_item=pending_reasoning_item,
             )
-            return [
-                *choices[:-1],
-                Choices(message=merged_message, finish_reason="tool_calls", index=last_choice.index),
-            ]
-
-        if accumulated_tool_calls:
-            msg = Message(
-                content=None,
-                tool_calls=accumulated_tool_calls,
-                reasoning_content=reasoning_content,
-                reasoning_items=cast(
-                    list[ChatCompletionReasoningItem] | None,
-                    ([pending_reasoning_item] if pending_reasoning_item is not None else None),
-                ),
-            )
-            choices.append(Choices(message=msg, finish_reason="tool_calls", index=index))
-            reasoning_content = None
-            pending_reasoning_item = None
-
-        return choices
+        ]
 
     @staticmethod
     def _build_empty_incomplete_choice(

litellm/responses/litellm_completion_transformation/transformation.py

@@ -2374,7 +2374,7 @@ class LiteLLMCompletionResponsesConfig:
             ),
             "function": function_dict,
             "type": "function",
-            "index": 0,
+            "index": index,
         }
 
         if provider_specific_fields:

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm"
-version = "1.103.4"
+version = "1.103.5"
 description = "Library to easily interface with LLM API providers"
 readme = "README.md"
 requires-python = ">=3.10, <3.15"
@@ -342,7 +342,7 @@ litellm-enterprise = { workspace = true }
 members = ["enterprise", "litellm-proxy-extras"]
 
 [tool.commitizen]
-version = "1.103.4"
+version = "1.103.5"
 version_files = [
     "pyproject.toml:^version",
 ]

tests/integration/_support/client.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import os
 import time
 import uuid
-from collections.abc import Callable, Iterator, Mapping
+from collections.abc import Callable, Iterator, Mapping, Sequence
 from contextlib import ExitStack, contextmanager
 from dataclasses import dataclass
 from hashlib import sha256
@@ -22,6 +22,11 @@ def object_value(value: JsonValue) -> dict[str, JsonValue]:
     return JSON_OBJECT.validate_python(value)
 
 
+def list_value(value: JsonValue) -> Sequence[JsonValue]:
+    assert isinstance(value, list), f"Expected a list, received {type(value).__name__}"
+    return value
+
+
 def string_value(value: JsonValue) -> str:
     assert isinstance(value, str), f"Expected a string, received {type(value).__name__}"
     return value

tests/integration/contracts.json

@@ -1605,6 +1605,30 @@
     ],
     "tests/integration/spend/test_stream_alias_billing.py::test_streamed_alias_matching_a_capability_rule_bills_the_deployment_price[anthropic-upstream]": [
       "quota_management.spend_tracking.streamed_alias_bills_deployment_price"
+    ],
+    "tests/integration/providers/test_responses_bridge_incomplete.py::test_chat_over_responses_deployment_folds_two_messages_and_function_call_into_one_choice": [
+      "other.provider_wire.responses_bridge.two_messages_and_function_call_fold_into_one_choice"
+    ],
+    "tests/integration/providers/test_responses_bridge_incomplete.py::test_chat_over_responses_deployment_folds_commentary_and_final_answer_into_one_choice": [
+      "other.provider_wire.responses_bridge.commentary_and_final_answer_fold_into_one_choice"
+    ],
+    "tests/integration/providers/test_responses_bridge_incomplete.py::test_chat_over_responses_deployment_folds_reasoning_two_messages_and_function_call_into_one_choice": [
+      "other.provider_wire.responses_bridge.reasoning_messages_and_call_fold_into_one_choice"
+    ],
+    "tests/integration/providers/test_responses_bridge_incomplete.py::test_chat_over_responses_deployment_folds_message_and_two_function_calls_into_one_choice": [
+      "other.provider_wire.responses_bridge.message_and_two_calls_fold_numbered_0_1"
+    ],
+    "tests/integration/providers/test_responses_bridge_incomplete.py::test_chat_stream_over_responses_deployment_folds_two_messages_and_function_call_into_one_choice": [
+      "other.provider_wire.responses_bridge.stream_two_messages_and_call_fold_into_one_choice"
+    ],
+    "tests/integration/providers/test_responses_bridge_incomplete.py::test_chat_stream_over_responses_deployment_folds_commentary_and_final_answer_into_one_choice": [
+      "other.provider_wire.responses_bridge.stream_commentary_and_final_answer_fold_into_one_choice"
+    ],
+    "tests/integration/providers/test_responses_bridge_incomplete.py::test_chat_stream_over_responses_deployment_folds_reasoning_two_messages_and_function_call_into_one_choice": [
+      "other.provider_wire.responses_bridge.stream_reasoning_messages_and_call_fold_into_one_choice"
+    ],
+    "tests/integration/providers/test_responses_bridge_incomplete.py::test_chat_stream_over_responses_deployment_folds_message_and_two_function_calls_into_one_choice": [
+      "other.provider_wire.responses_bridge.stream_message_and_two_calls_fold_numbered_0_1"
     ]
   },
   "browser": {

tests/integration/providers/test_responses_bridge_incomplete.py

@@ -0,0 +1,740 @@
+import json
+import uuid
+from typing import Final
+
+import httpx
+import pytest
+from integration._support.client import Gateway, list_value, object_value
+from integration._support.wire import Reply, Request, wire_server
+from pydantic import JsonValue
+
+
+_WEATHER_TOOL: Final[dict[str, JsonValue]] = {
+    "type": "function",
+    "function": {
+        "name": "get_weather",
+        "description": "Get the weather for a city.",
+        "parameters": {
+            "type": "object",
+            "properties": {"city": {"type": "string"}},
+            "required": ["city"],
+        },
+    },
+}
+
+
+def _message_item(item_id: str, text: str, phase: str | None = None) -> dict[str, JsonValue]:
+    return {
+        "type": "message",
+        "id": item_id,
+        "status": "completed",
+        "role": "assistant",
+        "content": [{"type": "output_text", "text": text, "annotations": []}],
+        **({} if phase is None else {"phase": phase}),
+    }
+
+
+_WEATHER_CALL: Final[dict[str, JsonValue]] = {
+    "type": "function_call",
+    "id": "fc_1",
+    "call_id": "call_1",
+    "name": "get_weather",
+    "arguments": '{"city":"Paris"}',
+    "status": "completed",
+}
+
+_TWO_MESSAGES_THEN_CALL: Final = (
+    _message_item("msg_first", "First message."),
+    _message_item("msg_second", "Second message."),
+    _WEATHER_CALL,
+)
+
+
+@pytest.mark.covers("other.provider_wire.responses_bridge.two_messages_and_function_call_fold_into_one_choice")
+def test_chat_over_responses_deployment_folds_two_messages_and_function_call_into_one_choice(
+    gateway: Gateway,
+) -> None:
+    identity: Final = "responses-bridge-two-messages-" + uuid.uuid4().hex
+
+    def respond(request: Request) -> Reply:
+        if request.method == "GET" and request.target == "/v1/models":
+            return Reply(body=b'{"object":"list","data":[]}')
+        assert request.method == "POST" and request.target == "/responses", request.target
+        return Reply(
+            body=json.dumps(
+                {
+                    "id": "resp_two_messages",
+                    "object": "response",
+                    "created_at": 1789788253,
+                    "status": "completed",
+                    "model": "gpt-6-sol",
+                    "output": list(_TWO_MESSAGES_THEN_CALL),
+                    "usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15},
+                }
+            ).encode()
+        )
+
+    with wire_server(respond) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(
+            model="openai/responses/gpt-6-sol", api_base=wire.url, api_key="synthetic-openai-key"
+        )
+        response: Final = gateway.request(
+            "POST",
+            "/v1/chat/completions",
+            {
+                "model": model,
+                "messages": [{"role": "user", "content": f"What is the weather in Paris? {identity}"}],
+                "tools": [_WEATHER_TOOL],
+                "cache": {"no-cache": True},
+            },
+        )
+        assert response.status_code == 200, response.text
+        body: Final = response.json()
+        assert body["choices"] == [
+            {
+                "finish_reason": "tool_calls",
+                "index": 0,
+                "message": {
+                    "role": "assistant",
+                    "content": "First message.Second message.",
+                    "tool_calls": [
+                        {
+                            "id": "fc_1",
+                            "type": "function",
+                            "function": {"name": "get_weather", "arguments": '{"city":"Paris"}'},
+                            "index": 0,
+                        }
+                    ],
+                },
+            }
+        ], response.text
+        assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/responses")]
+
+
+_COMMENTARY_THEN_FINAL_ANSWER: Final = (
+    _message_item("msg_commentary", "Let me answer that.", phase="commentary"),
+    _message_item("msg_final", '{"city": "Paris", "summary": "Sunny, 21C"}', phase="final_answer"),
+)
+
+
+@pytest.mark.covers("other.provider_wire.responses_bridge.commentary_and_final_answer_fold_into_one_choice")
+def test_chat_over_responses_deployment_folds_commentary_and_final_answer_into_one_choice(
+    gateway: Gateway,
+) -> None:
+    identity: Final = "responses-bridge-commentary-final-" + uuid.uuid4().hex
+
+    def respond(request: Request) -> Reply:
+        if request.method == "GET" and request.target == "/v1/models":
+            return Reply(body=b'{"object":"list","data":[]}')
+        assert request.method == "POST" and request.target == "/responses", request.target
+        return Reply(
+            body=json.dumps(
+                {
+                    "id": "resp_commentary_final",
+                    "object": "response",
+                    "created_at": 1789788253,
+                    "status": "completed",
+                    "model": "gpt-6-sol",
+                    "output": list(_COMMENTARY_THEN_FINAL_ANSWER),
+                    "usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15},
+                }
+            ).encode()
+        )
+
+    with wire_server(respond) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(
+            model="openai/responses/gpt-6-sol", api_base=wire.url, api_key="synthetic-openai-key"
+        )
+        response: Final = gateway.request(
+            "POST",
+            "/v1/chat/completions",
+            {
+                "model": model,
+                "messages": [{"role": "user", "content": f"Weather in Paris as JSON. {identity}"}],
+                "tools": [_WEATHER_TOOL],
+                "cache": {"no-cache": True},
+            },
+        )
+        assert response.status_code == 200, response.text
+        body: Final = response.json()
+        assert body["choices"] == [
+            {
+                "finish_reason": "stop",
+                "index": 0,
+                "message": {
+                    "role": "assistant",
+                    "content": 'Let me answer that.{"city": "Paris", "summary": "Sunny, 21C"}',
+                },
+            }
+        ], response.text
+        assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/responses")]
+
+
+_REASONING_ITEM: Final[dict[str, JsonValue]] = {
+    "type": "reasoning",
+    "id": "rs_fold",
+    "summary": [{"type": "summary_text", "text": "Checking the forecast."}],
+}
+
+_TIME_TOOL: Final[dict[str, JsonValue]] = {
+    "type": "function",
+    "function": {
+        "name": "get_time",
+        "description": "Get the local time for a city.",
+        "parameters": {
+            "type": "object",
+            "properties": {"city": {"type": "string"}},
+            "required": ["city"],
+        },
+    },
+}
+
+_TIME_CALL: Final[dict[str, JsonValue]] = {
+    "type": "function_call",
+    "id": "fc_2",
+    "call_id": "call_2",
+    "name": "get_time",
+    "arguments": '{"city":"Paris","tz":"CET"}',
+    "status": "completed",
+}
+
+_REASONING_THEN_TWO_MESSAGES_THEN_CALL: Final = (
+    _REASONING_ITEM,
+    _message_item("msg_first", "First message."),
+    _message_item("msg_second", "Second message."),
+    _WEATHER_CALL,
+)
+
+_MESSAGE_THEN_TWO_CALLS: Final = (
+    _message_item("msg_only", "Checking both."),
+    _WEATHER_CALL,
+    _TIME_CALL,
+)
+
+_TWO_CALLS_CHAT_SHAPE: Final = [
+    {
+        "id": "fc_1",
+        "type": "function",
+        "function": {"name": "get_weather", "arguments": '{"city":"Paris"}'},
+        "index": 0,
+    },
+    {
+        "id": "fc_2",
+        "type": "function",
+        "function": {"name": "get_time", "arguments": '{"city":"Paris","tz":"CET"}'},
+        "index": 1,
+    },
+]
+
+
+def _responses_body(response_id: str, output: tuple[dict[str, JsonValue], ...]) -> bytes:
+    return json.dumps(
+        {
+            "id": response_id,
+            "object": "response",
+            "created_at": 1789788253,
+            "status": "completed",
+            "model": "gpt-6-sol",
+            "output": list(output),
+            "usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15},
+        }
+    ).encode()
+
+
+@pytest.mark.covers("other.provider_wire.responses_bridge.reasoning_messages_and_call_fold_into_one_choice")
+def test_chat_over_responses_deployment_folds_reasoning_two_messages_and_function_call_into_one_choice(
+    gateway: Gateway,
+) -> None:
+    identity: Final = "responses-bridge-reasoning-two-messages-" + uuid.uuid4().hex
+
+    def respond(request: Request) -> Reply:
+        if request.method == "GET" and request.target == "/v1/models":
+            return Reply(body=b'{"object":"list","data":[]}')
+        assert request.method == "POST" and request.target == "/responses", request.target
+        return Reply(body=_responses_body("resp_reasoning_two_messages", _REASONING_THEN_TWO_MESSAGES_THEN_CALL))
+
+    with wire_server(respond) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(
+            model="openai/responses/gpt-6-sol", api_base=wire.url, api_key="synthetic-openai-key"
+        )
+        response: Final = gateway.request(
+            "POST",
+            "/v1/chat/completions",
+            {
+                "model": model,
+                "messages": [{"role": "user", "content": f"What is the weather in Paris? {identity}"}],
+                "tools": [_WEATHER_TOOL],
+                "cache": {"no-cache": True},
+            },
+        )
+        assert response.status_code == 200, response.text
+        body: Final = response.json()
+        assert body["choices"] == [
+            {
+                "finish_reason": "tool_calls",
+                "index": 0,
+                "message": {
+                    "role": "assistant",
+                    "content": "First message.Second message.",
+                    "reasoning_content": "Checking the forecast.",
+                    "reasoning_items": [_REASONING_ITEM],
+                    "tool_calls": _TWO_CALLS_CHAT_SHAPE[:1],
+                },
+            }
+        ], response.text
+        assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/responses")]
+
+
+@pytest.mark.covers("other.provider_wire.responses_bridge.message_and_two_calls_fold_numbered_0_1")
+def test_chat_over_responses_deployment_folds_message_and_two_function_calls_into_one_choice(
+    gateway: Gateway,
+) -> None:
+    identity: Final = "responses-bridge-two-calls-" + uuid.uuid4().hex
+
+    def respond(request: Request) -> Reply:
+        if request.method == "GET" and request.target == "/v1/models":
+            return Reply(body=b'{"object":"list","data":[]}')
+        assert request.method == "POST" and request.target == "/responses", request.target
+        return Reply(body=_responses_body("resp_two_calls", _MESSAGE_THEN_TWO_CALLS))
+
+    with wire_server(respond) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(
+            model="openai/responses/gpt-6-sol", api_base=wire.url, api_key="synthetic-openai-key"
+        )
+        response: Final = gateway.request(
+            "POST",
+            "/v1/chat/completions",
+            {
+                "model": model,
+                "messages": [{"role": "user", "content": f"Weather and time in Paris? {identity}"}],
+                "tools": [_WEATHER_TOOL, _TIME_TOOL],
+                "cache": {"no-cache": True},
+            },
+        )
+        assert response.status_code == 200, response.text
+        body: Final = response.json()
+        assert body["choices"] == [
+            {
+                "finish_reason": "tool_calls",
+                "index": 0,
+                "message": {
+                    "role": "assistant",
+                    "content": "Checking both.",
+                    "tool_calls": _TWO_CALLS_CHAT_SHAPE,
+                },
+            }
+        ], response.text
+        assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/responses")]
+
+
+def _responses_frame(event: str, payload: dict[str, JsonValue]) -> bytes:
+    return f"event: {event}\ndata: {json.dumps(payload, separators=(',', ':'))}\n\n".encode()
+
+
+def _message_item_frames(output_index: int, item: dict[str, JsonValue]) -> tuple[bytes, ...]:
+    item_id: Final = item["id"]
+    text: Final = object_value(list_value(item["content"])[0])["text"]
+    return (
+        _responses_frame(
+            "response.output_item.added",
+            {
+                "type": "response.output_item.added",
+                "output_index": output_index,
+                "item": {**item, "status": "in_progress", "content": []},
+            },
+        ),
+        _responses_frame(
+            "response.output_text.delta",
+            {
+                "type": "response.output_text.delta",
+                "item_id": item_id,
+                "output_index": output_index,
+                "content_index": 0,
+                "delta": text,
+            },
+        ),
+        _responses_frame(
+            "response.output_item.done",
+            {"type": "response.output_item.done", "output_index": output_index, "item": item},
+        ),
+    )
+
+
+def _function_call_frames(output_index: int, item: dict[str, JsonValue]) -> tuple[bytes, ...]:
+    return (
+        _responses_frame(
+            "response.output_item.added",
+            {
+                "type": "response.output_item.added",
+                "output_index": output_index,
+                "item": {**item, "status": "in_progress", "arguments": ""},
+            },
+        ),
+        _responses_frame(
+            "response.function_call_arguments.delta",
+            {
+                "type": "response.function_call_arguments.delta",
+                "item_id": item["id"],
+                "output_index": output_index,
+                "delta": item["arguments"],
+            },
+        ),
+        _responses_frame(
+            "response.function_call_arguments.done",
+            {
+                "type": "response.function_call_arguments.done",
+                "item_id": item["id"],
+                "output_index": output_index,
+                "arguments": item["arguments"],
+            },
+        ),
+        _responses_frame(
+            "response.output_item.done",
+            {"type": "response.output_item.done", "output_index": output_index, "item": item},
+        ),
+    )
+
+
+def _reasoning_item_frames(output_index: int, item: dict[str, JsonValue]) -> tuple[bytes, ...]:
+    text: Final = object_value(list_value(item["summary"])[0])["text"]
+    return (
+        _responses_frame(
+            "response.output_item.added",
+            {
+                "type": "response.output_item.added",
+                "output_index": output_index,
+                "item": {**item, "summary": []},
+            },
+        ),
+        _responses_frame(
+            "response.reasoning_summary_text.delta",
+            {
+                "type": "response.reasoning_summary_text.delta",
+                "item_id": item["id"],
+                "output_index": output_index,
+                "summary_index": 0,
+                "delta": text,
+            },
+        ),
+        _responses_frame(
+            "response.output_item.done",
+            {"type": "response.output_item.done", "output_index": output_index, "item": item},
+        ),
+    )
+
+
+def _sse_payloads(lines: list[str]) -> list[dict[str, JsonValue]]:
+    return [json.loads(line[6:]) for line in lines if line.startswith("data: ") and line != "data: [DONE]"]
+
+
+@pytest.mark.covers("other.provider_wire.responses_bridge.stream_two_messages_and_call_fold_into_one_choice")
+def test_chat_stream_over_responses_deployment_folds_two_messages_and_function_call_into_one_choice(
+    gateway: Gateway,
+) -> None:
+    identity: Final = "responses-bridge-two-messages-stream-" + uuid.uuid4().hex
+    response_payload: Final[dict[str, JsonValue]] = {
+        "id": "resp_two_messages_stream",
+        "object": "response",
+        "created_at": 1789788253,
+        "model": "gpt-6-sol",
+        "status": "in_progress",
+        "output": [],
+    }
+
+    def respond(request: Request) -> Reply:
+        if request.method == "GET" and request.target == "/v1/models":
+            return Reply(body=b'{"object":"list","data":[]}')
+        assert request.method == "POST" and request.target == "/responses", request.target
+        assert json.loads(request.body)["stream"] is True, request.body
+        return Reply(
+            content_type="text/event-stream",
+            chunks=(
+                _responses_frame("response.created", {"type": "response.created", "response": response_payload}),
+                *_message_item_frames(0, _TWO_MESSAGES_THEN_CALL[0]),
+                *_message_item_frames(1, _TWO_MESSAGES_THEN_CALL[1]),
+                *_function_call_frames(2, _TWO_MESSAGES_THEN_CALL[2]),
+                _responses_frame(
+                    "response.completed",
+                    {
+                        "type": "response.completed",
+                        "response": {
+                            **response_payload,
+                            "status": "completed",
+                            "output": list(_TWO_MESSAGES_THEN_CALL),
+                            "usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15},
+                        },
+                    },
+                ),
+            ),
+        )
+
+    with wire_server(respond) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(
+            model="openai/responses/gpt-6-sol", api_base=wire.url, api_key="synthetic-openai-key"
+        )
+        response: Final = gateway.request(
+            "POST",
+            "/v1/chat/completions",
+            {
+                "model": model,
+                "messages": [{"role": "user", "content": f"What is the weather in Paris? {identity}"}],
+                "tools": [_WEATHER_TOOL],
+                "stream": True,
+                "cache": {"no-cache": True},
+            },
+        )
+        assert response.status_code == 200, response.text
+        chunks: Final = _sse_payloads(list(response.iter_lines()))
+        choices: Final = [object_value(choice) for chunk in chunks for choice in list_value(chunk["choices"])]
+        assert [choice["index"] for choice in choices] == [0] * len(choices), chunks
+        deltas: Final = [object_value(choice["delta"]) for choice in choices]
+        assert (
+            "".join(str(delta["content"]) for delta in deltas if delta.get("content"))
+            == "First message.Second message."
+        )
+        tool_calls: Final = [
+            object_value(call)
+            for delta in deltas
+            if delta.get("tool_calls")
+            for call in list_value(delta["tool_calls"])
+        ]
+        assert [
+            object_value(call["function"])["name"] for call in tool_calls if object_value(call["function"]).get("name")
+        ] == ["get_weather"], chunks
+        assert (
+            "".join(str(object_value(call["function"]).get("arguments") or "") for call in tool_calls)
+            == '{"city":"Paris"}'
+        ), chunks
+        assert [choice["finish_reason"] for choice in choices if choice.get("finish_reason")] == ["tool_calls"], chunks
+        assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/responses")]
+
+
+@pytest.mark.covers("other.provider_wire.responses_bridge.stream_commentary_and_final_answer_fold_into_one_choice")
+def test_chat_stream_over_responses_deployment_folds_commentary_and_final_answer_into_one_choice(
+    gateway: Gateway,
+) -> None:
+    identity: Final = "responses-bridge-commentary-final-stream-" + uuid.uuid4().hex
+    response_payload: Final[dict[str, JsonValue]] = {
+        "id": "resp_commentary_final_stream",
+        "object": "response",
+        "created_at": 1789788253,
+        "model": "gpt-6-sol",
+        "status": "in_progress",
+        "output": [],
+    }
+
+    def respond(request: Request) -> Reply:
+        if request.method == "GET" and request.target == "/v1/models":
+            return Reply(body=b'{"object":"list","data":[]}')
+        assert request.method == "POST" and request.target == "/responses", request.target
+        assert json.loads(request.body)["stream"] is True, request.body
+        return Reply(
+            content_type="text/event-stream",
+            chunks=(
+                _responses_frame("response.created", {"type": "response.created", "response": response_payload}),
+                *_message_item_frames(0, _COMMENTARY_THEN_FINAL_ANSWER[0]),
+                *_message_item_frames(1, _COMMENTARY_THEN_FINAL_ANSWER[1]),
+                _responses_frame(
+                    "response.completed",
+                    {
+                        "type": "response.completed",
+                        "response": {
+                            **response_payload,
+                            "status": "completed",
+                            "output": list(_COMMENTARY_THEN_FINAL_ANSWER),
+                            "usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15},
+                        },
+                    },
+                ),
+            ),
+        )
+
+    with wire_server(respond) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(
+            model="openai/responses/gpt-6-sol", api_base=wire.url, api_key="synthetic-openai-key"
+        )
+        response: Final = gateway.request(
+            "POST",
+            "/v1/chat/completions",
+            {
+                "model": model,
+                "messages": [{"role": "user", "content": f"Weather in Paris as JSON. {identity}"}],
+                "tools": [_WEATHER_TOOL],
+                "stream": True,
+                "cache": {"no-cache": True},
+            },
+        )
+        assert response.status_code == 200, response.text
+        chunks: Final = _sse_payloads(list(response.iter_lines()))
+        choices: Final = [object_value(choice) for chunk in chunks for choice in list_value(chunk["choices"])]
+        assert [choice["index"] for choice in choices] == [0] * len(choices), chunks
+        deltas: Final = [object_value(choice["delta"]) for choice in choices]
+        assert (
+            "".join(str(delta["content"]) for delta in deltas if delta.get("content"))
+            == 'Let me answer that.{"city": "Paris", "summary": "Sunny, 21C"}'
+        ), chunks
+        assert not [delta for delta in deltas if delta.get("tool_calls")], chunks
+        assert [choice["finish_reason"] for choice in choices if choice.get("finish_reason")] == ["stop"], chunks
+        assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/responses")]
+
+
+def _stream_reply(response_id: str, output: tuple[dict[str, JsonValue], ...], frames: tuple[bytes, ...]) -> Reply:
+    response_payload: Final[dict[str, JsonValue]] = {
+        "id": response_id,
+        "object": "response",
+        "created_at": 1789788253,
+        "model": "gpt-6-sol",
+        "status": "in_progress",
+        "output": [],
+    }
+    return Reply(
+        content_type="text/event-stream",
+        chunks=(
+            _responses_frame("response.created", {"type": "response.created", "response": response_payload}),
+            *frames,
+            _responses_frame(
+                "response.completed",
+                {
+                    "type": "response.completed",
+                    "response": {
+                        **response_payload,
+                        "status": "completed",
+                        "output": list(output),
+                        "usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15},
+                    },
+                },
+            ),
+        ),
+    )
+
+
+def _stream_choices(response: httpx.Response) -> list[dict[str, JsonValue]]:
+    chunks: Final = _sse_payloads(list(response.iter_lines()))
+    return [object_value(choice) for chunk in chunks for choice in list_value(chunk["choices"])]
+
+
+def _stream_tool_calls(deltas: list[dict[str, JsonValue]]) -> list[dict[str, JsonValue]]:
+    return [
+        object_value(call) for delta in deltas if delta.get("tool_calls") for call in list_value(delta["tool_calls"])
+    ]
+
+
+@pytest.mark.covers("other.provider_wire.responses_bridge.stream_reasoning_messages_and_call_fold_into_one_choice")
+def test_chat_stream_over_responses_deployment_folds_reasoning_two_messages_and_function_call_into_one_choice(
+    gateway: Gateway,
+) -> None:
+    identity: Final = "responses-bridge-reasoning-two-messages-stream-" + uuid.uuid4().hex
+
+    def respond(request: Request) -> Reply:
+        if request.method == "GET" and request.target == "/v1/models":
+            return Reply(body=b'{"object":"list","data":[]}')
+        assert request.method == "POST" and request.target == "/responses", request.target
+        assert json.loads(request.body)["stream"] is True, request.body
+        return _stream_reply(
+            "resp_reasoning_two_messages_stream",
+            _REASONING_THEN_TWO_MESSAGES_THEN_CALL,
+            (
+                *_reasoning_item_frames(0, _REASONING_THEN_TWO_MESSAGES_THEN_CALL[0]),
+                *_message_item_frames(1, _REASONING_THEN_TWO_MESSAGES_THEN_CALL[1]),
+                *_message_item_frames(2, _REASONING_THEN_TWO_MESSAGES_THEN_CALL[2]),
+                *_function_call_frames(3, _REASONING_THEN_TWO_MESSAGES_THEN_CALL[3]),
+            ),
+        )
+
+    with wire_server(respond) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(
+            model="openai/responses/gpt-6-sol", api_base=wire.url, api_key="synthetic-openai-key"
+        )
+        response: Final = gateway.request(
+            "POST",
+            "/v1/chat/completions",
+            {
+                "model": model,
+                "messages": [{"role": "user", "content": f"What is the weather in Paris? {identity}"}],
+                "tools": [_WEATHER_TOOL],
+                "stream": True,
+                "cache": {"no-cache": True},
+            },
+        )
+        assert response.status_code == 200, response.text
+        choices: Final = _stream_choices(response)
+        assert [choice["index"] for choice in choices] == [0] * len(choices), choices
+        deltas: Final = [object_value(choice["delta"]) for choice in choices]
+        assert (
+            "".join(str(delta["reasoning_content"]) for delta in deltas if delta.get("reasoning_content"))
+            == "Checking the forecast."
+        ), choices
+        assert (
+            "".join(str(delta["content"]) for delta in deltas if delta.get("content"))
+            == "First message.Second message."
+        ), choices
+        tool_calls: Final = _stream_tool_calls(deltas)
+        assert [
+            object_value(call["function"])["name"] for call in tool_calls if object_value(call["function"]).get("name")
+        ] == ["get_weather"], choices
+        assert (
+            "".join(str(object_value(call["function"]).get("arguments") or "") for call in tool_calls)
+            == '{"city":"Paris"}'
+        ), choices
+        assert [choice["finish_reason"] for choice in choices if choice.get("finish_reason")] == ["tool_calls"], choices
+        assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/responses")]
+
+
+@pytest.mark.covers("other.provider_wire.responses_bridge.stream_message_and_two_calls_fold_numbered_0_1")
+def test_chat_stream_over_responses_deployment_folds_message_and_two_function_calls_into_one_choice(
+    gateway: Gateway,
+) -> None:
+    identity: Final = "responses-bridge-two-calls-stream-" + uuid.uuid4().hex
+
+    def respond(request: Request) -> Reply:
+        if request.method == "GET" and request.target == "/v1/models":
+            return Reply(body=b'{"object":"list","data":[]}')
+        assert request.method == "POST" and request.target == "/responses", request.target
+        assert json.loads(request.body)["stream"] is True, request.body
+        return _stream_reply(
+            "resp_two_calls_stream",
+            _MESSAGE_THEN_TWO_CALLS,
+            (
+                *_message_item_frames(0, _MESSAGE_THEN_TWO_CALLS[0]),
+                *_function_call_frames(1, _MESSAGE_THEN_TWO_CALLS[1]),
+                *_function_call_frames(2, _MESSAGE_THEN_TWO_CALLS[2]),
+            ),
+        )
+
+    with wire_server(respond) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(
+            model="openai/responses/gpt-6-sol", api_base=wire.url, api_key="synthetic-openai-key"
+        )
+        response: Final = gateway.request(
+            "POST",
+            "/v1/chat/completions",
+            {
+                "model": model,
+                "messages": [{"role": "user", "content": f"Weather and time in Paris? {identity}"}],
+                "tools": [_WEATHER_TOOL, _TIME_TOOL],
+                "stream": True,
+                "cache": {"no-cache": True},
+            },
+        )
+        assert response.status_code == 200, response.text
+        choices: Final = _stream_choices(response)
+        assert [choice["index"] for choice in choices] == [0] * len(choices), choices
+        deltas: Final = [object_value(choice["delta"]) for choice in choices]
+        assert "".join(str(delta["content"]) for delta in deltas if delta.get("content")) == "Checking both.", choices
+        tool_calls: Final = _stream_tool_calls(deltas)
+        assert [
+            (call["index"], object_value(call["function"])["name"])
+            for call in tool_calls
+            if object_value(call["function"]).get("name")
+        ] == [(0, "get_weather"), (1, "get_time")], choices
+        arguments_by_index: Final = {
+            index: "".join(
+                str(object_value(call["function"]).get("arguments") or "")
+                for call in tool_calls
+                if call["index"] == index
+            )
+            for index in (0, 1)
+        }
+        assert arguments_by_index == {0: '{"city":"Paris"}', 1: '{"city":"Paris","tz":"CET"}'}, choices
+        assert [choice["finish_reason"] for choice in choices if choice.get("finish_reason")] == ["tool_calls"], choices
+        assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/responses")]

tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py

@@ -4471,3 +4471,131 @@ def test_map_optional_params_verbosity_merges_into_text():
         verbosity_only_request,
     )
     assert verbosity_only_request["text"] == {"verbosity": "low"}
+
+
+def test_convert_response_output_folds_two_messages_and_function_call_into_one_choice() -> None:
+    first: Final = ResponseOutputMessage.model_validate(
+        {
+            "id": "msg_first",
+            "type": "message",
+            "role": "assistant",
+            "status": "completed",
+            "content": [{"type": "output_text", "text": "First message.", "annotations": []}],
+        }
+    )
+    second_annotation: Final = {
+        "type": "url_citation",
+        "start_index": 0,
+        "end_index": 6,
+        "title": "Second",
+        "url": "https://example.com/second",
+    }
+    second: Final = ResponseOutputMessage.model_validate(
+        {
+            "id": "msg_second",
+            "type": "message",
+            "role": "assistant",
+            "status": "completed",
+            "content": [{"type": "output_text", "text": "Second message.", "annotations": [second_annotation]}],
+        }
+    )
+    function_call: Final = ResponseFunctionToolCall(
+        id="fc_1",
+        type="function_call",
+        status="completed",
+        arguments='{"city":"Paris"}',
+        call_id="call_1",
+        name="get_weather",
+    )
+
+    choices: Final = LiteLLMResponsesTransformationHandler._convert_response_output_to_choices(  # pyright: ignore[reportPrivateUsage]  # exercises the converter directly
+        (first, second, function_call)
+    )
+
+    assert len(choices) == 1
+    assert choices[0].index == 0
+    assert choices[0].finish_reason == "tool_calls"
+    assert choices[0].message.content == "First message.Second message."
+    assert choices[0].message.annotations == [
+        {**second_annotation, "start_index": len("First message."), "end_index": len("First message.") + 6}
+    ]
+    tool_calls: Final = choices[0].message.tool_calls
+    assert tool_calls is not None
+    assert [call.model_dump(exclude_none=True) for call in tool_calls] == [
+        {
+            "id": "fc_1",
+            "type": "function",
+            "function": {"name": "get_weather", "arguments": '{"city":"Paris"}'},
+            "index": 0,
+        }
+    ]
+
+
+def test_convert_response_output_folds_commentary_and_final_answer_into_one_choice() -> None:
+    commentary: Final = ResponseOutputMessage.model_validate(
+        {
+            "id": "msg_commentary",
+            "type": "message",
+            "role": "assistant",
+            "status": "completed",
+            "phase": "commentary",
+            "content": [{"type": "output_text", "text": "Let me answer that.", "annotations": []}],
+        }
+    )
+    final_answer: Final = ResponseOutputMessage.model_validate(
+        {
+            "id": "msg_final",
+            "type": "message",
+            "role": "assistant",
+            "status": "completed",
+            "phase": "final_answer",
+            "content": [{"type": "output_text", "text": '{"city": "Paris"}', "annotations": []}],
+        }
+    )
+
+    choices: Final = LiteLLMResponsesTransformationHandler._convert_response_output_to_choices(  # pyright: ignore[reportPrivateUsage]  # exercises the converter directly
+        (commentary, final_answer)
+    )
+
+    assert len(choices) == 1
+    assert choices[0].index == 0
+    assert choices[0].finish_reason == "stop"
+    assert choices[0].message.content == 'Let me answer that.{"city": "Paris"}'
+    assert choices[0].message.tool_calls is None
+
+
+def test_convert_response_output_numbers_two_function_calls_zero_and_one() -> None:
+    message: Final = ResponseOutputMessage.model_validate(
+        {
+            "id": "msg_only",
+            "type": "message",
+            "role": "assistant",
+            "status": "completed",
+            "content": [{"type": "output_text", "text": "Checking both.", "annotations": []}],
+        }
+    )
+    weather: Final = ResponseFunctionToolCall(
+        id="fc_1",
+        type="function_call",
+        status="completed",
+        arguments='{"city":"Paris"}',
+        call_id="call_1",
+        name="get_weather",
+    )
+    time: Final = ResponseFunctionToolCall(
+        id="fc_2",
+        type="function_call",
+        status="completed",
+        arguments='{"city":"Paris"}',
+        call_id="call_2",
+        name="get_time",
+    )
+
+    choices: Final = LiteLLMResponsesTransformationHandler._convert_response_output_to_choices(  # pyright: ignore[reportPrivateUsage]  # the fold is the unit under test
+        (message, weather, time)
+    )
+
+    assert len(choices) == 1
+    tool_calls: Final = choices[0].message.tool_calls
+    assert tool_calls is not None
+    assert [(call.index, call.function.name) for call in tool_calls] == [(0, "get_weather"), (1, "get_time")]

tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py

@@ -3,6 +3,7 @@ from copy import deepcopy
 from typing import Final, Literal
 
 import pytest
+from openai.types.responses import ResponseFunctionToolCall
 from openai.types.responses.response_function_web_search import (
     ActionFind,
     ActionOpenPage,
@@ -5162,3 +5163,23 @@ async def test_bridge_rejects_untranslatable_tool_choice_with_a_400(stream: bool
         )
     assert exc_info.value.status_code == 400
     assert "tool_choice={'type': 'file_search'}" in str(exc_info.value)
+
+
+@pytest.mark.parametrize("index", [0, 1, 2])
+def test_convert_response_function_tool_call_keeps_the_index_it_is_given(index: int) -> None:
+    item: Final = ResponseFunctionToolCall(
+        id="fc_x",
+        type="function_call",
+        status="completed",
+        arguments='{"city":"Paris"}',
+        call_id="call_x",
+        name="get_weather",
+    )
+
+    tool_call: Final = (
+        LiteLLMCompletionResponsesConfig.convert_response_function_tool_call_to_chat_completion_tool_call(
+            tool_call_item=item, index=index
+        )
+    )
+
+    assert tool_call["index"] == index

uv.lock

@@ -4509,7 +4509,7 @@ wheels = [
 
 [[package]]
 name = "litellm"
-version = "1.103.4"
+version = "1.103.5"
 source = { editable = "." }
 dependencies = [
     { name = "aiohttp" },