scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-18 04:44:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 125

fix(responses): fold bridged Responses output into one chat choice (#46099)

Browse source 
Backport of #46099 to stable/1.103.x.
Cherry-picked from 4b7b8efd4c (main).
Adapted to the line: unit test lives under tests/test_litellm/, and only this
PR's own tests are carried into the integration file.
This commit is contained in:
devin-ai-integration[bot] 2026-10-16 13:34:12 -05:00 • committed by Mrinal Chanshetty
parent 69ff376019
commit a70f1310ad
3 changed files with 390 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

111
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -5,6 +5,7 @@ Handler for transforming /chat/completions api requests to litellm.responses req
import json
import os
from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence
from itertools import accumulate, chain
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, Union, cast, get_args
@ -70,7 +71,7 @@ if TYPE_CHECKING:
ChatCompletionToolReferenceObject,
OpenAIMessageContentListBlock,
)
from litellm.types.utils import Choices
from litellm.types.utils import Choices, Message
_CHAT_COMPLETION_FIELDS: Final = frozenset((*ModelResponse.model_fields, "usage"))
@ -197,6 +198,67 @@ def _as_chat_reasoning_items(
return cast(list[ChatCompletionReasoningItem], list(reasoning_items))
def _shift_annotation(annotation: ChatCompletionAnnotation, offset: int) -> ChatCompletionAnnotation:
shifted: Final = {
key: value + offset if key in ("start_index", "end_index") and isinstance(value, int) else value
for key, value in annotation.items()
}
return cast(ChatCompletionAnnotation, shifted)
def _message_annotations(message: "Message") -> Sequence[ChatCompletionAnnotation]:
annotations: Sequence[ChatCompletionAnnotation] | None = getattr(message, "annotations", None)
return annotations or ()
def _message_reasoning_content(message: "Message") -> str | None:
reasoning_content: str | None = getattr(message, "reasoning_content", None)
return reasoning_content
def _message_reasoning_items(message: "Message") -> Sequence[ChatCompletionReasoningItem]:
reasoning_items: Sequence[ChatCompletionReasoningItem] | None = getattr(message, "reasoning_items", None)
return reasoning_items or ()
def _shifted_annotations(messages: Sequence["Message"], offsets: Sequence[int]) -> Iterator[ChatCompletionAnnotation]:
for message, offset in zip(messages, offsets):
for annotation in _message_annotations(message):
yield _shift_annotation(annotation, offset)
def _fold_choices_into_one(
choices: Sequence["Choices"],
tool_calls: Sequence[Mapping[str, object]],
trailing_reasoning_content: str | None,
trailing_reasoning_item: _BuiltReasoningItem | None,
) -> "Choices":
"""Clients that only read choices[0] must still see every text and tool call."""
from litellm.types.utils import Choices, Message
messages: Final = tuple(choice.message for choice in choices)
texts: Final = tuple(str(message.content or "") for message in messages)
offsets: Final = (0, *accumulate(len(text) for text in texts))
annotations: Final = tuple(_shifted_annotations(messages, offsets))
reasoning_contents: Final = (*(_message_reasoning_content(m) for m in messages), trailing_reasoning_content)
reasoning_items: Final = (
*chain.from_iterable(_message_reasoning_items(m) for m in messages),
*(() if trailing_reasoning_item is None else (trailing_reasoning_item,)),
)
return Choices(
message=Message(
role=messages[0].role if messages else "assistant",
content="".join(texts) if messages else None,
annotations=list(annotations) or None,
tool_calls=list(tool_calls) or None,
reasoning_content=" ".join(value for value in reasoning_contents if value) or None,
reasoning_items=_as_chat_reasoning_items(reasoning_items),
),
finish_reason="tool_calls" if tool_calls else "stop",
index=0,
)
def _map_incomplete_reason_to_finish_reason(incomplete_reason: str | None) -> Literal["length", "content_filter"]:
if incomplete_reason == "content_filter":
return "content_filter"
@ -782,47 +844,16 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
else:
pass # don't fail request if item in list is not supported
if accumulated_tool_calls and choices:
last_choice: Final = choices[-1]
last_reasoning_content: Final = getattr(last_choice.message, "reasoning_content", None)
last_reasoning_items: Final = getattr(last_choice.message, "reasoning_items", None)
merged_reasoning_content: Final = (
" ".join(value for value in (last_reasoning_content, reasoning_content) if value) or None
)
merged_reasoning_items: Final = _as_chat_reasoning_items(
(
*(last_reasoning_items or ()),
*(() if pending_reasoning_item is None else (pending_reasoning_item,)),
)
)
merged_message: Final = Message(
role=last_choice.message.role,
content=last_choice.message.content,
annotations=getattr(last_choice.message, "annotations", None),
if not choices and not accumulated_tool_calls:
return choices
return [
_fold_choices_into_one(
choices=choices,
tool_calls=accumulated_tool_calls,
reasoning_content=merged_reasoning_content,
reasoning_items=merged_reasoning_items,
trailing_reasoning_content=reasoning_content,
trailing_reasoning_item=pending_reasoning_item,
)
return [
*choices[:-1],
Choices(message=merged_message, finish_reason="tool_calls", index=last_choice.index),
]
if accumulated_tool_calls:
msg = Message(
content=None,
tool_calls=accumulated_tool_calls,
reasoning_content=reasoning_content,
reasoning_items=cast(
list[ChatCompletionReasoningItem] | None,
([pending_reasoning_item] if pending_reasoning_item is not None else None),
),
)
choices.append(Choices(message=msg, finish_reason="tool_calls", index=index))
reasoning_content = None
pending_reasoning_item = None
return choices
]
@staticmethod
def _build_empty_incomplete_choice(

261
tests/integration/providers/test_responses_bridge_incomplete.py Normal file
View file

@ -0,0 +1,261 @@
import json
import uuid
from typing import Final
import pytest
from integration._support.client import Gateway, list_value, object_value
from integration._support.wire import Reply, Request, wire_server
from pydantic import JsonValue
_WEATHER_TOOL: Final[dict[str, JsonValue]] = {
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the weather for a city.",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
},
},
}
def _message_item(item_id: str, text: str) -> dict[str, JsonValue]:
return {
"type": "message",
"id": item_id,
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": text, "annotations": []}],
}
_WEATHER_CALL: Final[dict[str, JsonValue]] = {
"type": "function_call",
"id": "fc_1",
"call_id": "call_1",
"name": "get_weather",
"arguments": '{"city":"Paris"}',
"status": "completed",
}
_TWO_MESSAGES_THEN_CALL: Final = (
_message_item("msg_first", "First message."),
_message_item("msg_second", "Second message."),
_WEATHER_CALL,
)
def test_chat_over_responses_deployment_folds_two_messages_and_function_call_into_one_choice(
gateway: Gateway,
) -> None:
identity: Final = "responses-bridge-two-messages-" + uuid.uuid4().hex
def respond(request: Request) -> Reply:
if request.method == "GET" and request.target == "/v1/models":
return Reply(body=b'{"object":"list","data":[]}')
assert request.method == "POST" and request.target == "/responses", request.target
return Reply(
body=json.dumps(
{
"id": "resp_two_messages",
"object": "response",
"created_at": 1789788253,
"status": "completed",
"model": "gpt-6-sol",
"output": list(_TWO_MESSAGES_THEN_CALL),
"usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15},
}
).encode()
)
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(
model="openai/responses/gpt-6-sol", api_base=wire.url, api_key="synthetic-openai-key"
)
response: Final = gateway.request(
"POST",
"/v1/chat/completions",
{
"model": model,
"messages": [{"role": "user", "content": f"What is the weather in Paris? {identity}"}],
"tools": [_WEATHER_TOOL],
"cache": {"no-cache": True},
},
)
assert response.status_code == 200, response.text
body: Final = response.json()
assert body["choices"] == [
{
"finish_reason": "tool_calls",
"index": 0,
"message": {
"role": "assistant",
"content": "First message.Second message.",
"tool_calls": [
{
"id": "fc_1",
"type": "function",
"function": {"name": "get_weather", "arguments": '{"city":"Paris"}'},
"index": 0,
}
],
},
}
], response.text
assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/responses")]
def _responses_frame(event: str, payload: dict[str, JsonValue]) -> bytes:
return f"event: {event}\ndata: {json.dumps(payload, separators=(',', ':'))}\n\n".encode()
def _message_item_frames(output_index: int, item: dict[str, JsonValue]) -> tuple[bytes, ...]:
item_id: Final = item["id"]
text: Final = object_value(list_value(item["content"])[0])["text"]
return (
_responses_frame(
"response.output_item.added",
{
"type": "response.output_item.added",
"output_index": output_index,
"item": {**item, "status": "in_progress", "content": []},
},
),
_responses_frame(
"response.output_text.delta",
{
"type": "response.output_text.delta",
"item_id": item_id,
"output_index": output_index,
"content_index": 0,
"delta": text,
},
),
_responses_frame(
"response.output_item.done",
{"type": "response.output_item.done", "output_index": output_index, "item": item},
),
)
def _function_call_frames(output_index: int, item: dict[str, JsonValue]) -> tuple[bytes, ...]:
return (
_responses_frame(
"response.output_item.added",
{
"type": "response.output_item.added",
"output_index": output_index,
"item": {**item, "status": "in_progress", "arguments": ""},
},
),
_responses_frame(
"response.function_call_arguments.delta",
{
"type": "response.function_call_arguments.delta",
"item_id": item["id"],
"output_index": output_index,
"delta": item["arguments"],
},
),
_responses_frame(
"response.function_call_arguments.done",
{
"type": "response.function_call_arguments.done",
"item_id": item["id"],
"output_index": output_index,
"arguments": item["arguments"],
},
),
_responses_frame(
"response.output_item.done",
{"type": "response.output_item.done", "output_index": output_index, "item": item},
),
)
def _sse_payloads(lines: list[str]) -> list[dict[str, JsonValue]]:
return [json.loads(line[6:]) for line in lines if line.startswith("data: ") and line != "data: [DONE]"]
def test_chat_stream_over_responses_deployment_folds_two_messages_and_function_call_into_one_choice(
gateway: Gateway,
) -> None:
identity: Final = "responses-bridge-two-messages-stream-" + uuid.uuid4().hex
response_payload: Final[dict[str, JsonValue]] = {
"id": "resp_two_messages_stream",
"object": "response",
"created_at": 1789788253,
"model": "gpt-6-sol",
"status": "in_progress",
"output": [],
}
def respond(request: Request) -> Reply:
if request.method == "GET" and request.target == "/v1/models":
return Reply(body=b'{"object":"list","data":[]}')
assert request.method == "POST" and request.target == "/responses", request.target
assert json.loads(request.body)["stream"] is True, request.body
return Reply(
content_type="text/event-stream",
chunks=(
_responses_frame("response.created", {"type": "response.created", "response": response_payload}),
*_message_item_frames(0, _TWO_MESSAGES_THEN_CALL[0]),
*_message_item_frames(1, _TWO_MESSAGES_THEN_CALL[1]),
*_function_call_frames(2, _TWO_MESSAGES_THEN_CALL[2]),
_responses_frame(
"response.completed",
{
"type": "response.completed",
"response": {
**response_payload,
"status": "completed",
"output": list(_TWO_MESSAGES_THEN_CALL),
"usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15},
},
},
),
),
)
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(
model="openai/responses/gpt-6-sol", api_base=wire.url, api_key="synthetic-openai-key"
)
response: Final = gateway.request(
"POST",
"/v1/chat/completions",
{
"model": model,
"messages": [{"role": "user", "content": f"What is the weather in Paris? {identity}"}],
"tools": [_WEATHER_TOOL],
"stream": True,
"cache": {"no-cache": True},
},
)
assert response.status_code == 200, response.text
chunks: Final = _sse_payloads(list(response.iter_lines()))
choices: Final = [object_value(choice) for chunk in chunks for choice in list_value(chunk["choices"])]
assert [choice["index"] for choice in choices] == [0] * len(choices), chunks
deltas: Final = [object_value(choice["delta"]) for choice in choices]
assert (
"".join(str(delta["content"]) for delta in deltas if delta.get("content"))
== "First message.Second message."
)
tool_calls: Final = [
object_value(call)
for delta in deltas
if delta.get("tool_calls")
for call in list_value(delta["tool_calls"])
]
assert [
object_value(call["function"])["name"] for call in tool_calls if object_value(call["function"]).get("name")
] == ["get_weather"], chunks
assert (
"".join(str(object_value(call["function"]).get("arguments") or "") for call in tool_calls)
== '{"city":"Paris"}'
), chunks
assert [choice["finish_reason"] for choice in choices if choice.get("finish_reason")] == ["tool_calls"], chunks
assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/responses")]

58
tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py
View file

@ -4471,3 +4471,61 @@ def test_map_optional_params_verbosity_merges_into_text():
verbosity_only_request,
)
assert verbosity_only_request["text"] == {"verbosity": "low"}
def test_convert_response_output_folds_two_messages_and_function_call_into_one_choice() -> None:
first: Final = ResponseOutputMessage.model_validate(
{
"id": "msg_first",
"type": "message",
"role": "assistant",
"status": "completed",
"content": [{"type": "output_text", "text": "First message.", "annotations": []}],
}
)
second_annotation: Final = {
"type": "url_citation",
"start_index": 0,
"end_index": 6,
"title": "Second",
"url": "https://example.com/second",
}
second: Final = ResponseOutputMessage.model_validate(
{
"id": "msg_second",
"type": "message",
"role": "assistant",
"status": "completed",
"content": [{"type": "output_text", "text": "Second message.", "annotations": [second_annotation]}],
}
)
function_call: Final = ResponseFunctionToolCall(
id="fc_1",
type="function_call",
status="completed",
arguments='{"city":"Paris"}',
call_id="call_1",
name="get_weather",
)
choices: Final = LiteLLMResponsesTransformationHandler._convert_response_output_to_choices( # pyright: ignore[reportPrivateUsage] # exercises the converter directly
(first, second, function_call)
)
assert len(choices) == 1
assert choices[0].index == 0
assert choices[0].finish_reason == "tool_calls"
assert choices[0].message.content == "First message.Second message."
assert choices[0].message.annotations == [
{**second_annotation, "start_index": len("First message."), "end_index": len("First message.") + 6}
]
tool_calls: Final = choices[0].message.tool_calls
assert tool_calls is not None
assert [call.model_dump(exclude_none=True) for call in tool_calls] == [
{
"id": "fc_1",
"type": "function",
"function": {"name": "get_weather", "arguments": '{"city":"Paris"}'},
"index": 0,
}
]