mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-13 03:59:12 +00:00
fix(responses): fold bridged Responses output into one chat choice (#46099)
* fix(responses): fold bridged Responses output into one chat choice A Chat Completions request bridged to /responses got one choice per Responses message item, so a reply shaped [message, message, function_call] put the tool call on choices[1] and clients reading only choices[0] never saw it. Fold every message text, annotation, reasoning item and tool call into choices[0] Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(responses): drop mutable annotations and the extra cast in the choice fold Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(responses): remove a cast-ok marker that suppressed nothing Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(responses): join folded message texts with a newline Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * revert(responses): join folded message texts with no separator so stream and non-stream match Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot]
GitHub
parent
0c04d0bdb7
commit
4b7b8efd4c
3 changed files with 384 additions and 42 deletions
|
|
@ -74,7 +74,7 @@ if TYPE_CHECKING:
|
|||
ChatCompletionToolReferenceObject,
|
||||
OpenAIMessageContentListBlock,
|
||||
)
|
||||
from litellm.types.utils import Choices
|
||||
from litellm.types.utils import Choices, Message
|
||||
|
||||
|
||||
_CHAT_COMPLETION_FIELDS: Final = frozenset((*ModelResponse.model_fields, "usage"))
|
||||
|
|
@ -312,6 +312,67 @@ def _as_chat_reasoning_items(
|
|||
return cast(list[ChatCompletionReasoningItem], list(reasoning_items))
|
||||
|
||||
|
||||
def _shift_annotation(annotation: ChatCompletionAnnotation, offset: int) -> ChatCompletionAnnotation:
|
||||
shifted: Final = {
|
||||
key: value + offset if key in ("start_index", "end_index") and isinstance(value, int) else value
|
||||
for key, value in annotation.items()
|
||||
}
|
||||
return cast(ChatCompletionAnnotation, shifted)
|
||||
|
||||
|
||||
def _message_annotations(message: "Message") -> Sequence[ChatCompletionAnnotation]:
|
||||
annotations: Sequence[ChatCompletionAnnotation] | None = getattr(message, "annotations", None)
|
||||
return annotations or ()
|
||||
|
||||
|
||||
def _message_reasoning_content(message: "Message") -> str | None:
|
||||
reasoning_content: str | None = getattr(message, "reasoning_content", None)
|
||||
return reasoning_content
|
||||
|
||||
|
||||
def _message_reasoning_items(message: "Message") -> Sequence[ChatCompletionReasoningItem]:
|
||||
reasoning_items: Sequence[ChatCompletionReasoningItem] | None = getattr(message, "reasoning_items", None)
|
||||
return reasoning_items or ()
|
||||
|
||||
|
||||
def _shifted_annotations(messages: Sequence["Message"], offsets: Sequence[int]) -> Iterator[ChatCompletionAnnotation]:
|
||||
for message, offset in zip(messages, offsets):
|
||||
for annotation in _message_annotations(message):
|
||||
yield _shift_annotation(annotation, offset)
|
||||
|
||||
|
||||
def _fold_choices_into_one(
|
||||
choices: Sequence["Choices"],
|
||||
tool_calls: Sequence[Mapping[str, object]],
|
||||
trailing_reasoning_content: str | None,
|
||||
trailing_reasoning_item: _BuiltReasoningItem | None,
|
||||
) -> "Choices":
|
||||
"""Clients that only read choices[0] must still see every text and tool call."""
|
||||
from litellm.types.utils import Choices, Message
|
||||
|
||||
messages: Final = tuple(choice.message for choice in choices)
|
||||
texts: Final = tuple(str(message.content or "") for message in messages)
|
||||
offsets: Final = (0, *accumulate(len(text) for text in texts))
|
||||
annotations: Final = tuple(_shifted_annotations(messages, offsets))
|
||||
reasoning_contents: Final = (*(_message_reasoning_content(m) for m in messages), trailing_reasoning_content)
|
||||
reasoning_items: Final = (
|
||||
*chain.from_iterable(_message_reasoning_items(m) for m in messages),
|
||||
*(() if trailing_reasoning_item is None else (trailing_reasoning_item,)),
|
||||
)
|
||||
return Choices(
|
||||
message=Message(
|
||||
role=messages[0].role if messages else "assistant",
|
||||
content="".join(texts) if messages else None,
|
||||
annotations=list(annotations) or None,
|
||||
tool_calls=list(tool_calls) or None,
|
||||
reasoning_content=" ".join(value for value in reasoning_contents if value) or None,
|
||||
reasoning_items=_as_chat_reasoning_items(reasoning_items),
|
||||
),
|
||||
finish_reason="tool_calls" if tool_calls else "stop",
|
||||
index=0,
|
||||
)
|
||||
|
||||
|
||||
_ToolChoiceT = TypeVar("_ToolChoiceT")
|
||||
|
||||
|
||||
|
|
@ -958,47 +1019,16 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
|
|||
else:
|
||||
pass # don't fail request if item in list is not supported
|
||||
|
||||
if accumulated_tool_calls and choices:
|
||||
last_choice: Final = choices[-1]
|
||||
last_reasoning_content: Final = getattr(last_choice.message, "reasoning_content", None)
|
||||
last_reasoning_items: Final = getattr(last_choice.message, "reasoning_items", None)
|
||||
merged_reasoning_content: Final = (
|
||||
" ".join(value for value in (last_reasoning_content, reasoning_content) if value) or None
|
||||
)
|
||||
merged_reasoning_items: Final = _as_chat_reasoning_items(
|
||||
(
|
||||
*(last_reasoning_items or ()),
|
||||
*(() if pending_reasoning_item is None else (pending_reasoning_item,)),
|
||||
)
|
||||
)
|
||||
merged_message: Final = Message(
|
||||
role=last_choice.message.role,
|
||||
content=last_choice.message.content,
|
||||
annotations=getattr(last_choice.message, "annotations", None),
|
||||
if not choices and not accumulated_tool_calls:
|
||||
return choices
|
||||
return [
|
||||
_fold_choices_into_one(
|
||||
choices=choices,
|
||||
tool_calls=accumulated_tool_calls,
|
||||
reasoning_content=merged_reasoning_content,
|
||||
reasoning_items=merged_reasoning_items,
|
||||
trailing_reasoning_content=reasoning_content,
|
||||
trailing_reasoning_item=pending_reasoning_item,
|
||||
)
|
||||
return [
|
||||
*choices[:-1],
|
||||
Choices(message=merged_message, finish_reason="tool_calls", index=last_choice.index),
|
||||
]
|
||||
|
||||
if accumulated_tool_calls:
|
||||
msg = Message(
|
||||
content=None,
|
||||
tool_calls=accumulated_tool_calls,
|
||||
reasoning_content=reasoning_content,
|
||||
reasoning_items=cast(
|
||||
list[ChatCompletionReasoningItem] | None,
|
||||
([pending_reasoning_item] if pending_reasoning_item is not None else None),
|
||||
),
|
||||
)
|
||||
choices.append(Choices(message=msg, finish_reason="tool_calls", index=index))
|
||||
reasoning_content = None
|
||||
pending_reasoning_item = None
|
||||
|
||||
return choices
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _build_empty_incomplete_choice(
|
||||
|
|
|
|||
|
|
@ -3,8 +3,9 @@ import uuid
|
|||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.client import Gateway, list_value, object_value
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from pydantic import JsonValue
|
||||
|
||||
|
||||
@pytest.mark.covers("other.provider_wire.responses_bridge.max_output_tokens_incomplete_maps_to_length")
|
||||
|
|
@ -572,3 +573,256 @@ def test_chat_over_responses_deployment_merges_function_call_followed_by_message
|
|||
}
|
||||
], response.text
|
||||
assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/responses")]
|
||||
|
||||
|
||||
_WEATHER_TOOL: Final[dict[str, JsonValue]] = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get the weather for a city.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _message_item(item_id: str, text: str) -> dict[str, JsonValue]:
|
||||
return {
|
||||
"type": "message",
|
||||
"id": item_id,
|
||||
"status": "completed",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": text, "annotations": []}],
|
||||
}
|
||||
|
||||
|
||||
_WEATHER_CALL: Final[dict[str, JsonValue]] = {
|
||||
"type": "function_call",
|
||||
"id": "fc_1",
|
||||
"call_id": "call_1",
|
||||
"name": "get_weather",
|
||||
"arguments": '{"city":"Paris"}',
|
||||
"status": "completed",
|
||||
}
|
||||
|
||||
_TWO_MESSAGES_THEN_CALL: Final = (
|
||||
_message_item("msg_first", "First message."),
|
||||
_message_item("msg_second", "Second message."),
|
||||
_WEATHER_CALL,
|
||||
)
|
||||
|
||||
|
||||
def test_chat_over_responses_deployment_folds_two_messages_and_function_call_into_one_choice(
|
||||
gateway: Gateway,
|
||||
) -> None:
|
||||
identity: Final = "responses-bridge-two-messages-" + uuid.uuid4().hex
|
||||
|
||||
def respond(request: Request) -> Reply:
|
||||
if request.method == "GET" and request.target == "/v1/models":
|
||||
return Reply(body=b'{"object":"list","data":[]}')
|
||||
assert request.method == "POST" and request.target == "/responses", request.target
|
||||
return Reply(
|
||||
body=json.dumps(
|
||||
{
|
||||
"id": "resp_two_messages",
|
||||
"object": "response",
|
||||
"created_at": 1789788253,
|
||||
"status": "completed",
|
||||
"model": "gpt-6-sol",
|
||||
"output": list(_TWO_MESSAGES_THEN_CALL),
|
||||
"usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15},
|
||||
}
|
||||
).encode()
|
||||
)
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model="openai/responses/gpt-6-sol", api_base=wire.url, api_key="synthetic-openai-key"
|
||||
)
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": f"What is the weather in Paris? {identity}"}],
|
||||
"tools": [_WEATHER_TOOL],
|
||||
"cache": {"no-cache": True},
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
body: Final = response.json()
|
||||
assert body["choices"] == [
|
||||
{
|
||||
"finish_reason": "tool_calls",
|
||||
"index": 0,
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": "First message.Second message.",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "fc_1",
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather", "arguments": '{"city":"Paris"}'},
|
||||
"index": 0,
|
||||
}
|
||||
],
|
||||
},
|
||||
}
|
||||
], response.text
|
||||
assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/responses")]
|
||||
|
||||
|
||||
def _responses_frame(event: str, payload: dict[str, JsonValue]) -> bytes:
|
||||
return f"event: {event}\ndata: {json.dumps(payload, separators=(',', ':'))}\n\n".encode()
|
||||
|
||||
|
||||
def _message_item_frames(output_index: int, item: dict[str, JsonValue]) -> tuple[bytes, ...]:
|
||||
item_id: Final = item["id"]
|
||||
text: Final = object_value(list_value(item["content"])[0])["text"]
|
||||
return (
|
||||
_responses_frame(
|
||||
"response.output_item.added",
|
||||
{
|
||||
"type": "response.output_item.added",
|
||||
"output_index": output_index,
|
||||
"item": {**item, "status": "in_progress", "content": []},
|
||||
},
|
||||
),
|
||||
_responses_frame(
|
||||
"response.output_text.delta",
|
||||
{
|
||||
"type": "response.output_text.delta",
|
||||
"item_id": item_id,
|
||||
"output_index": output_index,
|
||||
"content_index": 0,
|
||||
"delta": text,
|
||||
},
|
||||
),
|
||||
_responses_frame(
|
||||
"response.output_item.done",
|
||||
{"type": "response.output_item.done", "output_index": output_index, "item": item},
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _function_call_frames(output_index: int, item: dict[str, JsonValue]) -> tuple[bytes, ...]:
|
||||
return (
|
||||
_responses_frame(
|
||||
"response.output_item.added",
|
||||
{
|
||||
"type": "response.output_item.added",
|
||||
"output_index": output_index,
|
||||
"item": {**item, "status": "in_progress", "arguments": ""},
|
||||
},
|
||||
),
|
||||
_responses_frame(
|
||||
"response.function_call_arguments.delta",
|
||||
{
|
||||
"type": "response.function_call_arguments.delta",
|
||||
"item_id": item["id"],
|
||||
"output_index": output_index,
|
||||
"delta": item["arguments"],
|
||||
},
|
||||
),
|
||||
_responses_frame(
|
||||
"response.function_call_arguments.done",
|
||||
{
|
||||
"type": "response.function_call_arguments.done",
|
||||
"item_id": item["id"],
|
||||
"output_index": output_index,
|
||||
"arguments": item["arguments"],
|
||||
},
|
||||
),
|
||||
_responses_frame(
|
||||
"response.output_item.done",
|
||||
{"type": "response.output_item.done", "output_index": output_index, "item": item},
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _sse_payloads(lines: list[str]) -> list[dict[str, JsonValue]]:
|
||||
return [json.loads(line[6:]) for line in lines if line.startswith("data: ") and line != "data: [DONE]"]
|
||||
|
||||
|
||||
def test_chat_stream_over_responses_deployment_folds_two_messages_and_function_call_into_one_choice(
|
||||
gateway: Gateway,
|
||||
) -> None:
|
||||
identity: Final = "responses-bridge-two-messages-stream-" + uuid.uuid4().hex
|
||||
response_payload: Final[dict[str, JsonValue]] = {
|
||||
"id": "resp_two_messages_stream",
|
||||
"object": "response",
|
||||
"created_at": 1789788253,
|
||||
"model": "gpt-6-sol",
|
||||
"status": "in_progress",
|
||||
"output": [],
|
||||
}
|
||||
|
||||
def respond(request: Request) -> Reply:
|
||||
if request.method == "GET" and request.target == "/v1/models":
|
||||
return Reply(body=b'{"object":"list","data":[]}')
|
||||
assert request.method == "POST" and request.target == "/responses", request.target
|
||||
assert json.loads(request.body)["stream"] is True, request.body
|
||||
return Reply(
|
||||
content_type="text/event-stream",
|
||||
chunks=(
|
||||
_responses_frame("response.created", {"type": "response.created", "response": response_payload}),
|
||||
*_message_item_frames(0, _TWO_MESSAGES_THEN_CALL[0]),
|
||||
*_message_item_frames(1, _TWO_MESSAGES_THEN_CALL[1]),
|
||||
*_function_call_frames(2, _TWO_MESSAGES_THEN_CALL[2]),
|
||||
_responses_frame(
|
||||
"response.completed",
|
||||
{
|
||||
"type": "response.completed",
|
||||
"response": {
|
||||
**response_payload,
|
||||
"status": "completed",
|
||||
"output": list(_TWO_MESSAGES_THEN_CALL),
|
||||
"usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15},
|
||||
},
|
||||
},
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model="openai/responses/gpt-6-sol", api_base=wire.url, api_key="synthetic-openai-key"
|
||||
)
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": f"What is the weather in Paris? {identity}"}],
|
||||
"tools": [_WEATHER_TOOL],
|
||||
"stream": True,
|
||||
"cache": {"no-cache": True},
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
chunks: Final = _sse_payloads(list(response.iter_lines()))
|
||||
choices: Final = [object_value(choice) for chunk in chunks for choice in list_value(chunk["choices"])]
|
||||
assert [choice["index"] for choice in choices] == [0] * len(choices), chunks
|
||||
deltas: Final = [object_value(choice["delta"]) for choice in choices]
|
||||
assert (
|
||||
"".join(str(delta["content"]) for delta in deltas if delta.get("content"))
|
||||
== "First message.Second message."
|
||||
)
|
||||
tool_calls: Final = [
|
||||
object_value(call)
|
||||
for delta in deltas
|
||||
if delta.get("tool_calls")
|
||||
for call in list_value(delta["tool_calls"])
|
||||
]
|
||||
assert [
|
||||
object_value(call["function"])["name"] for call in tool_calls if object_value(call["function"]).get("name")
|
||||
] == ["get_weather"], chunks
|
||||
assert (
|
||||
"".join(str(object_value(call["function"]).get("arguments") or "") for call in tool_calls)
|
||||
== '{"city":"Paris"}'
|
||||
), chunks
|
||||
assert [choice["finish_reason"] for choice in choices if choice.get("finish_reason")] == ["tool_calls"], chunks
|
||||
assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/responses")]
|
||||
|
|
|
|||
|
|
@ -5138,4 +5138,62 @@ def test_transform_request_drop_params_in_litellm_params_gates_the_prompt_cache_
|
|||
litellm_logging_obj=Mock(),
|
||||
)
|
||||
|
||||
assert "prompt_cache_breakpoint" not in result["input"][0]["content"][0]
|
||||
assert "prompt_cache_breakpoint" not in result["input"][0]["content"][0]
|
||||
|
||||
|
||||
def test_convert_response_output_folds_two_messages_and_function_call_into_one_choice() -> None:
|
||||
first: Final = ResponseOutputMessage.model_validate(
|
||||
{
|
||||
"id": "msg_first",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [{"type": "output_text", "text": "First message.", "annotations": []}],
|
||||
}
|
||||
)
|
||||
second_annotation: Final = {
|
||||
"type": "url_citation",
|
||||
"start_index": 0,
|
||||
"end_index": 6,
|
||||
"title": "Second",
|
||||
"url": "https://example.com/second",
|
||||
}
|
||||
second: Final = ResponseOutputMessage.model_validate(
|
||||
{
|
||||
"id": "msg_second",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [{"type": "output_text", "text": "Second message.", "annotations": [second_annotation]}],
|
||||
}
|
||||
)
|
||||
function_call: Final = ResponseFunctionToolCall(
|
||||
id="fc_1",
|
||||
type="function_call",
|
||||
status="completed",
|
||||
arguments='{"city":"Paris"}',
|
||||
call_id="call_1",
|
||||
name="get_weather",
|
||||
)
|
||||
|
||||
choices: Final = LiteLLMResponsesTransformationHandler._convert_response_output_to_choices( # pyright: ignore[reportPrivateUsage] # exercises the converter directly
|
||||
(first, second, function_call)
|
||||
)
|
||||
|
||||
assert len(choices) == 1
|
||||
assert choices[0].index == 0
|
||||
assert choices[0].finish_reason == "tool_calls"
|
||||
assert choices[0].message.content == "First message.Second message."
|
||||
assert choices[0].message.annotations == [
|
||||
{**second_annotation, "start_index": len("First message."), "end_index": len("First message.") + 6}
|
||||
]
|
||||
tool_calls: Final = choices[0].message.tool_calls
|
||||
assert tool_calls is not None
|
||||
assert [call.model_dump(exclude_none=True) for call in tool_calls] == [
|
||||
{
|
||||
"id": "fc_1",
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather", "arguments": '{"city":"Paris"}'},
|
||||
"index": 0,
|
||||
}
|
||||
]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue