fix(streaming): match MCP follow-ups and bridged summary text to the rebuilt choices

MCP auto-execute now answers only the first choice's tool calls, since its
follow-up turn continues that choice alone. The Responses bridge streams a
space before each reasoning summary part after the first, matching the
non-streamed reply. build_base_response now takes the per-choice streams
instead of having its choices replaced afterwards
This commit is contained in:
mateo-berri 2026-10-29 05:24:16 -07:00
parent 77357aed87
commit 551d835252
5 changed files with 60 additions and 16 deletions

View file

@ -2124,6 +2124,8 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
)
else:
raise ValueError(f"Chat provider: Invalid text delta {parsed_chunk}")
elif event_type == "response.reasoning_summary_part.added" and parsed_chunk.get("summary_index"):
return ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(reasoning_content=" "))])
elif event_type == "response.reasoning_summary_text.delta":
content_part = parsed_chunk.get("delta", None)
if content_part:

View file

@ -433,7 +433,11 @@ class ChunkProcessor:
None,
)
def build_base_response(self, chunks: Sequence["_BaseChunk"]) -> ModelResponse:
def build_base_response(
self,
chunks: Sequence["_BaseChunk"],
choice_streams: Sequence[tuple[int, Sequence["_BaseChunk"]]],
) -> ModelResponse:
chunk = self.first_chunk
id: Final = ChunkProcessor._get_chunk_id(chunks)
object: Final = chunk["object"]
@ -465,13 +469,17 @@ class ChunkProcessor:
if service_tier is not None
else MappingProxyType({})
),
"choices": [
{
"index": 0,
"message": {"role": role, "content": ""},
"finish_reason": finish_reason,
}
],
"choices": (
[ChunkProcessor.build_base_choice(index, choice_chunks) for index, choice_chunks in choice_streams]
if len(choice_streams) > 1
else [
{
"index": 0,
"message": {"role": role, "content": ""},
"finish_reason": finish_reason,
}
]
),
"usage": {
"prompt_tokens": 0, # Modify as needed
"completion_tokens": 0, # Modify as needed
@ -1747,12 +1755,8 @@ def stream_chunk_builder(
usage_model: Final = usage_tokenizer_model(
model, get_base_model_from_metadata(logging_obj.model_call_details) if logging_obj is not None else None
)
response: Final = processor.build_base_response(chunks)
choice_streams: Final = _choice_streams(chunks)
if len(choice_streams) > 1:
response.choices = [
ChunkProcessor.build_base_choice(index, choice_chunks) for index, choice_chunks in choice_streams
]
response: Final = processor.build_base_response(chunks, choice_streams)
rebuilt_choices: Final = tuple(zip(response.choices, (choice_chunks for _, choice_chunks in choice_streams)))
is_simple_text_stream: Final = all(

View file

@ -589,11 +589,11 @@ class LiteLLM_Proxy_MCP_Handler:
@staticmethod
def extract_tool_calls_from_chat_response(response: ModelResponse) -> list[object]:
"""Extract tool calls from a chat completion response."""
"""Extract the first choice's tool calls, since the follow-up turn continues that choice alone."""
tool_calls: Final[list[object]] = []
try:
for choice in response.choices:
for choice in response.choices[:1]:
message: Message | None = getattr(choice, "message", None)
if message is None:
continue

View file

@ -425,12 +425,15 @@ def test_bridged_reasoning_summary_parts_rebuild_into_the_one_choice_they_stream
iterator = OpenAiResponsesToChatCompletionStreamIterator(streaming_response=None, sync_stream=True)
events = (
{"type": "response.reasoning_summary_part.added", "output_index": 0, "summary_index": 0},
{
"type": "response.reasoning_summary_text.delta",
"delta": "First part.",
"output_index": 0,
"summary_index": 0,
},
{"type": "response.reasoning_summary_part.done", "output_index": 0, "summary_index": 0},
{"type": "response.reasoning_summary_part.added", "output_index": 0, "summary_index": 1},
{
"type": "response.reasoning_summary_text.delta",
"delta": "Second part.",
@ -443,7 +446,7 @@ def test_bridged_reasoning_summary_parts_rebuild_into_the_one_choice_they_stream
rebuilt = litellm.stream_chunk_builder(chunks=[iterator.chunk_parser(event) for event in events])
assert [(choice.index, choice.message.reasoning_content, choice.message.content) for choice in rebuilt.choices] == [
(0, "First part.Second part.", "Answer")
(0, "First part. Second part.", "Answer")
]

View file

@ -131,6 +131,41 @@ def test_extract_tool_calls_from_chat_response_handles_tool_calls():
assert tool_calls[0]["function"]["name"] == "foo"
def test_chat_follow_up_answers_only_tool_calls_in_its_assistant_turn():
response = ModelResponse(
id="resp-n2",
choices=[
{
"index": index,
"message": {
"role": "assistant",
"content": None,
"tool_calls": [
{"id": call_id, "type": "function", "function": {"name": "foo", "arguments": "{}"}}
],
},
"finish_reason": "tool_calls",
}
for index, call_id in enumerate(("call-a", "call-b"))
],
model="gpt",
created=0,
object="chat.completion",
)
tool_results = [
{"tool_call_id": tool_call["id"], "name": tool_call["function"]["name"], "result": "done"}
for tool_call in LiteLLM_Proxy_MCP_Handler.extract_tool_calls_from_chat_response(response)
]
follow_up = LiteLLM_Proxy_MCP_Handler.create_follow_up_messages_for_chat(
[{"role": "user", "content": "hi"}], response, tool_results
)
assistant_call_ids = [tool_call["id"] for tool_call in follow_up[1]["tool_calls"]]
answered_call_ids = [message["tool_call_id"] for message in follow_up if message["role"] == "tool"]
assert answered_call_ids == assistant_call_ids == ["call-a"]
def test_create_follow_up_messages_for_chat_appends_tool_results():
original_messages = [{"role": "user", "content": "hi"}]
response = ModelResponse(