mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-07 07:58:09 +00:00
fix(streaming): match MCP follow-ups and bridged summary text to the rebuilt choices
MCP auto-execute now answers only the first choice's tool calls, since its follow-up turn continues that choice alone. The Responses bridge streams a space before each reasoning summary part after the first, matching the non-streamed reply. build_base_response now takes the per-choice streams instead of having its choices replaced afterwards
This commit is contained in:
parent
77357aed87
commit
551d835252
5 changed files with 60 additions and 16 deletions
|
|
@ -2124,6 +2124,8 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
|
|||
)
|
||||
else:
|
||||
raise ValueError(f"Chat provider: Invalid text delta {parsed_chunk}")
|
||||
elif event_type == "response.reasoning_summary_part.added" and parsed_chunk.get("summary_index"):
|
||||
return ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(reasoning_content=" "))])
|
||||
elif event_type == "response.reasoning_summary_text.delta":
|
||||
content_part = parsed_chunk.get("delta", None)
|
||||
if content_part:
|
||||
|
|
|
|||
|
|
@ -433,7 +433,11 @@ class ChunkProcessor:
|
|||
None,
|
||||
)
|
||||
|
||||
def build_base_response(self, chunks: Sequence["_BaseChunk"]) -> ModelResponse:
|
||||
def build_base_response(
|
||||
self,
|
||||
chunks: Sequence["_BaseChunk"],
|
||||
choice_streams: Sequence[tuple[int, Sequence["_BaseChunk"]]],
|
||||
) -> ModelResponse:
|
||||
chunk = self.first_chunk
|
||||
id: Final = ChunkProcessor._get_chunk_id(chunks)
|
||||
object: Final = chunk["object"]
|
||||
|
|
@ -465,13 +469,17 @@ class ChunkProcessor:
|
|||
if service_tier is not None
|
||||
else MappingProxyType({})
|
||||
),
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": role, "content": ""},
|
||||
"finish_reason": finish_reason,
|
||||
}
|
||||
],
|
||||
"choices": (
|
||||
[ChunkProcessor.build_base_choice(index, choice_chunks) for index, choice_chunks in choice_streams]
|
||||
if len(choice_streams) > 1
|
||||
else [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": role, "content": ""},
|
||||
"finish_reason": finish_reason,
|
||||
}
|
||||
]
|
||||
),
|
||||
"usage": {
|
||||
"prompt_tokens": 0, # Modify as needed
|
||||
"completion_tokens": 0, # Modify as needed
|
||||
|
|
@ -1747,12 +1755,8 @@ def stream_chunk_builder(
|
|||
usage_model: Final = usage_tokenizer_model(
|
||||
model, get_base_model_from_metadata(logging_obj.model_call_details) if logging_obj is not None else None
|
||||
)
|
||||
response: Final = processor.build_base_response(chunks)
|
||||
choice_streams: Final = _choice_streams(chunks)
|
||||
if len(choice_streams) > 1:
|
||||
response.choices = [
|
||||
ChunkProcessor.build_base_choice(index, choice_chunks) for index, choice_chunks in choice_streams
|
||||
]
|
||||
response: Final = processor.build_base_response(chunks, choice_streams)
|
||||
rebuilt_choices: Final = tuple(zip(response.choices, (choice_chunks for _, choice_chunks in choice_streams)))
|
||||
|
||||
is_simple_text_stream: Final = all(
|
||||
|
|
|
|||
|
|
@ -589,11 +589,11 @@ class LiteLLM_Proxy_MCP_Handler:
|
|||
|
||||
@staticmethod
|
||||
def extract_tool_calls_from_chat_response(response: ModelResponse) -> list[object]:
|
||||
"""Extract tool calls from a chat completion response."""
|
||||
"""Extract the first choice's tool calls, since the follow-up turn continues that choice alone."""
|
||||
tool_calls: Final[list[object]] = []
|
||||
|
||||
try:
|
||||
for choice in response.choices:
|
||||
for choice in response.choices[:1]:
|
||||
message: Message | None = getattr(choice, "message", None)
|
||||
if message is None:
|
||||
continue
|
||||
|
|
|
|||
|
|
@ -425,12 +425,15 @@ def test_bridged_reasoning_summary_parts_rebuild_into_the_one_choice_they_stream
|
|||
|
||||
iterator = OpenAiResponsesToChatCompletionStreamIterator(streaming_response=None, sync_stream=True)
|
||||
events = (
|
||||
{"type": "response.reasoning_summary_part.added", "output_index": 0, "summary_index": 0},
|
||||
{
|
||||
"type": "response.reasoning_summary_text.delta",
|
||||
"delta": "First part.",
|
||||
"output_index": 0,
|
||||
"summary_index": 0,
|
||||
},
|
||||
{"type": "response.reasoning_summary_part.done", "output_index": 0, "summary_index": 0},
|
||||
{"type": "response.reasoning_summary_part.added", "output_index": 0, "summary_index": 1},
|
||||
{
|
||||
"type": "response.reasoning_summary_text.delta",
|
||||
"delta": "Second part.",
|
||||
|
|
@ -443,7 +446,7 @@ def test_bridged_reasoning_summary_parts_rebuild_into_the_one_choice_they_stream
|
|||
rebuilt = litellm.stream_chunk_builder(chunks=[iterator.chunk_parser(event) for event in events])
|
||||
|
||||
assert [(choice.index, choice.message.reasoning_content, choice.message.content) for choice in rebuilt.choices] == [
|
||||
(0, "First part.Second part.", "Answer")
|
||||
(0, "First part. Second part.", "Answer")
|
||||
]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -131,6 +131,41 @@ def test_extract_tool_calls_from_chat_response_handles_tool_calls():
|
|||
assert tool_calls[0]["function"]["name"] == "foo"
|
||||
|
||||
|
||||
def test_chat_follow_up_answers_only_tool_calls_in_its_assistant_turn():
|
||||
response = ModelResponse(
|
||||
id="resp-n2",
|
||||
choices=[
|
||||
{
|
||||
"index": index,
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{"id": call_id, "type": "function", "function": {"name": "foo", "arguments": "{}"}}
|
||||
],
|
||||
},
|
||||
"finish_reason": "tool_calls",
|
||||
}
|
||||
for index, call_id in enumerate(("call-a", "call-b"))
|
||||
],
|
||||
model="gpt",
|
||||
created=0,
|
||||
object="chat.completion",
|
||||
)
|
||||
tool_results = [
|
||||
{"tool_call_id": tool_call["id"], "name": tool_call["function"]["name"], "result": "done"}
|
||||
for tool_call in LiteLLM_Proxy_MCP_Handler.extract_tool_calls_from_chat_response(response)
|
||||
]
|
||||
|
||||
follow_up = LiteLLM_Proxy_MCP_Handler.create_follow_up_messages_for_chat(
|
||||
[{"role": "user", "content": "hi"}], response, tool_results
|
||||
)
|
||||
|
||||
assistant_call_ids = [tool_call["id"] for tool_call in follow_up[1]["tool_calls"]]
|
||||
answered_call_ids = [message["tool_call_id"] for message in follow_up if message["role"] == "tool"]
|
||||
assert answered_call_ids == assistant_call_ids == ["call-a"]
|
||||
|
||||
|
||||
def test_create_follow_up_messages_for_chat_appends_tool_results():
|
||||
original_messages = [{"role": "user", "content": "hi"}]
|
||||
response = ModelResponse(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue