diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 5542249d2c7..392c2be25fe 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -1277,9 +1277,14 @@ class CustomStreamWrapper: if _is_delta_empty: model_response.choices[0].delta = Delta(content=self._close_open_think_block()) # get any function call arguments - model_response.choices[0].finish_reason = map_finish_reason( + _mapped_finish_reason: Final = map_finish_reason( finish_reason=self.received_finish_reason ) # ensure consistent output to openai + model_response.choices[0].finish_reason = ( + # the calls were streamed in an earlier chunk, so the terminal + # reason says why the turn ended: tool_calls, not stop + "tool_calls" if _mapped_finish_reason == "stop" and self.tool_call else _mapped_finish_reason + ) self.sent_last_chunk = True @@ -1724,8 +1729,17 @@ class CustomStreamWrapper: return ## CHECK FOR TOOL USE - if "tool_calls" in completion_obj and len(completion_obj["tool_calls"]) > 0: - if self.is_function_call is True: # user passed in 'functions' param + # OpenAI-compatible providers deliver the calls on the delta, which is + # copied into model_response below the completion_obj in that branch, so + # checking completion_obj alone missed them and the stop -> tool_calls + # rewrite in finish_reason_handler() never fired for streamed calls. + tool_calls_in_completion_obj: Final = bool( + "tool_calls" in completion_obj and len(completion_obj["tool_calls"]) > 0 + ) + tool_calls_in_delta: Final = bool(model_response.choices and model_response.choices[0].delta.tool_calls) + + if tool_calls_in_completion_obj or tool_calls_in_delta: + if self.is_function_call is True and tool_calls_in_completion_obj: # user passed in 'functions' param completion_obj["function_call"] = completion_obj["tool_calls"][0]["function"] completion_obj["tool_calls"] = None diff --git a/tests/unit/litellm_core_utils/test_streaming_handler.py b/tests/unit/litellm_core_utils/test_streaming_handler.py index 41e8c50fa85..eb58452d289 100644 --- a/tests/unit/litellm_core_utils/test_streaming_handler.py +++ b/tests/unit/litellm_core_utils/test_streaming_handler.py @@ -8248,3 +8248,81 @@ def test_chunk_creator_usage_only_chunk_with_dict_usage_keeps_details( assert usage.prompt_tokens_details.cached_tokens == 80 assert usage.completion_tokens_details is not None assert usage.completion_tokens_details.reasoning_tokens == 5 + + +def _tool_call_chunk(*, finish_reason: str | None = None) -> ModelResponseStream: + from litellm.types.utils import ChatCompletionDeltaToolCall, Function + + return ModelResponseStream( + id="chatcmpl-tool-call", + created=1742056047, + model="gpt-4o-mini", + object="chat.completion.chunk", + choices=[ + StreamingChoices( + index=0, + delta=Delta( + role="assistant", + content=None, + tool_calls=[ + ChatCompletionDeltaToolCall( + id="call_1", + type="function", + function=Function(name="get_weather", arguments="{}"), + index=0, + ) + ], + ), + finish_reason=finish_reason, + ) + ], + ) + + +def _empty_stop_chunk() -> ModelResponseStream: + return ModelResponseStream( + id="chatcmpl-tool-call", + created=1742056047, + model="gpt-4o-mini", + object="chat.completion.chunk", + choices=[StreamingChoices(index=0, delta=Delta(content=""), finish_reason="stop")], + ) + + +@pytest.mark.parametrize( + "upstream_chunks", + [ + [_tool_call_chunk(), _empty_stop_chunk()], + [_tool_call_chunk(finish_reason="stop")], + ], + ids=["separate-chunks", "same-chunk"], +) +def test_streamed_tool_call_keeps_tool_calls_finish_reason( + logging_obj: Logging, upstream_chunks: list[ModelResponseStream] +): + """A streamed call ends with ``tool_calls``, like the non-streaming path. + + Providers that are OpenAI-compatible answer a call with + ``finish_reason: "stop"``. The non-streaming path rewrites that to + ``tool_calls``, and so does ``finish_reason_handler()`` for the synthetic + terminal chunk -- but an OpenAI-compatible stream never reached either: the + rewrite depended on ``self.tool_call``, which was only set from + ``completion_obj`` while those chunks carry the calls on the delta. + """ + wrapper: Final = CustomStreamWrapper( + completion_stream=ModelResponseListIterator(model_responses=upstream_chunks), + model="gpt-4o-mini", + logging_obj=logging_obj, + custom_llm_provider="openai", + ) + + emitted: Final = list(wrapper) + terminal: Final = [ + chunk.choices[0].finish_reason for chunk in emitted if chunk.choices and chunk.choices[0].finish_reason + ] + + assert terminal == ["tool_calls"] + + built: Final = litellm.stream_chunk_builder(chunks=wrapper.chunks) + assert built is not None + assert built.choices[0].finish_reason == "tool_calls" \ No newline at end of file