mirror of
https://github.com/BerriAI/litellm.git
synced 2026-08-28 05:25:59 +00:00
fix(ollama): end streamed tool calls with finish_reason tool_calls
ollama_chat streams tool calls in a chunk with done false and then sends a separate done chunk whose message is empty and whose done_reason is stop. The override that turns the finish reason into tool_calls only looked at the done chunk's own tool_calls field, which is None by then, so the stream ended in stop. Clients that gate tool execution on finish_reason == tool_calls therefore threw away the tool_calls delta they had just accumulated and treated the turn as ordinary text. Non-streaming was fine; it reads message.tool_calls off the full response. Track whether any chunk in the stream carried tool calls and use that on the done chunk. Streams that never produce a tool call keep whatever done_reason Ollama sent, so stop and length are unchanged.
This commit is contained in:
parent
a625d1e1ca
commit
cd7ac73013
2 changed files with 54 additions and 2 deletions
|
|
@ -425,6 +425,7 @@ class OllamaChatConfig(BaseConfig):
|
|||
class OllamaChatCompletionResponseIterator(BaseModelResponseIterator):
|
||||
started_reasoning_content: bool = False
|
||||
finished_reasoning_content: bool = False
|
||||
saw_tool_calls: bool = False
|
||||
|
||||
def _is_function_call_complete(self, function_args: str | dict) -> bool:
|
||||
if isinstance(function_args, dict):
|
||||
|
|
@ -470,6 +471,7 @@ class OllamaChatCompletionResponseIterator(BaseModelResponseIterator):
|
|||
# process tool calls - if complete function arg - add id to tool call
|
||||
tool_calls = chunk["message"].get("tool_calls")
|
||||
if tool_calls is not None:
|
||||
self.saw_tool_calls = True
|
||||
for tool_call in tool_calls:
|
||||
function_args = tool_call.get("function").get("arguments")
|
||||
if function_args is not None and len(function_args) > 0:
|
||||
|
|
@ -510,9 +512,13 @@ class OllamaChatCompletionResponseIterator(BaseModelResponseIterator):
|
|||
|
||||
if chunk["done"] is True:
|
||||
finish_reason = chunk.get("done_reason") or "stop"
|
||||
# Override finish_reason when tool_calls are present
|
||||
# Override finish_reason when tool_calls are present anywhere in
|
||||
# the stream. Ollama often sends them in a chunk with done false
|
||||
# and then a separate done chunk with an empty message, so keying
|
||||
# off this chunk's tool_calls alone leaves the stream ending in
|
||||
# "stop".
|
||||
# Fixes: https://github.com/BerriAI/litellm/issues/18922
|
||||
if tool_calls is not None:
|
||||
if self.saw_tool_calls:
|
||||
finish_reason = "tool_calls"
|
||||
choices = [
|
||||
StreamingChoices(
|
||||
|
|
|
|||
|
|
@ -597,6 +597,52 @@ class TestOllamaFinishReasonLength:
|
|||
result.choices[0].finish_reason == "length"
|
||||
), f"Expected 'length' when done_reason='length', got '{result.choices[0].finish_reason}'"
|
||||
|
||||
def test_finish_reason_tool_calls_when_split_across_chunks(self):
|
||||
"""Tool calls in an earlier chunk must still end the stream with 'tool_calls'.
|
||||
|
||||
Ollama commonly emits tool_calls in a chunk with done false and then a
|
||||
separate done chunk whose message is empty and whose done_reason is
|
||||
'stop'. Keying the override off the done chunk's own tool_calls left the
|
||||
stream finishing as 'stop', so spec-strict OpenAI clients dropped the
|
||||
accumulated tool_calls delta and never ran the tool.
|
||||
"""
|
||||
iterator = OllamaChatCompletionResponseIterator(
|
||||
streaming_response=iter([]),
|
||||
sync_stream=True,
|
||||
)
|
||||
|
||||
tool_call_chunk = {
|
||||
"model": "glm-4.7-flash",
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"tool_calls": [
|
||||
{
|
||||
"function": {
|
||||
"name": "list_files",
|
||||
"arguments": {"path": "."},
|
||||
}
|
||||
}
|
||||
],
|
||||
},
|
||||
"done": False,
|
||||
}
|
||||
done_chunk = {
|
||||
"model": "glm-4.7-flash",
|
||||
"message": {"role": "assistant", "content": ""},
|
||||
"done": True,
|
||||
"done_reason": "stop",
|
||||
}
|
||||
|
||||
tool_call_result = iterator.chunk_parser(tool_call_chunk)
|
||||
assert tool_call_result.choices[0].delta.tool_calls is not None
|
||||
assert tool_call_result.choices[0].finish_reason is None
|
||||
|
||||
done_result = iterator.chunk_parser(done_chunk)
|
||||
assert (
|
||||
done_result.choices[0].finish_reason == "tool_calls"
|
||||
), f"Expected 'tool_calls' after a tool call earlier in the stream, got '{done_result.choices[0].finish_reason}'"
|
||||
|
||||
def test_finish_reason_stop_streaming(self):
|
||||
"""Streaming: done_reason='stop' in final chunk must produce finish_reason='stop'."""
|
||||
iterator = OllamaChatCompletionResponseIterator(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue