mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-03 02:22:24 +00:00
fix(ollama): propagate done_reason='length' as finish_reason for max_tokens truncation
Ollama returns done_reason='length' when a response is cut off by num_predict
(the max_tokens limit). Previously, non-streaming responses hardcoded
finish_reason='stop', and streaming used chunk.get('done_reason', 'stop')
which also defaulted to 'stop' when done_reason was absent.
This meant callers (e.g. the Anthropic pass-through adapter, which maps
OpenAI 'length' -> Anthropic 'max_tokens') could never detect truncation,
making stop_reason always appear as 'end_turn' even for cut-off responses.
Fix: read done_reason from the response JSON in the non-streaming path and
use `chunk.get('done_reason') or 'stop'` in the streaming path, so Ollama's
actual done_reason passes through to the caller unchanged.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
72a461ba4a
commit
e29502660a
2 changed files with 147 additions and 2 deletions
|
|
@ -349,7 +349,8 @@ class OllamaChatConfig(BaseConfig):
|
|||
response_json = raw_response.json()
|
||||
|
||||
## RESPONSE OBJECT
|
||||
model_response.choices[0].finish_reason = "stop"
|
||||
_done_reason = response_json.get("done_reason", "stop")
|
||||
model_response.choices[0].finish_reason = _done_reason
|
||||
response_json_message = response_json.get("message")
|
||||
if response_json_message is not None:
|
||||
if "thinking" in response_json_message:
|
||||
|
|
@ -535,7 +536,7 @@ class OllamaChatCompletionResponseIterator(BaseModelResponseIterator):
|
|||
)
|
||||
|
||||
if chunk["done"] is True:
|
||||
finish_reason = chunk.get("done_reason", "stop")
|
||||
finish_reason = chunk.get("done_reason") or "stop"
|
||||
# Override finish_reason when tool_calls are present
|
||||
# Fixes: https://github.com/BerriAI/litellm/issues/18922
|
||||
if tool_calls is not None:
|
||||
|
|
|
|||
|
|
@ -476,6 +476,150 @@ class TestOllamaToolCalling:
|
|||
assert result.choices[0].message.tool_calls is None
|
||||
|
||||
|
||||
class TestOllamaFinishReasonLength:
|
||||
"""Tests for done_reason 'length' → finish_reason 'length' mapping.
|
||||
|
||||
Ollama returns done_reason='length' when a response is truncated by num_predict
|
||||
(max_tokens). Previously finish_reason was hardcoded to 'stop', hiding truncation.
|
||||
The Anthropic pass-through adapter then maps OpenAI 'length' → 'max_tokens'.
|
||||
"""
|
||||
|
||||
def test_finish_reason_length_non_streaming(self):
|
||||
"""Non-streaming: done_reason='length' must propagate as finish_reason='length'."""
|
||||
import json
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from litellm.types.utils import Choices, Message, ModelResponse
|
||||
|
||||
config = OllamaChatConfig()
|
||||
|
||||
ollama_response = {
|
||||
"model": "qwen3:2b",
|
||||
"created_at": "2025-01-11T00:00:00.000000Z",
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": "A neural network learns through",
|
||||
},
|
||||
"done": True,
|
||||
"done_reason": "length",
|
||||
"prompt_eval_count": 20,
|
||||
"eval_count": 20,
|
||||
}
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.json.return_value = ollama_response
|
||||
mock_response.text = json.dumps(ollama_response)
|
||||
|
||||
mock_logging = MagicMock()
|
||||
|
||||
model_response = ModelResponse()
|
||||
model_response.choices = [Choices(message=Message(content=""), index=0)]
|
||||
|
||||
result = config.transform_response(
|
||||
model="qwen3:2b",
|
||||
raw_response=mock_response,
|
||||
model_response=model_response,
|
||||
logging_obj=mock_logging,
|
||||
request_data={},
|
||||
messages=[{"role": "user", "content": "Explain neural networks."}],
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
encoding=None,
|
||||
api_key=None,
|
||||
json_mode=False,
|
||||
)
|
||||
|
||||
assert result.choices[0].finish_reason == "length", (
|
||||
f"Expected 'length' when done_reason='length', got '{result.choices[0].finish_reason}'"
|
||||
)
|
||||
|
||||
def test_finish_reason_stop_non_streaming(self):
|
||||
"""Non-streaming: done_reason='stop' (natural finish) must stay 'stop'."""
|
||||
import json
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from litellm.types.utils import Choices, Message, ModelResponse
|
||||
|
||||
config = OllamaChatConfig()
|
||||
|
||||
ollama_response = {
|
||||
"model": "qwen3:2b",
|
||||
"created_at": "2025-01-11T00:00:00.000000Z",
|
||||
"message": {"role": "assistant", "content": "2 + 2 = 4."},
|
||||
"done": True,
|
||||
"done_reason": "stop",
|
||||
"prompt_eval_count": 10,
|
||||
"eval_count": 8,
|
||||
}
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.json.return_value = ollama_response
|
||||
mock_response.text = json.dumps(ollama_response)
|
||||
|
||||
mock_logging = MagicMock()
|
||||
|
||||
model_response = ModelResponse()
|
||||
model_response.choices = [Choices(message=Message(content=""), index=0)]
|
||||
|
||||
result = config.transform_response(
|
||||
model="qwen3:2b",
|
||||
raw_response=mock_response,
|
||||
model_response=model_response,
|
||||
logging_obj=mock_logging,
|
||||
request_data={},
|
||||
messages=[{"role": "user", "content": "What is 2+2?"}],
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
encoding=None,
|
||||
api_key=None,
|
||||
json_mode=False,
|
||||
)
|
||||
|
||||
assert result.choices[0].finish_reason == "stop", (
|
||||
f"Expected 'stop' for natural finish, got '{result.choices[0].finish_reason}'"
|
||||
)
|
||||
|
||||
def test_finish_reason_length_streaming(self):
|
||||
"""Streaming: done_reason='length' in final chunk must produce finish_reason='length'."""
|
||||
iterator = OllamaChatCompletionResponseIterator(
|
||||
streaming_response=iter([]),
|
||||
sync_stream=True,
|
||||
)
|
||||
|
||||
done_chunk = {
|
||||
"model": "qwen3:2b",
|
||||
"message": {"role": "assistant", "content": "A neural network learns through"},
|
||||
"done": True,
|
||||
"done_reason": "length",
|
||||
}
|
||||
|
||||
result = iterator.chunk_parser(done_chunk)
|
||||
|
||||
assert result.choices[0].finish_reason == "length", (
|
||||
f"Expected 'length' when done_reason='length', got '{result.choices[0].finish_reason}'"
|
||||
)
|
||||
|
||||
def test_finish_reason_stop_streaming(self):
|
||||
"""Streaming: done_reason='stop' in final chunk must produce finish_reason='stop'."""
|
||||
iterator = OllamaChatCompletionResponseIterator(
|
||||
streaming_response=iter([]),
|
||||
sync_stream=True,
|
||||
)
|
||||
|
||||
done_chunk = {
|
||||
"model": "qwen3:2b",
|
||||
"message": {"role": "assistant", "content": "2 + 2 = 4."},
|
||||
"done": True,
|
||||
"done_reason": "stop",
|
||||
}
|
||||
|
||||
result = iterator.chunk_parser(done_chunk)
|
||||
|
||||
assert result.choices[0].finish_reason == "stop", (
|
||||
f"Expected 'stop' for natural finish, got '{result.choices[0].finish_reason}'"
|
||||
)
|
||||
|
||||
|
||||
class TestOllamaReasoningContentStreaming:
|
||||
"""Test that reasoning_content is properly extracted from all thinking chunks."""
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue