From e29502660a208044d2db53cba12b7d54923eb003 Mon Sep 17 00:00:00 2001 From: Jared Everett Date: Wed, 15 Apr 2026 18:17:39 -0600 Subject: [PATCH] fix(ollama): propagate done_reason='length' as finish_reason for max_tokens truncation Ollama returns done_reason='length' when a response is cut off by num_predict (the max_tokens limit). Previously, non-streaming responses hardcoded finish_reason='stop', and streaming used chunk.get('done_reason', 'stop') which also defaulted to 'stop' when done_reason was absent. This meant callers (e.g. the Anthropic pass-through adapter, which maps OpenAI 'length' -> Anthropic 'max_tokens') could never detect truncation, making stop_reason always appear as 'end_turn' even for cut-off responses. Fix: read done_reason from the response JSON in the non-streaming path and use `chunk.get('done_reason') or 'stop'` in the streaming path, so Ollama's actual done_reason passes through to the caller unchanged. Co-Authored-By: Claude Sonnet 4.6 --- litellm/llms/ollama/chat/transformation.py | 5 +- .../ollama/test_ollama_chat_transformation.py | 144 ++++++++++++++++++ 2 files changed, 147 insertions(+), 2 deletions(-) diff --git a/litellm/llms/ollama/chat/transformation.py b/litellm/llms/ollama/chat/transformation.py index 3d9618dfed0..518c3ee9bb6 100644 --- a/litellm/llms/ollama/chat/transformation.py +++ b/litellm/llms/ollama/chat/transformation.py @@ -349,7 +349,8 @@ class OllamaChatConfig(BaseConfig): response_json = raw_response.json() ## RESPONSE OBJECT - model_response.choices[0].finish_reason = "stop" + _done_reason = response_json.get("done_reason", "stop") + model_response.choices[0].finish_reason = _done_reason response_json_message = response_json.get("message") if response_json_message is not None: if "thinking" in response_json_message: @@ -535,7 +536,7 @@ class OllamaChatCompletionResponseIterator(BaseModelResponseIterator): ) if chunk["done"] is True: - finish_reason = chunk.get("done_reason", "stop") + finish_reason = chunk.get("done_reason") or "stop" # Override finish_reason when tool_calls are present # Fixes: https://github.com/BerriAI/litellm/issues/18922 if tool_calls is not None: diff --git a/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py b/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py index 02495106a84..84bfe7b56b9 100644 --- a/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py +++ b/tests/test_litellm/llms/ollama/test_ollama_chat_transformation.py @@ -476,6 +476,150 @@ class TestOllamaToolCalling: assert result.choices[0].message.tool_calls is None +class TestOllamaFinishReasonLength: + """Tests for done_reason 'length' → finish_reason 'length' mapping. + + Ollama returns done_reason='length' when a response is truncated by num_predict + (max_tokens). Previously finish_reason was hardcoded to 'stop', hiding truncation. + The Anthropic pass-through adapter then maps OpenAI 'length' → 'max_tokens'. + """ + + def test_finish_reason_length_non_streaming(self): + """Non-streaming: done_reason='length' must propagate as finish_reason='length'.""" + import json + from unittest.mock import MagicMock + + from litellm.types.utils import Choices, Message, ModelResponse + + config = OllamaChatConfig() + + ollama_response = { + "model": "qwen3:2b", + "created_at": "2025-01-11T00:00:00.000000Z", + "message": { + "role": "assistant", + "content": "A neural network learns through", + }, + "done": True, + "done_reason": "length", + "prompt_eval_count": 20, + "eval_count": 20, + } + + mock_response = MagicMock() + mock_response.json.return_value = ollama_response + mock_response.text = json.dumps(ollama_response) + + mock_logging = MagicMock() + + model_response = ModelResponse() + model_response.choices = [Choices(message=Message(content=""), index=0)] + + result = config.transform_response( + model="qwen3:2b", + raw_response=mock_response, + model_response=model_response, + logging_obj=mock_logging, + request_data={}, + messages=[{"role": "user", "content": "Explain neural networks."}], + optional_params={}, + litellm_params={}, + encoding=None, + api_key=None, + json_mode=False, + ) + + assert result.choices[0].finish_reason == "length", ( + f"Expected 'length' when done_reason='length', got '{result.choices[0].finish_reason}'" + ) + + def test_finish_reason_stop_non_streaming(self): + """Non-streaming: done_reason='stop' (natural finish) must stay 'stop'.""" + import json + from unittest.mock import MagicMock + + from litellm.types.utils import Choices, Message, ModelResponse + + config = OllamaChatConfig() + + ollama_response = { + "model": "qwen3:2b", + "created_at": "2025-01-11T00:00:00.000000Z", + "message": {"role": "assistant", "content": "2 + 2 = 4."}, + "done": True, + "done_reason": "stop", + "prompt_eval_count": 10, + "eval_count": 8, + } + + mock_response = MagicMock() + mock_response.json.return_value = ollama_response + mock_response.text = json.dumps(ollama_response) + + mock_logging = MagicMock() + + model_response = ModelResponse() + model_response.choices = [Choices(message=Message(content=""), index=0)] + + result = config.transform_response( + model="qwen3:2b", + raw_response=mock_response, + model_response=model_response, + logging_obj=mock_logging, + request_data={}, + messages=[{"role": "user", "content": "What is 2+2?"}], + optional_params={}, + litellm_params={}, + encoding=None, + api_key=None, + json_mode=False, + ) + + assert result.choices[0].finish_reason == "stop", ( + f"Expected 'stop' for natural finish, got '{result.choices[0].finish_reason}'" + ) + + def test_finish_reason_length_streaming(self): + """Streaming: done_reason='length' in final chunk must produce finish_reason='length'.""" + iterator = OllamaChatCompletionResponseIterator( + streaming_response=iter([]), + sync_stream=True, + ) + + done_chunk = { + "model": "qwen3:2b", + "message": {"role": "assistant", "content": "A neural network learns through"}, + "done": True, + "done_reason": "length", + } + + result = iterator.chunk_parser(done_chunk) + + assert result.choices[0].finish_reason == "length", ( + f"Expected 'length' when done_reason='length', got '{result.choices[0].finish_reason}'" + ) + + def test_finish_reason_stop_streaming(self): + """Streaming: done_reason='stop' in final chunk must produce finish_reason='stop'.""" + iterator = OllamaChatCompletionResponseIterator( + streaming_response=iter([]), + sync_stream=True, + ) + + done_chunk = { + "model": "qwen3:2b", + "message": {"role": "assistant", "content": "2 + 2 = 4."}, + "done": True, + "done_reason": "stop", + } + + result = iterator.chunk_parser(done_chunk) + + assert result.choices[0].finish_reason == "stop", ( + f"Expected 'stop' for natural finish, got '{result.choices[0].finish_reason}'" + ) + + class TestOllamaReasoningContentStreaming: """Test that reasoning_content is properly extracted from all thinking chunks."""